Home
last modified time | relevance | path

Searched full:xdp (Results 1 – 25 of 398) sorted by relevance

12345678910>>...16

/linux-6.8/include/net/
Dxdp.h2 /* include/net/xdp.h
15 * DOC: XDP RX-queue information
17 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
22 * reference to this xdp_rxq_info structure. This provides the XDP
32 * The struct is not directly tied to the XDP prog. A new XDP prog
43 MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
49 /* XDP flags for ndo_xdp_xmit */
74 XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
75 XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp paged memory is under
91 static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp) in xdp_buff_has_frags() argument
[all …]
Dxdp_sock_drv.h65 return pool->heads[0].xdp.rxq->napi_id; in xsk_pool_get_napi_id()
85 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_dma() argument
87 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_dma()
92 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_frame_dma() argument
94 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_frame_dma()
110 static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) in xsk_buff_alloc_batch() argument
112 return xp_alloc_batch(pool, xdp, max); in xsk_buff_alloc_batch()
120 static inline void xsk_buff_free(struct xdp_buff *xdp) in xsk_buff_free() argument
122 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_free()
126 if (likely(!xdp_buff_has_frags(xdp))) in xsk_buff_free()
[all …]
/linux-6.8/Documentation/bpf/
Dredirect.rst25 :doc: xdp redirect
29 those that do, not all of them support non-linear frames. Non-linear xdp
41 The following bpftrace command can be used to capture and count all XDP tracepoints:
45 sudo bpftrace -e 'tracepoint:xdp:* { @cnt[probe] = count(); }'
49 @cnt[tracepoint:xdp:mem_connect]: 18
50 @cnt[tracepoint:xdp:mem_disconnect]: 18
51 @cnt[tracepoint:xdp:xdp_exception]: 19605
52 @cnt[tracepoint:xdp:xdp_devmap_xmit]: 1393604
53 @cnt[tracepoint:xdp:xdp_redirect]: 22292200
56 The various xdp tracepoints can be found in ``source/include/trace/events/xdp.h``
[all …]
/linux-6.8/tools/testing/selftests/bpf/progs/
Dxdp_features.c65 xdp_process_echo_packet(struct xdp_md *xdp, bool dut) in xdp_process_echo_packet() argument
67 void *data_end = (void *)(long)xdp->data_end; in xdp_process_echo_packet()
68 void *data = (void *)(long)xdp->data; in xdp_process_echo_packet()
135 xdp_update_stats(struct xdp_md *xdp, bool tx, bool dut) in xdp_update_stats() argument
139 if (xdp_process_echo_packet(xdp, tx)) in xdp_update_stats()
155 SEC("xdp")
156 int xdp_tester_check_tx(struct xdp_md *xdp) in xdp_tester_check_tx() argument
158 xdp_update_stats(xdp, true, false); in xdp_tester_check_tx()
163 SEC("xdp")
164 int xdp_tester_check_rx(struct xdp_md *xdp) in xdp_tester_check_rx() argument
[all …]
Dverifier_xdp_direct_packet_access.c8 SEC("xdp")
9 __description("XDP pkt read, pkt_end mangling, bad access 1")
29 SEC("xdp")
30 __description("XDP pkt read, pkt_end mangling, bad access 2")
50 SEC("xdp")
51 __description("XDP pkt read, pkt_data' > pkt_end, corner case, good access")
70 SEC("xdp")
71 __description("XDP pkt read, pkt_data' > pkt_end, bad access 1")
91 SEC("xdp")
92 __description("XDP pkt read, pkt_data' > pkt_end, bad access 2")
[all …]
Dxsk_xdp_progs.c19 SEC("xdp.frags") int xsk_def_prog(struct xdp_md *xdp) in xsk_def_prog() argument
24 SEC("xdp.frags") int xsk_xdp_drop(struct xdp_md *xdp) in xsk_xdp_drop() argument
33 SEC("xdp.frags") int xsk_xdp_populate_metadata(struct xdp_md *xdp) in xsk_xdp_populate_metadata() argument
40 err = bpf_xdp_adjust_meta(xdp, -(int)sizeof(struct xdp_info)); in xsk_xdp_populate_metadata()
44 data = (void *)(long)xdp->data; in xsk_xdp_populate_metadata()
45 data_meta = (void *)(long)xdp->data_meta; in xsk_xdp_populate_metadata()
56 SEC("xdp") int xsk_xdp_shared_umem(struct xdp_md *xdp) in xsk_xdp_shared_umem() argument
58 void *data = (void *)(long)xdp->data; in xsk_xdp_shared_umem()
59 void *data_end = (void *)(long)xdp->data_end; in xsk_xdp_shared_umem()
Dtest_xdp_loop.c74 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
76 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
77 void *data = (void *)(long)xdp->data; in handle_ipv4()
107 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
110 data = (void *)(long)xdp->data; in handle_ipv4()
111 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
147 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
149 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
150 void *data = (void *)(long)xdp->data; in handle_ipv6()
177 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
Dtest_xdp.c78 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
80 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
81 void *data = (void *)(long)xdp->data; in handle_ipv4()
111 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
114 data = (void *)(long)xdp->data; in handle_ipv4()
115 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
151 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
153 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
154 void *data = (void *)(long)xdp->data; in handle_ipv6()
181 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
Dtest_xdp_do_redirect.c11 * @MARK_IN: frame is being processed by the input XDP prog.
28 SEC("xdp")
29 int xdp_redirect(struct xdp_md *xdp) in xdp_redirect() argument
31 __u32 *metadata = (void *)(long)xdp->data_meta; in xdp_redirect()
32 void *data_end = (void *)(long)xdp->data_end; in xdp_redirect()
33 void *data = (void *)(long)xdp->data; in xdp_redirect()
41 if (xdp->ingress_ifindex != (__u32)ifindex_in) in xdp_redirect()
55 if (bpf_xdp_adjust_meta(xdp, sizeof(__u64))) in xdp_redirect()
85 SEC("xdp")
86 int xdp_count_pkts(struct xdp_md *xdp) in xdp_count_pkts() argument
[all …]
/linux-6.8/drivers/net/ethernet/broadcom/bnxt/
Dbnxt_xdp.c28 struct xdp_buff *xdp) in bnxt_xmit_bd() argument
38 if (xdp && xdp_buff_has_frags(xdp)) { in bnxt_xmit_bd()
39 sinfo = xdp_get_shared_info_from_buff(xdp); in bnxt_xmit_bd()
47 if (xdp) in bnxt_xmit_bd()
48 tx_buf->page = virt_to_head_page(xdp->data); in bnxt_xmit_bd()
97 struct xdp_buff *xdp) in __bnxt_xmit_xdp() argument
101 tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp); in __bnxt_xmit_xdp()
184 struct xdp_buff *xdp) in bnxt_xdp_buff_init() argument
199 xdp_init_buff(xdp, buflen, &rxr->xdp_rxq); in bnxt_xdp_buff_init()
200 xdp_prepare_buff(xdp, data_ptr - offset, offset, len, false); in bnxt_xdp_buff_init()
[all …]
/linux-6.8/Documentation/networking/
Dxdp-rx-metadata.rst4 XDP RX Metadata
7 This document describes how an eXpress Data Path (XDP) program can access
14 XDP has access to a set of kfuncs to manipulate the metadata in an XDP frame.
16 implement these kfuncs. The set of kfuncs is declared in ``include/net/xdp.h``
22 .. kernel-doc:: net/core/xdp.c
25 .. kernel-doc:: net/core/xdp.c
28 .. kernel-doc:: net/core/xdp.c
31 An XDP program can use these kfuncs to read the metadata into stack
33 consumers, an XDP program can store it into the metadata area carried
42 Within an XDP frame, the metadata layout (accessed via ``xdp_buff``) is
[all …]
/linux-6.8/drivers/net/ethernet/intel/ice/
Dice_xsk.c6 #include <net/xdp.h>
355 * ice_realloc_zc_buf - reallocate XDP ZC queue pairs
360 * XDP requires more memory, than rx_buf provides.
438 * @xdp: SW ring of xdp_buff that will hold the buffers
449 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, in ice_fill_rx_descs() argument
456 buffs = xsk_buff_alloc_batch(pool, xdp, count); in ice_fill_rx_descs()
458 dma = xsk_buff_xdp_get_dma(*xdp); in ice_fill_rx_descs()
465 ice_xdp_meta_set_desc(*xdp, rx_desc); in ice_fill_rx_descs()
468 xdp++; in ice_fill_rx_descs()
491 struct xdp_buff **xdp; in __ice_alloc_rx_bufs_zc() local
[all …]
Dice_txrx_lib.h10 * @xdp: XDP buffer representing frame (linear and frags part)
12 * act: action to store onto Rx buffers related to XDP buffer parts
18 ice_set_rx_bufs_act(struct xdp_buff *xdp, const struct ice_rx_ring *rx_ring, in ice_set_rx_bufs_act() argument
21 u32 sinfo_frags = xdp_get_shared_info_from_buff(xdp)->nr_frags; in ice_set_rx_bufs_act()
35 /* adjust pagecnt_bias on frags freed by XDP prog */ in ice_set_rx_bufs_act()
124 * ice_xdp_ring_update_tail - Updates the XDP Tx ring tail register
125 * @xdp_ring: XDP Tx ring
127 * This function updates the XDP Tx ring tail register.
140 * @xdp_ring: XDP ring to produce the HW Tx descriptors on
157 int ice_xmit_xdp_buff(struct xdp_buff *xdp, struct ice_tx_ring *xdp_ring);
[all …]
Dice_txrx_lib.c263 * ice_clean_xdp_tx_buf - Free and unmap XDP Tx buffer
266 * @bq: XDP bulk flush struct
289 * ice_clean_xdp_irq - Reclaim resources after transmit completes on XDP ring
290 * @xdp_ring: XDP ring to clean
362 * __ice_xmit_xdp_ring - submit frame to XDP ring for transmission
363 * @xdp: XDP buffer to be placed onto Tx descriptors
364 * @xdp_ring: XDP ring for transmission
367 int __ice_xmit_xdp_ring(struct xdp_buff *xdp, struct ice_tx_ring *xdp_ring, in __ice_xmit_xdp_ring() argument
371 u32 size = xdp->data_end - xdp->data; in __ice_xmit_xdp_ring()
378 void *data = xdp->data; in __ice_xmit_xdp_ring()
[all …]
/linux-6.8/drivers/net/vmxnet3/
Dvmxnet3_xdp.c48 NL_SET_ERR_MSG_FMT_MOD(extack, "MTU %u too large for XDP", in vmxnet3_xdp_set()
54 NL_SET_ERR_MSG_MOD(extack, "LRO is not supported with XDP"); in vmxnet3_xdp_set()
86 "failed to re-create rx queues for XDP."); in vmxnet3_xdp_set()
92 "failed to activate device for XDP."); in vmxnet3_xdp_set()
100 /* This is the main xdp call used by kernel to set/unset eBPF program. */
148 } else { /* XDP buffer from page pool */ in vmxnet3_xdp_xmit_frame()
251 vmxnet3_run_xdp(struct vmxnet3_rx_queue *rq, struct xdp_buff *xdp, in vmxnet3_run_xdp() argument
260 act = bpf_prog_run_xdp(prog, xdp); in vmxnet3_run_xdp()
261 page = virt_to_page(xdp->data_hard_start); in vmxnet3_run_xdp()
267 err = xdp_do_redirect(rq->adapter->netdev, xdp, prog); in vmxnet3_run_xdp()
[all …]
/linux-6.8/drivers/net/ethernet/intel/ixgbe/
Dixgbe_xsk.c6 #include <net/xdp.h>
99 struct xdp_buff *xdp) in ixgbe_run_xdp_zc() argument
108 act = bpf_prog_run_xdp(xdp_prog, xdp); in ixgbe_run_xdp_zc()
111 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ixgbe_run_xdp_zc()
125 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc()
168 bi->xdp = xsk_buff_alloc(rx_ring->xsk_pool); in ixgbe_alloc_rx_buffers_zc()
169 if (!bi->xdp) { in ixgbe_alloc_rx_buffers_zc()
174 dma = xsk_buff_xdp_get_dma(bi->xdp); in ixgbe_alloc_rx_buffers_zc()
214 const struct xdp_buff *xdp) in ixgbe_construct_skb_zc() argument
216 unsigned int totalsize = xdp->data_end - xdp->data_meta; in ixgbe_construct_skb_zc()
[all …]
/linux-6.8/drivers/net/hyperv/
Dnetvsc_bpf.c17 #include <net/xdp.h>
25 struct xdp_buff *xdp) in netvsc_run_xdp() argument
35 xdp->data_hard_start = NULL; in netvsc_run_xdp()
56 xdp_init_buff(xdp, PAGE_SIZE, &nvchan->xdp_rxq); in netvsc_run_xdp()
57 xdp_prepare_buff(xdp, page_address(page), NETVSC_XDP_HDRM, len, false); in netvsc_run_xdp()
59 memcpy(xdp->data, data, len); in netvsc_run_xdp()
61 act = bpf_prog_run_xdp(prog, xdp); in netvsc_run_xdp()
73 if (!xdp_do_redirect(ndev, xdp, prog)) { in netvsc_run_xdp()
107 xdp->data_hard_start = NULL; in netvsc_run_xdp()
138 netdev_err(dev, "XDP: mtu:%u too large, buf_max:%u\n", in netvsc_xdp_set()
[all …]
/linux-6.8/include/trace/events/
Dxdp.h3 #define TRACE_SYSTEM xdp
12 #include <net/xdp.h>
32 const struct bpf_prog *xdp, u32 act),
34 TP_ARGS(dev, xdp, act),
43 __entry->prog_id = xdp->aux->id;
93 const struct bpf_prog *xdp,
98 TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index),
124 __entry->prog_id = xdp->aux->id;
143 const struct bpf_prog *xdp,
147 TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index)
[all …]
/linux-6.8/tools/bpf/bpftool/Documentation/
Dbpftool-net.rst33 | *ATTACH_TYPE* := { **xdp** | **xdpgeneric** | **xdpdrv** | **xdpoffload** }
40 Currently, device driver xdp attachments, tcx, netkit and old-style tc
54 The current output will start with all xdp program attachments, followed by
56 and finally netfilter programs. Both xdp programs and tcx/netkit/tc programs are
67 Currently, only XDP-related modes are supported for *ATTACH_TYPE*.
70 **xdp** - try native XDP and fallback to generic XDP if NIC driver does not support it;
71 …**xdpgeneric** - Generic XDP. runs at generic XDP hook when packet already enters receive path as …
72 **xdpdrv** - Native XDP. runs earliest point in driver's receive path;
73 **xdpoffload** - Offload XDP. runs directly on NIC on each packet reception;
79 Currently, only XDP-related modes are supported for *ATTACH_TYPE*.
[all …]
/linux-6.8/drivers/net/ethernet/microchip/lan966x/
Dlan966x_xdp.c9 static int lan966x_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp) in lan966x_xdp_setup() argument
18 NL_SET_ERR_MSG_MOD(xdp->extack, in lan966x_xdp_setup()
19 "Allow to set xdp only when using fdma"); in lan966x_xdp_setup()
24 old_prog = xchg(&port->xdp_prog, xdp->prog); in lan966x_xdp_setup()
43 int lan966x_xdp(struct net_device *dev, struct netdev_bpf *xdp) in lan966x_xdp() argument
45 switch (xdp->command) { in lan966x_xdp()
47 return lan966x_xdp_setup(dev, xdp); in lan966x_xdp()
79 struct xdp_buff xdp; in lan966x_xdp_run() local
82 xdp_init_buff(&xdp, PAGE_SIZE << lan966x->rx.page_order, in lan966x_xdp_run()
84 xdp_prepare_buff(&xdp, page_address(page), in lan966x_xdp_run()
[all …]
/linux-6.8/drivers/net/ethernet/intel/i40e/
Di40e_xsk.c56 * XDP requires more memory, than rx_buf provides.
190 * i40e_run_xdp_zc - Executes an XDP program on an xdp_buff
192 * @xdp: xdp_buff used as input to the XDP program
193 * @xdp_prog: XDP program to run
197 static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp, in i40e_run_xdp_zc() argument
204 act = bpf_prog_run_xdp(xdp_prog, xdp); in i40e_run_xdp_zc()
207 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in i40e_run_xdp_zc()
222 result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring); in i40e_run_xdp_zc()
244 struct xdp_buff **xdp; in i40e_alloc_rx_buffers_zc() local
249 xdp = i40e_rx_bi(rx_ring, ntu); in i40e_alloc_rx_buffers_zc()
[all …]
/linux-6.8/tools/testing/selftests/bpf/
Dtest_xdp_veth.sh5 # forward packets in-between using native XDP
64 if ! ip link set dev lo xdp off > /dev/null 2>&1; then
65 echo "selftests: $TESTNAME [SKIP] Could not run test without the ip xdp support"
104 xdp_redirect_map.bpf.o $BPF_DIR/progs type xdp \
109 ip link set dev veth1 xdp pinned $BPF_DIR/progs/xdp_redirect_map_0
110 ip link set dev veth2 xdp pinned $BPF_DIR/progs/xdp_redirect_map_1
111 ip link set dev veth3 xdp pinned $BPF_DIR/progs/xdp_redirect_map_2
113 ip -n ${NS1} link set dev veth11 xdp obj xdp_dummy.bpf.o sec xdp
114 ip -n ${NS2} link set dev veth22 xdp obj xdp_tx.bpf.o sec xdp
115 ip -n ${NS3} link set dev veth33 xdp obj xdp_dummy.bpf.o sec xdp
/linux-6.8/samples/bpf/
Dxdp_tx_iptunnel_kern.c77 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
79 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
80 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
184 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
Dxdp_adjust_tail_kern.c70 static __always_inline int send_icmp4_too_big(struct xdp_md *xdp) in send_icmp4_too_big() argument
74 if (bpf_xdp_adjust_head(xdp, 0 - headroom)) in send_icmp4_too_big()
76 void *data = (void *)(long)xdp->data; in send_icmp4_too_big()
77 void *data_end = (void *)(long)xdp->data_end; in send_icmp4_too_big()
120 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
122 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
123 void *data = (void *)(long)xdp->data; in handle_ipv4()
129 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in handle_ipv4()
131 return send_icmp4_too_big(xdp); in handle_ipv4()
137 int _xdp_icmp(struct xdp_md *xdp) in _xdp_icmp() argument
[all …]
/linux-6.8/drivers/net/ethernet/netronome/nfp/nfd3/
Dxsk.c35 txbuf->xdp = xrxbuf->xdp; in nfp_nfd3_xsk_tx_xdp()
87 skb_put_data(skb, xrxbuf->xdp->data, pkt_len); in nfp_nfd3_xsk_rx_skb()
105 xrxbuf->xdp->data - xrxbuf->xdp->data_meta); in nfp_nfd3_xsk_rx_skb()
184 xrxbuf->xdp->data += meta_len; in nfp_nfd3_xsk_rx()
185 xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len; in nfp_nfd3_xsk_rx()
186 xdp_set_data_meta_invalid(xrxbuf->xdp); in nfp_nfd3_xsk_rx()
187 xsk_buff_dma_sync_for_cpu(xrxbuf->xdp, r_vec->xsk_pool); in nfp_nfd3_xsk_rx()
188 net_prefetch(xrxbuf->xdp->data); in nfp_nfd3_xsk_rx()
192 xrxbuf->xdp->data - in nfp_nfd3_xsk_rx()
194 xrxbuf->xdp->data, in nfp_nfd3_xsk_rx()
[all …]

12345678910>>...16