Lines Matching +full:rx +full:- +full:tx +full:- +full:swap
1 // SPDX-License-Identifier: GPL-2.0
6 * RX:
7 * - UDP 9091 packets are diverted into AF_XDP
8 * - Metadata verified:
9 * - rx_timestamp
10 * - rx_hash
12 * TX:
13 * - UDP 9091 packets trigger TX reply
14 * - TX HW timestamp is requested and reported back upon completion
15 * - TX checksum is requested
16 * - TX launch time HW offload is requested for transmission
63 struct xsk_ring_prod tx; member
64 struct xsk_ring_cons rx; member
112 xsk->umem_area = mmap(NULL, UMEM_SIZE, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); in open_xsk()
113 if (xsk->umem_area == MAP_FAILED) in open_xsk()
114 return -ENOMEM; in open_xsk()
116 ret = xsk_umem__create(&xsk->umem, in open_xsk()
117 xsk->umem_area, UMEM_SIZE, in open_xsk()
118 &xsk->fill, in open_xsk()
119 &xsk->comp, in open_xsk()
124 ret = xsk_socket__create(&xsk->socket, ifindex, queue_id, in open_xsk()
125 xsk->umem, in open_xsk()
126 &xsk->rx, in open_xsk()
127 &xsk->tx, in open_xsk()
132 /* First half of umem is for TX. This way address matches 1-to-1 in open_xsk()
138 printf("%p: tx_desc[%d] -> %lx\n", xsk, i, addr); in open_xsk()
141 /* Second half of umem is for RX. */ in open_xsk()
143 ret = xsk_ring_prod__reserve(&xsk->fill, UMEM_NUM / 2, &idx); in open_xsk()
146 printf("%p: rx_desc[%d] -> %lx\n", xsk, i, addr); in open_xsk()
147 *xsk_ring_prod__fill_addr(&xsk->fill, idx + i) = addr; in open_xsk()
149 xsk_ring_prod__submit(&xsk->fill, ret); in open_xsk()
156 if (xsk->umem) in close_xsk()
157 xsk_umem__delete(xsk->umem); in close_xsk()
158 if (xsk->socket) in close_xsk()
159 xsk_socket__delete(xsk->socket); in close_xsk()
160 munmap(xsk->umem_area, UMEM_SIZE); in close_xsk()
167 if (xsk_ring_prod__reserve(&xsk->fill, 1, &idx) == 1) { in refill_rx()
168 printf("%p: complete rx idx=%u addr=%llx\n", xsk, idx, addr); in refill_rx()
169 *xsk_ring_prod__fill_addr(&xsk->fill, idx) = addr; in refill_rx()
170 xsk_ring_prod__submit(&xsk->fill, 1); in refill_rx()
176 return sendto(xsk_socket__fd(xsk->socket), NULL, 0, MSG_DONTWAIT, NULL, 0); in kick_tx()
181 return recvfrom(xsk_socket__fd(xsk->socket), NULL, 0, MSG_DONTWAIT, NULL, NULL); in kick_rx()
202 __s64 delta = (__s64)reference - (__s64)tstamp; in print_tstamp_delta()
226 meta = data - sizeof(*meta); in verify_xdp_metadata()
228 if (meta->hint_valid & XDP_META_FIELD_RSS) in verify_xdp_metadata()
230 meta->rx_hash, meta->rx_hash_type); in verify_xdp_metadata()
232 printf("No rx_hash, err=%d\n", meta->rx_hash_err); in verify_xdp_metadata()
234 if (meta->hint_valid & XDP_META_FIELD_TS) { in verify_xdp_metadata()
237 /* store received timestamps to calculate a delta at tx */ in verify_xdp_metadata()
238 last_hw_rx_timestamp = meta->rx_timestamp; in verify_xdp_metadata()
239 last_xdp_rx_timestamp = meta->xdp_timestamp; in verify_xdp_metadata()
241 print_tstamp_delta("HW RX-time", "User RX-time", in verify_xdp_metadata()
242 meta->rx_timestamp, ref_tstamp); in verify_xdp_metadata()
243 print_tstamp_delta("XDP RX-time", "User RX-time", in verify_xdp_metadata()
244 meta->xdp_timestamp, ref_tstamp); in verify_xdp_metadata()
246 printf("No rx_timestamp, err=%d\n", meta->rx_timestamp_err); in verify_xdp_metadata()
249 if (meta->hint_valid & XDP_META_FIELD_VLAN_TAG) { in verify_xdp_metadata()
250 printf("rx_vlan_proto: 0x%X\n", ntohs(meta->rx_vlan_proto)); in verify_xdp_metadata()
252 print_vlan_tci(meta->rx_vlan_tci); in verify_xdp_metadata()
255 meta->rx_vlan_tag_err); in verify_xdp_metadata()
284 if (cmsg->cmsg_level != SOL_SOCKET) in verify_skb_metadata()
287 switch (cmsg->cmsg_type) { in verify_skb_metadata()
290 if (ts->ts[2].tv_sec || ts->ts[2].tv_nsec) { in verify_skb_metadata()
292 ts->ts[2].tv_sec, ts->ts[2].tv_nsec); in verify_skb_metadata()
311 if (!xsk_ring_cons__peek(&xsk->comp, 1, &idx)) in complete_tx()
314 addr = *xsk_ring_cons__comp_addr(&xsk->comp, idx); in complete_tx()
315 data = xsk_umem__get_data(xsk->umem_area, addr); in complete_tx()
316 meta = data - sizeof(struct xsk_tx_metadata); in complete_tx()
318 printf("%p: complete tx idx=%u addr=%llx\n", xsk, idx, addr); in complete_tx()
320 if (meta->completion.tx_timestamp) { in complete_tx()
324 print_tstamp_delta("HW Launch-time", in complete_tx()
325 "HW TX-complete-time", in complete_tx()
327 meta->completion.tx_timestamp); in complete_tx()
329 print_tstamp_delta("HW TX-complete-time", "User TX-complete-time", in complete_tx()
330 meta->completion.tx_timestamp, ref_tstamp); in complete_tx()
331 print_tstamp_delta("XDP RX-time", "User TX-complete-time", in complete_tx()
333 print_tstamp_delta("HW RX-time", "HW TX-complete-time", in complete_tx()
334 last_hw_rx_timestamp, meta->completion.tx_timestamp); in complete_tx()
339 xsk_ring_cons__release(&xsk->comp, 1); in complete_tx()
344 #define swap(a, b, len) do { \ macro
366 ret = xsk_ring_prod__reserve(&xsk->tx, 1, &idx); in ping_pong()
368 printf("%p: failed to reserve tx slot\n", xsk); in ping_pong()
372 tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx); in ping_pong()
373 tx_desc->addr = idx % (UMEM_NUM / 2) * UMEM_FRAME_SIZE + sizeof(struct xsk_tx_metadata); in ping_pong()
374 data = xsk_umem__get_data(xsk->umem_area, tx_desc->addr); in ping_pong()
376 meta = data - sizeof(struct xsk_tx_metadata); in ping_pong()
378 meta->flags = XDP_TXMD_FLAGS_TIMESTAMP; in ping_pong()
382 if (eth->h_proto == htons(ETH_P_IP)) { in ping_pong()
385 } else if (eth->h_proto == htons(ETH_P_IPV6)) { in ping_pong()
389 printf("%p: failed to detect IP version for ping pong %04x\n", xsk, eth->h_proto); in ping_pong()
390 xsk_ring_prod__cancel(&xsk->tx, 1); in ping_pong()
396 len += sizeof(*ip6h) + ntohs(ip6h->payload_len); in ping_pong()
398 len += ntohs(iph->tot_len); in ping_pong()
400 swap(eth->h_dest, eth->h_source, ETH_ALEN); in ping_pong()
402 swap(&iph->saddr, &iph->daddr, 4); in ping_pong()
404 swap(&ip6h->saddr, &ip6h->daddr, 16); in ping_pong()
405 swap(&udph->source, &udph->dest, 2); in ping_pong()
407 want_csum = udph->check; in ping_pong()
409 udph->check = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, in ping_pong()
410 ntohs(udph->len), IPPROTO_UDP, 0); in ping_pong()
412 udph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, in ping_pong()
413 ntohs(udph->len), IPPROTO_UDP, 0); in ping_pong()
415 meta->flags |= XDP_TXMD_FLAGS_CHECKSUM; in ping_pong()
417 meta->request.csum_start = sizeof(*eth) + sizeof(*iph); in ping_pong()
419 meta->request.csum_start = sizeof(*eth) + sizeof(*ip6h); in ping_pong()
420 meta->request.csum_offset = offsetof(struct udphdr, check); in ping_pong()
422 printf("%p: ping-pong with csum=%04x (want %04x) csum_start=%d csum_offset=%d\n", in ping_pong()
423 xsk, ntohs(udph->check), ntohs(want_csum), in ping_pong()
424 meta->request.csum_start, meta->request.csum_offset); in ping_pong()
428 meta->flags |= XDP_TXMD_FLAGS_LAUNCH_TIME; in ping_pong()
429 meta->request.launch_time = last_hw_rx_timestamp + in ping_pong()
431 last_launch_time = meta->request.launch_time; in ping_pong()
432 print_tstamp_delta("HW RX-time", "HW Launch-time", in ping_pong()
434 meta->request.launch_time); in ping_pong()
438 tx_desc->options |= XDP_TX_METADATA; in ping_pong()
439 tx_desc->len = len; in ping_pong()
441 xsk_ring_prod__submit(&xsk->tx, 1); in ping_pong()
476 ret, errno, bpf_obj->bss->pkts_skip, in verify_metadata()
477 bpf_obj->bss->pkts_fail, bpf_obj->bss->pkts_redir); in verify_metadata()
495 ret = xsk_ring_cons__peek(&xsk->rx, 1, &idx); in verify_metadata()
500 rx_desc = xsk_ring_cons__rx_desc(&xsk->rx, idx); in verify_metadata()
501 comp_addr = xsk_umem__extract_addr(rx_desc->addr); in verify_metadata()
502 addr = xsk_umem__add_offset_to_addr(rx_desc->addr); in verify_metadata()
503 is_eop = !(rx_desc->options & XDP_PKT_CONTD); in verify_metadata()
504 printf("%p: rx_desc[%u]->addr=%llx addr=%llx comp_addr=%llx%s\n", in verify_metadata()
505 xsk, idx, rx_desc->addr, addr, comp_addr, is_eop ? " EoP" : ""); in verify_metadata()
507 verify_xdp_metadata(xsk_umem__get_data(xsk->umem_area, addr), in verify_metadata()
513 ping_pong(xsk, xsk_umem__get_data(xsk->umem_area, addr), in verify_metadata()
534 xsk_ring_cons__release(&xsk->rx, 1); in verify_metadata()
553 strncpy(ifr.ifr_name, ifname, IF_NAMESIZE - 1); in rxq_num()
574 strncpy(ifr.ifr_name, ifname, IF_NAMESIZE - 1); in hwtstamp_ioctl()
617 opts.old_prog_fd = bpf_program__fd(bpf_obj->progs.rx); in cleanup()
653 " -c Run in copy mode (zerocopy is default)\n" in print_usage()
654 " -h Display this help and exit\n\n" in print_usage()
655 " -m Enable multi-buffer XDP for larger MTU\n" in print_usage()
656 " -r Don't generate AF_XDP reply (rx metadata only)\n" in print_usage()
657 " -l Delta of launch time relative to HW RX-time in ns\n" in print_usage()
659 " -L Tx Queue to be enabled with launch time offload\n" in print_usage()
660 " default: 0 (Tx Queue 0)\n" in print_usage()
662 " echo -n xdp | nc -u -q1 <dst_ip> 9091\n"; in print_usage()
671 while ((opt = getopt(argc, argv, "chmrl:L:")) != -1) { in read_args()
695 fprintf(stderr, "Unknown option: -%c\n", optopt); in read_args()
699 error(-1, opterr, "Command line options error"); in read_args()
706 exit(-1); in read_args()
713 error(-1, errno, "Invalid interface name"); in read_args()
719 if (run_command("sudo tc qdisc show dev %s | grep -q 'qdisc mqprio 8001:'", ifname) == 0) in clean_existing_configurations()
723 if (run_command("sudo tc qdisc show dev %s | grep -q 'qdisc ingress ffff:'", ifname) == 0) in clean_existing_configurations()
727 if (run_command("sudo ethtool -n %s | grep -q 'Filter:'", ifname) == 0) { in clean_existing_configurations()
728 …run_command("sudo ethtool -n %s | grep 'Filter:' | awk '{print $2}' | xargs -n1 sudo ethtool -N %s… in clean_existing_configurations()
739 int server_fd = -1; in main()
761 /* Enable tx and rx hardware timestamping */ in main()
764 /* Prepare priority to traffic class map for tc-mqprio */ in main()
769 if (asprintf(&buf, "%d ", tc) == -1) { in main()
786 /* Prepare traffic class to hardware queue map for tc-mqprio */ in main()
788 if (asprintf(&buf, "1@%d ", i) == -1) { in main()
817 if (run_command("sudo ethtool -N %s flow-type ether vlan 0x2000 vlan-mask 0x1FFF action %d", in main()
825 run_command("sudo ethtool -K %s rxvlan on", ifname); in main()
836 error(1, -ret, "open_xsk"); in main()
838 printf("xsk_socket__fd() -> %d\n", xsk_socket__fd(rx_xsk[i].socket)); in main()
846 prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); in main()
853 error(1, -ret, "xdp_hw_metadata__load"); in main()
869 ret = bpf_map_update_elem(bpf_map__fd(bpf_obj->maps.xsk), &queue_id, &sock_fd, 0); in main()
871 error(1, -ret, "bpf_map_update_elem"); in main()
876 bpf_program__fd(bpf_obj->progs.rx), in main()
879 error(1, -ret, "bpf_xdp_attach"); in main()
886 error(1, -ret, "verify_metadata"); in main()