13a687befSWillem de Bruijn // SPDX-License-Identifier: GPL-2.0 23a687befSWillem de Bruijn 33a687befSWillem de Bruijn #define _GNU_SOURCE 43a687befSWillem de Bruijn 53a687befSWillem de Bruijn #include <arpa/inet.h> 63a687befSWillem de Bruijn #include <errno.h> 73a687befSWillem de Bruijn #include <error.h> 879ebc3c2SFred Klassen #include <linux/errqueue.h> 979ebc3c2SFred Klassen #include <linux/net_tstamp.h> 103a687befSWillem de Bruijn #include <netinet/if_ether.h> 113a687befSWillem de Bruijn #include <netinet/in.h> 123a687befSWillem de Bruijn #include <netinet/ip.h> 133a687befSWillem de Bruijn #include <netinet/ip6.h> 143a687befSWillem de Bruijn #include <netinet/udp.h> 153a687befSWillem de Bruijn #include <poll.h> 163a687befSWillem de Bruijn #include <sched.h> 173a687befSWillem de Bruijn #include <signal.h> 183a687befSWillem de Bruijn #include <stdbool.h> 193a687befSWillem de Bruijn #include <stdio.h> 203a687befSWillem de Bruijn #include <stdlib.h> 213a687befSWillem de Bruijn #include <string.h> 223a687befSWillem de Bruijn #include <sys/socket.h> 233a687befSWillem de Bruijn #include <sys/time.h> 2479ebc3c2SFred Klassen #include <sys/poll.h> 253a687befSWillem de Bruijn #include <sys/types.h> 263a687befSWillem de Bruijn #include <unistd.h> 273a687befSWillem de Bruijn 2822f1a38aSWillem de Bruijn #include "../kselftest.h" 2922f1a38aSWillem de Bruijn 303a687befSWillem de Bruijn #ifndef ETH_MAX_MTU 313a687befSWillem de Bruijn #define ETH_MAX_MTU 0xFFFFU 323a687befSWillem de Bruijn #endif 333a687befSWillem de Bruijn 343a687befSWillem de Bruijn #ifndef UDP_SEGMENT 353a687befSWillem de Bruijn #define UDP_SEGMENT 103 363a687befSWillem de Bruijn #endif 373a687befSWillem de Bruijn 383a687befSWillem de Bruijn #ifndef SO_ZEROCOPY 393a687befSWillem de Bruijn #define SO_ZEROCOPY 60 403a687befSWillem de Bruijn #endif 413a687befSWillem de Bruijn 4279ebc3c2SFred Klassen #ifndef SO_EE_ORIGIN_ZEROCOPY 4379ebc3c2SFred Klassen #define SO_EE_ORIGIN_ZEROCOPY 5 4479ebc3c2SFred Klassen #endif 4579ebc3c2SFred Klassen 463a687befSWillem de Bruijn #ifndef MSG_ZEROCOPY 473a687befSWillem de Bruijn #define MSG_ZEROCOPY 0x4000000 483a687befSWillem de Bruijn #endif 493a687befSWillem de Bruijn 5022f1a38aSWillem de Bruijn #ifndef ENOTSUPP 5122f1a38aSWillem de Bruijn #define ENOTSUPP 524 5222f1a38aSWillem de Bruijn #endif 5322f1a38aSWillem de Bruijn 543a687befSWillem de Bruijn #define NUM_PKT 100 553a687befSWillem de Bruijn 563a687befSWillem de Bruijn static bool cfg_cache_trash; 573a687befSWillem de Bruijn static int cfg_cpu = -1; 583a687befSWillem de Bruijn static int cfg_connected = true; 593a687befSWillem de Bruijn static int cfg_family = PF_UNSPEC; 603a687befSWillem de Bruijn static uint16_t cfg_mss; 613a687befSWillem de Bruijn static int cfg_payload_len = (1472 * 42); 623a687befSWillem de Bruijn static int cfg_port = 8000; 633a687befSWillem de Bruijn static int cfg_runtime_ms = -1; 6479ebc3c2SFred Klassen static bool cfg_poll; 653a687befSWillem de Bruijn static bool cfg_segment; 663a687befSWillem de Bruijn static bool cfg_sendmmsg; 673a687befSWillem de Bruijn static bool cfg_tcp; 6879ebc3c2SFred Klassen static uint32_t cfg_tx_ts = SOF_TIMESTAMPING_TX_SOFTWARE; 6979ebc3c2SFred Klassen static bool cfg_tx_tstamp; 7079ebc3c2SFred Klassen static bool cfg_audit; 7179ebc3c2SFred Klassen static bool cfg_verbose; 723a687befSWillem de Bruijn static bool cfg_zerocopy; 733327a9c4SPaolo Abeni static int cfg_msg_nr; 743327a9c4SPaolo Abeni static uint16_t cfg_gso_size; 7579ebc3c2SFred Klassen static unsigned long total_num_msgs; 7679ebc3c2SFred Klassen static unsigned long total_num_sends; 7779ebc3c2SFred Klassen static unsigned long stat_tx_ts; 7879ebc3c2SFred Klassen static unsigned long stat_tx_ts_errors; 7979ebc3c2SFred Klassen static unsigned long tstart; 8079ebc3c2SFred Klassen static unsigned long tend; 8179ebc3c2SFred Klassen static unsigned long stat_zcopies; 823a687befSWillem de Bruijn 833a687befSWillem de Bruijn static socklen_t cfg_alen; 843a687befSWillem de Bruijn static struct sockaddr_storage cfg_dst_addr; 853a687befSWillem de Bruijn 863a687befSWillem de Bruijn static bool interrupted; 873a687befSWillem de Bruijn static char buf[NUM_PKT][ETH_MAX_MTU]; 883a687befSWillem de Bruijn 893a687befSWillem de Bruijn static void sigint_handler(int signum) 903a687befSWillem de Bruijn { 913a687befSWillem de Bruijn if (signum == SIGINT) 923a687befSWillem de Bruijn interrupted = true; 933a687befSWillem de Bruijn } 943a687befSWillem de Bruijn 953a687befSWillem de Bruijn static unsigned long gettimeofday_ms(void) 963a687befSWillem de Bruijn { 973a687befSWillem de Bruijn struct timeval tv; 983a687befSWillem de Bruijn 993a687befSWillem de Bruijn gettimeofday(&tv, NULL); 1003a687befSWillem de Bruijn return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); 1013a687befSWillem de Bruijn } 1023a687befSWillem de Bruijn 1033a687befSWillem de Bruijn static int set_cpu(int cpu) 1043a687befSWillem de Bruijn { 1053a687befSWillem de Bruijn cpu_set_t mask; 1063a687befSWillem de Bruijn 1073a687befSWillem de Bruijn CPU_ZERO(&mask); 1083a687befSWillem de Bruijn CPU_SET(cpu, &mask); 1093a687befSWillem de Bruijn if (sched_setaffinity(0, sizeof(mask), &mask)) 1103a687befSWillem de Bruijn error(1, 0, "setaffinity %d", cpu); 1113a687befSWillem de Bruijn 1123a687befSWillem de Bruijn return 0; 1133a687befSWillem de Bruijn } 1143a687befSWillem de Bruijn 1153a687befSWillem de Bruijn static void setup_sockaddr(int domain, const char *str_addr, void *sockaddr) 1163a687befSWillem de Bruijn { 1173a687befSWillem de Bruijn struct sockaddr_in6 *addr6 = (void *) sockaddr; 1183a687befSWillem de Bruijn struct sockaddr_in *addr4 = (void *) sockaddr; 1193a687befSWillem de Bruijn 1203a687befSWillem de Bruijn switch (domain) { 1213a687befSWillem de Bruijn case PF_INET: 1223a687befSWillem de Bruijn addr4->sin_family = AF_INET; 1233a687befSWillem de Bruijn addr4->sin_port = htons(cfg_port); 1243a687befSWillem de Bruijn if (inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1) 1253a687befSWillem de Bruijn error(1, 0, "ipv4 parse error: %s", str_addr); 1263a687befSWillem de Bruijn break; 1273a687befSWillem de Bruijn case PF_INET6: 1283a687befSWillem de Bruijn addr6->sin6_family = AF_INET6; 1293a687befSWillem de Bruijn addr6->sin6_port = htons(cfg_port); 1303a687befSWillem de Bruijn if (inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1) 1313a687befSWillem de Bruijn error(1, 0, "ipv6 parse error: %s", str_addr); 1323a687befSWillem de Bruijn break; 1333a687befSWillem de Bruijn default: 1343a687befSWillem de Bruijn error(1, 0, "illegal domain"); 1353a687befSWillem de Bruijn } 1363a687befSWillem de Bruijn } 1373a687befSWillem de Bruijn 13879ebc3c2SFred Klassen static void flush_cmsg(struct cmsghdr *cmsg) 1393a687befSWillem de Bruijn { 14079ebc3c2SFred Klassen struct sock_extended_err *err; 14179ebc3c2SFred Klassen struct scm_timestamping *tss; 14279ebc3c2SFred Klassen __u32 lo; 14379ebc3c2SFred Klassen __u32 hi; 14479ebc3c2SFred Klassen int i; 14579ebc3c2SFred Klassen 14679ebc3c2SFred Klassen switch (cmsg->cmsg_level) { 14779ebc3c2SFred Klassen case SOL_SOCKET: 14879ebc3c2SFred Klassen if (cmsg->cmsg_type == SO_TIMESTAMPING) { 14979ebc3c2SFred Klassen i = (cfg_tx_ts == SOF_TIMESTAMPING_TX_HARDWARE) ? 2 : 0; 15079ebc3c2SFred Klassen tss = (struct scm_timestamping *)CMSG_DATA(cmsg); 15179ebc3c2SFred Klassen if (tss->ts[i].tv_sec == 0) 15279ebc3c2SFred Klassen stat_tx_ts_errors++; 15379ebc3c2SFred Klassen } else { 15479ebc3c2SFred Klassen error(1, 0, "unknown SOL_SOCKET cmsg type=%u\n", 15579ebc3c2SFred Klassen cmsg->cmsg_type); 15679ebc3c2SFred Klassen } 15779ebc3c2SFred Klassen break; 15879ebc3c2SFred Klassen case SOL_IP: 15979ebc3c2SFred Klassen case SOL_IPV6: 16079ebc3c2SFred Klassen switch (cmsg->cmsg_type) { 16179ebc3c2SFred Klassen case IP_RECVERR: 16279ebc3c2SFred Klassen case IPV6_RECVERR: 16379ebc3c2SFred Klassen { 16479ebc3c2SFred Klassen err = (struct sock_extended_err *)CMSG_DATA(cmsg); 16579ebc3c2SFred Klassen switch (err->ee_origin) { 16679ebc3c2SFred Klassen case SO_EE_ORIGIN_TIMESTAMPING: 16779ebc3c2SFred Klassen /* Got a TX timestamp from error queue */ 16879ebc3c2SFred Klassen stat_tx_ts++; 16979ebc3c2SFred Klassen break; 17079ebc3c2SFred Klassen case SO_EE_ORIGIN_ICMP: 17179ebc3c2SFred Klassen case SO_EE_ORIGIN_ICMP6: 17279ebc3c2SFred Klassen if (cfg_verbose) 17379ebc3c2SFred Klassen fprintf(stderr, 17479ebc3c2SFred Klassen "received ICMP error: type=%u, code=%u\n", 17579ebc3c2SFred Klassen err->ee_type, err->ee_code); 17679ebc3c2SFred Klassen break; 17779ebc3c2SFred Klassen case SO_EE_ORIGIN_ZEROCOPY: 17879ebc3c2SFred Klassen { 17979ebc3c2SFred Klassen lo = err->ee_info; 18079ebc3c2SFred Klassen hi = err->ee_data; 18179ebc3c2SFred Klassen /* range of IDs acknowledged */ 18279ebc3c2SFred Klassen stat_zcopies += hi - lo + 1; 18379ebc3c2SFred Klassen break; 18479ebc3c2SFred Klassen } 18579ebc3c2SFred Klassen case SO_EE_ORIGIN_LOCAL: 18679ebc3c2SFred Klassen if (cfg_verbose) 18779ebc3c2SFred Klassen fprintf(stderr, 18879ebc3c2SFred Klassen "received packet with local origin: %u\n", 18979ebc3c2SFred Klassen err->ee_origin); 19079ebc3c2SFred Klassen break; 19179ebc3c2SFred Klassen default: 19279ebc3c2SFred Klassen error(0, 1, "received packet with origin: %u", 19379ebc3c2SFred Klassen err->ee_origin); 19479ebc3c2SFred Klassen } 19579ebc3c2SFred Klassen break; 19679ebc3c2SFred Klassen } 19779ebc3c2SFred Klassen default: 19879ebc3c2SFred Klassen error(0, 1, "unknown IP msg type=%u\n", 19979ebc3c2SFred Klassen cmsg->cmsg_type); 20079ebc3c2SFred Klassen break; 20179ebc3c2SFred Klassen } 20279ebc3c2SFred Klassen break; 20379ebc3c2SFred Klassen default: 20479ebc3c2SFred Klassen error(0, 1, "unknown cmsg level=%u\n", 20579ebc3c2SFred Klassen cmsg->cmsg_level); 20679ebc3c2SFred Klassen } 20779ebc3c2SFred Klassen } 20879ebc3c2SFred Klassen 20979ebc3c2SFred Klassen static void flush_errqueue_recv(int fd) 21079ebc3c2SFred Klassen { 21179ebc3c2SFred Klassen char control[CMSG_SPACE(sizeof(struct scm_timestamping)) + 21279ebc3c2SFred Klassen CMSG_SPACE(sizeof(struct sock_extended_err)) + 21379ebc3c2SFred Klassen CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0}; 21479ebc3c2SFred Klassen struct msghdr msg = {0}; 21579ebc3c2SFred Klassen struct cmsghdr *cmsg; 2163a687befSWillem de Bruijn int ret; 2173a687befSWillem de Bruijn 2183a687befSWillem de Bruijn while (1) { 21979ebc3c2SFred Klassen msg.msg_control = control; 22079ebc3c2SFred Klassen msg.msg_controllen = sizeof(control); 2213a687befSWillem de Bruijn ret = recvmsg(fd, &msg, MSG_ERRQUEUE); 2223a687befSWillem de Bruijn if (ret == -1 && errno == EAGAIN) 2233a687befSWillem de Bruijn break; 2243a687befSWillem de Bruijn if (ret == -1) 2253a687befSWillem de Bruijn error(1, errno, "errqueue"); 22679ebc3c2SFred Klassen if (msg.msg_flags != MSG_ERRQUEUE) 2273a687befSWillem de Bruijn error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags); 22879ebc3c2SFred Klassen if (cfg_audit) { 22979ebc3c2SFred Klassen for (cmsg = CMSG_FIRSTHDR(&msg); 23079ebc3c2SFred Klassen cmsg; 23179ebc3c2SFred Klassen cmsg = CMSG_NXTHDR(&msg, cmsg)) 23279ebc3c2SFred Klassen flush_cmsg(cmsg); 23379ebc3c2SFred Klassen } 2343a687befSWillem de Bruijn msg.msg_flags = 0; 2353a687befSWillem de Bruijn } 2363a687befSWillem de Bruijn } 2373a687befSWillem de Bruijn 23879ebc3c2SFred Klassen static void flush_errqueue(int fd, const bool do_poll) 23979ebc3c2SFred Klassen { 24079ebc3c2SFred Klassen if (do_poll) { 24179ebc3c2SFred Klassen struct pollfd fds = {0}; 24279ebc3c2SFred Klassen int ret; 24379ebc3c2SFred Klassen 24479ebc3c2SFred Klassen fds.fd = fd; 24579ebc3c2SFred Klassen ret = poll(&fds, 1, 500); 24679ebc3c2SFred Klassen if (ret == 0) { 24779ebc3c2SFred Klassen if (cfg_verbose) 24879ebc3c2SFred Klassen fprintf(stderr, "poll timeout\n"); 24979ebc3c2SFred Klassen } else if (ret < 0) { 25079ebc3c2SFred Klassen error(1, errno, "poll"); 25179ebc3c2SFred Klassen } 25279ebc3c2SFred Klassen } 25379ebc3c2SFred Klassen 25479ebc3c2SFred Klassen flush_errqueue_recv(fd); 25579ebc3c2SFred Klassen } 25679ebc3c2SFred Klassen 2573a687befSWillem de Bruijn static int send_tcp(int fd, char *data) 2583a687befSWillem de Bruijn { 2593a687befSWillem de Bruijn int ret, done = 0, count = 0; 2603a687befSWillem de Bruijn 2613a687befSWillem de Bruijn while (done < cfg_payload_len) { 2623a687befSWillem de Bruijn ret = send(fd, data + done, cfg_payload_len - done, 2633a687befSWillem de Bruijn cfg_zerocopy ? MSG_ZEROCOPY : 0); 2643a687befSWillem de Bruijn if (ret == -1) 2653a687befSWillem de Bruijn error(1, errno, "write"); 2663a687befSWillem de Bruijn 2673a687befSWillem de Bruijn done += ret; 2683a687befSWillem de Bruijn count++; 2693a687befSWillem de Bruijn } 2703a687befSWillem de Bruijn 2713a687befSWillem de Bruijn return count; 2723a687befSWillem de Bruijn } 2733a687befSWillem de Bruijn 2743a687befSWillem de Bruijn static int send_udp(int fd, char *data) 2753a687befSWillem de Bruijn { 2763a687befSWillem de Bruijn int ret, total_len, len, count = 0; 2773a687befSWillem de Bruijn 2783a687befSWillem de Bruijn total_len = cfg_payload_len; 2793a687befSWillem de Bruijn 2803a687befSWillem de Bruijn while (total_len) { 2813a687befSWillem de Bruijn len = total_len < cfg_mss ? total_len : cfg_mss; 2823a687befSWillem de Bruijn 2833a687befSWillem de Bruijn ret = sendto(fd, data, len, cfg_zerocopy ? MSG_ZEROCOPY : 0, 2843a687befSWillem de Bruijn cfg_connected ? NULL : (void *)&cfg_dst_addr, 2853a687befSWillem de Bruijn cfg_connected ? 0 : cfg_alen); 2863a687befSWillem de Bruijn if (ret == -1) 2873a687befSWillem de Bruijn error(1, errno, "write"); 2883a687befSWillem de Bruijn if (ret != len) 2893a687befSWillem de Bruijn error(1, errno, "write: %uB != %uB\n", ret, len); 2903a687befSWillem de Bruijn 2913a687befSWillem de Bruijn total_len -= len; 2923a687befSWillem de Bruijn count++; 2933a687befSWillem de Bruijn } 2943a687befSWillem de Bruijn 2953a687befSWillem de Bruijn return count; 2963a687befSWillem de Bruijn } 2973a687befSWillem de Bruijn 29879ebc3c2SFred Klassen static void send_ts_cmsg(struct cmsghdr *cm) 29979ebc3c2SFred Klassen { 30079ebc3c2SFred Klassen uint32_t *valp; 30179ebc3c2SFred Klassen 30279ebc3c2SFred Klassen cm->cmsg_level = SOL_SOCKET; 30379ebc3c2SFred Klassen cm->cmsg_type = SO_TIMESTAMPING; 30479ebc3c2SFred Klassen cm->cmsg_len = CMSG_LEN(sizeof(cfg_tx_ts)); 30579ebc3c2SFred Klassen valp = (void *)CMSG_DATA(cm); 30679ebc3c2SFred Klassen *valp = cfg_tx_ts; 30779ebc3c2SFred Klassen } 30879ebc3c2SFred Klassen 3093a687befSWillem de Bruijn static int send_udp_sendmmsg(int fd, char *data) 3103a687befSWillem de Bruijn { 31179ebc3c2SFred Klassen char control[CMSG_SPACE(sizeof(cfg_tx_ts))] = {0}; 3123a687befSWillem de Bruijn const int max_nr_msg = ETH_MAX_MTU / ETH_DATA_LEN; 3133a687befSWillem de Bruijn struct mmsghdr mmsgs[max_nr_msg]; 3143a687befSWillem de Bruijn struct iovec iov[max_nr_msg]; 3153a687befSWillem de Bruijn unsigned int off = 0, left; 31679ebc3c2SFred Klassen size_t msg_controllen = 0; 3173a687befSWillem de Bruijn int i = 0, ret; 3183a687befSWillem de Bruijn 3193a687befSWillem de Bruijn memset(mmsgs, 0, sizeof(mmsgs)); 3203a687befSWillem de Bruijn 32179ebc3c2SFred Klassen if (cfg_tx_tstamp) { 32279ebc3c2SFred Klassen struct msghdr msg = {0}; 32379ebc3c2SFred Klassen struct cmsghdr *cmsg; 32479ebc3c2SFred Klassen 32579ebc3c2SFred Klassen msg.msg_control = control; 32679ebc3c2SFred Klassen msg.msg_controllen = sizeof(control); 32779ebc3c2SFred Klassen cmsg = CMSG_FIRSTHDR(&msg); 32879ebc3c2SFred Klassen send_ts_cmsg(cmsg); 32979ebc3c2SFred Klassen msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts)); 33079ebc3c2SFred Klassen } 33179ebc3c2SFred Klassen 3323a687befSWillem de Bruijn left = cfg_payload_len; 3333a687befSWillem de Bruijn while (left) { 3343a687befSWillem de Bruijn if (i == max_nr_msg) 3353a687befSWillem de Bruijn error(1, 0, "sendmmsg: exceeds max_nr_msg"); 3363a687befSWillem de Bruijn 3373a687befSWillem de Bruijn iov[i].iov_base = data + off; 3383a687befSWillem de Bruijn iov[i].iov_len = cfg_mss < left ? cfg_mss : left; 3393a687befSWillem de Bruijn 3403a687befSWillem de Bruijn mmsgs[i].msg_hdr.msg_iov = iov + i; 3413a687befSWillem de Bruijn mmsgs[i].msg_hdr.msg_iovlen = 1; 3423a687befSWillem de Bruijn 34379ebc3c2SFred Klassen mmsgs[i].msg_hdr.msg_name = (void *)&cfg_dst_addr; 34479ebc3c2SFred Klassen mmsgs[i].msg_hdr.msg_namelen = cfg_alen; 34579ebc3c2SFred Klassen if (msg_controllen) { 34679ebc3c2SFred Klassen mmsgs[i].msg_hdr.msg_control = control; 34779ebc3c2SFred Klassen mmsgs[i].msg_hdr.msg_controllen = msg_controllen; 34879ebc3c2SFred Klassen } 34979ebc3c2SFred Klassen 3503a687befSWillem de Bruijn off += iov[i].iov_len; 3513a687befSWillem de Bruijn left -= iov[i].iov_len; 3523a687befSWillem de Bruijn i++; 3533a687befSWillem de Bruijn } 3543a687befSWillem de Bruijn 3553a687befSWillem de Bruijn ret = sendmmsg(fd, mmsgs, i, cfg_zerocopy ? MSG_ZEROCOPY : 0); 3563a687befSWillem de Bruijn if (ret == -1) 3573a687befSWillem de Bruijn error(1, errno, "sendmmsg"); 3583a687befSWillem de Bruijn 3593a687befSWillem de Bruijn return ret; 3603a687befSWillem de Bruijn } 3613a687befSWillem de Bruijn 3623a687befSWillem de Bruijn static void send_udp_segment_cmsg(struct cmsghdr *cm) 3633a687befSWillem de Bruijn { 3643a687befSWillem de Bruijn uint16_t *valp; 3653a687befSWillem de Bruijn 3663a687befSWillem de Bruijn cm->cmsg_level = SOL_UDP; 3673a687befSWillem de Bruijn cm->cmsg_type = UDP_SEGMENT; 3683327a9c4SPaolo Abeni cm->cmsg_len = CMSG_LEN(sizeof(cfg_gso_size)); 3693a687befSWillem de Bruijn valp = (void *)CMSG_DATA(cm); 3703327a9c4SPaolo Abeni *valp = cfg_gso_size; 3713a687befSWillem de Bruijn } 3723a687befSWillem de Bruijn 3733a687befSWillem de Bruijn static int send_udp_segment(int fd, char *data) 3743a687befSWillem de Bruijn { 37579ebc3c2SFred Klassen char control[CMSG_SPACE(sizeof(cfg_gso_size)) + 37679ebc3c2SFred Klassen CMSG_SPACE(sizeof(cfg_tx_ts))] = {0}; 3773a687befSWillem de Bruijn struct msghdr msg = {0}; 3783a687befSWillem de Bruijn struct iovec iov = {0}; 37979ebc3c2SFred Klassen size_t msg_controllen; 38079ebc3c2SFred Klassen struct cmsghdr *cmsg; 3813a687befSWillem de Bruijn int ret; 3823a687befSWillem de Bruijn 3833a687befSWillem de Bruijn iov.iov_base = data; 3843a687befSWillem de Bruijn iov.iov_len = cfg_payload_len; 3853a687befSWillem de Bruijn 3863a687befSWillem de Bruijn msg.msg_iov = &iov; 3873a687befSWillem de Bruijn msg.msg_iovlen = 1; 3883a687befSWillem de Bruijn 3893a687befSWillem de Bruijn msg.msg_control = control; 3903a687befSWillem de Bruijn msg.msg_controllen = sizeof(control); 39179ebc3c2SFred Klassen cmsg = CMSG_FIRSTHDR(&msg); 39279ebc3c2SFred Klassen send_udp_segment_cmsg(cmsg); 39379ebc3c2SFred Klassen msg_controllen = CMSG_SPACE(sizeof(cfg_mss)); 39479ebc3c2SFred Klassen if (cfg_tx_tstamp) { 39579ebc3c2SFred Klassen cmsg = CMSG_NXTHDR(&msg, cmsg); 39679ebc3c2SFred Klassen send_ts_cmsg(cmsg); 39779ebc3c2SFred Klassen msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts)); 39879ebc3c2SFred Klassen } 3993a687befSWillem de Bruijn 40079ebc3c2SFred Klassen msg.msg_controllen = msg_controllen; 4013a687befSWillem de Bruijn msg.msg_name = (void *)&cfg_dst_addr; 4023a687befSWillem de Bruijn msg.msg_namelen = cfg_alen; 4033a687befSWillem de Bruijn 4043a687befSWillem de Bruijn ret = sendmsg(fd, &msg, cfg_zerocopy ? MSG_ZEROCOPY : 0); 4053a687befSWillem de Bruijn if (ret == -1) 4063a687befSWillem de Bruijn error(1, errno, "sendmsg"); 4073a687befSWillem de Bruijn if (ret != iov.iov_len) 408670cd684SMasami Hiramatsu error(1, 0, "sendmsg: %u != %llu\n", ret, 409670cd684SMasami Hiramatsu (unsigned long long)iov.iov_len); 4103a687befSWillem de Bruijn 4113a687befSWillem de Bruijn return 1; 4123a687befSWillem de Bruijn } 4133a687befSWillem de Bruijn 4143a687befSWillem de Bruijn static void usage(const char *filepath) 4153a687befSWillem de Bruijn { 41679ebc3c2SFred Klassen error(1, 0, "Usage: %s [-46acmHPtTuvz] [-C cpu] [-D dst ip] [-l secs] [-M messagenr] [-p port] [-s sendsize] [-S gsosize]", 4173a687befSWillem de Bruijn filepath); 4183a687befSWillem de Bruijn } 4193a687befSWillem de Bruijn 4203a687befSWillem de Bruijn static void parse_opts(int argc, char **argv) 4213a687befSWillem de Bruijn { 4229c1952aeSwujianguo const char *bind_addr = NULL; 4233a687befSWillem de Bruijn int max_len, hdrlen; 4243a687befSWillem de Bruijn int c; 4253a687befSWillem de Bruijn 42679ebc3c2SFred Klassen while ((c = getopt(argc, argv, "46acC:D:Hl:mM:p:s:PS:tTuvz")) != -1) { 4273a687befSWillem de Bruijn switch (c) { 4283a687befSWillem de Bruijn case '4': 4293a687befSWillem de Bruijn if (cfg_family != PF_UNSPEC) 4303a687befSWillem de Bruijn error(1, 0, "Pass one of -4 or -6"); 4313a687befSWillem de Bruijn cfg_family = PF_INET; 4323a687befSWillem de Bruijn cfg_alen = sizeof(struct sockaddr_in); 4333a687befSWillem de Bruijn break; 4343a687befSWillem de Bruijn case '6': 4353a687befSWillem de Bruijn if (cfg_family != PF_UNSPEC) 4363a687befSWillem de Bruijn error(1, 0, "Pass one of -4 or -6"); 4373a687befSWillem de Bruijn cfg_family = PF_INET6; 4383a687befSWillem de Bruijn cfg_alen = sizeof(struct sockaddr_in6); 4393a687befSWillem de Bruijn break; 44079ebc3c2SFred Klassen case 'a': 44179ebc3c2SFred Klassen cfg_audit = true; 44279ebc3c2SFred Klassen break; 4433a687befSWillem de Bruijn case 'c': 4443a687befSWillem de Bruijn cfg_cache_trash = true; 4453a687befSWillem de Bruijn break; 4463a687befSWillem de Bruijn case 'C': 4473a687befSWillem de Bruijn cfg_cpu = strtol(optarg, NULL, 0); 4483a687befSWillem de Bruijn break; 4493a687befSWillem de Bruijn case 'D': 4509c1952aeSwujianguo bind_addr = optarg; 4513a687befSWillem de Bruijn break; 4523a687befSWillem de Bruijn case 'l': 4533a687befSWillem de Bruijn cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000; 4543a687befSWillem de Bruijn break; 4553a687befSWillem de Bruijn case 'm': 4563a687befSWillem de Bruijn cfg_sendmmsg = true; 4573a687befSWillem de Bruijn break; 4583327a9c4SPaolo Abeni case 'M': 4593327a9c4SPaolo Abeni cfg_msg_nr = strtoul(optarg, NULL, 10); 4603327a9c4SPaolo Abeni break; 4613a687befSWillem de Bruijn case 'p': 4623a687befSWillem de Bruijn cfg_port = strtoul(optarg, NULL, 0); 4633a687befSWillem de Bruijn break; 46479ebc3c2SFred Klassen case 'P': 46579ebc3c2SFred Klassen cfg_poll = true; 46679ebc3c2SFred Klassen break; 4673a687befSWillem de Bruijn case 's': 4683a687befSWillem de Bruijn cfg_payload_len = strtoul(optarg, NULL, 0); 4693a687befSWillem de Bruijn break; 4703a687befSWillem de Bruijn case 'S': 4713327a9c4SPaolo Abeni cfg_gso_size = strtoul(optarg, NULL, 0); 4723a687befSWillem de Bruijn cfg_segment = true; 4733a687befSWillem de Bruijn break; 47479ebc3c2SFred Klassen case 'H': 47579ebc3c2SFred Klassen cfg_tx_ts = SOF_TIMESTAMPING_TX_HARDWARE; 47679ebc3c2SFred Klassen cfg_tx_tstamp = true; 47779ebc3c2SFred Klassen break; 4783a687befSWillem de Bruijn case 't': 4793a687befSWillem de Bruijn cfg_tcp = true; 4803a687befSWillem de Bruijn break; 48179ebc3c2SFred Klassen case 'T': 48279ebc3c2SFred Klassen cfg_tx_tstamp = true; 48379ebc3c2SFred Klassen break; 4843a687befSWillem de Bruijn case 'u': 4853a687befSWillem de Bruijn cfg_connected = false; 4863a687befSWillem de Bruijn break; 48779ebc3c2SFred Klassen case 'v': 48879ebc3c2SFred Klassen cfg_verbose = true; 48979ebc3c2SFred Klassen break; 4903a687befSWillem de Bruijn case 'z': 4913a687befSWillem de Bruijn cfg_zerocopy = true; 4923a687befSWillem de Bruijn break; 493*db9b47eeSAndrei Gherzan default: 494*db9b47eeSAndrei Gherzan exit(1); 4953a687befSWillem de Bruijn } 4963a687befSWillem de Bruijn } 4973a687befSWillem de Bruijn 4989c1952aeSwujianguo if (!bind_addr) 4999c1952aeSwujianguo bind_addr = cfg_family == PF_INET6 ? "::" : "0.0.0.0"; 5009c1952aeSwujianguo 5019c1952aeSwujianguo setup_sockaddr(cfg_family, bind_addr, &cfg_dst_addr); 5029c1952aeSwujianguo 5033a687befSWillem de Bruijn if (optind != argc) 5043a687befSWillem de Bruijn usage(argv[0]); 5053a687befSWillem de Bruijn 5063a687befSWillem de Bruijn if (cfg_family == PF_UNSPEC) 5073a687befSWillem de Bruijn error(1, 0, "must pass one of -4 or -6"); 5083a687befSWillem de Bruijn if (cfg_tcp && !cfg_connected) 5093a687befSWillem de Bruijn error(1, 0, "connectionless tcp makes no sense"); 5103a687befSWillem de Bruijn if (cfg_segment && cfg_sendmmsg) 5113a687befSWillem de Bruijn error(1, 0, "cannot combine segment offload and sendmmsg"); 51279ebc3c2SFred Klassen if (cfg_tx_tstamp && !(cfg_segment || cfg_sendmmsg)) 51379ebc3c2SFred Klassen error(1, 0, "Options -T and -H require either -S or -m option"); 5143a687befSWillem de Bruijn 5153a687befSWillem de Bruijn if (cfg_family == PF_INET) 5163a687befSWillem de Bruijn hdrlen = sizeof(struct iphdr) + sizeof(struct udphdr); 5173a687befSWillem de Bruijn else 5183a687befSWillem de Bruijn hdrlen = sizeof(struct ip6_hdr) + sizeof(struct udphdr); 5193a687befSWillem de Bruijn 5203a687befSWillem de Bruijn cfg_mss = ETH_DATA_LEN - hdrlen; 5213a687befSWillem de Bruijn max_len = ETH_MAX_MTU - hdrlen; 5223327a9c4SPaolo Abeni if (!cfg_gso_size) 5233327a9c4SPaolo Abeni cfg_gso_size = cfg_mss; 5243a687befSWillem de Bruijn 5253a687befSWillem de Bruijn if (cfg_payload_len > max_len) 5263a687befSWillem de Bruijn error(1, 0, "payload length %u exceeds max %u", 5273a687befSWillem de Bruijn cfg_payload_len, max_len); 5283a687befSWillem de Bruijn } 5293a687befSWillem de Bruijn 5303a687befSWillem de Bruijn static void set_pmtu_discover(int fd, bool is_ipv4) 5313a687befSWillem de Bruijn { 5323a687befSWillem de Bruijn int level, name, val; 5333a687befSWillem de Bruijn 5343a687befSWillem de Bruijn if (is_ipv4) { 5353a687befSWillem de Bruijn level = SOL_IP; 5363a687befSWillem de Bruijn name = IP_MTU_DISCOVER; 5373a687befSWillem de Bruijn val = IP_PMTUDISC_DO; 5383a687befSWillem de Bruijn } else { 5393a687befSWillem de Bruijn level = SOL_IPV6; 5403a687befSWillem de Bruijn name = IPV6_MTU_DISCOVER; 5413a687befSWillem de Bruijn val = IPV6_PMTUDISC_DO; 5423a687befSWillem de Bruijn } 5433a687befSWillem de Bruijn 5443a687befSWillem de Bruijn if (setsockopt(fd, level, name, &val, sizeof(val))) 5453a687befSWillem de Bruijn error(1, errno, "setsockopt path mtu"); 5463a687befSWillem de Bruijn } 5473a687befSWillem de Bruijn 54879ebc3c2SFred Klassen static void set_tx_timestamping(int fd) 54979ebc3c2SFred Klassen { 55079ebc3c2SFred Klassen int val = SOF_TIMESTAMPING_OPT_CMSG | SOF_TIMESTAMPING_OPT_ID | 55179ebc3c2SFred Klassen SOF_TIMESTAMPING_OPT_TSONLY; 55279ebc3c2SFred Klassen 55379ebc3c2SFred Klassen if (cfg_tx_ts == SOF_TIMESTAMPING_TX_SOFTWARE) 55479ebc3c2SFred Klassen val |= SOF_TIMESTAMPING_SOFTWARE; 55579ebc3c2SFred Klassen else 55679ebc3c2SFred Klassen val |= SOF_TIMESTAMPING_RAW_HARDWARE; 55779ebc3c2SFred Klassen 55879ebc3c2SFred Klassen if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val))) 55979ebc3c2SFred Klassen error(1, errno, "setsockopt tx timestamping"); 56079ebc3c2SFred Klassen } 56179ebc3c2SFred Klassen 56279ebc3c2SFred Klassen static void print_audit_report(unsigned long num_msgs, unsigned long num_sends) 56379ebc3c2SFred Klassen { 56479ebc3c2SFred Klassen unsigned long tdelta; 56579ebc3c2SFred Klassen 56679ebc3c2SFred Klassen tdelta = tend - tstart; 56779ebc3c2SFred Klassen if (!tdelta) 56879ebc3c2SFred Klassen return; 56979ebc3c2SFred Klassen 57079ebc3c2SFred Klassen fprintf(stderr, "Summary over %lu.%03lu seconds...\n", 57179ebc3c2SFred Klassen tdelta / 1000, tdelta % 1000); 57279ebc3c2SFred Klassen fprintf(stderr, 57379ebc3c2SFred Klassen "sum %s tx: %6lu MB/s %10lu calls (%lu/s) %10lu msgs (%lu/s)\n", 57479ebc3c2SFred Klassen cfg_tcp ? "tcp" : "udp", 57579ebc3c2SFred Klassen ((num_msgs * cfg_payload_len) >> 10) / tdelta, 57679ebc3c2SFred Klassen num_sends, num_sends * 1000 / tdelta, 57779ebc3c2SFred Klassen num_msgs, num_msgs * 1000 / tdelta); 57879ebc3c2SFred Klassen 57979ebc3c2SFred Klassen if (cfg_tx_tstamp) { 58079ebc3c2SFred Klassen if (stat_tx_ts_errors) 58179ebc3c2SFred Klassen error(1, 0, 58279ebc3c2SFred Klassen "Expected clean TX Timestamps: %9lu msgs received %6lu errors", 58379ebc3c2SFred Klassen stat_tx_ts, stat_tx_ts_errors); 58479ebc3c2SFred Klassen if (stat_tx_ts != num_sends) 58579ebc3c2SFred Klassen error(1, 0, 58679ebc3c2SFred Klassen "Unexpected number of TX Timestamps: %9lu expected %9lu received", 58779ebc3c2SFred Klassen num_sends, stat_tx_ts); 58879ebc3c2SFred Klassen fprintf(stderr, 58979ebc3c2SFred Klassen "Tx Timestamps: %19lu received %17lu errors\n", 59079ebc3c2SFred Klassen stat_tx_ts, stat_tx_ts_errors); 59179ebc3c2SFred Klassen } 59279ebc3c2SFred Klassen 59379ebc3c2SFred Klassen if (cfg_zerocopy) { 59479ebc3c2SFred Klassen if (stat_zcopies != num_sends) 59579ebc3c2SFred Klassen error(1, 0, "Unexpected number of Zerocopy completions: %9lu expected %9lu received", 59679ebc3c2SFred Klassen num_sends, stat_zcopies); 59779ebc3c2SFred Klassen fprintf(stderr, 59879ebc3c2SFred Klassen "Zerocopy acks: %19lu\n", 59979ebc3c2SFred Klassen stat_zcopies); 60079ebc3c2SFred Klassen } 60179ebc3c2SFred Klassen } 60279ebc3c2SFred Klassen 60379ebc3c2SFred Klassen static void print_report(unsigned long num_msgs, unsigned long num_sends) 60479ebc3c2SFred Klassen { 60579ebc3c2SFred Klassen fprintf(stderr, 60679ebc3c2SFred Klassen "%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n", 60779ebc3c2SFred Klassen cfg_tcp ? "tcp" : "udp", 60879ebc3c2SFred Klassen (num_msgs * cfg_payload_len) >> 20, 60979ebc3c2SFred Klassen num_sends, num_msgs); 61079ebc3c2SFred Klassen 61179ebc3c2SFred Klassen if (cfg_audit) { 61279ebc3c2SFred Klassen total_num_msgs += num_msgs; 61379ebc3c2SFred Klassen total_num_sends += num_sends; 61479ebc3c2SFred Klassen } 61579ebc3c2SFred Klassen } 61679ebc3c2SFred Klassen 6173a687befSWillem de Bruijn int main(int argc, char **argv) 6183a687befSWillem de Bruijn { 6193a687befSWillem de Bruijn unsigned long num_msgs, num_sends; 6203a687befSWillem de Bruijn unsigned long tnow, treport, tstop; 62122f1a38aSWillem de Bruijn int fd, i, val, ret; 6223a687befSWillem de Bruijn 6233a687befSWillem de Bruijn parse_opts(argc, argv); 6243a687befSWillem de Bruijn 6253a687befSWillem de Bruijn if (cfg_cpu > 0) 6263a687befSWillem de Bruijn set_cpu(cfg_cpu); 6273a687befSWillem de Bruijn 6283a687befSWillem de Bruijn for (i = 0; i < sizeof(buf[0]); i++) 6293a687befSWillem de Bruijn buf[0][i] = 'a' + (i % 26); 6303a687befSWillem de Bruijn for (i = 1; i < NUM_PKT; i++) 6313a687befSWillem de Bruijn memcpy(buf[i], buf[0], sizeof(buf[0])); 6323a687befSWillem de Bruijn 6333a687befSWillem de Bruijn signal(SIGINT, sigint_handler); 6343a687befSWillem de Bruijn 6353a687befSWillem de Bruijn fd = socket(cfg_family, cfg_tcp ? SOCK_STREAM : SOCK_DGRAM, 0); 6363a687befSWillem de Bruijn if (fd == -1) 6373a687befSWillem de Bruijn error(1, errno, "socket"); 6383a687befSWillem de Bruijn 6393a687befSWillem de Bruijn if (cfg_zerocopy) { 6403a687befSWillem de Bruijn val = 1; 64122f1a38aSWillem de Bruijn 64222f1a38aSWillem de Bruijn ret = setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, 64322f1a38aSWillem de Bruijn &val, sizeof(val)); 64422f1a38aSWillem de Bruijn if (ret) { 64522f1a38aSWillem de Bruijn if (errno == ENOPROTOOPT || errno == ENOTSUPP) { 64622f1a38aSWillem de Bruijn fprintf(stderr, "SO_ZEROCOPY not supported"); 64722f1a38aSWillem de Bruijn exit(KSFT_SKIP); 64822f1a38aSWillem de Bruijn } 6493a687befSWillem de Bruijn error(1, errno, "setsockopt zerocopy"); 6503a687befSWillem de Bruijn } 65122f1a38aSWillem de Bruijn } 6523a687befSWillem de Bruijn 6533a687befSWillem de Bruijn if (cfg_connected && 6543a687befSWillem de Bruijn connect(fd, (void *)&cfg_dst_addr, cfg_alen)) 6553a687befSWillem de Bruijn error(1, errno, "connect"); 6563a687befSWillem de Bruijn 6573a687befSWillem de Bruijn if (cfg_segment) 6583a687befSWillem de Bruijn set_pmtu_discover(fd, cfg_family == PF_INET); 6593a687befSWillem de Bruijn 66079ebc3c2SFred Klassen if (cfg_tx_tstamp) 66179ebc3c2SFred Klassen set_tx_timestamping(fd); 66279ebc3c2SFred Klassen 6633a687befSWillem de Bruijn num_msgs = num_sends = 0; 6643a687befSWillem de Bruijn tnow = gettimeofday_ms(); 66579ebc3c2SFred Klassen tstart = tnow; 66679ebc3c2SFred Klassen tend = tnow; 6673a687befSWillem de Bruijn tstop = tnow + cfg_runtime_ms; 6683a687befSWillem de Bruijn treport = tnow + 1000; 6693a687befSWillem de Bruijn 6703a687befSWillem de Bruijn i = 0; 6713a687befSWillem de Bruijn do { 6723a687befSWillem de Bruijn if (cfg_tcp) 6733a687befSWillem de Bruijn num_sends += send_tcp(fd, buf[i]); 6743a687befSWillem de Bruijn else if (cfg_segment) 6753a687befSWillem de Bruijn num_sends += send_udp_segment(fd, buf[i]); 6763a687befSWillem de Bruijn else if (cfg_sendmmsg) 6773a687befSWillem de Bruijn num_sends += send_udp_sendmmsg(fd, buf[i]); 6783a687befSWillem de Bruijn else 6793a687befSWillem de Bruijn num_sends += send_udp(fd, buf[i]); 6803a687befSWillem de Bruijn num_msgs++; 68179ebc3c2SFred Klassen if ((cfg_zerocopy && ((num_msgs & 0xF) == 0)) || cfg_tx_tstamp) 68279ebc3c2SFred Klassen flush_errqueue(fd, cfg_poll); 6833a687befSWillem de Bruijn 6843327a9c4SPaolo Abeni if (cfg_msg_nr && num_msgs >= cfg_msg_nr) 6853327a9c4SPaolo Abeni break; 6863327a9c4SPaolo Abeni 6873a687befSWillem de Bruijn tnow = gettimeofday_ms(); 68879ebc3c2SFred Klassen if (tnow >= treport) { 68979ebc3c2SFred Klassen print_report(num_msgs, num_sends); 6903a687befSWillem de Bruijn num_msgs = num_sends = 0; 6913a687befSWillem de Bruijn treport = tnow + 1000; 6923a687befSWillem de Bruijn } 6933a687befSWillem de Bruijn 6943a687befSWillem de Bruijn /* cold cache when writing buffer */ 6953a687befSWillem de Bruijn if (cfg_cache_trash) 6963a687befSWillem de Bruijn i = ++i < NUM_PKT ? i : 0; 6973a687befSWillem de Bruijn 6983a687befSWillem de Bruijn } while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop)); 6993a687befSWillem de Bruijn 70079ebc3c2SFred Klassen if (cfg_zerocopy || cfg_tx_tstamp) 70179ebc3c2SFred Klassen flush_errqueue(fd, true); 70279ebc3c2SFred Klassen 7033a687befSWillem de Bruijn if (close(fd)) 7043a687befSWillem de Bruijn error(1, errno, "close"); 7053a687befSWillem de Bruijn 70679ebc3c2SFred Klassen if (cfg_audit) { 70779ebc3c2SFred Klassen tend = tnow; 70879ebc3c2SFred Klassen total_num_msgs += num_msgs; 70979ebc3c2SFred Klassen total_num_sends += num_sends; 71079ebc3c2SFred Klassen print_audit_report(total_num_msgs, total_num_sends); 71179ebc3c2SFred Klassen } 71279ebc3c2SFred Klassen 7133a687befSWillem de Bruijn return 0; 7143a687befSWillem de Bruijn } 715