1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * tcpdevmem netcat. Works similarly to netcat but does device memory TCP
4 * instead of regular TCP. Uses udmabuf to mock a dmabuf provider.
5 *
6 * Usage:
7 *
8 * On server:
9 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201
10 *
11 * On client:
12 * echo -n "hello\nworld" | \
13 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
14 *
15 * Note this is compatible with regular netcat. i.e. the sender or receiver can
16 * be replaced with regular netcat to test the RX or TX path in isolation.
17 *
18 * Test data validation (devmem TCP on RX only):
19 *
20 * On server:
21 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 -v 7
22 *
23 * On client:
24 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06) | \
25 * head -c 1G | \
26 * nc <server IP> 5201 -p 5201
27 *
28 * Test data validation (devmem TCP on RX and TX, validation happens on RX):
29 *
30 * On server:
31 * ncdevmem -s <server IP> [-c <client IP>] -l -p 5201 -v 8 -f eth1
32 *
33 * On client:
34 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06\\x07) | \
35 * head -c 1M | \
36 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
37 */
38 #define _GNU_SOURCE
39 #define __EXPORTED_HEADERS__
40
41 #include <linux/uio.h>
42 #include <stdio.h>
43 #include <stdlib.h>
44 #include <unistd.h>
45 #include <stdbool.h>
46 #include <string.h>
47 #include <errno.h>
48 #define __iovec_defined
49 #include <fcntl.h>
50 #include <malloc.h>
51 #include <error.h>
52 #include <poll.h>
53
54 #include <arpa/inet.h>
55 #include <sys/socket.h>
56 #include <sys/mman.h>
57 #include <sys/ioctl.h>
58 #include <sys/syscall.h>
59 #include <sys/time.h>
60
61 #include <linux/memfd.h>
62 #include <linux/dma-buf.h>
63 #include <linux/errqueue.h>
64 #include <linux/udmabuf.h>
65 #include <linux/types.h>
66 #include <linux/netlink.h>
67 #include <linux/genetlink.h>
68 #include <linux/netdev.h>
69 #include <linux/ethtool_netlink.h>
70 #include <time.h>
71 #include <net/if.h>
72
73 #include "netdev-user.h"
74 #include "ethtool-user.h"
75 #include <ynl.h>
76
77 #define PAGE_SHIFT 12
78 #define TEST_PREFIX "ncdevmem"
79 #define NUM_PAGES 16000
80
81 #ifndef MSG_SOCK_DEVMEM
82 #define MSG_SOCK_DEVMEM 0x2000000
83 #endif
84
85 #define MAX_IOV 1024
86
87 static size_t max_chunk;
88 static char *server_ip;
89 static char *client_ip;
90 static char *port;
91 static size_t do_validation;
92 static int start_queue = -1;
93 static int num_queues = -1;
94 static char *ifname;
95 static unsigned int ifindex;
96 static unsigned int dmabuf_id;
97 static uint32_t tx_dmabuf_id;
98 static int waittime_ms = 500;
99
100 struct memory_buffer {
101 int fd;
102 size_t size;
103
104 int devfd;
105 int memfd;
106 char *buf_mem;
107 };
108
109 struct memory_provider {
110 struct memory_buffer *(*alloc)(size_t size);
111 void (*free)(struct memory_buffer *ctx);
112 void (*memcpy_to_device)(struct memory_buffer *dst, size_t off,
113 void *src, int n);
114 void (*memcpy_from_device)(void *dst, struct memory_buffer *src,
115 size_t off, int n);
116 };
117
udmabuf_alloc(size_t size)118 static struct memory_buffer *udmabuf_alloc(size_t size)
119 {
120 struct udmabuf_create create;
121 struct memory_buffer *ctx;
122 int ret;
123
124 ctx = malloc(sizeof(*ctx));
125 if (!ctx)
126 error(1, ENOMEM, "malloc failed");
127
128 ctx->size = size;
129
130 ctx->devfd = open("/dev/udmabuf", O_RDWR);
131 if (ctx->devfd < 0)
132 error(1, errno,
133 "%s: [skip,no-udmabuf: Unable to access DMA buffer device file]\n",
134 TEST_PREFIX);
135
136 ctx->memfd = memfd_create("udmabuf-test", MFD_ALLOW_SEALING);
137 if (ctx->memfd < 0)
138 error(1, errno, "%s: [skip,no-memfd]\n", TEST_PREFIX);
139
140 ret = fcntl(ctx->memfd, F_ADD_SEALS, F_SEAL_SHRINK);
141 if (ret < 0)
142 error(1, errno, "%s: [skip,fcntl-add-seals]\n", TEST_PREFIX);
143
144 ret = ftruncate(ctx->memfd, size);
145 if (ret == -1)
146 error(1, errno, "%s: [FAIL,memfd-truncate]\n", TEST_PREFIX);
147
148 memset(&create, 0, sizeof(create));
149
150 create.memfd = ctx->memfd;
151 create.offset = 0;
152 create.size = size;
153 ctx->fd = ioctl(ctx->devfd, UDMABUF_CREATE, &create);
154 if (ctx->fd < 0)
155 error(1, errno, "%s: [FAIL, create udmabuf]\n", TEST_PREFIX);
156
157 ctx->buf_mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED,
158 ctx->fd, 0);
159 if (ctx->buf_mem == MAP_FAILED)
160 error(1, errno, "%s: [FAIL, map udmabuf]\n", TEST_PREFIX);
161
162 return ctx;
163 }
164
udmabuf_free(struct memory_buffer * ctx)165 static void udmabuf_free(struct memory_buffer *ctx)
166 {
167 munmap(ctx->buf_mem, ctx->size);
168 close(ctx->fd);
169 close(ctx->memfd);
170 close(ctx->devfd);
171 free(ctx);
172 }
173
udmabuf_memcpy_to_device(struct memory_buffer * dst,size_t off,void * src,int n)174 static void udmabuf_memcpy_to_device(struct memory_buffer *dst, size_t off,
175 void *src, int n)
176 {
177 struct dma_buf_sync sync = {};
178
179 sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE;
180 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
181
182 memcpy(dst->buf_mem + off, src, n);
183
184 sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE;
185 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
186 }
187
udmabuf_memcpy_from_device(void * dst,struct memory_buffer * src,size_t off,int n)188 static void udmabuf_memcpy_from_device(void *dst, struct memory_buffer *src,
189 size_t off, int n)
190 {
191 struct dma_buf_sync sync = {};
192
193 sync.flags = DMA_BUF_SYNC_START;
194 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
195
196 memcpy(dst, src->buf_mem + off, n);
197
198 sync.flags = DMA_BUF_SYNC_END;
199 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
200 }
201
202 static struct memory_provider udmabuf_memory_provider = {
203 .alloc = udmabuf_alloc,
204 .free = udmabuf_free,
205 .memcpy_to_device = udmabuf_memcpy_to_device,
206 .memcpy_from_device = udmabuf_memcpy_from_device,
207 };
208
209 static struct memory_provider *provider = &udmabuf_memory_provider;
210
print_nonzero_bytes(void * ptr,size_t size)211 static void print_nonzero_bytes(void *ptr, size_t size)
212 {
213 unsigned char *p = ptr;
214 unsigned int i;
215
216 for (i = 0; i < size; i++)
217 putchar(p[i]);
218 }
219
validate_buffer(void * line,size_t size)220 void validate_buffer(void *line, size_t size)
221 {
222 static unsigned char seed = 1;
223 unsigned char *ptr = line;
224 unsigned char expected;
225 static int errors;
226 size_t i;
227
228 for (i = 0; i < size; i++) {
229 expected = seed ? seed : '\n';
230 if (ptr[i] != expected) {
231 fprintf(stderr,
232 "Failed validation: expected=%u, actual=%u, index=%lu\n",
233 expected, ptr[i], i);
234 errors++;
235 if (errors > 20)
236 error(1, 0, "validation failed.");
237 }
238 seed++;
239 if (seed == do_validation)
240 seed = 0;
241 }
242
243 fprintf(stdout, "Validated buffer\n");
244 }
245
rxq_num(int ifindex)246 static int rxq_num(int ifindex)
247 {
248 struct ethtool_channels_get_req *req;
249 struct ethtool_channels_get_rsp *rsp;
250 struct ynl_error yerr;
251 struct ynl_sock *ys;
252 int num = -1;
253
254 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
255 if (!ys) {
256 fprintf(stderr, "YNL: %s\n", yerr.msg);
257 return -1;
258 }
259
260 req = ethtool_channels_get_req_alloc();
261 ethtool_channels_get_req_set_header_dev_index(req, ifindex);
262 rsp = ethtool_channels_get(ys, req);
263 if (rsp)
264 num = rsp->rx_count + rsp->combined_count;
265 ethtool_channels_get_req_free(req);
266 ethtool_channels_get_rsp_free(rsp);
267
268 ynl_sock_destroy(ys);
269
270 return num;
271 }
272
273 #define run_command(cmd, ...) \
274 ({ \
275 char command[256]; \
276 memset(command, 0, sizeof(command)); \
277 snprintf(command, sizeof(command), cmd, ##__VA_ARGS__); \
278 fprintf(stderr, "Running: %s\n", command); \
279 system(command); \
280 })
281
reset_flow_steering(void)282 static int reset_flow_steering(void)
283 {
284 /* Depending on the NIC, toggling ntuple off and on might not
285 * be allowed. Additionally, attempting to delete existing filters
286 * will fail if no filters are present. Therefore, do not enforce
287 * the exit status.
288 */
289
290 run_command("sudo ethtool -K %s ntuple off >&2", ifname);
291 run_command("sudo ethtool -K %s ntuple on >&2", ifname);
292 run_command(
293 "sudo ethtool -n %s | grep 'Filter:' | awk '{print $2}' | xargs -n1 ethtool -N %s delete >&2",
294 ifname, ifname);
295 return 0;
296 }
297
tcp_data_split_str(int val)298 static const char *tcp_data_split_str(int val)
299 {
300 switch (val) {
301 case 0:
302 return "off";
303 case 1:
304 return "auto";
305 case 2:
306 return "on";
307 default:
308 return "?";
309 }
310 }
311
configure_headersplit(bool on)312 static int configure_headersplit(bool on)
313 {
314 struct ethtool_rings_get_req *get_req;
315 struct ethtool_rings_get_rsp *get_rsp;
316 struct ethtool_rings_set_req *req;
317 struct ynl_error yerr;
318 struct ynl_sock *ys;
319 int ret;
320
321 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
322 if (!ys) {
323 fprintf(stderr, "YNL: %s\n", yerr.msg);
324 return -1;
325 }
326
327 req = ethtool_rings_set_req_alloc();
328 ethtool_rings_set_req_set_header_dev_index(req, ifindex);
329 /* 0 - off, 1 - auto, 2 - on */
330 ethtool_rings_set_req_set_tcp_data_split(req, on ? 2 : 0);
331 ret = ethtool_rings_set(ys, req);
332 if (ret < 0)
333 fprintf(stderr, "YNL failed: %s\n", ys->err.msg);
334 ethtool_rings_set_req_free(req);
335
336 if (ret == 0) {
337 get_req = ethtool_rings_get_req_alloc();
338 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
339 get_rsp = ethtool_rings_get(ys, get_req);
340 ethtool_rings_get_req_free(get_req);
341 if (get_rsp)
342 fprintf(stderr, "TCP header split: %s\n",
343 tcp_data_split_str(get_rsp->tcp_data_split));
344 ethtool_rings_get_rsp_free(get_rsp);
345 }
346
347 ynl_sock_destroy(ys);
348
349 return ret;
350 }
351
configure_rss(void)352 static int configure_rss(void)
353 {
354 return run_command("sudo ethtool -X %s equal %d >&2", ifname, start_queue);
355 }
356
configure_channels(unsigned int rx,unsigned int tx)357 static int configure_channels(unsigned int rx, unsigned int tx)
358 {
359 return run_command("sudo ethtool -L %s rx %u tx %u", ifname, rx, tx);
360 }
361
configure_flow_steering(struct sockaddr_in6 * server_sin)362 static int configure_flow_steering(struct sockaddr_in6 *server_sin)
363 {
364 const char *type = "tcp6";
365 const char *server_addr;
366 char buf[40];
367
368 inet_ntop(AF_INET6, &server_sin->sin6_addr, buf, sizeof(buf));
369 server_addr = buf;
370
371 if (IN6_IS_ADDR_V4MAPPED(&server_sin->sin6_addr)) {
372 type = "tcp4";
373 server_addr = strrchr(server_addr, ':') + 1;
374 }
375
376 /* Try configure 5-tuple */
377 if (run_command("sudo ethtool -N %s flow-type %s %s %s dst-ip %s %s %s dst-port %s queue %d >&2",
378 ifname,
379 type,
380 client_ip ? "src-ip" : "",
381 client_ip ?: "",
382 server_addr,
383 client_ip ? "src-port" : "",
384 client_ip ? port : "",
385 port, start_queue))
386 /* If that fails, try configure 3-tuple */
387 if (run_command("sudo ethtool -N %s flow-type %s dst-ip %s dst-port %s queue %d >&2",
388 ifname,
389 type,
390 server_addr,
391 port, start_queue))
392 /* If that fails, return error */
393 return -1;
394
395 return 0;
396 }
397
bind_rx_queue(unsigned int ifindex,unsigned int dmabuf_fd,struct netdev_queue_id * queues,unsigned int n_queue_index,struct ynl_sock ** ys)398 static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
399 struct netdev_queue_id *queues,
400 unsigned int n_queue_index, struct ynl_sock **ys)
401 {
402 struct netdev_bind_rx_req *req = NULL;
403 struct netdev_bind_rx_rsp *rsp = NULL;
404 struct ynl_error yerr;
405
406 *ys = ynl_sock_create(&ynl_netdev_family, &yerr);
407 if (!*ys) {
408 fprintf(stderr, "YNL: %s\n", yerr.msg);
409 return -1;
410 }
411
412 req = netdev_bind_rx_req_alloc();
413 netdev_bind_rx_req_set_ifindex(req, ifindex);
414 netdev_bind_rx_req_set_fd(req, dmabuf_fd);
415 __netdev_bind_rx_req_set_queues(req, queues, n_queue_index);
416
417 rsp = netdev_bind_rx(*ys, req);
418 if (!rsp) {
419 perror("netdev_bind_rx");
420 goto err_close;
421 }
422
423 if (!rsp->_present.id) {
424 perror("id not present");
425 goto err_close;
426 }
427
428 fprintf(stderr, "got dmabuf id=%d\n", rsp->id);
429 dmabuf_id = rsp->id;
430
431 netdev_bind_rx_req_free(req);
432 netdev_bind_rx_rsp_free(rsp);
433
434 return 0;
435
436 err_close:
437 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
438 netdev_bind_rx_req_free(req);
439 ynl_sock_destroy(*ys);
440 return -1;
441 }
442
bind_tx_queue(unsigned int ifindex,unsigned int dmabuf_fd,struct ynl_sock ** ys)443 static int bind_tx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
444 struct ynl_sock **ys)
445 {
446 struct netdev_bind_tx_req *req = NULL;
447 struct netdev_bind_tx_rsp *rsp = NULL;
448 struct ynl_error yerr;
449
450 *ys = ynl_sock_create(&ynl_netdev_family, &yerr);
451 if (!*ys) {
452 fprintf(stderr, "YNL: %s\n", yerr.msg);
453 return -1;
454 }
455
456 req = netdev_bind_tx_req_alloc();
457 netdev_bind_tx_req_set_ifindex(req, ifindex);
458 netdev_bind_tx_req_set_fd(req, dmabuf_fd);
459
460 rsp = netdev_bind_tx(*ys, req);
461 if (!rsp) {
462 perror("netdev_bind_tx");
463 goto err_close;
464 }
465
466 if (!rsp->_present.id) {
467 perror("id not present");
468 goto err_close;
469 }
470
471 fprintf(stderr, "got tx dmabuf id=%d\n", rsp->id);
472 tx_dmabuf_id = rsp->id;
473
474 netdev_bind_tx_req_free(req);
475 netdev_bind_tx_rsp_free(rsp);
476
477 return 0;
478
479 err_close:
480 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
481 netdev_bind_tx_req_free(req);
482 ynl_sock_destroy(*ys);
483 return -1;
484 }
485
enable_reuseaddr(int fd)486 static void enable_reuseaddr(int fd)
487 {
488 int opt = 1;
489 int ret;
490
491 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));
492 if (ret)
493 error(1, errno, "%s: [FAIL, SO_REUSEPORT]\n", TEST_PREFIX);
494
495 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
496 if (ret)
497 error(1, errno, "%s: [FAIL, SO_REUSEADDR]\n", TEST_PREFIX);
498 }
499
parse_address(const char * str,int port,struct sockaddr_in6 * sin6)500 static int parse_address(const char *str, int port, struct sockaddr_in6 *sin6)
501 {
502 int ret;
503
504 sin6->sin6_family = AF_INET6;
505 sin6->sin6_port = htons(port);
506
507 ret = inet_pton(sin6->sin6_family, str, &sin6->sin6_addr);
508 if (ret != 1) {
509 /* fallback to plain IPv4 */
510 ret = inet_pton(AF_INET, str, &sin6->sin6_addr.s6_addr32[3]);
511 if (ret != 1)
512 return -1;
513
514 /* add ::ffff prefix */
515 sin6->sin6_addr.s6_addr32[0] = 0;
516 sin6->sin6_addr.s6_addr32[1] = 0;
517 sin6->sin6_addr.s6_addr16[4] = 0;
518 sin6->sin6_addr.s6_addr16[5] = 0xffff;
519 }
520
521 return 0;
522 }
523
create_queues(void)524 static struct netdev_queue_id *create_queues(void)
525 {
526 struct netdev_queue_id *queues;
527 size_t i = 0;
528
529 queues = netdev_queue_id_alloc(num_queues);
530 for (i = 0; i < num_queues; i++) {
531 netdev_queue_id_set_type(&queues[i], NETDEV_QUEUE_TYPE_RX);
532 netdev_queue_id_set_id(&queues[i], start_queue + i);
533 }
534
535 return queues;
536 }
537
do_server(struct memory_buffer * mem)538 static int do_server(struct memory_buffer *mem)
539 {
540 char ctrl_data[sizeof(int) * 20000];
541 size_t non_page_aligned_frags = 0;
542 struct sockaddr_in6 client_addr;
543 struct sockaddr_in6 server_sin;
544 size_t page_aligned_frags = 0;
545 size_t total_received = 0;
546 socklen_t client_addr_len;
547 bool is_devmem = false;
548 char *tmp_mem = NULL;
549 struct ynl_sock *ys;
550 char iobuf[819200];
551 char buffer[256];
552 int socket_fd;
553 int client_fd;
554 int ret;
555
556 ret = parse_address(server_ip, atoi(port), &server_sin);
557 if (ret < 0)
558 error(1, 0, "parse server address");
559
560 if (reset_flow_steering())
561 error(1, 0, "Failed to reset flow steering\n");
562
563 if (configure_headersplit(1))
564 error(1, 0, "Failed to enable TCP header split\n");
565
566 /* Configure RSS to divert all traffic from our devmem queues */
567 if (configure_rss())
568 error(1, 0, "Failed to configure rss\n");
569
570 /* Flow steer our devmem flows to start_queue */
571 if (configure_flow_steering(&server_sin))
572 error(1, 0, "Failed to configure flow steering\n");
573
574 sleep(1);
575
576 if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys))
577 error(1, 0, "Failed to bind\n");
578
579 tmp_mem = malloc(mem->size);
580 if (!tmp_mem)
581 error(1, ENOMEM, "malloc failed");
582
583 socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
584 if (socket_fd < 0)
585 error(1, errno, "%s: [FAIL, create socket]\n", TEST_PREFIX);
586
587 enable_reuseaddr(socket_fd);
588
589 fprintf(stderr, "binding to address %s:%d\n", server_ip,
590 ntohs(server_sin.sin6_port));
591
592 ret = bind(socket_fd, &server_sin, sizeof(server_sin));
593 if (ret)
594 error(1, errno, "%s: [FAIL, bind]\n", TEST_PREFIX);
595
596 ret = listen(socket_fd, 1);
597 if (ret)
598 error(1, errno, "%s: [FAIL, listen]\n", TEST_PREFIX);
599
600 client_addr_len = sizeof(client_addr);
601
602 inet_ntop(AF_INET6, &server_sin.sin6_addr, buffer,
603 sizeof(buffer));
604 fprintf(stderr, "Waiting or connection on %s:%d\n", buffer,
605 ntohs(server_sin.sin6_port));
606 client_fd = accept(socket_fd, &client_addr, &client_addr_len);
607
608 inet_ntop(AF_INET6, &client_addr.sin6_addr, buffer,
609 sizeof(buffer));
610 fprintf(stderr, "Got connection from %s:%d\n", buffer,
611 ntohs(client_addr.sin6_port));
612
613 while (1) {
614 struct iovec iov = { .iov_base = iobuf,
615 .iov_len = sizeof(iobuf) };
616 struct dmabuf_cmsg *dmabuf_cmsg = NULL;
617 struct cmsghdr *cm = NULL;
618 struct msghdr msg = { 0 };
619 struct dmabuf_token token;
620 ssize_t ret;
621
622 is_devmem = false;
623
624 msg.msg_iov = &iov;
625 msg.msg_iovlen = 1;
626 msg.msg_control = ctrl_data;
627 msg.msg_controllen = sizeof(ctrl_data);
628 ret = recvmsg(client_fd, &msg, MSG_SOCK_DEVMEM);
629 fprintf(stderr, "recvmsg ret=%ld\n", ret);
630 if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
631 continue;
632 if (ret < 0) {
633 perror("recvmsg");
634 continue;
635 }
636 if (ret == 0) {
637 fprintf(stderr, "client exited\n");
638 goto cleanup;
639 }
640
641 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
642 if (cm->cmsg_level != SOL_SOCKET ||
643 (cm->cmsg_type != SCM_DEVMEM_DMABUF &&
644 cm->cmsg_type != SCM_DEVMEM_LINEAR)) {
645 fprintf(stderr, "skipping non-devmem cmsg\n");
646 continue;
647 }
648
649 dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm);
650 is_devmem = true;
651
652 if (cm->cmsg_type == SCM_DEVMEM_LINEAR) {
653 /* TODO: process data copied from skb's linear
654 * buffer.
655 */
656 fprintf(stderr,
657 "SCM_DEVMEM_LINEAR. dmabuf_cmsg->frag_size=%u\n",
658 dmabuf_cmsg->frag_size);
659
660 continue;
661 }
662
663 token.token_start = dmabuf_cmsg->frag_token;
664 token.token_count = 1;
665
666 total_received += dmabuf_cmsg->frag_size;
667 fprintf(stderr,
668 "received frag_page=%llu, in_page_offset=%llu, frag_offset=%llu, frag_size=%u, token=%u, total_received=%lu, dmabuf_id=%u\n",
669 dmabuf_cmsg->frag_offset >> PAGE_SHIFT,
670 dmabuf_cmsg->frag_offset % getpagesize(),
671 dmabuf_cmsg->frag_offset,
672 dmabuf_cmsg->frag_size, dmabuf_cmsg->frag_token,
673 total_received, dmabuf_cmsg->dmabuf_id);
674
675 if (dmabuf_cmsg->dmabuf_id != dmabuf_id)
676 error(1, 0,
677 "received on wrong dmabuf_id: flow steering error\n");
678
679 if (dmabuf_cmsg->frag_size % getpagesize())
680 non_page_aligned_frags++;
681 else
682 page_aligned_frags++;
683
684 provider->memcpy_from_device(tmp_mem, mem,
685 dmabuf_cmsg->frag_offset,
686 dmabuf_cmsg->frag_size);
687
688 if (do_validation)
689 validate_buffer(tmp_mem,
690 dmabuf_cmsg->frag_size);
691 else
692 print_nonzero_bytes(tmp_mem,
693 dmabuf_cmsg->frag_size);
694
695 ret = setsockopt(client_fd, SOL_SOCKET,
696 SO_DEVMEM_DONTNEED, &token,
697 sizeof(token));
698 if (ret != 1)
699 error(1, 0,
700 "SO_DEVMEM_DONTNEED not enough tokens");
701 }
702 if (!is_devmem)
703 error(1, 0, "flow steering error\n");
704
705 fprintf(stderr, "total_received=%lu\n", total_received);
706 }
707
708 fprintf(stderr, "%s: ok\n", TEST_PREFIX);
709
710 fprintf(stderr, "page_aligned_frags=%lu, non_page_aligned_frags=%lu\n",
711 page_aligned_frags, non_page_aligned_frags);
712
713 cleanup:
714
715 free(tmp_mem);
716 close(client_fd);
717 close(socket_fd);
718 ynl_sock_destroy(ys);
719
720 return 0;
721 }
722
run_devmem_tests(void)723 void run_devmem_tests(void)
724 {
725 struct memory_buffer *mem;
726 struct ynl_sock *ys;
727
728 mem = provider->alloc(getpagesize() * NUM_PAGES);
729
730 /* Configure RSS to divert all traffic from our devmem queues */
731 if (configure_rss())
732 error(1, 0, "rss error\n");
733
734 if (configure_headersplit(1))
735 error(1, 0, "Failed to configure header split\n");
736
737 if (!bind_rx_queue(ifindex, mem->fd,
738 calloc(num_queues, sizeof(struct netdev_queue_id)),
739 num_queues, &ys))
740 error(1, 0, "Binding empty queues array should have failed\n");
741
742 if (configure_headersplit(0))
743 error(1, 0, "Failed to configure header split\n");
744
745 if (!bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys))
746 error(1, 0, "Configure dmabuf with header split off should have failed\n");
747
748 if (configure_headersplit(1))
749 error(1, 0, "Failed to configure header split\n");
750
751 if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys))
752 error(1, 0, "Failed to bind\n");
753
754 /* Deactivating a bound queue should not be legal */
755 if (!configure_channels(num_queues, num_queues - 1))
756 error(1, 0, "Deactivating a bound queue should be illegal.\n");
757
758 /* Closing the netlink socket does an implicit unbind */
759 ynl_sock_destroy(ys);
760
761 provider->free(mem);
762 }
763
gettimeofday_ms(void)764 static uint64_t gettimeofday_ms(void)
765 {
766 struct timeval tv;
767
768 gettimeofday(&tv, NULL);
769 return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000ULL);
770 }
771
do_poll(int fd)772 static int do_poll(int fd)
773 {
774 struct pollfd pfd;
775 int ret;
776
777 pfd.revents = 0;
778 pfd.fd = fd;
779
780 ret = poll(&pfd, 1, waittime_ms);
781 if (ret == -1)
782 error(1, errno, "poll");
783
784 return ret && (pfd.revents & POLLERR);
785 }
786
wait_compl(int fd)787 static void wait_compl(int fd)
788 {
789 int64_t tstop = gettimeofday_ms() + waittime_ms;
790 char control[CMSG_SPACE(100)] = {};
791 struct sock_extended_err *serr;
792 struct msghdr msg = {};
793 struct cmsghdr *cm;
794 __u32 hi, lo;
795 int ret;
796
797 msg.msg_control = control;
798 msg.msg_controllen = sizeof(control);
799
800 while (gettimeofday_ms() < tstop) {
801 if (!do_poll(fd))
802 continue;
803
804 ret = recvmsg(fd, &msg, MSG_ERRQUEUE);
805 if (ret < 0) {
806 if (errno == EAGAIN)
807 continue;
808 error(1, errno, "recvmsg(MSG_ERRQUEUE)");
809 return;
810 }
811 if (msg.msg_flags & MSG_CTRUNC)
812 error(1, 0, "MSG_CTRUNC\n");
813
814 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
815 if (cm->cmsg_level != SOL_IP &&
816 cm->cmsg_level != SOL_IPV6)
817 continue;
818 if (cm->cmsg_level == SOL_IP &&
819 cm->cmsg_type != IP_RECVERR)
820 continue;
821 if (cm->cmsg_level == SOL_IPV6 &&
822 cm->cmsg_type != IPV6_RECVERR)
823 continue;
824
825 serr = (void *)CMSG_DATA(cm);
826 if (serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY)
827 error(1, 0, "wrong origin %u", serr->ee_origin);
828 if (serr->ee_errno != 0)
829 error(1, 0, "wrong errno %d", serr->ee_errno);
830
831 hi = serr->ee_data;
832 lo = serr->ee_info;
833
834 fprintf(stderr, "tx complete [%d,%d]\n", lo, hi);
835 return;
836 }
837 }
838
839 error(1, 0, "did not receive tx completion");
840 }
841
do_client(struct memory_buffer * mem)842 static int do_client(struct memory_buffer *mem)
843 {
844 char ctrl_data[CMSG_SPACE(sizeof(__u32))];
845 struct sockaddr_in6 server_sin;
846 struct sockaddr_in6 client_sin;
847 struct ynl_sock *ys = NULL;
848 struct iovec iov[MAX_IOV];
849 struct msghdr msg = {};
850 ssize_t line_size = 0;
851 struct cmsghdr *cmsg;
852 char *line = NULL;
853 size_t len = 0;
854 int socket_fd;
855 __u32 ddmabuf;
856 int opt = 1;
857 int ret;
858
859 ret = parse_address(server_ip, atoi(port), &server_sin);
860 if (ret < 0)
861 error(1, 0, "parse server address");
862
863 socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
864 if (socket_fd < 0)
865 error(1, socket_fd, "create socket");
866
867 enable_reuseaddr(socket_fd);
868
869 ret = setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, ifname,
870 strlen(ifname) + 1);
871 if (ret)
872 error(1, errno, "bindtodevice");
873
874 if (bind_tx_queue(ifindex, mem->fd, &ys))
875 error(1, 0, "Failed to bind\n");
876
877 if (client_ip) {
878 ret = parse_address(client_ip, atoi(port), &client_sin);
879 if (ret < 0)
880 error(1, 0, "parse client address");
881
882 ret = bind(socket_fd, &client_sin, sizeof(client_sin));
883 if (ret)
884 error(1, errno, "bind");
885 }
886
887 ret = setsockopt(socket_fd, SOL_SOCKET, SO_ZEROCOPY, &opt, sizeof(opt));
888 if (ret)
889 error(1, errno, "set sock opt");
890
891 fprintf(stderr, "Connect to %s %d (via %s)\n", server_ip,
892 ntohs(server_sin.sin6_port), ifname);
893
894 ret = connect(socket_fd, &server_sin, sizeof(server_sin));
895 if (ret)
896 error(1, errno, "connect");
897
898 while (1) {
899 free(line);
900 line = NULL;
901 line_size = getline(&line, &len, stdin);
902
903 if (line_size < 0)
904 break;
905
906 if (max_chunk) {
907 msg.msg_iovlen =
908 (line_size + max_chunk - 1) / max_chunk;
909 if (msg.msg_iovlen > MAX_IOV)
910 error(1, 0,
911 "can't partition %zd bytes into maximum of %d chunks",
912 line_size, MAX_IOV);
913
914 for (int i = 0; i < msg.msg_iovlen; i++) {
915 iov[i].iov_base = (void *)(i * max_chunk);
916 iov[i].iov_len = max_chunk;
917 }
918
919 iov[msg.msg_iovlen - 1].iov_len =
920 line_size - (msg.msg_iovlen - 1) * max_chunk;
921 } else {
922 iov[0].iov_base = 0;
923 iov[0].iov_len = line_size;
924 msg.msg_iovlen = 1;
925 }
926
927 msg.msg_iov = iov;
928 provider->memcpy_to_device(mem, 0, line, line_size);
929
930 msg.msg_control = ctrl_data;
931 msg.msg_controllen = sizeof(ctrl_data);
932
933 cmsg = CMSG_FIRSTHDR(&msg);
934 cmsg->cmsg_level = SOL_SOCKET;
935 cmsg->cmsg_type = SCM_DEVMEM_DMABUF;
936 cmsg->cmsg_len = CMSG_LEN(sizeof(__u32));
937
938 ddmabuf = tx_dmabuf_id;
939
940 *((__u32 *)CMSG_DATA(cmsg)) = ddmabuf;
941
942 ret = sendmsg(socket_fd, &msg, MSG_ZEROCOPY);
943 if (ret < 0)
944 error(1, errno, "Failed sendmsg");
945
946 fprintf(stderr, "sendmsg_ret=%d\n", ret);
947
948 if (ret != line_size)
949 error(1, errno, "Did not send all bytes %d vs %zd", ret,
950 line_size);
951
952 wait_compl(socket_fd);
953 }
954
955 fprintf(stderr, "%s: tx ok\n", TEST_PREFIX);
956
957 free(line);
958 close(socket_fd);
959
960 if (ys)
961 ynl_sock_destroy(ys);
962
963 return 0;
964 }
965
main(int argc,char * argv[])966 int main(int argc, char *argv[])
967 {
968 struct memory_buffer *mem;
969 int is_server = 0, opt;
970 int ret;
971
972 while ((opt = getopt(argc, argv, "ls:c:p:v:q:t:f:z:")) != -1) {
973 switch (opt) {
974 case 'l':
975 is_server = 1;
976 break;
977 case 's':
978 server_ip = optarg;
979 break;
980 case 'c':
981 client_ip = optarg;
982 break;
983 case 'p':
984 port = optarg;
985 break;
986 case 'v':
987 do_validation = atoll(optarg);
988 break;
989 case 'q':
990 num_queues = atoi(optarg);
991 break;
992 case 't':
993 start_queue = atoi(optarg);
994 break;
995 case 'f':
996 ifname = optarg;
997 break;
998 case 'z':
999 max_chunk = atoi(optarg);
1000 break;
1001 case '?':
1002 fprintf(stderr, "unknown option: %c\n", optopt);
1003 break;
1004 }
1005 }
1006
1007 if (!ifname)
1008 error(1, 0, "Missing -f argument\n");
1009
1010 ifindex = if_nametoindex(ifname);
1011
1012 fprintf(stderr, "using ifindex=%u\n", ifindex);
1013
1014 if (!server_ip && !client_ip) {
1015 if (start_queue < 0 && num_queues < 0) {
1016 num_queues = rxq_num(ifindex);
1017 if (num_queues < 0)
1018 error(1, 0, "couldn't detect number of queues\n");
1019 if (num_queues < 2)
1020 error(1, 0,
1021 "number of device queues is too low\n");
1022 /* make sure can bind to multiple queues */
1023 start_queue = num_queues / 2;
1024 num_queues /= 2;
1025 }
1026
1027 if (start_queue < 0 || num_queues < 0)
1028 error(1, 0, "Both -t and -q are required\n");
1029
1030 run_devmem_tests();
1031 return 0;
1032 }
1033
1034 if (start_queue < 0 && num_queues < 0) {
1035 num_queues = rxq_num(ifindex);
1036 if (num_queues < 2)
1037 error(1, 0, "number of device queues is too low\n");
1038
1039 num_queues = 1;
1040 start_queue = rxq_num(ifindex) - num_queues;
1041
1042 if (start_queue < 0)
1043 error(1, 0, "couldn't detect number of queues\n");
1044
1045 fprintf(stderr, "using queues %d..%d\n", start_queue, start_queue + num_queues);
1046 }
1047
1048 for (; optind < argc; optind++)
1049 fprintf(stderr, "extra arguments: %s\n", argv[optind]);
1050
1051 if (start_queue < 0)
1052 error(1, 0, "Missing -t argument\n");
1053
1054 if (num_queues < 0)
1055 error(1, 0, "Missing -q argument\n");
1056
1057 if (!server_ip)
1058 error(1, 0, "Missing -s argument\n");
1059
1060 if (!port)
1061 error(1, 0, "Missing -p argument\n");
1062
1063 mem = provider->alloc(getpagesize() * NUM_PAGES);
1064 ret = is_server ? do_server(mem) : do_client(mem);
1065 provider->free(mem);
1066
1067 return ret;
1068 }
1069