projects/linux·blob master · refs

1574 lines · 34.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * tcpdevmem netcat. Works similarly to netcat but does device memory TCP
4 * instead of regular TCP. Uses udmabuf to mock a dmabuf provider.
5 *
6 * Usage:
7 *
8 * On server:
9 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201
10 *
11 * On client:
12 * echo -n "hello\nworld" | \
13 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
14 *
15 * Note this is compatible with regular netcat. i.e. the sender or receiver can
16 * be replaced with regular netcat to test the RX or TX path in isolation.
17 *
18 * Test data validation (devmem TCP on RX only):
19 *
20 * On server:
21 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 -v 7
22 *
23 * On client:
24 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06) | \
25 * head -c 1G | \
26 * nc <server IP> 5201 -p 5201
27 *
28 * Test data validation (devmem TCP on RX and TX, validation happens on RX):
29 *
30 * On server:
31 * ncdevmem -s <server IP> [-c <client IP>] -l -p 5201 -v 8 -f eth1
32 *
33 * On client:
34 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06\\x07) | \
35 * head -c 1M | \
36 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
37 */
38#define _GNU_SOURCE
39#define __EXPORTED_HEADERS__
40
41#include <linux/uio.h>
42#include <stdarg.h>
43#include <stdint.h>
44#include <stdio.h>
45#include <stdlib.h>
46#include <unistd.h>
47#include <stdbool.h>
48#include <string.h>
49#include <errno.h>
50#define __iovec_defined
51#include <fcntl.h>
52#include <limits.h>
53#include <malloc.h>
54#include <error.h>
55#include <poll.h>
56
57#include <arpa/inet.h>
58#include <sys/socket.h>
59#include <sys/mman.h>
60#include <sys/ioctl.h>
61#include <sys/syscall.h>
62#include <sys/time.h>
63
64#include <linux/memfd.h>
65#include <sys/param.h>
66#include <linux/dma-buf.h>
67#include <linux/errqueue.h>
68#include <linux/udmabuf.h>
69#include <linux/types.h>
70#include <linux/netlink.h>
71#include <linux/genetlink.h>
72#include <linux/netdev.h>
73#include <linux/ethtool_netlink.h>
74#include <time.h>
75#include <net/if.h>
76
77#include "netdev-user.h"
78#include "ethtool-user.h"
79#include <ynl.h>
80
81#define PAGE_SHIFT 12
82#define TEST_PREFIX "ncdevmem"
83#define NUM_PAGES 16000
84#define MB(x) ((x) << 20)
85
86#ifndef MSG_SOCK_DEVMEM
87#define MSG_SOCK_DEVMEM 0x2000000
88#endif
89
90#define MAX_IOV 1024
91
92static size_t max_chunk;
93static char *server_ip;
94static char *client_ip;
95static char *port;
96static size_t do_validation;
97static int start_queue = -1;
98static int num_queues = -1;
99static int skip_config;
100static char *ifname;
101static unsigned int ifindex;
102static unsigned int dmabuf_id;
103static uint32_t tx_dmabuf_id;
104static int waittime_ms = 500;
105static bool fail_on_linear;
106static uint32_t rx_page_size;
107
108/* System state loaded by current_config_load() */
109#define MAX_FLOWS 8
110static int ntuple_ids[MAX_FLOWS] = { -1, -1, -1, -1, -1, -1, -1, -1, };
111
112struct memory_buffer {
113 int fd;
114 size_t size;
115
116 int devfd;
117 int memfd;
118 char *buf_mem;
119};
120
121struct memory_provider {
122 struct memory_buffer *(*alloc)(size_t size);
123 void (*free)(struct memory_buffer *ctx);
124 void (*memcpy_to_device)(struct memory_buffer *dst, size_t off,
125 void *src, int n);
126 void (*memcpy_from_device)(void *dst, struct memory_buffer *src,
127 size_t off, int n);
128};
129
130static void pr_err(const char *fmt, ...)
131{
132 va_list args;
133
134 fprintf(stderr, "%s: ", TEST_PREFIX);
135
136 va_start(args, fmt);
137 vfprintf(stderr, fmt, args);
138 va_end(args);
139
140 if (errno != 0)
141 fprintf(stderr, ": %s", strerror(errno));
142 fprintf(stderr, "\n");
143}
144
145static struct memory_buffer *udmabuf_alloc(size_t size)
146{
147 struct udmabuf_create create;
148 struct memory_buffer *ctx;
149 unsigned int memfd_flags;
150 int ret;
151
152 ctx = malloc(sizeof(*ctx));
153 if (!ctx)
154 return NULL;
155
156 ctx->size = size;
157
158 ctx->devfd = open("/dev/udmabuf", O_RDONLY);
159 if (ctx->devfd < 0) {
160 pr_err("[skip,no-udmabuf: Unable to access DMA buffer device file]");
161 goto err_free_ctx;
162 }
163
164 memfd_flags = MFD_ALLOW_SEALING;
165 if (rx_page_size > getpagesize())
166 memfd_flags |= MFD_HUGETLB | MFD_HUGE_2MB;
167
168 ctx->memfd = memfd_create("udmabuf-test", memfd_flags);
169 if (ctx->memfd < 0) {
170 pr_err("[skip,no-memfd%s]",
171 (memfd_flags & MFD_HUGETLB) ? " (need hugepages)" : "");
172 goto err_close_dev;
173 }
174
175 ret = fcntl(ctx->memfd, F_ADD_SEALS, F_SEAL_SHRINK);
176 if (ret < 0) {
177 pr_err("[skip,fcntl-add-seals]");
178 goto err_close_memfd;
179 }
180
181 if (memfd_flags & MFD_HUGETLB) {
182 size = roundup(size, MB(2));
183 ctx->size = size;
184 }
185
186 ret = ftruncate(ctx->memfd, size);
187 if (ret == -1) {
188 pr_err("[FAIL,memfd-truncate]");
189 goto err_close_memfd;
190 }
191
192 memset(&create, 0, sizeof(create));
193
194 create.memfd = ctx->memfd;
195 create.offset = 0;
196 create.size = size;
197 ctx->fd = ioctl(ctx->devfd, UDMABUF_CREATE, &create);
198 if (ctx->fd < 0) {
199 pr_err("[FAIL, create udmabuf]");
200 goto err_close_fd;
201 }
202
203 ctx->buf_mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED,
204 ctx->fd, 0);
205 if (ctx->buf_mem == MAP_FAILED) {
206 pr_err("[FAIL, map udmabuf]");
207 goto err_close_fd;
208 }
209
210 return ctx;
211
212err_close_fd:
213 close(ctx->fd);
214err_close_memfd:
215 close(ctx->memfd);
216err_close_dev:
217 close(ctx->devfd);
218err_free_ctx:
219 free(ctx);
220 return NULL;
221}
222
223static void udmabuf_free(struct memory_buffer *ctx)
224{
225 munmap(ctx->buf_mem, ctx->size);
226 close(ctx->fd);
227 close(ctx->memfd);
228 close(ctx->devfd);
229 free(ctx);
230}
231
232static void udmabuf_memcpy_to_device(struct memory_buffer *dst, size_t off,
233 void *src, int n)
234{
235 struct dma_buf_sync sync = {};
236
237 sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE;
238 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
239
240 memcpy(dst->buf_mem + off, src, n);
241
242 sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE;
243 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
244}
245
246static void udmabuf_memcpy_from_device(void *dst, struct memory_buffer *src,
247 size_t off, int n)
248{
249 struct dma_buf_sync sync = {};
250
251 sync.flags = DMA_BUF_SYNC_START;
252 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
253
254 memcpy(dst, src->buf_mem + off, n);
255
256 sync.flags = DMA_BUF_SYNC_END;
257 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
258}
259
260static struct memory_provider udmabuf_memory_provider = {
261 .alloc = udmabuf_alloc,
262 .free = udmabuf_free,
263 .memcpy_to_device = udmabuf_memcpy_to_device,
264 .memcpy_from_device = udmabuf_memcpy_from_device,
265};
266
267static struct memory_provider *provider = &udmabuf_memory_provider;
268
269static void print_nonzero_bytes(void *ptr, size_t size)
270{
271 unsigned char *p = ptr;
272 unsigned int i;
273
274 for (i = 0; i < size; i++)
275 putchar(p[i]);
276}
277
278int validate_buffer(void *line, size_t size)
279{
280 static unsigned char seed = 1;
281 unsigned char *ptr = line;
282 unsigned char expected;
283 static int errors;
284 size_t i;
285
286 for (i = 0; i < size; i++) {
287 expected = seed ? seed : '\n';
288 if (ptr[i] != expected) {
289 fprintf(stderr,
290 "Failed validation: expected=%u, actual=%u, index=%lu\n",
291 expected, ptr[i], i);
292 errors++;
293 if (errors > 20) {
294 pr_err("validation failed");
295 return -1;
296 }
297 }
298 seed++;
299 if (seed == do_validation)
300 seed = 0;
301 }
302
303 fprintf(stdout, "Validated buffer\n");
304 return 0;
305}
306
307static int
308__run_command(char *out, size_t outlen, const char *cmd, va_list args)
309{
310 char command[256];
311 FILE *fp;
312
313 vsnprintf(command, sizeof(command), cmd, args);
314
315 fprintf(stderr, "Running: %s\n", command);
316 fp = popen(command, "r");
317 if (!fp)
318 return -1;
319 if (out) {
320 size_t len;
321
322 if (!fgets(out, outlen, fp))
323 return -1;
324
325 /* Remove trailing newline if present */
326 len = strlen(out);
327 if (len && out[len - 1] == '\n')
328 out[len - 1] = '\0';
329 }
330 return pclose(fp);
331}
332
333static int run_command(const char *cmd, ...)
334{
335 va_list args;
336 int ret;
337
338 va_start(args, cmd);
339 ret = __run_command(NULL, 0, cmd, args);
340 va_end(args);
341
342 return ret;
343}
344
345static int ethtool_add_flow(const char *format, ...)
346{
347 char local_output[256], cmd[256];
348 const char *id_start;
349 int flow_idx, ret;
350 char *endptr;
351 long flow_id;
352 va_list args;
353
354 for (flow_idx = 0; flow_idx < MAX_FLOWS; flow_idx++)
355 if (ntuple_ids[flow_idx] == -1)
356 break;
357 if (flow_idx == MAX_FLOWS) {
358 fprintf(stderr, "Error: too many flows\n");
359 return -1;
360 }
361
362 snprintf(cmd, sizeof(cmd), "ethtool -N %s %s", ifname, format);
363
364 va_start(args, format);
365 ret = __run_command(local_output, sizeof(local_output), cmd, args);
366 va_end(args);
367
368 if (ret != 0)
369 return ret;
370
371 /* Extract the ID from the output */
372 id_start = strstr(local_output, "Added rule with ID ");
373 if (!id_start)
374 return -1;
375 id_start += strlen("Added rule with ID ");
376
377 flow_id = strtol(id_start, &endptr, 10);
378 if (endptr == id_start || flow_id < 0 || flow_id > INT_MAX)
379 return -1;
380
381 fprintf(stderr, "Added flow rule with ID %ld\n", flow_id);
382 ntuple_ids[flow_idx] = flow_id;
383 return flow_id;
384}
385
386static int rxq_num(int ifindex)
387{
388 struct ethtool_channels_get_req *req;
389 struct ethtool_channels_get_rsp *rsp;
390 struct ynl_error yerr;
391 struct ynl_sock *ys;
392 int num = -1;
393
394 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
395 if (!ys) {
396 fprintf(stderr, "YNL: %s\n", yerr.msg);
397 return -1;
398 }
399
400 req = ethtool_channels_get_req_alloc();
401 ethtool_channels_get_req_set_header_dev_index(req, ifindex);
402 rsp = ethtool_channels_get(ys, req);
403 if (rsp)
404 num = rsp->rx_count + rsp->combined_count;
405 ethtool_channels_get_req_free(req);
406 ethtool_channels_get_rsp_free(rsp);
407
408 ynl_sock_destroy(ys);
409
410 return num;
411}
412
413static void reset_flow_steering(void)
414{
415 int i;
416
417 for (i = 0; i < MAX_FLOWS; i++) {
418 if (ntuple_ids[i] == -1)
419 continue;
420 run_command("ethtool -N %s delete %d",
421 ifname, ntuple_ids[i]);
422 ntuple_ids[i] = -1;
423 }
424}
425
426static const char *tcp_data_split_str(int val)
427{
428 switch (val) {
429 case 0:
430 return "off";
431 case 1:
432 return "auto";
433 case 2:
434 return "on";
435 default:
436 return "?";
437 }
438}
439
440static struct ethtool_rings_get_rsp *get_ring_config(void)
441{
442 struct ethtool_rings_get_req *get_req;
443 struct ethtool_rings_get_rsp *get_rsp;
444 struct ynl_error yerr;
445 struct ynl_sock *ys;
446
447 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
448 if (!ys) {
449 fprintf(stderr, "YNL: %s\n", yerr.msg);
450 return NULL;
451 }
452
453 get_req = ethtool_rings_get_req_alloc();
454 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
455 get_rsp = ethtool_rings_get(ys, get_req);
456 ethtool_rings_get_req_free(get_req);
457
458 ynl_sock_destroy(ys);
459
460 return get_rsp;
461}
462
463static void restore_ring_config(const struct ethtool_rings_get_rsp *config)
464{
465 struct ethtool_rings_get_req *get_req;
466 struct ethtool_rings_get_rsp *get_rsp;
467 struct ethtool_rings_set_req *req;
468 struct ynl_error yerr;
469 struct ynl_sock *ys;
470 int ret;
471
472 if (!config)
473 return;
474
475 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
476 if (!ys) {
477 fprintf(stderr, "YNL: %s\n", yerr.msg);
478 return;
479 }
480
481 req = ethtool_rings_set_req_alloc();
482 ethtool_rings_set_req_set_header_dev_index(req, ifindex);
483 ethtool_rings_set_req_set_tcp_data_split(req,
484 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN);
485 if (config->_present.hds_thresh)
486 ethtool_rings_set_req_set_hds_thresh(req, config->hds_thresh);
487
488 ret = ethtool_rings_set(ys, req);
489 if (ret < 0)
490 fprintf(stderr, "YNL restoring HDS cfg: %s\n", ys->err.msg);
491
492 get_req = ethtool_rings_get_req_alloc();
493 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
494 get_rsp = ethtool_rings_get(ys, get_req);
495 ethtool_rings_get_req_free(get_req);
496
497 /* use explicit value if UKNOWN didn't give us the previous */
498 if (get_rsp->tcp_data_split != config->tcp_data_split) {
499 ethtool_rings_set_req_set_tcp_data_split(req,
500 config->tcp_data_split);
501 ret = ethtool_rings_set(ys, req);
502 if (ret < 0)
503 fprintf(stderr, "YNL restoring expl HDS cfg: %s\n",
504 ys->err.msg);
505 }
506
507 ethtool_rings_get_rsp_free(get_rsp);
508 ethtool_rings_set_req_free(req);
509
510 ynl_sock_destroy(ys);
511}
512
513static int
514configure_headersplit(const struct ethtool_rings_get_rsp *old, bool on)
515{
516 struct ethtool_rings_get_req *get_req;
517 struct ethtool_rings_get_rsp *get_rsp;
518 struct ethtool_rings_set_req *req;
519 struct ynl_error yerr;
520 struct ynl_sock *ys;
521 int ret;
522
523 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
524 if (!ys) {
525 fprintf(stderr, "YNL: %s\n", yerr.msg);
526 return -1;
527 }
528
529 req = ethtool_rings_set_req_alloc();
530 ethtool_rings_set_req_set_header_dev_index(req, ifindex);
531 if (on) {
532 ethtool_rings_set_req_set_tcp_data_split(req,
533 ETHTOOL_TCP_DATA_SPLIT_ENABLED);
534 if (old->_present.hds_thresh)
535 ethtool_rings_set_req_set_hds_thresh(req, 0);
536 } else {
537 ethtool_rings_set_req_set_tcp_data_split(req,
538 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN);
539 }
540 ret = ethtool_rings_set(ys, req);
541 if (ret < 0)
542 fprintf(stderr, "YNL failed: %s\n", ys->err.msg);
543 ethtool_rings_set_req_free(req);
544
545 if (ret == 0) {
546 get_req = ethtool_rings_get_req_alloc();
547 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
548 get_rsp = ethtool_rings_get(ys, get_req);
549 ethtool_rings_get_req_free(get_req);
550 if (get_rsp)
551 fprintf(stderr, "TCP header split: %s\n",
552 tcp_data_split_str(get_rsp->tcp_data_split));
553 ethtool_rings_get_rsp_free(get_rsp);
554 }
555
556 ynl_sock_destroy(ys);
557
558 return ret;
559}
560
561static int configure_rss(void)
562{
563 return run_command("ethtool -X %s equal %d >&2", ifname, start_queue);
564}
565
566static void reset_rss(void)
567{
568 run_command("ethtool -X %s default >&2", ifname, start_queue);
569}
570
571static int check_changing_channels(unsigned int rx, unsigned int tx)
572{
573 struct ethtool_channels_get_req *gchan;
574 struct ethtool_channels_set_req *schan;
575 struct ethtool_channels_get_rsp *chan;
576 struct ynl_error yerr;
577 struct ynl_sock *ys;
578 int ret;
579
580 fprintf(stderr, "setting channel count rx:%u tx:%u\n", rx, tx);
581
582 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
583 if (!ys) {
584 fprintf(stderr, "YNL: %s\n", yerr.msg);
585 return -1;
586 }
587
588 gchan = ethtool_channels_get_req_alloc();
589 if (!gchan) {
590 ret = -1;
591 goto exit_close_sock;
592 }
593
594 ethtool_channels_get_req_set_header_dev_index(gchan, ifindex);
595 chan = ethtool_channels_get(ys, gchan);
596 ethtool_channels_get_req_free(gchan);
597 if (!chan) {
598 fprintf(stderr, "YNL get channels: %s\n", ys->err.msg);
599 ret = -1;
600 goto exit_close_sock;
601 }
602
603 schan = ethtool_channels_set_req_alloc();
604 if (!schan) {
605 ret = -1;
606 goto exit_free_chan;
607 }
608
609 ethtool_channels_set_req_set_header_dev_index(schan, ifindex);
610
611 if (chan->_present.combined_count) {
612 if (chan->_present.rx_count || chan->_present.tx_count) {
613 ethtool_channels_set_req_set_rx_count(schan, 0);
614 ethtool_channels_set_req_set_tx_count(schan, 0);
615 }
616
617 if (rx == tx) {
618 ethtool_channels_set_req_set_combined_count(schan, rx);
619 } else if (rx > tx) {
620 ethtool_channels_set_req_set_combined_count(schan, tx);
621 ethtool_channels_set_req_set_rx_count(schan, rx - tx);
622 } else {
623 ethtool_channels_set_req_set_combined_count(schan, rx);
624 ethtool_channels_set_req_set_tx_count(schan, tx - rx);
625 }
626
627 } else if (chan->_present.rx_count) {
628 ethtool_channels_set_req_set_rx_count(schan, rx);
629 ethtool_channels_set_req_set_tx_count(schan, tx);
630 } else {
631 fprintf(stderr, "Error: device has neither combined nor rx channels\n");
632 ret = -1;
633 goto exit_free_schan;
634 }
635
636 ret = ethtool_channels_set(ys, schan);
637 if (ret) {
638 fprintf(stderr, "YNL set channels: %s\n", ys->err.msg);
639 } else {
640 /* We were expecting a failure, go back to previous settings */
641 ethtool_channels_set_req_set_combined_count(schan,
642 chan->combined_count);
643 ethtool_channels_set_req_set_rx_count(schan, chan->rx_count);
644 ethtool_channels_set_req_set_tx_count(schan, chan->tx_count);
645
646 ret = ethtool_channels_set(ys, schan);
647 if (ret)
648 fprintf(stderr, "YNL un-setting channels: %s\n",
649 ys->err.msg);
650 }
651
652exit_free_schan:
653 ethtool_channels_set_req_free(schan);
654exit_free_chan:
655 ethtool_channels_get_rsp_free(chan);
656exit_close_sock:
657 ynl_sock_destroy(ys);
658
659 return ret;
660}
661
662static int configure_flow_steering(struct sockaddr_in6 *server_sin)
663{
664 const char *type = "tcp6";
665 const char *server_addr;
666 char buf[40];
667 int flow_id;
668
669 inet_ntop(AF_INET6, &server_sin->sin6_addr, buf, sizeof(buf));
670 server_addr = buf;
671
672 if (IN6_IS_ADDR_V4MAPPED(&server_sin->sin6_addr)) {
673 type = "tcp4";
674 server_addr = strrchr(server_addr, ':') + 1;
675 }
676
677 /* Try configure 5-tuple */
678 flow_id = ethtool_add_flow("flow-type %s %s %s dst-ip %s %s %s dst-port %s queue %d",
679 type,
680 client_ip ? "src-ip" : "",
681 client_ip ?: "",
682 server_addr,
683 client_ip ? "src-port" : "",
684 client_ip ? port : "",
685 port, start_queue);
686 if (flow_id < 0) {
687 /* If that fails, try configure 3-tuple */
688 flow_id = ethtool_add_flow("flow-type %s dst-ip %s dst-port %s queue %d",
689 type, server_addr, port, start_queue);
690 if (flow_id < 0)
691 /* If that fails, return error */
692 return -1;
693 }
694
695 return 0;
696}
697
698static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
699 struct netdev_queue_id *queues,
700 unsigned int n_queue_index, struct ynl_sock **ys)
701{
702 struct netdev_bind_rx_req *req = NULL;
703 struct netdev_bind_rx_rsp *rsp = NULL;
704 struct ynl_error yerr;
705
706 *ys = ynl_sock_create(&ynl_netdev_family, &yerr);
707 if (!*ys) {
708 netdev_queue_id_free(queues);
709 fprintf(stderr, "YNL: %s\n", yerr.msg);
710 return -1;
711 }
712
713 req = netdev_bind_rx_req_alloc();
714 netdev_bind_rx_req_set_ifindex(req, ifindex);
715 netdev_bind_rx_req_set_fd(req, dmabuf_fd);
716 __netdev_bind_rx_req_set_queues(req, queues, n_queue_index);
717 if (rx_page_size)
718 netdev_bind_rx_req_set_rx_page_size(req, rx_page_size);
719
720 rsp = netdev_bind_rx(*ys, req);
721 if (!rsp) {
722 perror("netdev_bind_rx");
723 goto err_close;
724 }
725
726 if (!rsp->_present.id) {
727 perror("id not present");
728 goto err_close;
729 }
730
731 fprintf(stderr, "got dmabuf id=%d\n", rsp->id);
732 dmabuf_id = rsp->id;
733
734 netdev_bind_rx_req_free(req);
735 netdev_bind_rx_rsp_free(rsp);
736
737 return 0;
738
739err_close:
740 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
741 netdev_bind_rx_req_free(req);
742 ynl_sock_destroy(*ys);
743 return -1;
744}
745
746static int bind_tx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
747 struct ynl_sock **ys)
748{
749 struct netdev_bind_tx_req *req = NULL;
750 struct netdev_bind_tx_rsp *rsp = NULL;
751 struct ynl_error yerr;
752
753 *ys = ynl_sock_create(&ynl_netdev_family, &yerr);
754 if (!*ys) {
755 fprintf(stderr, "YNL: %s\n", yerr.msg);
756 return -1;
757 }
758
759 req = netdev_bind_tx_req_alloc();
760 netdev_bind_tx_req_set_ifindex(req, ifindex);
761 netdev_bind_tx_req_set_fd(req, dmabuf_fd);
762
763 rsp = netdev_bind_tx(*ys, req);
764 if (!rsp) {
765 perror("netdev_bind_tx");
766 goto err_close;
767 }
768
769 if (!rsp->_present.id) {
770 perror("id not present");
771 goto err_close;
772 }
773
774 fprintf(stderr, "got tx dmabuf id=%d\n", rsp->id);
775 tx_dmabuf_id = rsp->id;
776
777 netdev_bind_tx_req_free(req);
778 netdev_bind_tx_rsp_free(rsp);
779
780 return 0;
781
782err_close:
783 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
784 netdev_bind_tx_req_free(req);
785 ynl_sock_destroy(*ys);
786 return -1;
787}
788
789static int enable_reuseaddr(int fd)
790{
791 int opt = 1;
792 int ret;
793
794 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));
795 if (ret) {
796 pr_err("SO_REUSEPORT failed");
797 return -1;
798 }
799
800 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
801 if (ret) {
802 pr_err("SO_REUSEADDR failed");
803 return -1;
804 }
805
806 return 0;
807}
808
809static int parse_address(const char *str, int port, struct sockaddr_in6 *sin6)
810{
811 int ret;
812
813 sin6->sin6_family = AF_INET6;
814 sin6->sin6_port = htons(port);
815
816 ret = inet_pton(sin6->sin6_family, str, &sin6->sin6_addr);
817 if (ret != 1) {
818 /* fallback to plain IPv4 */
819 ret = inet_pton(AF_INET, str, &sin6->sin6_addr.s6_addr32[3]);
820 if (ret != 1)
821 return -1;
822
823 /* add ::ffff prefix */
824 sin6->sin6_addr.s6_addr32[0] = 0;
825 sin6->sin6_addr.s6_addr32[1] = 0;
826 sin6->sin6_addr.s6_addr16[4] = 0;
827 sin6->sin6_addr.s6_addr16[5] = 0xffff;
828 }
829
830 return 0;
831}
832
833static struct netdev_queue_id *create_queues(void)
834{
835 struct netdev_queue_id *queues;
836 size_t i = 0;
837
838 queues = netdev_queue_id_alloc(num_queues);
839 for (i = 0; i < num_queues; i++) {
840 netdev_queue_id_set_type(&queues[i], NETDEV_QUEUE_TYPE_RX);
841 netdev_queue_id_set_id(&queues[i], start_queue + i);
842 }
843
844 return queues;
845}
846
847static int do_server(struct memory_buffer *mem)
848{
849 struct ethtool_rings_get_rsp *ring_config = NULL;
850 char ctrl_data[sizeof(int) * 20000];
851 size_t non_page_aligned_frags = 0;
852 struct sockaddr_in6 client_addr;
853 struct sockaddr_in6 server_sin;
854 size_t page_aligned_frags = 0;
855 size_t total_received = 0;
856 socklen_t client_addr_len;
857 bool is_devmem = false;
858 char *tmp_mem = NULL;
859 struct ynl_sock *ys;
860 char iobuf[819200];
861 int ret, err = -1;
862 char buffer[256];
863 int socket_fd;
864 int client_fd;
865
866 ret = parse_address(server_ip, atoi(port), &server_sin);
867 if (ret < 0) {
868 pr_err("parse server address");
869 return -1;
870 }
871
872 if (!skip_config) {
873 ring_config = get_ring_config();
874 if (!ring_config) {
875 pr_err("Failed to get current ring configuration");
876 return -1;
877 }
878
879 if (configure_headersplit(ring_config, 1)) {
880 pr_err("Failed to enable TCP header split");
881 goto err_free_ring_config;
882 }
883
884 /* Configure RSS to divert all traffic from our devmem queues */
885 if (configure_rss()) {
886 pr_err("Failed to configure rss");
887 goto err_reset_headersplit;
888 }
889
890 /* Flow steer our devmem flows to start_queue */
891 if (configure_flow_steering(&server_sin)) {
892 pr_err("Failed to configure flow steering");
893 goto err_reset_rss;
894 }
895 }
896
897 if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys)) {
898 pr_err("Failed to bind");
899 goto err_reset_flow_steering;
900 }
901
902 tmp_mem = malloc(mem->size);
903 if (!tmp_mem)
904 goto err_unbind;
905
906 socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
907 if (socket_fd < 0) {
908 pr_err("Failed to create socket");
909 goto err_free_tmp;
910 }
911
912 if (enable_reuseaddr(socket_fd))
913 goto err_close_socket;
914
915 fprintf(stderr, "binding to address %s:%d\n", server_ip,
916 ntohs(server_sin.sin6_port));
917
918 ret = bind(socket_fd, &server_sin, sizeof(server_sin));
919 if (ret) {
920 pr_err("Failed to bind");
921 goto err_close_socket;
922 }
923
924 ret = listen(socket_fd, 1);
925 if (ret) {
926 pr_err("Failed to listen");
927 goto err_close_socket;
928 }
929
930 client_addr_len = sizeof(client_addr);
931
932 inet_ntop(AF_INET6, &server_sin.sin6_addr, buffer,
933 sizeof(buffer));
934 fprintf(stderr, "Waiting or connection on %s:%d\n", buffer,
935 ntohs(server_sin.sin6_port));
936 client_fd = accept(socket_fd, &client_addr, &client_addr_len);
937 if (client_fd < 0) {
938 pr_err("Failed to accept");
939 goto err_close_socket;
940 }
941
942 inet_ntop(AF_INET6, &client_addr.sin6_addr, buffer,
943 sizeof(buffer));
944 fprintf(stderr, "Got connection from %s:%d\n", buffer,
945 ntohs(client_addr.sin6_port));
946
947 while (1) {
948 struct iovec iov = { .iov_base = iobuf,
949 .iov_len = sizeof(iobuf) };
950 struct dmabuf_cmsg *dmabuf_cmsg = NULL;
951 struct cmsghdr *cm = NULL;
952 struct msghdr msg = { 0 };
953 struct dmabuf_token token;
954 ssize_t ret;
955
956 is_devmem = false;
957
958 msg.msg_iov = &iov;
959 msg.msg_iovlen = 1;
960 msg.msg_control = ctrl_data;
961 msg.msg_controllen = sizeof(ctrl_data);
962 ret = recvmsg(client_fd, &msg, MSG_SOCK_DEVMEM);
963 fprintf(stderr, "recvmsg ret=%ld\n", ret);
964 if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
965 continue;
966 if (ret < 0) {
967 perror("recvmsg");
968 if (errno == EFAULT) {
969 pr_err("received EFAULT, won't recover");
970 goto err_close_client;
971 }
972 continue;
973 }
974 if (ret == 0) {
975 errno = 0;
976 pr_err("client exited");
977 goto cleanup;
978 }
979
980 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
981 if (cm->cmsg_level != SOL_SOCKET ||
982 (cm->cmsg_type != SCM_DEVMEM_DMABUF &&
983 cm->cmsg_type != SCM_DEVMEM_LINEAR)) {
984 fprintf(stderr, "skipping non-devmem cmsg\n");
985 continue;
986 }
987
988 dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm);
989 is_devmem = true;
990
991 if (cm->cmsg_type == SCM_DEVMEM_LINEAR) {
992 /* TODO: process data copied from skb's linear
993 * buffer.
994 */
995 fprintf(stderr,
996 "SCM_DEVMEM_LINEAR. dmabuf_cmsg->frag_size=%u\n",
997 dmabuf_cmsg->frag_size);
998
999 if (fail_on_linear) {
1000 pr_err("received SCM_DEVMEM_LINEAR but --fail-on-linear (-L) set");
1001 goto err_close_client;
1002 }
1003
1004 continue;
1005 }
1006
1007 token.token_start = dmabuf_cmsg->frag_token;
1008 token.token_count = 1;
1009
1010 total_received += dmabuf_cmsg->frag_size;
1011 fprintf(stderr,
1012 "received frag_page=%llu, in_page_offset=%llu, frag_offset=%llu, frag_size=%u, token=%u, total_received=%lu, dmabuf_id=%u\n",
1013 dmabuf_cmsg->frag_offset >> PAGE_SHIFT,
1014 dmabuf_cmsg->frag_offset % getpagesize(),
1015 dmabuf_cmsg->frag_offset,
1016 dmabuf_cmsg->frag_size, dmabuf_cmsg->frag_token,
1017 total_received, dmabuf_cmsg->dmabuf_id);
1018
1019 if (dmabuf_cmsg->dmabuf_id != dmabuf_id) {
1020 pr_err("received on wrong dmabuf_id: flow steering error");
1021 goto err_close_client;
1022 }
1023
1024 if (dmabuf_cmsg->frag_size % getpagesize())
1025 non_page_aligned_frags++;
1026 else
1027 page_aligned_frags++;
1028
1029 provider->memcpy_from_device(tmp_mem, mem,
1030 dmabuf_cmsg->frag_offset,
1031 dmabuf_cmsg->frag_size);
1032
1033 if (do_validation) {
1034 if (validate_buffer(tmp_mem,
1035 dmabuf_cmsg->frag_size))
1036 goto err_close_client;
1037 } else {
1038 print_nonzero_bytes(tmp_mem,
1039 dmabuf_cmsg->frag_size);
1040 }
1041
1042 ret = setsockopt(client_fd, SOL_SOCKET,
1043 SO_DEVMEM_DONTNEED, &token,
1044 sizeof(token));
1045 if (ret != 1) {
1046 pr_err("SO_DEVMEM_DONTNEED not enough tokens");
1047 goto err_close_client;
1048 }
1049 }
1050 if (!is_devmem) {
1051 pr_err("flow steering error");
1052 goto err_close_client;
1053 }
1054
1055 fprintf(stderr, "total_received=%lu\n", total_received);
1056 }
1057
1058 fprintf(stderr, "%s: ok\n", TEST_PREFIX);
1059
1060 fprintf(stderr, "page_aligned_frags=%lu, non_page_aligned_frags=%lu\n",
1061 page_aligned_frags, non_page_aligned_frags);
1062
1063cleanup:
1064 err = 0;
1065
1066err_close_client:
1067 close(client_fd);
1068err_close_socket:
1069 close(socket_fd);
1070err_free_tmp:
1071 free(tmp_mem);
1072err_unbind:
1073 ynl_sock_destroy(ys);
1074err_reset_flow_steering:
1075 if (!skip_config)
1076 reset_flow_steering();
1077err_reset_rss:
1078 if (!skip_config)
1079 reset_rss();
1080err_reset_headersplit:
1081 if (!skip_config)
1082 restore_ring_config(ring_config);
1083err_free_ring_config:
1084 if (!skip_config)
1085 ethtool_rings_get_rsp_free(ring_config);
1086 return err;
1087}
1088
1089int run_devmem_tests(void)
1090{
1091 struct ethtool_rings_get_rsp *ring_config;
1092 struct netdev_queue_id *queues;
1093 struct memory_buffer *mem;
1094 struct ynl_sock *ys;
1095 int err = -1;
1096
1097 mem = provider->alloc(getpagesize() * NUM_PAGES);
1098 if (!mem) {
1099 pr_err("Failed to allocate memory buffer");
1100 return -1;
1101 }
1102
1103 ring_config = get_ring_config();
1104 if (!ring_config) {
1105 pr_err("Failed to get current ring configuration");
1106 goto err_free_mem;
1107 }
1108
1109 /* Configure RSS to divert all traffic from our devmem queues */
1110 if (configure_rss()) {
1111 pr_err("rss error");
1112 goto err_free_ring_config;
1113 }
1114
1115 if (configure_headersplit(ring_config, 1)) {
1116 pr_err("Failed to configure header split");
1117 goto err_reset_rss;
1118 }
1119
1120 queues = netdev_queue_id_alloc(num_queues);
1121 if (!queues) {
1122 pr_err("Failed to allocate empty queues array");
1123 goto err_reset_headersplit;
1124 }
1125
1126 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) {
1127 pr_err("Binding empty queues array should have failed");
1128 goto err_unbind;
1129 }
1130
1131 if (configure_headersplit(ring_config, 0)) {
1132 pr_err("Failed to configure header split");
1133 goto err_reset_headersplit;
1134 }
1135
1136 queues = create_queues();
1137 if (!queues) {
1138 pr_err("Failed to create queues");
1139 goto err_reset_headersplit;
1140 }
1141
1142 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) {
1143 pr_err("Configure dmabuf with header split off should have failed");
1144 goto err_unbind;
1145 }
1146
1147 if (configure_headersplit(ring_config, 1)) {
1148 pr_err("Failed to configure header split");
1149 goto err_reset_headersplit;
1150 }
1151
1152 queues = create_queues();
1153 if (!queues) {
1154 pr_err("Failed to create queues");
1155 goto err_reset_headersplit;
1156 }
1157
1158 if (bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) {
1159 pr_err("Failed to bind");
1160 goto err_reset_headersplit;
1161 }
1162
1163 /* Deactivating a bound queue should not be legal */
1164 if (!check_changing_channels(num_queues, num_queues)) {
1165 pr_err("Deactivating a bound queue should be illegal");
1166 goto err_unbind;
1167 }
1168
1169 err = 0;
1170 goto err_unbind;
1171
1172err_unbind:
1173 ynl_sock_destroy(ys);
1174err_reset_headersplit:
1175 restore_ring_config(ring_config);
1176err_reset_rss:
1177 reset_rss();
1178err_free_ring_config:
1179 ethtool_rings_get_rsp_free(ring_config);
1180err_free_mem:
1181 provider->free(mem);
1182 return err;
1183}
1184
1185static uint64_t gettimeofday_ms(void)
1186{
1187 struct timeval tv;
1188
1189 gettimeofday(&tv, NULL);
1190 return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000ULL);
1191}
1192
1193static int do_poll(int fd)
1194{
1195 struct pollfd pfd;
1196 int ret;
1197
1198 pfd.revents = 0;
1199 pfd.fd = fd;
1200
1201 ret = poll(&pfd, 1, waittime_ms);
1202 if (ret == -1) {
1203 pr_err("poll");
1204 return -1;
1205 }
1206
1207 return ret && (pfd.revents & POLLERR);
1208}
1209
1210static int wait_compl(int fd)
1211{
1212 int64_t tstop = gettimeofday_ms() + waittime_ms;
1213 char control[CMSG_SPACE(100)] = {};
1214 struct sock_extended_err *serr;
1215 struct msghdr msg = {};
1216 struct cmsghdr *cm;
1217 __u32 hi, lo;
1218 int ret;
1219
1220 msg.msg_control = control;
1221 msg.msg_controllen = sizeof(control);
1222
1223 while (gettimeofday_ms() < tstop) {
1224 ret = do_poll(fd);
1225 if (ret < 0)
1226 return ret;
1227 if (!ret)
1228 continue;
1229
1230 ret = recvmsg(fd, &msg, MSG_ERRQUEUE);
1231 if (ret < 0) {
1232 if (errno == EAGAIN)
1233 continue;
1234 pr_err("recvmsg(MSG_ERRQUEUE)");
1235 return -1;
1236 }
1237 if (msg.msg_flags & MSG_CTRUNC) {
1238 pr_err("MSG_CTRUNC");
1239 return -1;
1240 }
1241
1242 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
1243 if (cm->cmsg_level != SOL_IP &&
1244 cm->cmsg_level != SOL_IPV6)
1245 continue;
1246 if (cm->cmsg_level == SOL_IP &&
1247 cm->cmsg_type != IP_RECVERR)
1248 continue;
1249 if (cm->cmsg_level == SOL_IPV6 &&
1250 cm->cmsg_type != IPV6_RECVERR)
1251 continue;
1252
1253 serr = (void *)CMSG_DATA(cm);
1254 if (serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) {
1255 pr_err("wrong origin %u", serr->ee_origin);
1256 return -1;
1257 }
1258 if (serr->ee_errno != 0) {
1259 pr_err("wrong errno %d", serr->ee_errno);
1260 return -1;
1261 }
1262
1263 hi = serr->ee_data;
1264 lo = serr->ee_info;
1265
1266 fprintf(stderr, "tx complete [%d,%d]\n", lo, hi);
1267 return 0;
1268 }
1269 }
1270
1271 pr_err("did not receive tx completion");
1272 return -1;
1273}
1274
1275static int do_client(struct memory_buffer *mem)
1276{
1277 char ctrl_data[CMSG_SPACE(sizeof(__u32))];
1278 struct sockaddr_in6 server_sin;
1279 struct sockaddr_in6 client_sin;
1280 struct ynl_sock *ys = NULL;
1281 struct iovec iov[MAX_IOV];
1282 struct msghdr msg = {};
1283 ssize_t line_size = 0;
1284 struct cmsghdr *cmsg;
1285 char *line = NULL;
1286 int ret, err = -1;
1287 size_t len = 0;
1288 int socket_fd;
1289 __u32 ddmabuf;
1290 int opt = 1;
1291
1292 ret = parse_address(server_ip, atoi(port), &server_sin);
1293 if (ret < 0) {
1294 pr_err("parse server address");
1295 return -1;
1296 }
1297
1298 if (client_ip) {
1299 ret = parse_address(client_ip, atoi(port), &client_sin);
1300 if (ret < 0) {
1301 pr_err("parse client address");
1302 return ret;
1303 }
1304 }
1305
1306 socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
1307 if (socket_fd < 0) {
1308 pr_err("create socket");
1309 return -1;
1310 }
1311
1312 if (enable_reuseaddr(socket_fd))
1313 goto err_close_socket;
1314
1315 ret = setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, ifname,
1316 strlen(ifname) + 1);
1317 if (ret) {
1318 pr_err("bindtodevice");
1319 goto err_close_socket;
1320 }
1321
1322 if (bind_tx_queue(ifindex, mem->fd, &ys)) {
1323 pr_err("Failed to bind");
1324 goto err_close_socket;
1325 }
1326
1327 if (client_ip) {
1328 ret = bind(socket_fd, &client_sin, sizeof(client_sin));
1329 if (ret) {
1330 pr_err("bind");
1331 goto err_unbind;
1332 }
1333 }
1334
1335 ret = setsockopt(socket_fd, SOL_SOCKET, SO_ZEROCOPY, &opt, sizeof(opt));
1336 if (ret) {
1337 pr_err("set sock opt");
1338 goto err_unbind;
1339 }
1340
1341 fprintf(stderr, "Connect to %s %d (via %s)\n", server_ip,
1342 ntohs(server_sin.sin6_port), ifname);
1343
1344 ret = connect(socket_fd, &server_sin, sizeof(server_sin));
1345 if (ret) {
1346 pr_err("connect");
1347 goto err_unbind;
1348 }
1349
1350 while (1) {
1351 free(line);
1352 line = NULL;
1353 line_size = getline(&line, &len, stdin);
1354
1355 if (line_size < 0)
1356 break;
1357
1358 if (max_chunk) {
1359 msg.msg_iovlen =
1360 (line_size + max_chunk - 1) / max_chunk;
1361 if (msg.msg_iovlen > MAX_IOV) {
1362 pr_err("can't partition %zd bytes into maximum of %d chunks",
1363 line_size, MAX_IOV);
1364 goto err_free_line;
1365 }
1366
1367 for (int i = 0; i < msg.msg_iovlen; i++) {
1368 iov[i].iov_base = (void *)(i * max_chunk);
1369 iov[i].iov_len = max_chunk;
1370 }
1371
1372 iov[msg.msg_iovlen - 1].iov_len =
1373 line_size - (msg.msg_iovlen - 1) * max_chunk;
1374 } else {
1375 iov[0].iov_base = 0;
1376 iov[0].iov_len = line_size;
1377 msg.msg_iovlen = 1;
1378 }
1379
1380 msg.msg_iov = iov;
1381 provider->memcpy_to_device(mem, 0, line, line_size);
1382
1383 msg.msg_control = ctrl_data;
1384 msg.msg_controllen = sizeof(ctrl_data);
1385
1386 cmsg = CMSG_FIRSTHDR(&msg);
1387 cmsg->cmsg_level = SOL_SOCKET;
1388 cmsg->cmsg_type = SCM_DEVMEM_DMABUF;
1389 cmsg->cmsg_len = CMSG_LEN(sizeof(__u32));
1390
1391 ddmabuf = tx_dmabuf_id;
1392
1393 *((__u32 *)CMSG_DATA(cmsg)) = ddmabuf;
1394
1395 ret = sendmsg(socket_fd, &msg, MSG_ZEROCOPY);
1396 if (ret < 0) {
1397 pr_err("Failed sendmsg");
1398 goto err_free_line;
1399 }
1400
1401 fprintf(stderr, "sendmsg_ret=%d\n", ret);
1402
1403 if (ret != line_size) {
1404 pr_err("Did not send all bytes %d vs %zd", ret, line_size);
1405 goto err_free_line;
1406 }
1407
1408 if (wait_compl(socket_fd))
1409 goto err_free_line;
1410 }
1411
1412 fprintf(stderr, "%s: tx ok\n", TEST_PREFIX);
1413
1414 err = 0;
1415
1416err_free_line:
1417 free(line);
1418err_unbind:
1419 ynl_sock_destroy(ys);
1420err_close_socket:
1421 close(socket_fd);
1422 return err;
1423}
1424
1425int main(int argc, char *argv[])
1426{
1427 struct memory_buffer *mem;
1428 int is_server = 0, opt;
1429 int ret, err = 1;
1430
1431 while ((opt = getopt(argc, argv, "Lls:c:p:v:q:t:f:z:nb:")) != -1) {
1432 switch (opt) {
1433 case 'L':
1434 fail_on_linear = true;
1435 break;
1436 case 'l':
1437 is_server = 1;
1438 break;
1439 case 's':
1440 server_ip = optarg;
1441 break;
1442 case 'c':
1443 client_ip = optarg;
1444 break;
1445 case 'p':
1446 port = optarg;
1447 break;
1448 case 'v':
1449 do_validation = atoll(optarg);
1450 break;
1451 case 'q':
1452 num_queues = atoi(optarg);
1453 break;
1454 case 't':
1455 start_queue = atoi(optarg);
1456 break;
1457 case 'f':
1458 ifname = optarg;
1459 break;
1460 case 'z':
1461 max_chunk = atoi(optarg);
1462 break;
1463 case 'n':
1464 skip_config = 1;
1465 break;
1466 case 'b': {
1467 unsigned long val;
1468
1469 errno = 0;
1470 val = strtoul(optarg, NULL, 0);
1471 if ((val == ULONG_MAX && errno == ERANGE) ||
1472 val > UINT32_MAX) {
1473 pr_err("invalid rx_page_size: %s", optarg);
1474 return 1;
1475 }
1476 rx_page_size = val;
1477 break;
1478 }
1479 case '?':
1480 fprintf(stderr, "unknown option: %c\n", optopt);
1481 break;
1482 }
1483 }
1484
1485 if (!ifname) {
1486 pr_err("Missing -f argument");
1487 return 1;
1488 }
1489
1490 ifindex = if_nametoindex(ifname);
1491
1492 fprintf(stderr, "using ifindex=%u\n", ifindex);
1493
1494 if (!server_ip && !client_ip) {
1495 if (start_queue < 0 && num_queues < 0) {
1496 num_queues = rxq_num(ifindex);
1497 if (num_queues < 0) {
1498 pr_err("couldn't detect number of queues");
1499 return 1;
1500 }
1501 if (num_queues < 2) {
1502 pr_err("number of device queues is too low");
1503 return 1;
1504 }
1505 /* make sure can bind to multiple queues */
1506 start_queue = num_queues / 2;
1507 num_queues /= 2;
1508 }
1509
1510 if (start_queue < 0 || num_queues < 0) {
1511 pr_err("Both -t and -q are required");
1512 return 1;
1513 }
1514
1515 return run_devmem_tests();
1516 }
1517
1518 if (start_queue < 0 && num_queues < 0) {
1519 num_queues = rxq_num(ifindex);
1520 if (num_queues < 2) {
1521 pr_err("number of device queues is too low");
1522 return 1;
1523 }
1524
1525 num_queues = 1;
1526 start_queue = rxq_num(ifindex) - num_queues;
1527
1528 if (start_queue < 0) {
1529 pr_err("couldn't detect number of queues");
1530 return 1;
1531 }
1532
1533 fprintf(stderr, "using queues %d..%d\n", start_queue, start_queue + num_queues);
1534 }
1535
1536 for (; optind < argc; optind++)
1537 fprintf(stderr, "extra arguments: %s\n", argv[optind]);
1538
1539 if (start_queue < 0) {
1540 pr_err("Missing -t argument");
1541 return 1;
1542 }
1543
1544 if (num_queues < 0) {
1545 pr_err("Missing -q argument");
1546 return 1;
1547 }
1548
1549 if (!server_ip) {
1550 pr_err("Missing -s argument");
1551 return 1;
1552 }
1553
1554 if (!port) {
1555 pr_err("Missing -p argument");
1556 return 1;
1557 }
1558
1559 mem = provider->alloc(getpagesize() * NUM_PAGES);
1560 if (!mem) {
1561 pr_err("Failed to allocate memory buffer");
1562 return 1;
1563 }
1564
1565 ret = is_server ? do_server(mem) : do_client(mem);
1566 if (ret)
1567 goto err_free_mem;
1568
1569 err = 0;
1570
1571err_free_mem:
1572 provider->free(mem);
1573 return err;
1574}