| 1 | |
| 2 | |
| 3 | #include <stdio.h>
|
| 4 | #include <stdlib.h>
|
| 5 | #include <sys/socket.h>
|
| 6 | #include <sys/ioctl.h>
|
| 7 | #include <sys/select.h>
|
| 8 | #include <netinet/in.h>
|
| 9 | #include <arpa/inet.h>
|
| 10 | #include <unistd.h>
|
| 11 | #include <string.h>
|
| 12 | #include <errno.h>
|
| 13 | #include <stdbool.h>
|
| 14 | #include <signal.h>
|
| 15 | #include <fcntl.h>
|
| 16 | #include <sys/wait.h>
|
| 17 | #include <time.h>
|
| 18 | #include <sched.h>
|
| 19 |
|
| 20 | #include <sys/time.h>
|
| 21 | #include <sys/types.h>
|
| 22 | #include <sys/sendfile.h>
|
| 23 |
|
| 24 | #include <linux/netlink.h>
|
| 25 | #include <linux/socket.h>
|
| 26 | #include <linux/sock_diag.h>
|
| 27 | #include <linux/bpf.h>
|
| 28 | #include <linux/if_link.h>
|
| 29 | #include <assert.h>
|
| 30 | #include <libgen.h>
|
| 31 |
|
| 32 | #include <getopt.h>
|
| 33 |
|
| 34 | #include <bpf/bpf.h>
|
| 35 | #include <bpf/libbpf.h>
|
| 36 |
|
| 37 | #include "bpf_util.h"
|
| 38 | #include "cgroup_helpers.h"
|
| 39 |
|
| 40 | int running;
|
| 41 | static void running_handler(int a);
|
| 42 |
|
| 43 |
|
| 44 | #define S1_PORT 10000
|
| 45 | #define S2_PORT 10001
|
| 46 |
|
| 47 | #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"
|
| 48 | #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
|
| 49 | #define CG_PATH "/sockmap"
|
| 50 |
|
| 51 | #define EDATAINTEGRITY 2001
|
| 52 |
|
| 53 |
|
| 54 | int s1, s2, c1, c2, p1, p2;
|
| 55 | int test_cnt;
|
| 56 | int passed;
|
| 57 | int failed;
|
| 58 | int map_fd[8];
|
| 59 | struct bpf_map *maps[8];
|
| 60 | struct bpf_program *progs[8];
|
| 61 | struct bpf_link *links[8];
|
| 62 |
|
| 63 | int txmsg_pass;
|
| 64 | int txmsg_redir;
|
| 65 | int txmsg_drop;
|
| 66 | int txmsg_apply;
|
| 67 | int txmsg_cork;
|
| 68 | int txmsg_start;
|
| 69 | int txmsg_end;
|
| 70 | int txmsg_start_push;
|
| 71 | int txmsg_end_push;
|
| 72 | int txmsg_start_pop;
|
| 73 | int txmsg_pop;
|
| 74 | int txmsg_ingress;
|
| 75 | int txmsg_redir_skb;
|
| 76 | int peek_flag;
|
| 77 | int skb_use_parser;
|
| 78 | int txmsg_omit_skb_parser;
|
| 79 | int verify_push_start;
|
| 80 | int verify_push_len;
|
| 81 | int verify_pop_start;
|
| 82 | int verify_pop_len;
|
| 83 |
|
| 84 | static const struct option long_options[] = {
|
| 85 | {"help", no_argument, NULL, 'h' },
|
| 86 | {"cgroup", required_argument, NULL, 'c' },
|
| 87 | {"rate", required_argument, NULL, 'r' },
|
| 88 | {"verbose", optional_argument, NULL, 'v' },
|
| 89 | {"iov_count", required_argument, NULL, 'i' },
|
| 90 | {"length", required_argument, NULL, 'l' },
|
| 91 | {"test", required_argument, NULL, 't' },
|
| 92 | {"data_test", no_argument, NULL, 'd' },
|
| 93 | {"txmsg", no_argument, &txmsg_pass, 1 },
|
| 94 | {"txmsg_redir", no_argument, &txmsg_redir, 1 },
|
| 95 | {"txmsg_drop", no_argument, &txmsg_drop, 1 },
|
| 96 | {"txmsg_apply", required_argument, NULL, 'a'},
|
| 97 | {"txmsg_cork", required_argument, NULL, 'k'},
|
| 98 | {"txmsg_start", required_argument, NULL, 's'},
|
| 99 | {"txmsg_end", required_argument, NULL, 'e'},
|
| 100 | {"txmsg_start_push", required_argument, NULL, 'p'},
|
| 101 | {"txmsg_end_push", required_argument, NULL, 'q'},
|
| 102 | {"txmsg_start_pop", required_argument, NULL, 'w'},
|
| 103 | {"txmsg_pop", required_argument, NULL, 'x'},
|
| 104 | {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
|
| 105 | {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
|
| 106 | {"peek", no_argument, &peek_flag, 1 },
|
| 107 | {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
|
| 108 | {"whitelist", required_argument, NULL, 'n' },
|
| 109 | {"blacklist", required_argument, NULL, 'b' },
|
| 110 | {0, 0, NULL, 0 }
|
| 111 | };
|
| 112 |
|
| 113 | struct test_env {
|
| 114 | const char *type;
|
| 115 | const char *subtest;
|
| 116 | const char *prepend;
|
| 117 |
|
| 118 | int test_num;
|
| 119 | int subtest_num;
|
| 120 |
|
| 121 | int succ_cnt;
|
| 122 | int fail_cnt;
|
| 123 | int fail_last;
|
| 124 | };
|
| 125 |
|
| 126 | struct test_env env;
|
| 127 |
|
| 128 | struct sockmap_options {
|
| 129 | int verbose;
|
| 130 | bool base;
|
| 131 | bool sendpage;
|
| 132 | bool data_test;
|
| 133 | bool drop_expected;
|
| 134 | bool check_recved_len;
|
| 135 | bool tx_wait_mem;
|
| 136 | int iov_count;
|
| 137 | int iov_length;
|
| 138 | int rate;
|
| 139 | char *map;
|
| 140 | char *whitelist;
|
| 141 | char *blacklist;
|
| 142 | char *prepend;
|
| 143 | };
|
| 144 |
|
| 145 | struct _test {
|
| 146 | char *title;
|
| 147 | void (*tester)(int cg_fd, struct sockmap_options *opt);
|
| 148 | };
|
| 149 |
|
| 150 | static void test_start(void)
|
| 151 | {
|
| 152 | env.subtest_num++;
|
| 153 | }
|
| 154 |
|
| 155 | static void test_fail(void)
|
| 156 | {
|
| 157 | env.fail_cnt++;
|
| 158 | }
|
| 159 |
|
| 160 | static void test_pass(void)
|
| 161 | {
|
| 162 | env.succ_cnt++;
|
| 163 | }
|
| 164 |
|
| 165 | static void test_reset(void)
|
| 166 | {
|
| 167 | txmsg_start = txmsg_end = 0;
|
| 168 | txmsg_start_pop = txmsg_pop = 0;
|
| 169 | txmsg_start_push = txmsg_end_push = 0;
|
| 170 | txmsg_pass = txmsg_drop = txmsg_redir = 0;
|
| 171 | txmsg_apply = txmsg_cork = 0;
|
| 172 | txmsg_ingress = txmsg_redir_skb = 0;
|
| 173 | txmsg_omit_skb_parser = 0;
|
| 174 | skb_use_parser = 0;
|
| 175 | }
|
| 176 |
|
| 177 | static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
|
| 178 | {
|
| 179 | env.type = o->map;
|
| 180 | env.subtest = t->title;
|
| 181 | env.prepend = o->prepend;
|
| 182 | env.test_num++;
|
| 183 | env.subtest_num = 0;
|
| 184 | env.fail_last = env.fail_cnt;
|
| 185 | test_reset();
|
| 186 | return 0;
|
| 187 | }
|
| 188 |
|
| 189 | static void test_end_subtest(void)
|
| 190 | {
|
| 191 | int error = env.fail_cnt - env.fail_last;
|
| 192 | int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
|
| 193 |
|
| 194 | if (!error)
|
| 195 | test_pass();
|
| 196 |
|
| 197 | fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
|
| 198 | env.test_num, env.subtest_num,
|
| 199 | !type ? "sockmap" : "sockhash",
|
| 200 | env.prepend ? : "",
|
| 201 | env.subtest, error ? "FAIL" : "OK");
|
| 202 | }
|
| 203 |
|
| 204 | static void test_print_results(void)
|
| 205 | {
|
| 206 | fprintf(stdout, "Pass: %d Fail: %d\n",
|
| 207 | env.succ_cnt, env.fail_cnt);
|
| 208 | }
|
| 209 |
|
| 210 | static void usage(char *argv[])
|
| 211 | {
|
| 212 | int i;
|
| 213 |
|
| 214 | printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
|
| 215 | printf(" options:\n");
|
| 216 | for (i = 0; long_options[i].name != 0; i++) {
|
| 217 | printf(" --%-12s", long_options[i].name);
|
| 218 | if (long_options[i].flag != NULL)
|
| 219 | printf(" flag (internal value:%d)\n",
|
| 220 | *long_options[i].flag);
|
| 221 | else
|
| 222 | printf(" -%c\n", long_options[i].val);
|
| 223 | }
|
| 224 | printf("\n");
|
| 225 | }
|
| 226 |
|
| 227 | static int sockmap_init_sockets(int verbose)
|
| 228 | {
|
| 229 | int i, err, one = 1;
|
| 230 | struct sockaddr_in addr;
|
| 231 | int *fds[4] = {&s1, &s2, &c1, &c2};
|
| 232 |
|
| 233 | s1 = s2 = p1 = p2 = c1 = c2 = 0;
|
| 234 |
|
| 235 |
|
| 236 | for (i = 0; i < 4; i++) {
|
| 237 | *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
|
| 238 | if (*fds[i] < 0) {
|
| 239 | perror("socket s1 failed()");
|
| 240 | return errno;
|
| 241 | }
|
| 242 | }
|
| 243 |
|
| 244 |
|
| 245 | for (i = 0; i < 2; i++) {
|
| 246 | err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
|
| 247 | (char *)&one, sizeof(one));
|
| 248 | if (err) {
|
| 249 | perror("setsockopt failed()");
|
| 250 | return errno;
|
| 251 | }
|
| 252 | }
|
| 253 |
|
| 254 |
|
| 255 | for (i = 0; i < 2; i++) {
|
| 256 | err = ioctl(*fds[i], FIONBIO, (char *)&one);
|
| 257 | if (err < 0) {
|
| 258 | perror("ioctl s1 failed()");
|
| 259 | return errno;
|
| 260 | }
|
| 261 | }
|
| 262 |
|
| 263 |
|
| 264 | memset(&addr, 0, sizeof(struct sockaddr_in));
|
| 265 | addr.sin_family = AF_INET;
|
| 266 | addr.sin_addr.s_addr = inet_addr("127.0.0.1");
|
| 267 |
|
| 268 | addr.sin_port = htons(S1_PORT);
|
| 269 | err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
|
| 270 | if (err < 0) {
|
| 271 | perror("bind s1 failed()");
|
| 272 | return errno;
|
| 273 | }
|
| 274 |
|
| 275 | addr.sin_port = htons(S2_PORT);
|
| 276 | err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
|
| 277 | if (err < 0) {
|
| 278 | perror("bind s2 failed()");
|
| 279 | return errno;
|
| 280 | }
|
| 281 |
|
| 282 |
|
| 283 | addr.sin_port = htons(S1_PORT);
|
| 284 | err = listen(s1, 32);
|
| 285 | if (err < 0) {
|
| 286 | perror("listen s1 failed()");
|
| 287 | return errno;
|
| 288 | }
|
| 289 |
|
| 290 | addr.sin_port = htons(S2_PORT);
|
| 291 | err = listen(s2, 32);
|
| 292 | if (err < 0) {
|
| 293 | perror("listen s1 failed()");
|
| 294 | return errno;
|
| 295 | }
|
| 296 |
|
| 297 |
|
| 298 | addr.sin_port = htons(S1_PORT);
|
| 299 | err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
|
| 300 | if (err < 0 && errno != EINPROGRESS) {
|
| 301 | perror("connect c1 failed()");
|
| 302 | return errno;
|
| 303 | }
|
| 304 |
|
| 305 | addr.sin_port = htons(S2_PORT);
|
| 306 | err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
|
| 307 | if (err < 0 && errno != EINPROGRESS) {
|
| 308 | perror("connect c2 failed()");
|
| 309 | return errno;
|
| 310 | } else if (err < 0) {
|
| 311 | err = 0;
|
| 312 | }
|
| 313 |
|
| 314 |
|
| 315 | p1 = accept(s1, NULL, NULL);
|
| 316 | if (p1 < 0) {
|
| 317 | perror("accept s1 failed()");
|
| 318 | return errno;
|
| 319 | }
|
| 320 |
|
| 321 | p2 = accept(s2, NULL, NULL);
|
| 322 | if (p2 < 0) {
|
| 323 | perror("accept s1 failed()");
|
| 324 | return errno;
|
| 325 | }
|
| 326 |
|
| 327 | if (verbose > 1) {
|
| 328 | printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
|
| 329 | printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
|
| 330 | c1, s1, c2, s2);
|
| 331 | }
|
| 332 | return 0;
|
| 333 | }
|
| 334 |
|
| 335 | struct msg_stats {
|
| 336 | size_t bytes_sent;
|
| 337 | size_t bytes_recvd;
|
| 338 | struct timespec start;
|
| 339 | struct timespec end;
|
| 340 | };
|
| 341 |
|
| 342 | static int msg_loop_sendpage(int fd, int iov_length, int cnt,
|
| 343 | struct msg_stats *s,
|
| 344 | struct sockmap_options *opt)
|
| 345 | {
|
| 346 | bool drop = opt->drop_expected;
|
| 347 | unsigned char k = 0;
|
| 348 | int i, j, fp;
|
| 349 | FILE *file;
|
| 350 |
|
| 351 | file = tmpfile();
|
| 352 | if (!file) {
|
| 353 | perror("create file for sendpage");
|
| 354 | return 1;
|
| 355 | }
|
| 356 | for (i = 0; i < cnt; i++, k = 0) {
|
| 357 | for (j = 0; j < iov_length; j++, k++)
|
| 358 | fwrite(&k, sizeof(char), 1, file);
|
| 359 | }
|
| 360 | fflush(file);
|
| 361 | fseek(file, 0, SEEK_SET);
|
| 362 |
|
| 363 | fp = fileno(file);
|
| 364 |
|
| 365 | clock_gettime(CLOCK_MONOTONIC, &s->start);
|
| 366 | for (i = 0; i < cnt; i++) {
|
| 367 | int sent;
|
| 368 |
|
| 369 | errno = 0;
|
| 370 | sent = sendfile(fd, fp, NULL, iov_length);
|
| 371 |
|
| 372 | if (!drop && sent < 0) {
|
| 373 | perror("sendpage loop error");
|
| 374 | fclose(file);
|
| 375 | return sent;
|
| 376 | } else if (drop && sent >= 0) {
|
| 377 | printf("sendpage loop error expected: %i errno %i\n",
|
| 378 | sent, errno);
|
| 379 | fclose(file);
|
| 380 | return -EIO;
|
| 381 | }
|
| 382 |
|
| 383 | if (sent > 0)
|
| 384 | s->bytes_sent += sent;
|
| 385 | }
|
| 386 | clock_gettime(CLOCK_MONOTONIC, &s->end);
|
| 387 | fclose(file);
|
| 388 | return 0;
|
| 389 | }
|
| 390 |
|
| 391 | static void msg_free_iov(struct msghdr *msg)
|
| 392 | {
|
| 393 | int i;
|
| 394 |
|
| 395 | for (i = 0; i < msg->msg_iovlen; i++)
|
| 396 | free(msg->msg_iov[i].iov_base);
|
| 397 | free(msg->msg_iov);
|
| 398 | msg->msg_iov = NULL;
|
| 399 | msg->msg_iovlen = 0;
|
| 400 | }
|
| 401 |
|
| 402 | static int msg_alloc_iov(struct msghdr *msg,
|
| 403 | int iov_count, int iov_length,
|
| 404 | bool data, bool xmit)
|
| 405 | {
|
| 406 | unsigned char k = 0;
|
| 407 | struct iovec *iov;
|
| 408 | int i;
|
| 409 |
|
| 410 | iov = calloc(iov_count, sizeof(struct iovec));
|
| 411 | if (!iov)
|
| 412 | return errno;
|
| 413 |
|
| 414 | for (i = 0; i < iov_count; i++) {
|
| 415 | unsigned char *d = calloc(iov_length, sizeof(char));
|
| 416 |
|
| 417 | if (!d) {
|
| 418 | fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
|
| 419 | goto unwind_iov;
|
| 420 | }
|
| 421 | iov[i].iov_base = d;
|
| 422 | iov[i].iov_len = iov_length;
|
| 423 |
|
| 424 | if (data && xmit) {
|
| 425 | int j;
|
| 426 |
|
| 427 | for (j = 0; j < iov_length; j++)
|
| 428 | d[j] = k++;
|
| 429 | }
|
| 430 | }
|
| 431 |
|
| 432 | msg->msg_iov = iov;
|
| 433 | msg->msg_iovlen = iov_count;
|
| 434 |
|
| 435 | return 0;
|
| 436 | unwind_iov:
|
| 437 | for (i--; i >= 0 ; i--)
|
| 438 | free(iov[i].iov_base);
|
| 439 | free(iov);
|
| 440 | return -ENOMEM;
|
| 441 | }
|
| 442 |
|
| 443 |
|
| 444 | static void msg_verify_date_prep(void)
|
| 445 | {
|
| 446 | int push_range_end = txmsg_start_push + txmsg_end_push - 1;
|
| 447 | int pop_range_end = txmsg_start_pop + txmsg_pop - 1;
|
| 448 |
|
| 449 | if (txmsg_end_push && txmsg_pop &&
|
| 450 | txmsg_start_push <= pop_range_end && txmsg_start_pop <= push_range_end) {
|
| 451 |
|
| 452 | int overlap_len;
|
| 453 |
|
| 454 | verify_push_start = txmsg_start_push;
|
| 455 | verify_pop_start = txmsg_start_pop;
|
| 456 | if (txmsg_start_push < txmsg_start_pop)
|
| 457 | overlap_len = min(push_range_end - txmsg_start_pop + 1, txmsg_pop);
|
| 458 | else
|
| 459 | overlap_len = min(pop_range_end - txmsg_start_push + 1, txmsg_end_push);
|
| 460 | verify_push_len = max(txmsg_end_push - overlap_len, 0);
|
| 461 | verify_pop_len = max(txmsg_pop - overlap_len, 0);
|
| 462 | } else {
|
| 463 |
|
| 464 | verify_push_start = txmsg_start_push;
|
| 465 | verify_pop_start = txmsg_start_pop;
|
| 466 | verify_push_len = txmsg_end_push;
|
| 467 | verify_pop_len = txmsg_pop;
|
| 468 | }
|
| 469 | }
|
| 470 |
|
| 471 | static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz,
|
| 472 | unsigned char *k_p, int *bytes_cnt_p,
|
| 473 | int *check_cnt_p, int *push_p)
|
| 474 | {
|
| 475 | int bytes_cnt = *bytes_cnt_p, check_cnt = *check_cnt_p, push = *push_p;
|
| 476 | unsigned char k = *k_p;
|
| 477 | int i, j;
|
| 478 |
|
| 479 | for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) {
|
| 480 | unsigned char *d = msg->msg_iov[i].iov_base;
|
| 481 |
|
| 482 | for (; j < msg->msg_iov[i].iov_len && size; j++) {
|
| 483 | if (push > 0 &&
|
| 484 | check_cnt == verify_push_start + verify_push_len - push) {
|
| 485 | int skipped;
|
| 486 | revisit_push:
|
| 487 | skipped = push;
|
| 488 | if (j + push >= msg->msg_iov[i].iov_len)
|
| 489 | skipped = msg->msg_iov[i].iov_len - j;
|
| 490 | push -= skipped;
|
| 491 | size -= skipped;
|
| 492 | j += skipped - 1;
|
| 493 | check_cnt += skipped;
|
| 494 | continue;
|
| 495 | }
|
| 496 |
|
| 497 | if (verify_pop_len > 0 && check_cnt == verify_pop_start) {
|
| 498 | bytes_cnt += verify_pop_len;
|
| 499 | check_cnt += verify_pop_len;
|
| 500 | k += verify_pop_len;
|
| 501 |
|
| 502 | if (bytes_cnt == chunk_sz) {
|
| 503 | k = 0;
|
| 504 | bytes_cnt = 0;
|
| 505 | check_cnt = 0;
|
| 506 | push = verify_push_len;
|
| 507 | }
|
| 508 |
|
| 509 | if (push > 0 &&
|
| 510 | check_cnt == verify_push_start + verify_push_len - push)
|
| 511 | goto revisit_push;
|
| 512 | }
|
| 513 |
|
| 514 | if (d[j] != k++) {
|
| 515 | fprintf(stderr,
|
| 516 | "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
|
| 517 | i, j, d[j], k - 1, d[j+1], k);
|
| 518 | return -EDATAINTEGRITY;
|
| 519 | }
|
| 520 | bytes_cnt++;
|
| 521 | check_cnt++;
|
| 522 | if (bytes_cnt == chunk_sz) {
|
| 523 | k = 0;
|
| 524 | bytes_cnt = 0;
|
| 525 | check_cnt = 0;
|
| 526 | push = verify_push_len;
|
| 527 | }
|
| 528 | size--;
|
| 529 | }
|
| 530 | }
|
| 531 | *k_p = k;
|
| 532 | *bytes_cnt_p = bytes_cnt;
|
| 533 | *check_cnt_p = check_cnt;
|
| 534 | *push_p = push;
|
| 535 | return 0;
|
| 536 | }
|
| 537 |
|
| 538 | static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
|
| 539 | struct msg_stats *s, bool tx,
|
| 540 | struct sockmap_options *opt)
|
| 541 | {
|
| 542 | struct msghdr msg = {0}, msg_peek = {0};
|
| 543 | int err, i, flags = MSG_NOSIGNAL;
|
| 544 | bool drop = opt->drop_expected;
|
| 545 | bool data = opt->data_test;
|
| 546 | int iov_alloc_length = iov_length;
|
| 547 |
|
| 548 | if (!tx && opt->check_recved_len)
|
| 549 | iov_alloc_length *= 2;
|
| 550 |
|
| 551 | err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
|
| 552 | if (err)
|
| 553 | goto out_errno;
|
| 554 | if (peek_flag) {
|
| 555 | err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
|
| 556 | if (err)
|
| 557 | goto out_errno;
|
| 558 | }
|
| 559 |
|
| 560 | if (tx) {
|
| 561 | clock_gettime(CLOCK_MONOTONIC, &s->start);
|
| 562 | for (i = 0; i < cnt; i++) {
|
| 563 | int sent;
|
| 564 |
|
| 565 | errno = 0;
|
| 566 | sent = sendmsg(fd, &msg, flags);
|
| 567 |
|
| 568 | if (!drop && sent < 0) {
|
| 569 | if (opt->tx_wait_mem && errno == EACCES) {
|
| 570 | errno = 0;
|
| 571 | goto out_errno;
|
| 572 | }
|
| 573 | perror("sendmsg loop error");
|
| 574 | goto out_errno;
|
| 575 | } else if (drop && sent >= 0) {
|
| 576 | fprintf(stderr,
|
| 577 | "sendmsg loop error expected: %i errno %i\n",
|
| 578 | sent, errno);
|
| 579 | errno = -EIO;
|
| 580 | goto out_errno;
|
| 581 | }
|
| 582 | if (sent > 0)
|
| 583 | s->bytes_sent += sent;
|
| 584 | }
|
| 585 | clock_gettime(CLOCK_MONOTONIC, &s->end);
|
| 586 | } else {
|
| 587 | float total_bytes, txmsg_pop_total, txmsg_push_total;
|
| 588 | int slct, recvp = 0, recv, max_fd = fd;
|
| 589 | int fd_flags = O_NONBLOCK;
|
| 590 | struct timeval timeout;
|
| 591 | unsigned char k = 0;
|
| 592 | int bytes_cnt = 0;
|
| 593 | int check_cnt = 0;
|
| 594 | int push = 0;
|
| 595 | fd_set w;
|
| 596 |
|
| 597 | fcntl(fd, fd_flags);
|
| 598 | |
| 599 | * call msg_pop_data helper so we need to account for this
|
| 600 | * by calculating the number of apply iterations. Note user
|
| 601 | * of the tool can create cases where no data is sent by
|
| 602 | * manipulating pop/push/pull/etc. For example txmsg_apply 1
|
| 603 | * with txmsg_pop 1 will try to apply 1B at a time but each
|
| 604 | * iteration will then pop 1B so no data will ever be sent.
|
| 605 | * This is really only useful for testing edge cases in code
|
| 606 | * paths.
|
| 607 |
|
| 608 | total_bytes = (float)iov_length * (float)cnt;
|
| 609 | if (!opt->sendpage)
|
| 610 | total_bytes *= (float)iov_count;
|
| 611 | if (txmsg_apply) {
|
| 612 | txmsg_push_total = txmsg_end_push * (total_bytes / txmsg_apply);
|
| 613 | txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
|
| 614 | } else {
|
| 615 | txmsg_push_total = txmsg_end_push * cnt;
|
| 616 | txmsg_pop_total = txmsg_pop * cnt;
|
| 617 | }
|
| 618 | total_bytes += txmsg_push_total;
|
| 619 | total_bytes -= txmsg_pop_total;
|
| 620 | if (data) {
|
| 621 | msg_verify_date_prep();
|
| 622 | push = verify_push_len;
|
| 623 | }
|
| 624 | err = clock_gettime(CLOCK_MONOTONIC, &s->start);
|
| 625 | if (err < 0)
|
| 626 | perror("recv start time");
|
| 627 | while (s->bytes_recvd < total_bytes) {
|
| 628 | if (txmsg_cork) {
|
| 629 | timeout.tv_sec = 0;
|
| 630 | timeout.tv_usec = 300000;
|
| 631 | } else {
|
| 632 | timeout.tv_sec = 3;
|
| 633 | timeout.tv_usec = 0;
|
| 634 | }
|
| 635 |
|
| 636 |
|
| 637 | FD_ZERO(&w);
|
| 638 | FD_SET(fd, &w);
|
| 639 |
|
| 640 | slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
|
| 641 | if (slct == -1) {
|
| 642 | perror("select()");
|
| 643 | clock_gettime(CLOCK_MONOTONIC, &s->end);
|
| 644 | goto out_errno;
|
| 645 | } else if (!slct) {
|
| 646 | if (opt->verbose)
|
| 647 | fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
|
| 648 | errno = -EIO;
|
| 649 | clock_gettime(CLOCK_MONOTONIC, &s->end);
|
| 650 | goto out_errno;
|
| 651 | }
|
| 652 |
|
| 653 | if (opt->tx_wait_mem) {
|
| 654 | FD_ZERO(&w);
|
| 655 | FD_SET(fd, &w);
|
| 656 | slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
|
| 657 | errno = 0;
|
| 658 | close(fd);
|
| 659 | goto out_errno;
|
| 660 | }
|
| 661 |
|
| 662 | errno = 0;
|
| 663 | if (peek_flag) {
|
| 664 | flags |= MSG_PEEK;
|
| 665 | recvp = recvmsg(fd, &msg_peek, flags);
|
| 666 | if (recvp < 0) {
|
| 667 | if (errno != EWOULDBLOCK) {
|
| 668 | clock_gettime(CLOCK_MONOTONIC, &s->end);
|
| 669 | goto out_errno;
|
| 670 | }
|
| 671 | }
|
| 672 | flags = 0;
|
| 673 | }
|
| 674 |
|
| 675 | recv = recvmsg(fd, &msg, flags);
|
| 676 | if (recv < 0) {
|
| 677 | if (errno != EWOULDBLOCK) {
|
| 678 | clock_gettime(CLOCK_MONOTONIC, &s->end);
|
| 679 | perror("recv failed()");
|
| 680 | goto out_errno;
|
| 681 | }
|
| 682 | }
|
| 683 |
|
| 684 | if (recv > 0)
|
| 685 | s->bytes_recvd += recv;
|
| 686 |
|
| 687 | if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
|
| 688 | errno = EMSGSIZE;
|
| 689 | fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
|
| 690 | s->bytes_recvd, total_bytes);
|
| 691 | goto out_errno;
|
| 692 | }
|
| 693 |
|
| 694 | if (data) {
|
| 695 | int chunk_sz = opt->sendpage ?
|
| 696 | iov_length :
|
| 697 | iov_length * iov_count;
|
| 698 |
|
| 699 | errno = msg_verify_data(&msg, recv, chunk_sz, &k, &bytes_cnt,
|
| 700 | &check_cnt, &push);
|
| 701 | if (errno) {
|
| 702 | perror("data verify msg failed");
|
| 703 | goto out_errno;
|
| 704 | }
|
| 705 | if (recvp) {
|
| 706 | errno = msg_verify_data(&msg_peek,
|
| 707 | recvp,
|
| 708 | chunk_sz,
|
| 709 | &k,
|
| 710 | &bytes_cnt,
|
| 711 | &check_cnt,
|
| 712 | &push);
|
| 713 | if (errno) {
|
| 714 | perror("data verify msg_peek failed");
|
| 715 | goto out_errno;
|
| 716 | }
|
| 717 | }
|
| 718 | }
|
| 719 | }
|
| 720 | clock_gettime(CLOCK_MONOTONIC, &s->end);
|
| 721 | }
|
| 722 |
|
| 723 | msg_free_iov(&msg);
|
| 724 | msg_free_iov(&msg_peek);
|
| 725 | return err;
|
| 726 | out_errno:
|
| 727 | msg_free_iov(&msg);
|
| 728 | msg_free_iov(&msg_peek);
|
| 729 | return errno;
|
| 730 | }
|
| 731 |
|
| 732 | static float giga = 1000000000;
|
| 733 |
|
| 734 | static inline float sentBps(struct msg_stats s)
|
| 735 | {
|
| 736 | return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
|
| 737 | }
|
| 738 |
|
| 739 | static inline float recvdBps(struct msg_stats s)
|
| 740 | {
|
| 741 | return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
|
| 742 | }
|
| 743 |
|
| 744 | static int sendmsg_test(struct sockmap_options *opt)
|
| 745 | {
|
| 746 | float sent_Bps = 0, recvd_Bps = 0;
|
| 747 | int rx_fd, txpid, rxpid, err = 0;
|
| 748 | struct msg_stats s = {0};
|
| 749 | int iov_count = opt->iov_count;
|
| 750 | int iov_buf = opt->iov_length;
|
| 751 | int rx_status, tx_status;
|
| 752 | int cnt = opt->rate;
|
| 753 |
|
| 754 | errno = 0;
|
| 755 |
|
| 756 | if (opt->base)
|
| 757 | rx_fd = p1;
|
| 758 | else
|
| 759 | rx_fd = p2;
|
| 760 |
|
| 761 | if (opt->tx_wait_mem) {
|
| 762 | struct timeval timeout;
|
| 763 | int rxtx_buf_len = 1024;
|
| 764 |
|
| 765 | timeout.tv_sec = 3;
|
| 766 | timeout.tv_usec = 0;
|
| 767 |
|
| 768 | err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
|
| 769 | err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
|
| 770 | err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
|
| 771 | if (err) {
|
| 772 | perror("setsockopt failed()");
|
| 773 | return errno;
|
| 774 | }
|
| 775 | }
|
| 776 |
|
| 777 | rxpid = fork();
|
| 778 | if (rxpid == 0) {
|
| 779 | if (opt->drop_expected)
|
| 780 | _exit(0);
|
| 781 |
|
| 782 | if (!iov_buf)
|
| 783 | _exit(0);
|
| 784 |
|
| 785 | if (opt->sendpage)
|
| 786 | iov_count = 1;
|
| 787 | err = msg_loop(rx_fd, iov_count, iov_buf,
|
| 788 | cnt, &s, false, opt);
|
| 789 | if (opt->verbose > 1)
|
| 790 | fprintf(stderr,
|
| 791 | "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
|
| 792 | iov_count, iov_buf, cnt, err);
|
| 793 | if (s.end.tv_sec - s.start.tv_sec) {
|
| 794 | sent_Bps = sentBps(s);
|
| 795 | recvd_Bps = recvdBps(s);
|
| 796 | }
|
| 797 | if (opt->verbose > 1)
|
| 798 | fprintf(stdout,
|
| 799 | "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
|
| 800 | s.bytes_sent, sent_Bps, sent_Bps/giga,
|
| 801 | s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
|
| 802 | peek_flag ? "(peek_msg)" : "");
|
| 803 | if (err && err != -EDATAINTEGRITY && txmsg_cork)
|
| 804 | err = 0;
|
| 805 | exit(err ? 1 : 0);
|
| 806 | } else if (rxpid == -1) {
|
| 807 | perror("msg_loop_rx");
|
| 808 | return errno;
|
| 809 | }
|
| 810 |
|
| 811 | if (opt->tx_wait_mem)
|
| 812 | close(c2);
|
| 813 |
|
| 814 | txpid = fork();
|
| 815 | if (txpid == 0) {
|
| 816 | if (opt->sendpage)
|
| 817 | err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
|
| 818 | else
|
| 819 | err = msg_loop(c1, iov_count, iov_buf,
|
| 820 | cnt, &s, true, opt);
|
| 821 |
|
| 822 | if (err)
|
| 823 | fprintf(stderr,
|
| 824 | "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
|
| 825 | iov_count, iov_buf, cnt, err);
|
| 826 | if (s.end.tv_sec - s.start.tv_sec) {
|
| 827 | sent_Bps = sentBps(s);
|
| 828 | recvd_Bps = recvdBps(s);
|
| 829 | }
|
| 830 | if (opt->verbose > 1)
|
| 831 | fprintf(stdout,
|
| 832 | "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
|
| 833 | s.bytes_sent, sent_Bps, sent_Bps/giga,
|
| 834 | s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
|
| 835 | exit(err ? 1 : 0);
|
| 836 | } else if (txpid == -1) {
|
| 837 | perror("msg_loop_tx");
|
| 838 | return errno;
|
| 839 | }
|
| 840 |
|
| 841 | assert(waitpid(rxpid, &rx_status, 0) == rxpid);
|
| 842 | assert(waitpid(txpid, &tx_status, 0) == txpid);
|
| 843 | if (WIFEXITED(rx_status)) {
|
| 844 | err = WEXITSTATUS(rx_status);
|
| 845 | if (err) {
|
| 846 | fprintf(stderr, "rx thread exited with err %d.\n", err);
|
| 847 | goto out;
|
| 848 | }
|
| 849 | }
|
| 850 | if (WIFEXITED(tx_status)) {
|
| 851 | err = WEXITSTATUS(tx_status);
|
| 852 | if (err)
|
| 853 | fprintf(stderr, "tx thread exited with err %d.\n", err);
|
| 854 | }
|
| 855 | out:
|
| 856 | return err;
|
| 857 | }
|
| 858 |
|
| 859 | static int forever_ping_pong(int rate, struct sockmap_options *opt)
|
| 860 | {
|
| 861 | struct timeval timeout;
|
| 862 | char buf[1024] = {0};
|
| 863 | int sc;
|
| 864 |
|
| 865 | timeout.tv_sec = 10;
|
| 866 | timeout.tv_usec = 0;
|
| 867 |
|
| 868 |
|
| 869 | sc = send(c1, buf, sizeof(buf), 0);
|
| 870 | if (sc < 0) {
|
| 871 | perror("send failed()");
|
| 872 | return sc;
|
| 873 | }
|
| 874 |
|
| 875 | do {
|
| 876 | int s, rc, i, max_fd = p2;
|
| 877 | fd_set w;
|
| 878 |
|
| 879 |
|
| 880 | FD_ZERO(&w);
|
| 881 | FD_SET(c1, &w);
|
| 882 | FD_SET(c2, &w);
|
| 883 | FD_SET(p1, &w);
|
| 884 | FD_SET(p2, &w);
|
| 885 |
|
| 886 | s = select(max_fd + 1, &w, NULL, NULL, &timeout);
|
| 887 | if (s == -1) {
|
| 888 | perror("select()");
|
| 889 | break;
|
| 890 | } else if (!s) {
|
| 891 | fprintf(stderr, "unexpected timeout\n");
|
| 892 | break;
|
| 893 | }
|
| 894 |
|
| 895 | for (i = 0; i <= max_fd && s > 0; ++i) {
|
| 896 | if (!FD_ISSET(i, &w))
|
| 897 | continue;
|
| 898 |
|
| 899 | s--;
|
| 900 |
|
| 901 | rc = recv(i, buf, sizeof(buf), 0);
|
| 902 | if (rc < 0) {
|
| 903 | if (errno != EWOULDBLOCK) {
|
| 904 | perror("recv failed()");
|
| 905 | return rc;
|
| 906 | }
|
| 907 | }
|
| 908 |
|
| 909 | if (rc == 0) {
|
| 910 | close(i);
|
| 911 | break;
|
| 912 | }
|
| 913 |
|
| 914 | sc = send(i, buf, rc, 0);
|
| 915 | if (sc < 0) {
|
| 916 | perror("send failed()");
|
| 917 | return sc;
|
| 918 | }
|
| 919 | }
|
| 920 |
|
| 921 | if (rate)
|
| 922 | sleep(rate);
|
| 923 |
|
| 924 | if (opt->verbose) {
|
| 925 | printf(".");
|
| 926 | fflush(stdout);
|
| 927 |
|
| 928 | }
|
| 929 | } while (running);
|
| 930 |
|
| 931 | return 0;
|
| 932 | }
|
| 933 |
|
| 934 | enum {
|
| 935 | SELFTESTS,
|
| 936 | PING_PONG,
|
| 937 | SENDMSG,
|
| 938 | BASE,
|
| 939 | BASE_SENDPAGE,
|
| 940 | SENDPAGE,
|
| 941 | };
|
| 942 |
|
| 943 | static int run_options(struct sockmap_options *options, int cg_fd, int test)
|
| 944 | {
|
| 945 | int i, key, next_key, err, zero = 0;
|
| 946 | struct bpf_program *tx_prog;
|
| 947 |
|
| 948 |
|
| 949 | if (test == BASE || test == BASE_SENDPAGE)
|
| 950 | goto run;
|
| 951 |
|
| 952 |
|
| 953 | if (!txmsg_omit_skb_parser) {
|
| 954 | links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]);
|
| 955 | if (!links[0]) {
|
| 956 | fprintf(stderr,
|
| 957 | "ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n",
|
| 958 | bpf_program__fd(progs[0]), map_fd[0], strerror(errno));
|
| 959 | return -1;
|
| 960 | }
|
| 961 | }
|
| 962 |
|
| 963 | links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]);
|
| 964 | if (!links[1]) {
|
| 965 | fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n",
|
| 966 | strerror(errno));
|
| 967 | return -1;
|
| 968 | }
|
| 969 |
|
| 970 |
|
| 971 | err = bpf_prog_attach(bpf_program__fd(progs[2]), cg_fd, BPF_CGROUP_SOCK_OPS, 0);
|
| 972 | if (err) {
|
| 973 | fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
|
| 974 | err, strerror(errno));
|
| 975 | return err;
|
| 976 | }
|
| 977 |
|
| 978 | run:
|
| 979 | err = sockmap_init_sockets(options->verbose);
|
| 980 | if (err) {
|
| 981 | fprintf(stderr, "ERROR: test socket failed: %d\n", err);
|
| 982 | goto out;
|
| 983 | }
|
| 984 |
|
| 985 |
|
| 986 | if (txmsg_pass)
|
| 987 | tx_prog = progs[3];
|
| 988 | else if (txmsg_redir)
|
| 989 | tx_prog = progs[4];
|
| 990 | else if (txmsg_apply)
|
| 991 | tx_prog = progs[5];
|
| 992 | else if (txmsg_cork)
|
| 993 | tx_prog = progs[6];
|
| 994 | else if (txmsg_drop)
|
| 995 | tx_prog = progs[7];
|
| 996 | else
|
| 997 | tx_prog = NULL;
|
| 998 |
|
| 999 | if (tx_prog) {
|
| 1000 | int redir_fd;
|
| 1001 |
|
| 1002 | links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]);
|
| 1003 | if (!links[4]) {
|
| 1004 | fprintf(stderr,
|
| 1005 | "ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n",
|
| 1006 | strerror(errno));
|
| 1007 | err = -1;
|
| 1008 | goto out;
|
| 1009 | }
|
| 1010 |
|
| 1011 | i = 0;
|
| 1012 | err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
|
| 1013 | if (err) {
|
| 1014 | fprintf(stderr,
|
| 1015 | "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
|
| 1016 | err, strerror(errno));
|
| 1017 | goto out;
|
| 1018 | }
|
| 1019 |
|
| 1020 | if (txmsg_redir)
|
| 1021 | redir_fd = c2;
|
| 1022 | else
|
| 1023 | redir_fd = c1;
|
| 1024 |
|
| 1025 | err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
|
| 1026 | if (err) {
|
| 1027 | fprintf(stderr,
|
| 1028 | "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
|
| 1029 | err, strerror(errno));
|
| 1030 | goto out;
|
| 1031 | }
|
| 1032 |
|
| 1033 | if (txmsg_apply) {
|
| 1034 | err = bpf_map_update_elem(map_fd[3],
|
| 1035 | &i, &txmsg_apply, BPF_ANY);
|
| 1036 | if (err) {
|
| 1037 | fprintf(stderr,
|
| 1038 | "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
|
| 1039 | err, strerror(errno));
|
| 1040 | goto out;
|
| 1041 | }
|
| 1042 | }
|
| 1043 |
|
| 1044 | if (txmsg_cork) {
|
| 1045 | err = bpf_map_update_elem(map_fd[4],
|
| 1046 | &i, &txmsg_cork, BPF_ANY);
|
| 1047 | if (err) {
|
| 1048 | fprintf(stderr,
|
| 1049 | "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
|
| 1050 | err, strerror(errno));
|
| 1051 | goto out;
|
| 1052 | }
|
| 1053 | }
|
| 1054 |
|
| 1055 | if (txmsg_start) {
|
| 1056 | err = bpf_map_update_elem(map_fd[5],
|
| 1057 | &i, &txmsg_start, BPF_ANY);
|
| 1058 | if (err) {
|
| 1059 | fprintf(stderr,
|
| 1060 | "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
|
| 1061 | err, strerror(errno));
|
| 1062 | goto out;
|
| 1063 | }
|
| 1064 | }
|
| 1065 |
|
| 1066 | if (txmsg_end) {
|
| 1067 | i = 1;
|
| 1068 | err = bpf_map_update_elem(map_fd[5],
|
| 1069 | &i, &txmsg_end, BPF_ANY);
|
| 1070 | if (err) {
|
| 1071 | fprintf(stderr,
|
| 1072 | "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
|
| 1073 | err, strerror(errno));
|
| 1074 | goto out;
|
| 1075 | }
|
| 1076 | }
|
| 1077 |
|
| 1078 | if (txmsg_start_push) {
|
| 1079 | i = 2;
|
| 1080 | err = bpf_map_update_elem(map_fd[5],
|
| 1081 | &i, &txmsg_start_push, BPF_ANY);
|
| 1082 | if (err) {
|
| 1083 | fprintf(stderr,
|
| 1084 | "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
|
| 1085 | err, strerror(errno));
|
| 1086 | goto out;
|
| 1087 | }
|
| 1088 | }
|
| 1089 |
|
| 1090 | if (txmsg_end_push) {
|
| 1091 | i = 3;
|
| 1092 | err = bpf_map_update_elem(map_fd[5],
|
| 1093 | &i, &txmsg_end_push, BPF_ANY);
|
| 1094 | if (err) {
|
| 1095 | fprintf(stderr,
|
| 1096 | "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
|
| 1097 | txmsg_end_push, i, err, strerror(errno));
|
| 1098 | goto out;
|
| 1099 | }
|
| 1100 | }
|
| 1101 |
|
| 1102 | if (txmsg_start_pop) {
|
| 1103 | i = 4;
|
| 1104 | err = bpf_map_update_elem(map_fd[5],
|
| 1105 | &i, &txmsg_start_pop, BPF_ANY);
|
| 1106 | if (err) {
|
| 1107 | fprintf(stderr,
|
| 1108 | "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
|
| 1109 | txmsg_start_pop, i, err, strerror(errno));
|
| 1110 | goto out;
|
| 1111 | }
|
| 1112 | } else {
|
| 1113 | i = 4;
|
| 1114 | bpf_map_update_elem(map_fd[5],
|
| 1115 | &i, &txmsg_start_pop, BPF_ANY);
|
| 1116 | }
|
| 1117 |
|
| 1118 | if (txmsg_pop) {
|
| 1119 | i = 5;
|
| 1120 | err = bpf_map_update_elem(map_fd[5],
|
| 1121 | &i, &txmsg_pop, BPF_ANY);
|
| 1122 | if (err) {
|
| 1123 | fprintf(stderr,
|
| 1124 | "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
|
| 1125 | txmsg_pop, i, err, strerror(errno));
|
| 1126 | goto out;
|
| 1127 | }
|
| 1128 | } else {
|
| 1129 | i = 5;
|
| 1130 | bpf_map_update_elem(map_fd[5],
|
| 1131 | &i, &txmsg_pop, BPF_ANY);
|
| 1132 |
|
| 1133 | }
|
| 1134 |
|
| 1135 | if (txmsg_ingress) {
|
| 1136 | int in = BPF_F_INGRESS;
|
| 1137 |
|
| 1138 | i = 0;
|
| 1139 | err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
|
| 1140 | if (err) {
|
| 1141 | fprintf(stderr,
|
| 1142 | "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
|
| 1143 | err, strerror(errno));
|
| 1144 | }
|
| 1145 | i = 1;
|
| 1146 | err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
|
| 1147 | if (err) {
|
| 1148 | fprintf(stderr,
|
| 1149 | "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
|
| 1150 | err, strerror(errno));
|
| 1151 | }
|
| 1152 | err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
|
| 1153 | if (err) {
|
| 1154 | fprintf(stderr,
|
| 1155 | "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
|
| 1156 | err, strerror(errno));
|
| 1157 | }
|
| 1158 |
|
| 1159 | i = 2;
|
| 1160 | err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
|
| 1161 | if (err) {
|
| 1162 | fprintf(stderr,
|
| 1163 | "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
|
| 1164 | err, strerror(errno));
|
| 1165 | }
|
| 1166 | }
|
| 1167 |
|
| 1168 | if (txmsg_redir_skb) {
|
| 1169 | int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
|
| 1170 | p2 : p1;
|
| 1171 | int ingress = BPF_F_INGRESS;
|
| 1172 |
|
| 1173 | i = 0;
|
| 1174 | err = bpf_map_update_elem(map_fd[7],
|
| 1175 | &i, &ingress, BPF_ANY);
|
| 1176 | if (err) {
|
| 1177 | fprintf(stderr,
|
| 1178 | "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
|
| 1179 | err, strerror(errno));
|
| 1180 | }
|
| 1181 |
|
| 1182 | i = 3;
|
| 1183 | err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
|
| 1184 | if (err) {
|
| 1185 | fprintf(stderr,
|
| 1186 | "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
|
| 1187 | err, strerror(errno));
|
| 1188 | }
|
| 1189 | }
|
| 1190 | }
|
| 1191 |
|
| 1192 | if (skb_use_parser) {
|
| 1193 | i = 2;
|
| 1194 | err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
|
| 1195 | }
|
| 1196 |
|
| 1197 | if (txmsg_drop)
|
| 1198 | options->drop_expected = true;
|
| 1199 |
|
| 1200 | if (test == PING_PONG)
|
| 1201 | err = forever_ping_pong(options->rate, options);
|
| 1202 | else if (test == SENDMSG) {
|
| 1203 | options->base = false;
|
| 1204 | options->sendpage = false;
|
| 1205 | err = sendmsg_test(options);
|
| 1206 | } else if (test == SENDPAGE) {
|
| 1207 | options->base = false;
|
| 1208 | options->sendpage = true;
|
| 1209 | err = sendmsg_test(options);
|
| 1210 | } else if (test == BASE) {
|
| 1211 | options->base = true;
|
| 1212 | options->sendpage = false;
|
| 1213 | err = sendmsg_test(options);
|
| 1214 | } else if (test == BASE_SENDPAGE) {
|
| 1215 | options->base = true;
|
| 1216 | options->sendpage = true;
|
| 1217 | err = sendmsg_test(options);
|
| 1218 | } else
|
| 1219 | fprintf(stderr, "unknown test\n");
|
| 1220 | out:
|
| 1221 |
|
| 1222 | bpf_prog_detach2(bpf_program__fd(progs[2]), cg_fd, BPF_CGROUP_SOCK_OPS);
|
| 1223 |
|
| 1224 | for (i = 0; i < ARRAY_SIZE(links); i++) {
|
| 1225 | if (links[i])
|
| 1226 | bpf_link__detach(links[i]);
|
| 1227 | }
|
| 1228 |
|
| 1229 | for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
|
| 1230 | key = next_key = 0;
|
| 1231 | bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
|
| 1232 | while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
|
| 1233 | bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
|
| 1234 | key = next_key;
|
| 1235 | }
|
| 1236 | }
|
| 1237 |
|
| 1238 | close(s1);
|
| 1239 | close(s2);
|
| 1240 | close(p1);
|
| 1241 | close(p2);
|
| 1242 | close(c1);
|
| 1243 | close(c2);
|
| 1244 | return err;
|
| 1245 | }
|
| 1246 |
|
| 1247 | static char *test_to_str(int test)
|
| 1248 | {
|
| 1249 | switch (test) {
|
| 1250 | case SENDMSG:
|
| 1251 | return "sendmsg";
|
| 1252 | case SENDPAGE:
|
| 1253 | return "sendpage";
|
| 1254 | }
|
| 1255 | return "unknown";
|
| 1256 | }
|
| 1257 |
|
| 1258 | static void append_str(char *dst, const char *src, size_t dst_cap)
|
| 1259 | {
|
| 1260 | size_t avail = dst_cap - strlen(dst);
|
| 1261 |
|
| 1262 | if (avail <= 1)
|
| 1263 | return;
|
| 1264 |
|
| 1265 | strncat(dst, src, avail - 1);
|
| 1266 | }
|
| 1267 |
|
| 1268 | #define OPTSTRING 60
|
| 1269 | static void test_options(char *options)
|
| 1270 | {
|
| 1271 | char tstr[OPTSTRING];
|
| 1272 |
|
| 1273 | memset(options, 0, OPTSTRING);
|
| 1274 |
|
| 1275 | if (txmsg_pass)
|
| 1276 | append_str(options, "pass,", OPTSTRING);
|
| 1277 | if (txmsg_redir)
|
| 1278 | append_str(options, "redir,", OPTSTRING);
|
| 1279 | if (txmsg_drop)
|
| 1280 | append_str(options, "drop,", OPTSTRING);
|
| 1281 | if (txmsg_apply) {
|
| 1282 | snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
|
| 1283 | append_str(options, tstr, OPTSTRING);
|
| 1284 | }
|
| 1285 | if (txmsg_cork) {
|
| 1286 | snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
|
| 1287 | append_str(options, tstr, OPTSTRING);
|
| 1288 | }
|
| 1289 | if (txmsg_start) {
|
| 1290 | snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
|
| 1291 | append_str(options, tstr, OPTSTRING);
|
| 1292 | }
|
| 1293 | if (txmsg_end) {
|
| 1294 | snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
|
| 1295 | append_str(options, tstr, OPTSTRING);
|
| 1296 | }
|
| 1297 | if (txmsg_start_pop) {
|
| 1298 | snprintf(tstr, OPTSTRING, "pop (%d,%d),",
|
| 1299 | txmsg_start_pop, txmsg_start_pop + txmsg_pop);
|
| 1300 | append_str(options, tstr, OPTSTRING);
|
| 1301 | }
|
| 1302 | if (txmsg_ingress)
|
| 1303 | append_str(options, "ingress,", OPTSTRING);
|
| 1304 | if (txmsg_redir_skb)
|
| 1305 | append_str(options, "redir_skb,", OPTSTRING);
|
| 1306 | if (peek_flag)
|
| 1307 | append_str(options, "peek,", OPTSTRING);
|
| 1308 | }
|
| 1309 |
|
| 1310 | static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
|
| 1311 | {
|
| 1312 | char *options = calloc(OPTSTRING, sizeof(char));
|
| 1313 | int err;
|
| 1314 |
|
| 1315 | if (test == SENDPAGE)
|
| 1316 | opt->sendpage = true;
|
| 1317 | else
|
| 1318 | opt->sendpage = false;
|
| 1319 |
|
| 1320 | if (txmsg_drop)
|
| 1321 | opt->drop_expected = true;
|
| 1322 | else
|
| 1323 | opt->drop_expected = false;
|
| 1324 |
|
| 1325 | test_options(options);
|
| 1326 |
|
| 1327 | if (opt->verbose) {
|
| 1328 | fprintf(stdout,
|
| 1329 | " [TEST %i]: (%i, %i, %i, %s, %s): ",
|
| 1330 | test_cnt, opt->rate, opt->iov_count, opt->iov_length,
|
| 1331 | test_to_str(test), options);
|
| 1332 | fflush(stdout);
|
| 1333 | }
|
| 1334 | err = run_options(opt, cgrp, test);
|
| 1335 | if (opt->verbose)
|
| 1336 | fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
|
| 1337 | test_cnt++;
|
| 1338 | !err ? passed++ : failed++;
|
| 1339 | free(options);
|
| 1340 | return err;
|
| 1341 | }
|
| 1342 |
|
| 1343 | static void test_exec(int cgrp, struct sockmap_options *opt)
|
| 1344 | {
|
| 1345 | int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
|
| 1346 | int err;
|
| 1347 |
|
| 1348 | if (type == 0) {
|
| 1349 | test_start();
|
| 1350 | err = __test_exec(cgrp, SENDMSG, opt);
|
| 1351 | if (err)
|
| 1352 | test_fail();
|
| 1353 | } else {
|
| 1354 | test_start();
|
| 1355 | err = __test_exec(cgrp, SENDPAGE, opt);
|
| 1356 | if (err)
|
| 1357 | test_fail();
|
| 1358 | }
|
| 1359 | }
|
| 1360 |
|
| 1361 | static void test_send_one(struct sockmap_options *opt, int cgrp)
|
| 1362 | {
|
| 1363 | opt->iov_length = 1;
|
| 1364 | opt->iov_count = 1;
|
| 1365 | opt->rate = 1;
|
| 1366 | test_exec(cgrp, opt);
|
| 1367 |
|
| 1368 | opt->iov_length = 1;
|
| 1369 | opt->iov_count = 1024;
|
| 1370 | opt->rate = 1;
|
| 1371 | test_exec(cgrp, opt);
|
| 1372 |
|
| 1373 | opt->iov_length = 1024;
|
| 1374 | opt->iov_count = 1;
|
| 1375 | opt->rate = 1;
|
| 1376 | test_exec(cgrp, opt);
|
| 1377 |
|
| 1378 | }
|
| 1379 |
|
| 1380 | static void test_send_many(struct sockmap_options *opt, int cgrp)
|
| 1381 | {
|
| 1382 | opt->iov_length = 3;
|
| 1383 | opt->iov_count = 1;
|
| 1384 | opt->rate = 512;
|
| 1385 | test_exec(cgrp, opt);
|
| 1386 |
|
| 1387 | opt->rate = 100;
|
| 1388 | opt->iov_count = 1;
|
| 1389 | opt->iov_length = 5;
|
| 1390 | test_exec(cgrp, opt);
|
| 1391 | }
|
| 1392 |
|
| 1393 | static void test_send_large(struct sockmap_options *opt, int cgrp)
|
| 1394 | {
|
| 1395 | opt->iov_length = 8192;
|
| 1396 | opt->iov_count = 32;
|
| 1397 | opt->rate = 2;
|
| 1398 | test_exec(cgrp, opt);
|
| 1399 | }
|
| 1400 |
|
| 1401 | static void test_send(struct sockmap_options *opt, int cgrp)
|
| 1402 | {
|
| 1403 | test_send_one(opt, cgrp);
|
| 1404 | test_send_many(opt, cgrp);
|
| 1405 | test_send_large(opt, cgrp);
|
| 1406 | sched_yield();
|
| 1407 | }
|
| 1408 |
|
| 1409 | static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
|
| 1410 | {
|
| 1411 |
|
| 1412 | txmsg_pass = 1;
|
| 1413 | test_send(opt, cgrp);
|
| 1414 | }
|
| 1415 |
|
| 1416 | static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
|
| 1417 | {
|
| 1418 | txmsg_redir = 1;
|
| 1419 | test_send(opt, cgrp);
|
| 1420 | }
|
| 1421 |
|
| 1422 | static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
|
| 1423 | {
|
| 1424 | opt->tx_wait_mem = true;
|
| 1425 | txmsg_redir = 1;
|
| 1426 | test_send_large(opt, cgrp);
|
| 1427 |
|
| 1428 | txmsg_redir = 1;
|
| 1429 | txmsg_apply = 4097;
|
| 1430 | test_send_large(opt, cgrp);
|
| 1431 | opt->tx_wait_mem = false;
|
| 1432 | }
|
| 1433 |
|
| 1434 | static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
|
| 1435 | {
|
| 1436 | txmsg_drop = 1;
|
| 1437 | test_send(opt, cgrp);
|
| 1438 | }
|
| 1439 |
|
| 1440 | static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
|
| 1441 | {
|
| 1442 | txmsg_pass = txmsg_drop = 0;
|
| 1443 | txmsg_ingress = txmsg_redir = 1;
|
| 1444 | test_send(opt, cgrp);
|
| 1445 | }
|
| 1446 |
|
| 1447 | |
| 1448 | * cork can leave data on the ring if user program is buggy and
|
| 1449 | * doesn't flush them somehow. They do take some time however
|
| 1450 | * because they wait for a timeout. Test pass, redir and cork with
|
| 1451 | * apply logic. Use cork size of 4097 with send_large to avoid
|
| 1452 | * aligning cork size with send size.
|
| 1453 |
|
| 1454 | static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
|
| 1455 | {
|
| 1456 | txmsg_pass = 1;
|
| 1457 | txmsg_redir = 0;
|
| 1458 | txmsg_cork = 4097;
|
| 1459 | txmsg_apply = 4097;
|
| 1460 | test_send_large(opt, cgrp);
|
| 1461 |
|
| 1462 | txmsg_pass = 0;
|
| 1463 | txmsg_redir = 1;
|
| 1464 | txmsg_apply = 0;
|
| 1465 | txmsg_cork = 4097;
|
| 1466 | test_send_large(opt, cgrp);
|
| 1467 |
|
| 1468 | txmsg_pass = 0;
|
| 1469 | txmsg_redir = 1;
|
| 1470 | txmsg_apply = 4097;
|
| 1471 | txmsg_cork = 4097;
|
| 1472 | test_send_large(opt, cgrp);
|
| 1473 | }
|
| 1474 |
|
| 1475 | static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
|
| 1476 | {
|
| 1477 |
|
| 1478 | txmsg_pass = 1;
|
| 1479 | txmsg_start = 1;
|
| 1480 | txmsg_end = 2;
|
| 1481 | test_send(opt, cgrp);
|
| 1482 |
|
| 1483 |
|
| 1484 | txmsg_pass = 1;
|
| 1485 | txmsg_start = 4096;
|
| 1486 | txmsg_end = 9182;
|
| 1487 | test_send_large(opt, cgrp);
|
| 1488 |
|
| 1489 |
|
| 1490 | txmsg_redir = 1;
|
| 1491 | txmsg_start = 1;
|
| 1492 | txmsg_end = 2;
|
| 1493 | test_send(opt, cgrp);
|
| 1494 |
|
| 1495 |
|
| 1496 | txmsg_redir = 0;
|
| 1497 | txmsg_cork = 512;
|
| 1498 | txmsg_start = 1;
|
| 1499 | txmsg_end = 2;
|
| 1500 | test_send_many(opt, cgrp);
|
| 1501 |
|
| 1502 |
|
| 1503 | txmsg_redir = 1;
|
| 1504 | txmsg_cork = 512;
|
| 1505 | txmsg_start = 1;
|
| 1506 | txmsg_end = 2;
|
| 1507 | test_send_many(opt, cgrp);
|
| 1508 | }
|
| 1509 |
|
| 1510 | static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
|
| 1511 | {
|
| 1512 | bool data = opt->data_test;
|
| 1513 |
|
| 1514 |
|
| 1515 | txmsg_pass = 1;
|
| 1516 | txmsg_start_pop = 1;
|
| 1517 | txmsg_pop = 2;
|
| 1518 | test_send_many(opt, cgrp);
|
| 1519 |
|
| 1520 |
|
| 1521 | txmsg_pass = 1;
|
| 1522 | txmsg_start_pop = 4096;
|
| 1523 | txmsg_pop = 4096;
|
| 1524 | test_send_large(opt, cgrp);
|
| 1525 |
|
| 1526 |
|
| 1527 | txmsg_redir = 1;
|
| 1528 | txmsg_start_pop = 1;
|
| 1529 | txmsg_pop = 2;
|
| 1530 | test_send_many(opt, cgrp);
|
| 1531 |
|
| 1532 | |
| 1533 | * - It makes the layout of the received data difficult
|
| 1534 | * - It makes it hard to calculate the total_bytes in the recvmsg
|
| 1535 | * Temporarily skip the data integrity test for this case now.
|
| 1536 |
|
| 1537 | opt->data_test = false;
|
| 1538 |
|
| 1539 | txmsg_redir = 0;
|
| 1540 | txmsg_cork = 512;
|
| 1541 | txmsg_start_pop = 1;
|
| 1542 | txmsg_pop = 2;
|
| 1543 | test_send_many(opt, cgrp);
|
| 1544 |
|
| 1545 |
|
| 1546 | txmsg_redir = 1;
|
| 1547 | txmsg_cork = 4;
|
| 1548 | txmsg_start_pop = 1;
|
| 1549 | txmsg_pop = 2;
|
| 1550 | test_send_many(opt, cgrp);
|
| 1551 | opt->data_test = data;
|
| 1552 | }
|
| 1553 |
|
| 1554 | static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
|
| 1555 | {
|
| 1556 | bool data = opt->data_test;
|
| 1557 |
|
| 1558 |
|
| 1559 | txmsg_pass = 1;
|
| 1560 | txmsg_start_push = 1;
|
| 1561 | txmsg_end_push = 1;
|
| 1562 | test_send(opt, cgrp);
|
| 1563 |
|
| 1564 |
|
| 1565 | txmsg_pass = 1;
|
| 1566 | txmsg_start_push = 4096;
|
| 1567 | txmsg_end_push = 4096;
|
| 1568 | test_send_large(opt, cgrp);
|
| 1569 |
|
| 1570 |
|
| 1571 | txmsg_redir = 1;
|
| 1572 | txmsg_start_push = 1;
|
| 1573 | txmsg_end_push = 2;
|
| 1574 | test_send_many(opt, cgrp);
|
| 1575 |
|
| 1576 | |
| 1577 | * - It makes the layout of the received data difficult
|
| 1578 | * - It makes it hard to calculate the total_bytes in the recvmsg
|
| 1579 | * Temporarily skip the data integrity test for this case now.
|
| 1580 |
|
| 1581 | opt->data_test = false;
|
| 1582 |
|
| 1583 | txmsg_redir = 0;
|
| 1584 | txmsg_cork = 512;
|
| 1585 | txmsg_start_push = 1;
|
| 1586 | txmsg_end_push = 2;
|
| 1587 | test_send_many(opt, cgrp);
|
| 1588 | opt->data_test = data;
|
| 1589 | }
|
| 1590 |
|
| 1591 | static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
|
| 1592 | {
|
| 1593 |
|
| 1594 | txmsg_pass = 1;
|
| 1595 | txmsg_start_push = 1;
|
| 1596 | txmsg_end_push = 10;
|
| 1597 | txmsg_start_pop = 5;
|
| 1598 | txmsg_pop = 4;
|
| 1599 | test_send_large(opt, cgrp);
|
| 1600 |
|
| 1601 | txmsg_pass = 1;
|
| 1602 | txmsg_start_push = 1;
|
| 1603 | txmsg_end_push = 10;
|
| 1604 | txmsg_start_pop = 5;
|
| 1605 | txmsg_pop = 16;
|
| 1606 | test_send_large(opt, cgrp);
|
| 1607 |
|
| 1608 | txmsg_pass = 1;
|
| 1609 | txmsg_start_push = 5;
|
| 1610 | txmsg_end_push = 4;
|
| 1611 | txmsg_start_pop = 1;
|
| 1612 | txmsg_pop = 10;
|
| 1613 | test_send_large(opt, cgrp);
|
| 1614 |
|
| 1615 | txmsg_pass = 1;
|
| 1616 | txmsg_start_push = 5;
|
| 1617 | txmsg_end_push = 16;
|
| 1618 | txmsg_start_pop = 1;
|
| 1619 | txmsg_pop = 10;
|
| 1620 | test_send_large(opt, cgrp);
|
| 1621 |
|
| 1622 |
|
| 1623 | txmsg_pass = 1;
|
| 1624 | txmsg_start_push = 1;
|
| 1625 | txmsg_end_push = 10;
|
| 1626 | txmsg_start_pop = 16;
|
| 1627 | txmsg_pop = 4;
|
| 1628 | test_send_large(opt, cgrp);
|
| 1629 |
|
| 1630 | txmsg_pass = 1;
|
| 1631 | txmsg_start_push = 16;
|
| 1632 | txmsg_end_push = 10;
|
| 1633 | txmsg_start_pop = 5;
|
| 1634 | txmsg_pop = 4;
|
| 1635 | test_send_large(opt, cgrp);
|
| 1636 | }
|
| 1637 |
|
| 1638 | static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
|
| 1639 | {
|
| 1640 | txmsg_pass = 1;
|
| 1641 | txmsg_redir = 0;
|
| 1642 | txmsg_ingress = 0;
|
| 1643 | txmsg_apply = 1;
|
| 1644 | txmsg_cork = 0;
|
| 1645 | test_send_one(opt, cgrp);
|
| 1646 |
|
| 1647 | txmsg_pass = 0;
|
| 1648 | txmsg_redir = 1;
|
| 1649 | txmsg_ingress = 0;
|
| 1650 | txmsg_apply = 1;
|
| 1651 | txmsg_cork = 0;
|
| 1652 | test_send_one(opt, cgrp);
|
| 1653 |
|
| 1654 | txmsg_pass = 0;
|
| 1655 | txmsg_redir = 1;
|
| 1656 | txmsg_ingress = 1;
|
| 1657 | txmsg_apply = 1;
|
| 1658 | txmsg_cork = 0;
|
| 1659 | test_send_one(opt, cgrp);
|
| 1660 |
|
| 1661 | txmsg_pass = 1;
|
| 1662 | txmsg_redir = 0;
|
| 1663 | txmsg_ingress = 0;
|
| 1664 | txmsg_apply = 1024;
|
| 1665 | txmsg_cork = 0;
|
| 1666 | test_send_large(opt, cgrp);
|
| 1667 |
|
| 1668 | txmsg_pass = 0;
|
| 1669 | txmsg_redir = 1;
|
| 1670 | txmsg_ingress = 0;
|
| 1671 | txmsg_apply = 1024;
|
| 1672 | txmsg_cork = 0;
|
| 1673 | test_send_large(opt, cgrp);
|
| 1674 |
|
| 1675 | txmsg_pass = 0;
|
| 1676 | txmsg_redir = 1;
|
| 1677 | txmsg_ingress = 1;
|
| 1678 | txmsg_apply = 1024;
|
| 1679 | txmsg_cork = 0;
|
| 1680 | test_send_large(opt, cgrp);
|
| 1681 | }
|
| 1682 |
|
| 1683 | static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
|
| 1684 | {
|
| 1685 | txmsg_pass = 1;
|
| 1686 | txmsg_redir = 0;
|
| 1687 | txmsg_apply = 0;
|
| 1688 | txmsg_cork = 1;
|
| 1689 | test_send(opt, cgrp);
|
| 1690 |
|
| 1691 | txmsg_pass = 1;
|
| 1692 | txmsg_redir = 0;
|
| 1693 | txmsg_apply = 1;
|
| 1694 | txmsg_cork = 1;
|
| 1695 | test_send(opt, cgrp);
|
| 1696 | }
|
| 1697 |
|
| 1698 | static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
|
| 1699 | {
|
| 1700 | txmsg_pass = 1;
|
| 1701 | skb_use_parser = 512;
|
| 1702 | opt->iov_length = 256;
|
| 1703 | opt->iov_count = 1;
|
| 1704 | opt->rate = 2;
|
| 1705 | test_exec(cgrp, opt);
|
| 1706 | }
|
| 1707 |
|
| 1708 | static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
|
| 1709 | {
|
| 1710 | skb_use_parser = 10;
|
| 1711 | opt->iov_length = 20;
|
| 1712 | opt->iov_count = 1;
|
| 1713 | opt->rate = 1;
|
| 1714 | opt->check_recved_len = true;
|
| 1715 | test_exec(cgrp, opt);
|
| 1716 | opt->check_recved_len = false;
|
| 1717 | }
|
| 1718 |
|
| 1719 | char *map_names[] = {
|
| 1720 | "sock_map",
|
| 1721 | "sock_map_txmsg",
|
| 1722 | "sock_map_redir",
|
| 1723 | "sock_apply_bytes",
|
| 1724 | "sock_cork_bytes",
|
| 1725 | "sock_bytes",
|
| 1726 | "sock_redir_flags",
|
| 1727 | "sock_skb_opts",
|
| 1728 | };
|
| 1729 |
|
| 1730 | static int populate_progs(char *bpf_file)
|
| 1731 | {
|
| 1732 | struct bpf_program *prog;
|
| 1733 | struct bpf_object *obj;
|
| 1734 | int i = 0;
|
| 1735 | long err;
|
| 1736 |
|
| 1737 | obj = bpf_object__open(bpf_file);
|
| 1738 | err = libbpf_get_error(obj);
|
| 1739 | if (err) {
|
| 1740 | char err_buf[256];
|
| 1741 |
|
| 1742 | libbpf_strerror(err, err_buf, sizeof(err_buf));
|
| 1743 | printf("Unable to load eBPF objects in file '%s' : %s\n",
|
| 1744 | bpf_file, err_buf);
|
| 1745 | return -1;
|
| 1746 | }
|
| 1747 |
|
| 1748 | i = bpf_object__load(obj);
|
| 1749 | i = 0;
|
| 1750 | bpf_object__for_each_program(prog, obj) {
|
| 1751 | progs[i] = prog;
|
| 1752 | i++;
|
| 1753 | }
|
| 1754 |
|
| 1755 | for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
|
| 1756 | maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
|
| 1757 | map_fd[i] = bpf_map__fd(maps[i]);
|
| 1758 | if (map_fd[i] < 0) {
|
| 1759 | fprintf(stderr, "load_bpf_file: (%i) %s\n",
|
| 1760 | map_fd[i], strerror(errno));
|
| 1761 | return -1;
|
| 1762 | }
|
| 1763 | }
|
| 1764 |
|
| 1765 | for (i = 0; i < ARRAY_SIZE(links); i++)
|
| 1766 | links[i] = NULL;
|
| 1767 |
|
| 1768 | return 0;
|
| 1769 | }
|
| 1770 |
|
| 1771 | struct _test test[] = {
|
| 1772 | {"txmsg test passthrough", test_txmsg_pass},
|
| 1773 | {"txmsg test redirect", test_txmsg_redir},
|
| 1774 | {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
|
| 1775 | {"txmsg test drop", test_txmsg_drop},
|
| 1776 | {"txmsg test ingress redirect", test_txmsg_ingress_redir},
|
| 1777 | {"txmsg test apply", test_txmsg_apply},
|
| 1778 | {"txmsg test cork", test_txmsg_cork},
|
| 1779 | {"txmsg test hanging corks", test_txmsg_cork_hangs},
|
| 1780 | {"txmsg test push_data", test_txmsg_push},
|
| 1781 | {"txmsg test pull-data", test_txmsg_pull},
|
| 1782 | {"txmsg test pop-data", test_txmsg_pop},
|
| 1783 | {"txmsg test push/pop data", test_txmsg_push_pop},
|
| 1784 | {"txmsg test ingress parser", test_txmsg_ingress_parser},
|
| 1785 | {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
|
| 1786 | };
|
| 1787 |
|
| 1788 | static int check_whitelist(struct _test *t, struct sockmap_options *opt)
|
| 1789 | {
|
| 1790 | char *entry, *ptr;
|
| 1791 |
|
| 1792 | if (!opt->whitelist)
|
| 1793 | return 0;
|
| 1794 | ptr = strdup(opt->whitelist);
|
| 1795 | if (!ptr)
|
| 1796 | return -ENOMEM;
|
| 1797 | entry = strtok(ptr, ",");
|
| 1798 | while (entry) {
|
| 1799 | if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
|
| 1800 | strstr(opt->map, entry) != 0 ||
|
| 1801 | strstr(t->title, entry) != 0) {
|
| 1802 | free(ptr);
|
| 1803 | return 0;
|
| 1804 | }
|
| 1805 | entry = strtok(NULL, ",");
|
| 1806 | }
|
| 1807 | free(ptr);
|
| 1808 | return -EINVAL;
|
| 1809 | }
|
| 1810 |
|
| 1811 | static int check_blacklist(struct _test *t, struct sockmap_options *opt)
|
| 1812 | {
|
| 1813 | char *entry, *ptr;
|
| 1814 |
|
| 1815 | if (!opt->blacklist)
|
| 1816 | return -EINVAL;
|
| 1817 | ptr = strdup(opt->blacklist);
|
| 1818 | if (!ptr)
|
| 1819 | return -ENOMEM;
|
| 1820 | entry = strtok(ptr, ",");
|
| 1821 | while (entry) {
|
| 1822 | if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
|
| 1823 | strstr(opt->map, entry) != 0 ||
|
| 1824 | strstr(t->title, entry) != 0) {
|
| 1825 | free(ptr);
|
| 1826 | return 0;
|
| 1827 | }
|
| 1828 | entry = strtok(NULL, ",");
|
| 1829 | }
|
| 1830 | free(ptr);
|
| 1831 | return -EINVAL;
|
| 1832 | }
|
| 1833 |
|
| 1834 | static int __test_selftests(int cg_fd, struct sockmap_options *opt)
|
| 1835 | {
|
| 1836 | int i, err;
|
| 1837 |
|
| 1838 | err = populate_progs(opt->map);
|
| 1839 | if (err < 0) {
|
| 1840 | fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
|
| 1841 | return err;
|
| 1842 | }
|
| 1843 |
|
| 1844 |
|
| 1845 | for (i = 0; i < ARRAY_SIZE(test); i++) {
|
| 1846 | struct _test t = test[i];
|
| 1847 |
|
| 1848 | if (check_whitelist(&t, opt) != 0)
|
| 1849 | continue;
|
| 1850 | if (check_blacklist(&t, opt) == 0)
|
| 1851 | continue;
|
| 1852 |
|
| 1853 | test_start_subtest(&t, opt);
|
| 1854 | t.tester(cg_fd, opt);
|
| 1855 | test_end_subtest();
|
| 1856 | }
|
| 1857 |
|
| 1858 | return err;
|
| 1859 | }
|
| 1860 |
|
| 1861 | static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
|
| 1862 | {
|
| 1863 | opt->map = BPF_SOCKMAP_FILENAME;
|
| 1864 | __test_selftests(cg_fd, opt);
|
| 1865 | }
|
| 1866 |
|
| 1867 | static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
|
| 1868 | {
|
| 1869 | opt->map = BPF_SOCKHASH_FILENAME;
|
| 1870 | __test_selftests(cg_fd, opt);
|
| 1871 | }
|
| 1872 |
|
| 1873 | static int test_selftest(int cg_fd, struct sockmap_options *opt)
|
| 1874 | {
|
| 1875 | test_selftests_sockmap(cg_fd, opt);
|
| 1876 | test_selftests_sockhash(cg_fd, opt);
|
| 1877 | test_print_results();
|
| 1878 | return 0;
|
| 1879 | }
|
| 1880 |
|
| 1881 | int main(int argc, char **argv)
|
| 1882 | {
|
| 1883 | int iov_count = 1, length = 1024, rate = 1;
|
| 1884 | struct sockmap_options options = {0};
|
| 1885 | int opt, longindex, err, cg_fd = 0;
|
| 1886 | char *bpf_file = BPF_SOCKMAP_FILENAME;
|
| 1887 | int test = SELFTESTS;
|
| 1888 | bool cg_created = 0;
|
| 1889 |
|
| 1890 | while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
|
| 1891 | long_options, &longindex)) != -1) {
|
| 1892 | switch (opt) {
|
| 1893 | case 's':
|
| 1894 | txmsg_start = atoi(optarg);
|
| 1895 | break;
|
| 1896 | case 'e':
|
| 1897 | txmsg_end = atoi(optarg);
|
| 1898 | break;
|
| 1899 | case 'p':
|
| 1900 | txmsg_start_push = atoi(optarg);
|
| 1901 | break;
|
| 1902 | case 'q':
|
| 1903 | txmsg_end_push = atoi(optarg);
|
| 1904 | break;
|
| 1905 | case 'w':
|
| 1906 | txmsg_start_pop = atoi(optarg);
|
| 1907 | break;
|
| 1908 | case 'x':
|
| 1909 | txmsg_pop = atoi(optarg);
|
| 1910 | break;
|
| 1911 | case 'a':
|
| 1912 | txmsg_apply = atoi(optarg);
|
| 1913 | break;
|
| 1914 | case 'k':
|
| 1915 | txmsg_cork = atoi(optarg);
|
| 1916 | break;
|
| 1917 | case 'c':
|
| 1918 | cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
|
| 1919 | if (cg_fd < 0) {
|
| 1920 | fprintf(stderr,
|
| 1921 | "ERROR: (%i) open cg path failed: %s\n",
|
| 1922 | cg_fd, optarg);
|
| 1923 | return cg_fd;
|
| 1924 | }
|
| 1925 | break;
|
| 1926 | case 'r':
|
| 1927 | rate = atoi(optarg);
|
| 1928 | break;
|
| 1929 | case 'v':
|
| 1930 | options.verbose = 1;
|
| 1931 | if (optarg)
|
| 1932 | options.verbose = atoi(optarg);
|
| 1933 | break;
|
| 1934 | case 'i':
|
| 1935 | iov_count = atoi(optarg);
|
| 1936 | break;
|
| 1937 | case 'l':
|
| 1938 | length = atoi(optarg);
|
| 1939 | break;
|
| 1940 | case 'd':
|
| 1941 | options.data_test = true;
|
| 1942 | break;
|
| 1943 | case 't':
|
| 1944 | if (strcmp(optarg, "ping") == 0) {
|
| 1945 | test = PING_PONG;
|
| 1946 | } else if (strcmp(optarg, "sendmsg") == 0) {
|
| 1947 | test = SENDMSG;
|
| 1948 | } else if (strcmp(optarg, "base") == 0) {
|
| 1949 | test = BASE;
|
| 1950 | } else if (strcmp(optarg, "base_sendpage") == 0) {
|
| 1951 | test = BASE_SENDPAGE;
|
| 1952 | } else if (strcmp(optarg, "sendpage") == 0) {
|
| 1953 | test = SENDPAGE;
|
| 1954 | } else {
|
| 1955 | usage(argv);
|
| 1956 | return -1;
|
| 1957 | }
|
| 1958 | break;
|
| 1959 | case 'n':
|
| 1960 | options.whitelist = strdup(optarg);
|
| 1961 | if (!options.whitelist)
|
| 1962 | return -ENOMEM;
|
| 1963 | break;
|
| 1964 | case 'b':
|
| 1965 | options.blacklist = strdup(optarg);
|
| 1966 | if (!options.blacklist)
|
| 1967 | return -ENOMEM;
|
| 1968 | case 0:
|
| 1969 | break;
|
| 1970 | case 'h':
|
| 1971 | default:
|
| 1972 | usage(argv);
|
| 1973 | return -1;
|
| 1974 | }
|
| 1975 | }
|
| 1976 |
|
| 1977 | if (!cg_fd) {
|
| 1978 | cg_fd = cgroup_setup_and_join(CG_PATH);
|
| 1979 | if (cg_fd < 0)
|
| 1980 | return cg_fd;
|
| 1981 | cg_created = 1;
|
| 1982 | }
|
| 1983 |
|
| 1984 |
|
| 1985 | libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
|
| 1986 |
|
| 1987 | if (test == SELFTESTS) {
|
| 1988 | err = test_selftest(cg_fd, &options);
|
| 1989 | goto out;
|
| 1990 | }
|
| 1991 |
|
| 1992 | err = populate_progs(bpf_file);
|
| 1993 | if (err) {
|
| 1994 | fprintf(stderr, "populate program: (%s) %s\n",
|
| 1995 | bpf_file, strerror(errno));
|
| 1996 | return 1;
|
| 1997 | }
|
| 1998 | running = 1;
|
| 1999 |
|
| 2000 |
|
| 2001 | signal(SIGINT, running_handler);
|
| 2002 |
|
| 2003 | options.iov_count = iov_count;
|
| 2004 | options.iov_length = length;
|
| 2005 | options.rate = rate;
|
| 2006 |
|
| 2007 | err = run_options(&options, cg_fd, test);
|
| 2008 | out:
|
| 2009 | if (options.whitelist)
|
| 2010 | free(options.whitelist);
|
| 2011 | if (options.blacklist)
|
| 2012 | free(options.blacklist);
|
| 2013 | close(cg_fd);
|
| 2014 | if (cg_created)
|
| 2015 | cleanup_cgroup_environment();
|
| 2016 | return err;
|
| 2017 | }
|
| 2018 |
|
| 2019 | void running_handler(int a)
|
| 2020 | {
|
| 2021 | running = 0;
|
| 2022 | }
|