projects/linux·blob master · refs

1441 lines · 39.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2020 Cloudflare
3#include <error.h>
4#include <linux/tcp.h>
5#include <linux/socket.h>
6#include <sys/epoll.h>
7
8#include "test_progs.h"
9#include "test_skmsg_load_helpers.skel.h"
10#include "test_sockmap_invalid_update.skel.h"
11#include "test_sockmap_skb_verdict_attach.skel.h"
12#include "test_sockmap_progs_query.skel.h"
13#include "test_sockmap_pass_prog.skel.h"
14#include "test_sockmap_drop_prog.skel.h"
15#include "test_sockmap_change_tail.skel.h"
16#include "test_sockmap_msg_pop_data.skel.h"
17#include "bpf_iter_sockmap.skel.h"
18
19#include "sockmap_helpers.h"
20
21#define TCP_REPAIR 19 /* TCP sock is under repair right now */
22
23#define TCP_REPAIR_ON 1
24#define TCP_REPAIR_OFF_NO_WP -1 /* Turn off without window probes */
25
26/**
27 * SOL_TCP is defined in <netinet/tcp.h> (glibc), but the copybuf_address
28 * field of tcp_zerocopy_receive is not yet included in older versions.
29 * This workaround remains necessary until the glibc update propagates.
30 */
31#ifndef SOL_TCP
32#define SOL_TCP 6
33#endif
34
35static int connected_socket_v4(void)
36{
37 struct sockaddr_in addr = {
38 .sin_family = AF_INET,
39 .sin_port = htons(80),
40 .sin_addr = { inet_addr("127.0.0.1") },
41 };
42 socklen_t len = sizeof(addr);
43 int s, repair, err;
44
45 s = socket(AF_INET, SOCK_STREAM, 0);
46 if (!ASSERT_GE(s, 0, "socket"))
47 goto error;
48
49 repair = TCP_REPAIR_ON;
50 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair));
51 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)"))
52 goto error;
53
54 err = connect(s, (struct sockaddr *)&addr, len);
55 if (!ASSERT_OK(err, "connect"))
56 goto error;
57
58 repair = TCP_REPAIR_OFF_NO_WP;
59 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair));
60 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)"))
61 goto error;
62
63 return s;
64error:
65 perror(__func__);
66 close(s);
67 return -1;
68}
69
70static void compare_cookies(struct bpf_map *src, struct bpf_map *dst)
71{
72 __u32 i, max_entries = bpf_map__max_entries(src);
73 int err, src_fd, dst_fd;
74
75 src_fd = bpf_map__fd(src);
76 dst_fd = bpf_map__fd(dst);
77
78 for (i = 0; i < max_entries; i++) {
79 __u64 src_cookie, dst_cookie;
80
81 err = bpf_map_lookup_elem(src_fd, &i, &src_cookie);
82 if (err && errno == ENOENT) {
83 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie);
84 ASSERT_ERR(err, "map_lookup_elem(dst)");
85 ASSERT_EQ(errno, ENOENT, "map_lookup_elem(dst)");
86 continue;
87 }
88 if (!ASSERT_OK(err, "lookup_elem(src)"))
89 continue;
90
91 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie);
92 if (!ASSERT_OK(err, "lookup_elem(dst)"))
93 continue;
94
95 ASSERT_EQ(dst_cookie, src_cookie, "cookie mismatch");
96 }
97}
98
99/* Create a map, populate it with one socket, and free the map. */
100static void test_sockmap_create_update_free(enum bpf_map_type map_type)
101{
102 const int zero = 0;
103 int s, map, err;
104
105 s = connected_socket_v4();
106 if (!ASSERT_GE(s, 0, "connected_socket_v4"))
107 return;
108
109 map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL);
110 if (!ASSERT_GE(map, 0, "bpf_map_create"))
111 goto out;
112
113 err = bpf_map_update_elem(map, &zero, &s, BPF_NOEXIST);
114 if (!ASSERT_OK(err, "bpf_map_update"))
115 goto out;
116
117out:
118 close(map);
119 close(s);
120}
121
122static void test_sockmap_vsock_delete_on_close(void)
123{
124 int map, c, p, err, zero = 0;
125
126 map = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(int),
127 sizeof(int), 1, NULL);
128 if (!ASSERT_OK_FD(map, "bpf_map_create"))
129 return;
130
131 err = create_pair(AF_VSOCK, SOCK_STREAM, &c, &p);
132 if (!ASSERT_OK(err, "create_pair"))
133 goto close_map;
134
135 if (xbpf_map_update_elem(map, &zero, &c, BPF_NOEXIST))
136 goto close_socks;
137
138 xclose(c);
139 xclose(p);
140
141 err = create_pair(AF_VSOCK, SOCK_STREAM, &c, &p);
142 if (!ASSERT_OK(err, "create_pair"))
143 goto close_map;
144
145 err = bpf_map_update_elem(map, &zero, &c, BPF_NOEXIST);
146 ASSERT_OK(err, "after close(), bpf_map_update");
147
148close_socks:
149 xclose(c);
150 xclose(p);
151close_map:
152 xclose(map);
153}
154
155static void test_skmsg_helpers(enum bpf_map_type map_type)
156{
157 struct test_skmsg_load_helpers *skel;
158 int err, map, verdict;
159
160 skel = test_skmsg_load_helpers__open_and_load();
161 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load"))
162 return;
163
164 verdict = bpf_program__fd(skel->progs.prog_msg_verdict);
165 map = bpf_map__fd(skel->maps.sock_map);
166
167 err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0);
168 if (!ASSERT_OK(err, "bpf_prog_attach"))
169 goto out;
170
171 err = bpf_prog_detach2(verdict, map, BPF_SK_MSG_VERDICT);
172 if (!ASSERT_OK(err, "bpf_prog_detach2"))
173 goto out;
174out:
175 test_skmsg_load_helpers__destroy(skel);
176}
177
178static void test_skmsg_helpers_with_link(enum bpf_map_type map_type)
179{
180 struct bpf_program *prog, *prog_clone, *prog_clone2;
181 DECLARE_LIBBPF_OPTS(bpf_link_update_opts, opts);
182 struct test_skmsg_load_helpers *skel;
183 struct bpf_link *link, *link2;
184 int err, map;
185
186 skel = test_skmsg_load_helpers__open_and_load();
187 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load"))
188 return;
189
190 prog = skel->progs.prog_msg_verdict;
191 prog_clone = skel->progs.prog_msg_verdict_clone;
192 prog_clone2 = skel->progs.prog_msg_verdict_clone2;
193 map = bpf_map__fd(skel->maps.sock_map);
194
195 link = bpf_program__attach_sockmap(prog, map);
196 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap"))
197 goto out;
198
199 /* Fail since bpf_link for the same prog has been created. */
200 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_MSG_VERDICT, 0);
201 if (!ASSERT_ERR(err, "bpf_prog_attach"))
202 goto out;
203
204 /* Fail since bpf_link for the same prog type has been created. */
205 link2 = bpf_program__attach_sockmap(prog_clone, map);
206 if (!ASSERT_ERR_PTR(link2, "bpf_program__attach_sockmap")) {
207 bpf_link__destroy(link2);
208 goto out;
209 }
210
211 err = bpf_link__update_program(link, prog_clone);
212 if (!ASSERT_OK(err, "bpf_link__update_program"))
213 goto out;
214
215 /* Fail since a prog with different type attempts to do update. */
216 err = bpf_link__update_program(link, skel->progs.prog_skb_verdict);
217 if (!ASSERT_ERR(err, "bpf_link__update_program"))
218 goto out;
219
220 /* Fail since the old prog does not match the one in the kernel. */
221 opts.old_prog_fd = bpf_program__fd(prog_clone2);
222 opts.flags = BPF_F_REPLACE;
223 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts);
224 if (!ASSERT_ERR(err, "bpf_link_update"))
225 goto out;
226
227 opts.old_prog_fd = bpf_program__fd(prog_clone);
228 opts.flags = BPF_F_REPLACE;
229 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts);
230 if (!ASSERT_OK(err, "bpf_link_update"))
231 goto out;
232out:
233 bpf_link__destroy(link);
234 test_skmsg_load_helpers__destroy(skel);
235}
236
237static void test_sockmap_invalid_update(void)
238{
239 struct test_sockmap_invalid_update *skel;
240
241 skel = test_sockmap_invalid_update__open_and_load();
242 if (!ASSERT_NULL(skel, "open_and_load"))
243 test_sockmap_invalid_update__destroy(skel);
244}
245
246static void test_sockmap_copy(enum bpf_map_type map_type)
247{
248 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
249 int err, len, src_fd, iter_fd;
250 union bpf_iter_link_info linfo = {};
251 __u32 i, num_sockets, num_elems;
252 struct bpf_iter_sockmap *skel;
253 __s64 *sock_fd = NULL;
254 struct bpf_link *link;
255 struct bpf_map *src;
256 char buf[64];
257
258 skel = bpf_iter_sockmap__open_and_load();
259 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
260 return;
261
262 if (map_type == BPF_MAP_TYPE_SOCKMAP) {
263 src = skel->maps.sockmap;
264 num_elems = bpf_map__max_entries(src);
265 num_sockets = num_elems - 1;
266 } else {
267 src = skel->maps.sockhash;
268 num_elems = bpf_map__max_entries(src) - 1;
269 num_sockets = num_elems;
270 }
271
272 sock_fd = calloc(num_sockets, sizeof(*sock_fd));
273 if (!ASSERT_OK_PTR(sock_fd, "calloc(sock_fd)"))
274 goto out;
275
276 for (i = 0; i < num_sockets; i++)
277 sock_fd[i] = -1;
278
279 src_fd = bpf_map__fd(src);
280
281 for (i = 0; i < num_sockets; i++) {
282 sock_fd[i] = connected_socket_v4();
283 if (!ASSERT_NEQ(sock_fd[i], -1, "connected_socket_v4"))
284 goto out;
285
286 err = bpf_map_update_elem(src_fd, &i, &sock_fd[i], BPF_NOEXIST);
287 if (!ASSERT_OK(err, "map_update"))
288 goto out;
289 }
290
291 linfo.map.map_fd = src_fd;
292 opts.link_info = &linfo;
293 opts.link_info_len = sizeof(linfo);
294 link = bpf_program__attach_iter(skel->progs.copy, &opts);
295 if (!ASSERT_OK_PTR(link, "attach_iter"))
296 goto out;
297
298 iter_fd = bpf_iter_create(bpf_link__fd(link));
299 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
300 goto free_link;
301
302 /* do some tests */
303 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
304 ;
305 if (!ASSERT_GE(len, 0, "read"))
306 goto close_iter;
307
308 /* test results */
309 if (!ASSERT_EQ(skel->bss->elems, num_elems, "elems"))
310 goto close_iter;
311
312 if (!ASSERT_EQ(skel->bss->socks, num_sockets, "socks"))
313 goto close_iter;
314
315 compare_cookies(src, skel->maps.dst);
316
317close_iter:
318 close(iter_fd);
319free_link:
320 bpf_link__destroy(link);
321out:
322 for (i = 0; sock_fd && i < num_sockets; i++)
323 if (sock_fd[i] >= 0)
324 close(sock_fd[i]);
325 if (sock_fd)
326 free(sock_fd);
327 bpf_iter_sockmap__destroy(skel);
328}
329
330static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first,
331 enum bpf_attach_type second)
332{
333 struct test_sockmap_skb_verdict_attach *skel;
334 int err, map, verdict;
335
336 skel = test_sockmap_skb_verdict_attach__open_and_load();
337 if (!ASSERT_OK_PTR(skel, "open_and_load"))
338 return;
339
340 verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
341 map = bpf_map__fd(skel->maps.sock_map);
342
343 err = bpf_prog_attach(verdict, map, first, 0);
344 if (!ASSERT_OK(err, "bpf_prog_attach"))
345 goto out;
346
347 err = bpf_prog_attach(verdict, map, second, 0);
348 ASSERT_EQ(err, -EBUSY, "prog_attach_fail");
349
350 err = bpf_prog_detach2(verdict, map, first);
351 if (!ASSERT_OK(err, "bpf_prog_detach2"))
352 goto out;
353out:
354 test_sockmap_skb_verdict_attach__destroy(skel);
355}
356
357static void test_sockmap_skb_verdict_attach_with_link(void)
358{
359 struct test_sockmap_skb_verdict_attach *skel;
360 struct bpf_program *prog;
361 struct bpf_link *link;
362 int err, map;
363
364 skel = test_sockmap_skb_verdict_attach__open_and_load();
365 if (!ASSERT_OK_PTR(skel, "open_and_load"))
366 return;
367 prog = skel->progs.prog_skb_verdict;
368 map = bpf_map__fd(skel->maps.sock_map);
369 link = bpf_program__attach_sockmap(prog, map);
370 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap"))
371 goto out;
372
373 bpf_link__destroy(link);
374
375 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0);
376 if (!ASSERT_OK(err, "bpf_prog_attach"))
377 goto out;
378
379 /* Fail since attaching with the same prog/map has been done. */
380 link = bpf_program__attach_sockmap(prog, map);
381 if (!ASSERT_ERR_PTR(link, "bpf_program__attach_sockmap"))
382 bpf_link__destroy(link);
383
384 err = bpf_prog_detach2(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT);
385 if (!ASSERT_OK(err, "bpf_prog_detach2"))
386 goto out;
387out:
388 test_sockmap_skb_verdict_attach__destroy(skel);
389}
390
391static __u32 query_prog_id(int prog_fd)
392{
393 struct bpf_prog_info info = {};
394 __u32 info_len = sizeof(info);
395 int err;
396
397 err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len);
398 if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd") ||
399 !ASSERT_EQ(info_len, sizeof(info), "bpf_prog_get_info_by_fd"))
400 return 0;
401
402 return info.id;
403}
404
405static void test_sockmap_progs_query(enum bpf_attach_type attach_type)
406{
407 struct test_sockmap_progs_query *skel;
408 int err, map_fd, verdict_fd;
409 __u32 attach_flags = 0;
410 __u32 prog_ids[3] = {};
411 __u32 prog_cnt = 3;
412
413 skel = test_sockmap_progs_query__open_and_load();
414 if (!ASSERT_OK_PTR(skel, "test_sockmap_progs_query__open_and_load"))
415 return;
416
417 map_fd = bpf_map__fd(skel->maps.sock_map);
418
419 if (attach_type == BPF_SK_MSG_VERDICT)
420 verdict_fd = bpf_program__fd(skel->progs.prog_skmsg_verdict);
421 else
422 verdict_fd = bpf_program__fd(skel->progs.prog_skb_verdict);
423
424 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */,
425 &attach_flags, prog_ids, &prog_cnt);
426 ASSERT_OK(err, "bpf_prog_query failed");
427 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query");
428 ASSERT_EQ(prog_cnt, 0, "wrong program count on query");
429
430 err = bpf_prog_attach(verdict_fd, map_fd, attach_type, 0);
431 if (!ASSERT_OK(err, "bpf_prog_attach failed"))
432 goto out;
433
434 prog_cnt = 1;
435 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */,
436 &attach_flags, prog_ids, &prog_cnt);
437 ASSERT_OK(err, "bpf_prog_query failed");
438 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query");
439 ASSERT_EQ(prog_cnt, 1, "wrong program count on query");
440 ASSERT_EQ(prog_ids[0], query_prog_id(verdict_fd),
441 "wrong prog_ids on query");
442
443 bpf_prog_detach2(verdict_fd, map_fd, attach_type);
444out:
445 test_sockmap_progs_query__destroy(skel);
446}
447
448#define MAX_EVENTS 10
449static void test_sockmap_skb_verdict_shutdown(void)
450{
451 int n, err, map, verdict, c1 = -1, p1 = -1;
452 struct epoll_event ev, events[MAX_EVENTS];
453 struct test_sockmap_pass_prog *skel;
454 int zero = 0;
455 int epollfd;
456 char b;
457
458 skel = test_sockmap_pass_prog__open_and_load();
459 if (!ASSERT_OK_PTR(skel, "open_and_load"))
460 return;
461
462 verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
463 map = bpf_map__fd(skel->maps.sock_map_rx);
464
465 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0);
466 if (!ASSERT_OK(err, "bpf_prog_attach"))
467 goto out;
468
469 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1);
470 if (err < 0)
471 goto out;
472
473 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST);
474 if (err < 0)
475 goto out_close;
476
477 shutdown(p1, SHUT_WR);
478
479 ev.events = EPOLLIN;
480 ev.data.fd = c1;
481
482 epollfd = epoll_create1(0);
483 if (!ASSERT_GT(epollfd, -1, "epoll_create(0)"))
484 goto out_close;
485 err = epoll_ctl(epollfd, EPOLL_CTL_ADD, c1, &ev);
486 if (!ASSERT_OK(err, "epoll_ctl(EPOLL_CTL_ADD)"))
487 goto out_close;
488 err = epoll_wait(epollfd, events, MAX_EVENTS, -1);
489 if (!ASSERT_EQ(err, 1, "epoll_wait(fd)"))
490 goto out_close;
491
492 n = recv(c1, &b, 1, MSG_DONTWAIT);
493 ASSERT_EQ(n, 0, "recv(fin)");
494out_close:
495 close(c1);
496 close(p1);
497out:
498 test_sockmap_pass_prog__destroy(skel);
499}
500
501
502static void do_test_sockmap_skb_verdict_fionread(int sotype, bool pass_prog)
503{
504 int err, map, verdict, c0 = -1, c1 = -1, p0 = -1, p1 = -1;
505 int expected, zero = 0, sent, recvd, avail;
506 struct test_sockmap_pass_prog *pass = NULL;
507 struct test_sockmap_drop_prog *drop = NULL;
508 char buf[256] = "0123456789";
509 int split_len = sizeof(buf) / 2;
510
511 if (pass_prog) {
512 pass = test_sockmap_pass_prog__open_and_load();
513 if (!ASSERT_OK_PTR(pass, "open_and_load"))
514 return;
515 verdict = bpf_program__fd(pass->progs.prog_skb_verdict);
516 map = bpf_map__fd(pass->maps.sock_map_rx);
517 if (sotype == SOCK_DGRAM)
518 expected = split_len; /* FIONREAD for UDP is different from TCP */
519 else
520 expected = sizeof(buf);
521 } else {
522 drop = test_sockmap_drop_prog__open_and_load();
523 if (!ASSERT_OK_PTR(drop, "open_and_load"))
524 return;
525 verdict = bpf_program__fd(drop->progs.prog_skb_verdict);
526 map = bpf_map__fd(drop->maps.sock_map_rx);
527 /* On drop data is consumed immediately and copied_seq inc'd */
528 expected = 0;
529 }
530
531
532 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0);
533 if (!ASSERT_OK(err, "bpf_prog_attach"))
534 goto out;
535
536 err = create_socket_pairs(AF_INET, sotype, &c0, &c1, &p0, &p1);
537 if (!ASSERT_OK(err, "create_socket_pairs()"))
538 goto out;
539
540 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST);
541 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)"))
542 goto out_close;
543
544 sent = xsend(p1, &buf, split_len, 0);
545 sent += xsend(p1, &buf, sizeof(buf) - split_len, 0);
546 ASSERT_EQ(sent, sizeof(buf), "xsend(p1)");
547 err = ioctl(c1, FIONREAD, &avail);
548 ASSERT_OK(err, "ioctl(FIONREAD) error");
549 ASSERT_EQ(avail, expected, "ioctl(FIONREAD)");
550 /* On DROP test there will be no data to read */
551 if (pass_prog) {
552 recvd = recv_timeout(c1, &buf, sizeof(buf), MSG_DONTWAIT, IO_TIMEOUT_SEC);
553 ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(c0)");
554 }
555
556out_close:
557 close(c0);
558 close(p0);
559 close(c1);
560 close(p1);
561out:
562 if (pass_prog)
563 test_sockmap_pass_prog__destroy(pass);
564 else
565 test_sockmap_drop_prog__destroy(drop);
566}
567
568static void test_sockmap_skb_verdict_fionread(bool pass_prog)
569{
570 do_test_sockmap_skb_verdict_fionread(SOCK_STREAM, pass_prog);
571 do_test_sockmap_skb_verdict_fionread(SOCK_DGRAM, pass_prog);
572}
573
574static void test_sockmap_skb_verdict_change_tail(void)
575{
576 struct test_sockmap_change_tail *skel;
577 int err, map, verdict;
578 int c1, p1, sent, recvd;
579 int zero = 0;
580 char buf[2];
581
582 skel = test_sockmap_change_tail__open_and_load();
583 if (!ASSERT_OK_PTR(skel, "open_and_load"))
584 return;
585 verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
586 map = bpf_map__fd(skel->maps.sock_map_rx);
587
588 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0);
589 if (!ASSERT_OK(err, "bpf_prog_attach"))
590 goto out;
591 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1);
592 if (!ASSERT_OK(err, "create_pair()"))
593 goto out;
594 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST);
595 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)"))
596 goto out_close;
597 sent = xsend(p1, "Tr", 2, 0);
598 ASSERT_EQ(sent, 2, "xsend(p1)");
599 recvd = recv(c1, buf, 2, 0);
600 ASSERT_EQ(recvd, 1, "recv(c1)");
601 ASSERT_EQ(skel->data->change_tail_ret, 0, "change_tail_ret");
602
603 sent = xsend(p1, "G", 1, 0);
604 ASSERT_EQ(sent, 1, "xsend(p1)");
605 recvd = recv(c1, buf, 2, 0);
606 ASSERT_EQ(recvd, 2, "recv(c1)");
607 ASSERT_EQ(skel->data->change_tail_ret, 0, "change_tail_ret");
608
609 sent = xsend(p1, "E", 1, 0);
610 ASSERT_EQ(sent, 1, "xsend(p1)");
611 recvd = recv(c1, buf, 1, 0);
612 ASSERT_EQ(recvd, 1, "recv(c1)");
613 ASSERT_EQ(skel->data->change_tail_ret, -EINVAL, "change_tail_ret");
614
615out_close:
616 close(c1);
617 close(p1);
618out:
619 test_sockmap_change_tail__destroy(skel);
620}
621
622static void test_sockmap_msg_verdict_pop_data(void)
623{
624 struct test_sockmap_msg_pop_data *skel;
625 int err, map, verdict;
626 int c1 = -1, p1 = -1, sent;
627 int zero = 0;
628 char *buf;
629 const size_t len = 32 * 1024;
630
631 skel = test_sockmap_msg_pop_data__open_and_load();
632 if (!ASSERT_OK_PTR(skel, "open_and_load"))
633 return;
634
635 verdict = bpf_program__fd(skel->progs.prog_msg_pop_data);
636 map = bpf_map__fd(skel->maps.sock_map);
637
638 err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0);
639 if (!ASSERT_OK(err, "bpf_prog_attach"))
640 goto out;
641
642 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1);
643 if (!ASSERT_OK(err, "create_pair"))
644 goto out;
645
646 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST);
647 if (!ASSERT_OK(err, "bpf_map_update_elem"))
648 goto out_close;
649
650 buf = calloc(len, 1);
651 if (!ASSERT_OK_PTR(buf, "calloc"))
652 goto out_close;
653
654 sent = xsend(c1, buf, len, 0);
655 ASSERT_EQ(sent, (ssize_t)len, "xsend");
656 ASSERT_EQ(skel->data->pop_data_ret, -EINVAL, "pop_data_rejects overflow");
657
658 free(buf);
659
660out_close:
661 close(c1);
662 close(p1);
663out:
664 test_sockmap_msg_pop_data__destroy(skel);
665}
666
667static void test_sockmap_skb_verdict_peek_helper(int map)
668{
669 int err, c1, p1, zero = 0, sent, recvd, avail;
670 char snd[256] = "0123456789";
671 char rcv[256] = "0";
672
673 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1);
674 if (!ASSERT_OK(err, "create_pair()"))
675 return;
676
677 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST);
678 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)"))
679 goto out_close;
680
681 sent = xsend(p1, snd, sizeof(snd), 0);
682 ASSERT_EQ(sent, sizeof(snd), "xsend(p1)");
683 recvd = recv(c1, rcv, sizeof(rcv), MSG_PEEK);
684 ASSERT_EQ(recvd, sizeof(rcv), "recv(c1)");
685 err = ioctl(c1, FIONREAD, &avail);
686 ASSERT_OK(err, "ioctl(FIONREAD) error");
687 ASSERT_EQ(avail, sizeof(snd), "after peek ioctl(FIONREAD)");
688 recvd = recv(c1, rcv, sizeof(rcv), 0);
689 ASSERT_EQ(recvd, sizeof(rcv), "recv(p0)");
690 err = ioctl(c1, FIONREAD, &avail);
691 ASSERT_OK(err, "ioctl(FIONREAD) error");
692 ASSERT_EQ(avail, 0, "after read ioctl(FIONREAD)");
693
694out_close:
695 close(c1);
696 close(p1);
697}
698
699static void test_sockmap_skb_verdict_peek(void)
700{
701 struct test_sockmap_pass_prog *pass;
702 int err, map, verdict;
703
704 pass = test_sockmap_pass_prog__open_and_load();
705 if (!ASSERT_OK_PTR(pass, "open_and_load"))
706 return;
707 verdict = bpf_program__fd(pass->progs.prog_skb_verdict);
708 map = bpf_map__fd(pass->maps.sock_map_rx);
709
710 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0);
711 if (!ASSERT_OK(err, "bpf_prog_attach"))
712 goto out;
713
714 test_sockmap_skb_verdict_peek_helper(map);
715
716out:
717 test_sockmap_pass_prog__destroy(pass);
718}
719
720static void test_sockmap_skb_verdict_peek_with_link(void)
721{
722 struct test_sockmap_pass_prog *pass;
723 struct bpf_program *prog;
724 struct bpf_link *link;
725 int err, map;
726
727 pass = test_sockmap_pass_prog__open_and_load();
728 if (!ASSERT_OK_PTR(pass, "open_and_load"))
729 return;
730 prog = pass->progs.prog_skb_verdict;
731 map = bpf_map__fd(pass->maps.sock_map_rx);
732 link = bpf_program__attach_sockmap(prog, map);
733 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap"))
734 goto out;
735
736 err = bpf_link__update_program(link, pass->progs.prog_skb_verdict_clone);
737 if (!ASSERT_OK(err, "bpf_link__update_program"))
738 goto out;
739
740 /* Fail since a prog with different attach type attempts to do update. */
741 err = bpf_link__update_program(link, pass->progs.prog_skb_parser);
742 if (!ASSERT_ERR(err, "bpf_link__update_program"))
743 goto out;
744
745 test_sockmap_skb_verdict_peek_helper(map);
746 ASSERT_EQ(pass->bss->clone_called, 1, "clone_called");
747out:
748 bpf_link__destroy(link);
749 test_sockmap_pass_prog__destroy(pass);
750}
751
752static void test_sockmap_unconnected_unix(void)
753{
754 int err, map, stream = -1, dgram = -1, zero = 0;
755 struct test_sockmap_pass_prog *skel;
756
757 skel = test_sockmap_pass_prog__open_and_load();
758 if (!ASSERT_OK_PTR(skel, "open_and_load"))
759 return;
760
761 map = bpf_map__fd(skel->maps.sock_map_rx);
762
763 stream = xsocket(AF_UNIX, SOCK_STREAM, 0);
764 if (stream < 0)
765 goto out;
766
767 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0);
768 if (dgram < 0)
769 goto out;
770
771 err = bpf_map_update_elem(map, &zero, &stream, BPF_ANY);
772 if (!ASSERT_ERR(err, "bpf_map_update_elem(stream)"))
773 goto out;
774
775 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY);
776 ASSERT_OK(err, "bpf_map_update_elem(dgram)");
777out:
778 close(stream);
779 close(dgram);
780 test_sockmap_pass_prog__destroy(skel);
781}
782
783static void test_sockmap_many_socket(void)
784{
785 struct test_sockmap_pass_prog *skel;
786 int stream[2], dgram, udp, tcp;
787 int i, err, map, entry = 0;
788
789 skel = test_sockmap_pass_prog__open_and_load();
790 if (!ASSERT_OK_PTR(skel, "open_and_load"))
791 return;
792
793 map = bpf_map__fd(skel->maps.sock_map_rx);
794
795 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0);
796 if (dgram < 0) {
797 test_sockmap_pass_prog__destroy(skel);
798 return;
799 }
800
801 tcp = connected_socket_v4();
802 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) {
803 close(dgram);
804 test_sockmap_pass_prog__destroy(skel);
805 return;
806 }
807
808 udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK);
809 if (udp < 0) {
810 close(dgram);
811 close(tcp);
812 test_sockmap_pass_prog__destroy(skel);
813 return;
814 }
815
816 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream);
817 ASSERT_OK(err, "socketpair(af_unix, sock_stream)");
818 if (err)
819 goto out;
820
821 for (i = 0; i < 2; i++, entry++) {
822 err = bpf_map_update_elem(map, &entry, &stream[0], BPF_ANY);
823 ASSERT_OK(err, "bpf_map_update_elem(stream)");
824 }
825 for (i = 0; i < 2; i++, entry++) {
826 err = bpf_map_update_elem(map, &entry, &dgram, BPF_ANY);
827 ASSERT_OK(err, "bpf_map_update_elem(dgram)");
828 }
829 for (i = 0; i < 2; i++, entry++) {
830 err = bpf_map_update_elem(map, &entry, &udp, BPF_ANY);
831 ASSERT_OK(err, "bpf_map_update_elem(udp)");
832 }
833 for (i = 0; i < 2; i++, entry++) {
834 err = bpf_map_update_elem(map, &entry, &tcp, BPF_ANY);
835 ASSERT_OK(err, "bpf_map_update_elem(tcp)");
836 }
837 for (entry--; entry >= 0; entry--) {
838 err = bpf_map_delete_elem(map, &entry);
839 ASSERT_OK(err, "bpf_map_delete_elem(entry)");
840 }
841
842 close(stream[0]);
843 close(stream[1]);
844out:
845 close(dgram);
846 close(tcp);
847 close(udp);
848 test_sockmap_pass_prog__destroy(skel);
849}
850
851static void test_sockmap_many_maps(void)
852{
853 struct test_sockmap_pass_prog *skel;
854 int stream[2], dgram, udp, tcp;
855 int i, err, map[2], entry = 0;
856
857 skel = test_sockmap_pass_prog__open_and_load();
858 if (!ASSERT_OK_PTR(skel, "open_and_load"))
859 return;
860
861 map[0] = bpf_map__fd(skel->maps.sock_map_rx);
862 map[1] = bpf_map__fd(skel->maps.sock_map_tx);
863
864 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0);
865 if (dgram < 0) {
866 test_sockmap_pass_prog__destroy(skel);
867 return;
868 }
869
870 tcp = connected_socket_v4();
871 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) {
872 close(dgram);
873 test_sockmap_pass_prog__destroy(skel);
874 return;
875 }
876
877 udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK);
878 if (udp < 0) {
879 close(dgram);
880 close(tcp);
881 test_sockmap_pass_prog__destroy(skel);
882 return;
883 }
884
885 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream);
886 ASSERT_OK(err, "socketpair(af_unix, sock_stream)");
887 if (err)
888 goto out;
889
890 for (i = 0; i < 2; i++, entry++) {
891 err = bpf_map_update_elem(map[i], &entry, &stream[0], BPF_ANY);
892 ASSERT_OK(err, "bpf_map_update_elem(stream)");
893 }
894 for (i = 0; i < 2; i++, entry++) {
895 err = bpf_map_update_elem(map[i], &entry, &dgram, BPF_ANY);
896 ASSERT_OK(err, "bpf_map_update_elem(dgram)");
897 }
898 for (i = 0; i < 2; i++, entry++) {
899 err = bpf_map_update_elem(map[i], &entry, &udp, BPF_ANY);
900 ASSERT_OK(err, "bpf_map_update_elem(udp)");
901 }
902 for (i = 0; i < 2; i++, entry++) {
903 err = bpf_map_update_elem(map[i], &entry, &tcp, BPF_ANY);
904 ASSERT_OK(err, "bpf_map_update_elem(tcp)");
905 }
906 for (entry--; entry >= 0; entry--) {
907 err = bpf_map_delete_elem(map[1], &entry);
908 entry--;
909 ASSERT_OK(err, "bpf_map_delete_elem(entry)");
910 err = bpf_map_delete_elem(map[0], &entry);
911 ASSERT_OK(err, "bpf_map_delete_elem(entry)");
912 }
913
914 close(stream[0]);
915 close(stream[1]);
916out:
917 close(dgram);
918 close(tcp);
919 close(udp);
920 test_sockmap_pass_prog__destroy(skel);
921}
922
923static void test_sockmap_same_sock(void)
924{
925 struct test_sockmap_pass_prog *skel;
926 int stream[2], dgram, udp, tcp;
927 int i, err, map, zero = 0;
928
929 skel = test_sockmap_pass_prog__open_and_load();
930 if (!ASSERT_OK_PTR(skel, "open_and_load"))
931 return;
932
933 map = bpf_map__fd(skel->maps.sock_map_rx);
934
935 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0);
936 if (dgram < 0) {
937 test_sockmap_pass_prog__destroy(skel);
938 return;
939 }
940
941 tcp = connected_socket_v4();
942 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) {
943 close(dgram);
944 test_sockmap_pass_prog__destroy(skel);
945 return;
946 }
947
948 udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK);
949 if (udp < 0) {
950 close(dgram);
951 close(tcp);
952 test_sockmap_pass_prog__destroy(skel);
953 return;
954 }
955
956 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream);
957 ASSERT_OK(err, "socketpair(af_unix, sock_stream)");
958 if (err) {
959 close(tcp);
960 goto out;
961 }
962
963 for (i = 0; i < 2; i++) {
964 err = bpf_map_update_elem(map, &zero, &stream[0], BPF_ANY);
965 ASSERT_OK(err, "bpf_map_update_elem(stream)");
966 }
967 for (i = 0; i < 2; i++) {
968 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY);
969 ASSERT_OK(err, "bpf_map_update_elem(dgram)");
970 }
971 for (i = 0; i < 2; i++) {
972 err = bpf_map_update_elem(map, &zero, &udp, BPF_ANY);
973 ASSERT_OK(err, "bpf_map_update_elem(udp)");
974 }
975 for (i = 0; i < 2; i++) {
976 err = bpf_map_update_elem(map, &zero, &tcp, BPF_ANY);
977 ASSERT_OK(err, "bpf_map_update_elem(tcp)");
978 }
979
980 close(tcp);
981 err = bpf_map_delete_elem(map, &zero);
982 ASSERT_ERR(err, "bpf_map_delete_elem(entry)");
983
984 close(stream[0]);
985 close(stream[1]);
986out:
987 close(dgram);
988 close(udp);
989 test_sockmap_pass_prog__destroy(skel);
990}
991
992static void test_sockmap_skb_verdict_vsock_poll(void)
993{
994 struct test_sockmap_pass_prog *skel;
995 int err, map, conn, peer;
996 struct bpf_program *prog;
997 struct bpf_link *link;
998 char buf = 'x';
999 int zero = 0;
1000
1001 skel = test_sockmap_pass_prog__open_and_load();
1002 if (!ASSERT_OK_PTR(skel, "open_and_load"))
1003 return;
1004
1005 if (create_pair(AF_VSOCK, SOCK_STREAM, &conn, &peer))
1006 goto destroy;
1007
1008 prog = skel->progs.prog_skb_verdict;
1009 map = bpf_map__fd(skel->maps.sock_map_rx);
1010 link = bpf_program__attach_sockmap(prog, map);
1011 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap"))
1012 goto close;
1013
1014 err = bpf_map_update_elem(map, &zero, &conn, BPF_ANY);
1015 if (!ASSERT_OK(err, "bpf_map_update_elem"))
1016 goto detach;
1017
1018 if (xsend(peer, &buf, 1, 0) != 1)
1019 goto detach;
1020
1021 err = poll_read(conn, IO_TIMEOUT_SEC);
1022 if (!ASSERT_OK(err, "poll"))
1023 goto detach;
1024
1025 if (xrecv_nonblock(conn, &buf, 1, 0) != 1)
1026 FAIL("xrecv_nonblock");
1027detach:
1028 bpf_link__destroy(link);
1029close:
1030 xclose(conn);
1031 xclose(peer);
1032destroy:
1033 test_sockmap_pass_prog__destroy(skel);
1034}
1035
1036static void test_sockmap_vsock_unconnected(void)
1037{
1038 struct sockaddr_storage addr;
1039 int map, s, zero = 0;
1040 socklen_t alen;
1041
1042 map = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(int),
1043 sizeof(int), 1, NULL);
1044 if (!ASSERT_OK_FD(map, "bpf_map_create"))
1045 return;
1046
1047 s = xsocket(AF_VSOCK, SOCK_STREAM, 0);
1048 if (s < 0)
1049 goto close_map;
1050
1051 /* Fail connect(), but trigger transport assignment. */
1052 init_addr_loopback(AF_VSOCK, &addr, &alen);
1053 if (!ASSERT_ERR(connect(s, sockaddr(&addr), alen), "connect"))
1054 goto close_sock;
1055
1056 ASSERT_ERR(bpf_map_update_elem(map, &zero, &s, BPF_ANY), "map_update");
1057
1058close_sock:
1059 xclose(s);
1060close_map:
1061 xclose(map);
1062}
1063
1064/* it is used to reproduce WARNING */
1065static void test_sockmap_zc(void)
1066{
1067 int map, err, sent, recvd, zero = 0, one = 1, on = 1;
1068 char buf[10] = "0123456789", rcv[11], addr[100];
1069 struct test_sockmap_pass_prog *skel = NULL;
1070 int c0 = -1, p0 = -1, c1 = -1, p1 = -1;
1071 struct tcp_zerocopy_receive zc;
1072 socklen_t zc_len = sizeof(zc);
1073 struct bpf_program *prog;
1074
1075 skel = test_sockmap_pass_prog__open_and_load();
1076 if (!ASSERT_OK_PTR(skel, "open_and_load"))
1077 return;
1078
1079 if (create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1))
1080 goto end;
1081
1082 prog = skel->progs.prog_skb_verdict_ingress;
1083 map = bpf_map__fd(skel->maps.sock_map_rx);
1084
1085 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0);
1086 if (!ASSERT_OK(err, "bpf_prog_attach"))
1087 goto end;
1088
1089 err = bpf_map_update_elem(map, &zero, &p0, BPF_ANY);
1090 if (!ASSERT_OK(err, "bpf_map_update_elem"))
1091 goto end;
1092
1093 err = bpf_map_update_elem(map, &one, &p1, BPF_ANY);
1094 if (!ASSERT_OK(err, "bpf_map_update_elem"))
1095 goto end;
1096
1097 sent = xsend(c0, buf, sizeof(buf), 0);
1098 if (!ASSERT_EQ(sent, sizeof(buf), "xsend"))
1099 goto end;
1100
1101 /* trigger tcp_bpf_recvmsg_parser and inc copied_seq of p1 */
1102 recvd = recv_timeout(p1, rcv, sizeof(rcv), MSG_DONTWAIT, 1);
1103 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p1)"))
1104 goto end;
1105
1106 /* uninstall sockmap of p1 */
1107 bpf_map_delete_elem(map, &one);
1108
1109 /* trigger tcp stack and the rcv_nxt of p1 is less than copied_seq */
1110 sent = xsend(c1, buf, sizeof(buf) - 1, 0);
1111 if (!ASSERT_EQ(sent, sizeof(buf) - 1, "xsend"))
1112 goto end;
1113
1114 err = setsockopt(p1, SOL_SOCKET, SO_ZEROCOPY, &on, sizeof(on));
1115 if (!ASSERT_OK(err, "setsockopt"))
1116 goto end;
1117
1118 memset(&zc, 0, sizeof(zc));
1119 zc.copybuf_address = (__u64)((unsigned long)addr);
1120 zc.copybuf_len = sizeof(addr);
1121
1122 err = getsockopt(p1, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, &zc, &zc_len);
1123 if (!ASSERT_OK(err, "getsockopt"))
1124 goto end;
1125
1126end:
1127 if (c0 >= 0)
1128 close(c0);
1129 if (p0 >= 0)
1130 close(p0);
1131 if (c1 >= 0)
1132 close(c1);
1133 if (p1 >= 0)
1134 close(p1);
1135 test_sockmap_pass_prog__destroy(skel);
1136}
1137
1138/* it is used to check whether copied_seq of sk is correct */
1139static void test_sockmap_copied_seq(bool strp)
1140{
1141 int i, map, err, sent, recvd, zero = 0, one = 1;
1142 struct test_sockmap_pass_prog *skel = NULL;
1143 int c0 = -1, p0 = -1, c1 = -1, p1 = -1;
1144 char buf[10] = "0123456789", rcv[11];
1145 struct bpf_program *prog;
1146
1147 skel = test_sockmap_pass_prog__open_and_load();
1148 if (!ASSERT_OK_PTR(skel, "open_and_load"))
1149 return;
1150
1151 if (create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1))
1152 goto end;
1153
1154 prog = skel->progs.prog_skb_verdict_ingress;
1155 map = bpf_map__fd(skel->maps.sock_map_rx);
1156
1157 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0);
1158 if (!ASSERT_OK(err, "bpf_prog_attach verdict"))
1159 goto end;
1160
1161 if (strp) {
1162 prog = skel->progs.prog_skb_verdict_ingress_strp;
1163 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_PARSER, 0);
1164 if (!ASSERT_OK(err, "bpf_prog_attach parser"))
1165 goto end;
1166 }
1167
1168 err = bpf_map_update_elem(map, &zero, &p0, BPF_ANY);
1169 if (!ASSERT_OK(err, "bpf_map_update_elem(p0)"))
1170 goto end;
1171
1172 err = bpf_map_update_elem(map, &one, &p1, BPF_ANY);
1173 if (!ASSERT_OK(err, "bpf_map_update_elem(p1)"))
1174 goto end;
1175
1176 /* just trigger sockamp: data sent by c0 will be received by p1 */
1177 sent = xsend(c0, buf, sizeof(buf), 0);
1178 if (!ASSERT_EQ(sent, sizeof(buf), "xsend(c0), bpf"))
1179 goto end;
1180
1181 /* do partial read */
1182 recvd = recv_timeout(p1, rcv, 1, MSG_DONTWAIT, 1);
1183 recvd += recv_timeout(p1, rcv + 1, sizeof(rcv) - 1, MSG_DONTWAIT, 1);
1184 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p1), bpf") ||
1185 !ASSERT_OK(memcmp(buf, rcv, recvd), "data mismatch"))
1186 goto end;
1187
1188 /* uninstall sockmap of p1 and p0 */
1189 err = bpf_map_delete_elem(map, &one);
1190 if (!ASSERT_OK(err, "bpf_map_delete_elem(1)"))
1191 goto end;
1192
1193 err = bpf_map_delete_elem(map, &zero);
1194 if (!ASSERT_OK(err, "bpf_map_delete_elem(0)"))
1195 goto end;
1196
1197 /* now all sockets become plain socket, they should still work */
1198 for (i = 0; i < 5; i++) {
1199 /* test copied_seq of p1 by running tcp native stack */
1200 sent = xsend(c1, buf, sizeof(buf), 0);
1201 if (!ASSERT_EQ(sent, sizeof(buf), "xsend(c1), native"))
1202 goto end;
1203
1204 recvd = recv(p1, rcv, sizeof(rcv), MSG_DONTWAIT);
1205 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p1), native"))
1206 goto end;
1207
1208 /* p0 previously redirected skb to p1, we also check copied_seq of p0 */
1209 sent = xsend(c0, buf, sizeof(buf), 0);
1210 if (!ASSERT_EQ(sent, sizeof(buf), "xsend(c0), native"))
1211 goto end;
1212
1213 recvd = recv(p0, rcv, sizeof(rcv), MSG_DONTWAIT);
1214 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p0), native"))
1215 goto end;
1216 }
1217
1218end:
1219 if (c0 >= 0)
1220 close(c0);
1221 if (p0 >= 0)
1222 close(p0);
1223 if (c1 >= 0)
1224 close(c1);
1225 if (p1 >= 0)
1226 close(p1);
1227 test_sockmap_pass_prog__destroy(skel);
1228}
1229
1230/* Wait until FIONREAD returns the expected value or timeout */
1231static int wait_for_fionread(int fd, int expected, unsigned int timeout_ms)
1232{
1233 unsigned int elapsed = 0;
1234 int avail = 0;
1235
1236 while (elapsed < timeout_ms) {
1237 if (ioctl(fd, FIONREAD, &avail) < 0)
1238 return -errno;
1239 if (avail >= expected)
1240 return avail;
1241 usleep(1000);
1242 elapsed++;
1243 }
1244 return avail;
1245}
1246
1247/* it is used to send data to via native stack and BPF redirecting */
1248static void test_sockmap_multi_channels(int sotype)
1249{
1250 int map, err, sent, recvd, zero = 0, one = 1, avail = 0, expected;
1251 struct test_sockmap_pass_prog *skel = NULL;
1252 int c0 = -1, p0 = -1, c1 = -1, p1 = -1;
1253 char buf[10] = "0123456789", rcv[11];
1254 struct bpf_program *prog;
1255
1256 skel = test_sockmap_pass_prog__open_and_load();
1257 if (!ASSERT_OK_PTR(skel, "open_and_load"))
1258 return;
1259
1260 err = create_socket_pairs(AF_INET, sotype, &c0, &c1, &p0, &p1);
1261 if (err)
1262 goto end;
1263
1264 prog = skel->progs.prog_skb_verdict_ingress;
1265 map = bpf_map__fd(skel->maps.sock_map_rx);
1266
1267 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0);
1268 if (!ASSERT_OK(err, "bpf_prog_attach verdict"))
1269 goto end;
1270
1271 err = bpf_map_update_elem(map, &zero, &p0, BPF_ANY);
1272 if (!ASSERT_OK(err, "bpf_map_update_elem(p0)"))
1273 goto end;
1274
1275 err = bpf_map_update_elem(map, &one, &p1, BPF_ANY);
1276 if (!ASSERT_OK(err, "bpf_map_update_elem"))
1277 goto end;
1278
1279 /* send data to p1 via native stack */
1280 sent = xsend(c1, buf, 2, 0);
1281 if (!ASSERT_EQ(sent, 2, "xsend(2)"))
1282 goto end;
1283
1284 avail = wait_for_fionread(p1, 2, IO_TIMEOUT_SEC);
1285 ASSERT_EQ(avail, 2, "ioctl(FIONREAD) partial return");
1286
1287 /* send data to p1 via bpf redirecting */
1288 sent = xsend(c0, buf + 2, sizeof(buf) - 2, 0);
1289 if (!ASSERT_EQ(sent, sizeof(buf) - 2, "xsend(remain-data)"))
1290 goto end;
1291
1292 /* Poll FIONREAD until expected bytes arrive, poll_read() is unreliable
1293 * here since it may return immediately if prior data is already queued.
1294 */
1295 expected = sotype == SOCK_DGRAM ? 2 : sizeof(buf);
1296 avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC);
1297 ASSERT_EQ(avail, expected, "ioctl(FIONREAD) full return");
1298
1299 recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1);
1300 if (!ASSERT_EQ(recvd, expected, "recv_timeout(p1)") ||
1301 !ASSERT_OK(memcmp(buf, rcv, recvd), "data mismatch"))
1302 goto end;
1303
1304 /* process remaining data for udp if secondary data is available */
1305 expected = sizeof(buf) - expected;
1306 if (expected) {
1307 avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC);
1308 ASSERT_EQ(avail, expected, "second ioctl(FIONREAD) full return");
1309
1310 recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1);
1311 if (!ASSERT_EQ(recvd, expected, "second recv_timeout(p1)") ||
1312 !ASSERT_OK(memcmp(buf + sizeof(buf) - expected, rcv, recvd),
1313 "second data mismatch"))
1314 goto end;
1315 }
1316end:
1317 if (c0 >= 0)
1318 close(c0);
1319 if (p0 >= 0)
1320 close(p0);
1321 if (c1 >= 0)
1322 close(c1);
1323 if (p1 >= 0)
1324 close(p1);
1325 test_sockmap_pass_prog__destroy(skel);
1326}
1327
1328/* A socket in a sockmap without a verdict program keeps its ingress data
1329 * in sk_receive_queue: FIONREAD must account for it.
1330 */
1331static void test_sockmap_no_verdict_fionread(void)
1332{
1333 int err, map, zero = 0, sent, avail;
1334 int c0 = -1, c1 = -1, p0 = -1, p1 = -1;
1335 struct test_sockmap_pass_prog *skel;
1336 char buf[256] = "0123456789";
1337
1338 skel = test_sockmap_pass_prog__open_and_load();
1339 if (!ASSERT_OK_PTR(skel, "open_and_load"))
1340 return;
1341 map = bpf_map__fd(skel->maps.sock_map_rx);
1342
1343 err = create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1);
1344 if (!ASSERT_OK(err, "create_socket_pairs()"))
1345 goto out;
1346
1347 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST);
1348 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)"))
1349 goto out_close;
1350
1351 sent = xsend(p1, &buf, sizeof(buf), 0);
1352 ASSERT_EQ(sent, sizeof(buf), "xsend(p1)");
1353 avail = wait_for_fionread(c1, sizeof(buf), IO_TIMEOUT_SEC);
1354 ASSERT_EQ(avail, sizeof(buf), "ioctl(FIONREAD)");
1355
1356out_close:
1357 close(c0);
1358 close(p0);
1359 close(c1);
1360 close(p1);
1361out:
1362 test_sockmap_pass_prog__destroy(skel);
1363}
1364
1365void test_sockmap_basic(void)
1366{
1367 if (test__start_subtest("sockmap create_update_free"))
1368 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKMAP);
1369 if (test__start_subtest("sockhash create_update_free"))
1370 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKHASH);
1371 if (test__start_subtest("sockmap vsock delete on close"))
1372 test_sockmap_vsock_delete_on_close();
1373 if (test__start_subtest("sockmap sk_msg load helpers"))
1374 test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP);
1375 if (test__start_subtest("sockhash sk_msg load helpers"))
1376 test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH);
1377 if (test__start_subtest("sockmap update in unsafe context"))
1378 test_sockmap_invalid_update();
1379 if (test__start_subtest("sockmap copy"))
1380 test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP);
1381 if (test__start_subtest("sockhash copy"))
1382 test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH);
1383 if (test__start_subtest("sockmap skb_verdict attach")) {
1384 test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT,
1385 BPF_SK_SKB_STREAM_VERDICT);
1386 test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT,
1387 BPF_SK_SKB_VERDICT);
1388 }
1389 if (test__start_subtest("sockmap skb_verdict attach_with_link"))
1390 test_sockmap_skb_verdict_attach_with_link();
1391 if (test__start_subtest("sockmap msg_verdict progs query"))
1392 test_sockmap_progs_query(BPF_SK_MSG_VERDICT);
1393 if (test__start_subtest("sockmap stream_parser progs query"))
1394 test_sockmap_progs_query(BPF_SK_SKB_STREAM_PARSER);
1395 if (test__start_subtest("sockmap stream_verdict progs query"))
1396 test_sockmap_progs_query(BPF_SK_SKB_STREAM_VERDICT);
1397 if (test__start_subtest("sockmap skb_verdict progs query"))
1398 test_sockmap_progs_query(BPF_SK_SKB_VERDICT);
1399 if (test__start_subtest("sockmap skb_verdict shutdown"))
1400 test_sockmap_skb_verdict_shutdown();
1401 if (test__start_subtest("sockmap skb_verdict fionread"))
1402 test_sockmap_skb_verdict_fionread(true);
1403 if (test__start_subtest("sockmap no_verdict fionread"))
1404 test_sockmap_no_verdict_fionread();
1405 if (test__start_subtest("sockmap skb_verdict fionread on drop"))
1406 test_sockmap_skb_verdict_fionread(false);
1407 if (test__start_subtest("sockmap skb_verdict change tail"))
1408 test_sockmap_skb_verdict_change_tail();
1409 if (test__start_subtest("sockmap msg_verdict pop_data overflow"))
1410 test_sockmap_msg_verdict_pop_data();
1411 if (test__start_subtest("sockmap skb_verdict msg_f_peek"))
1412 test_sockmap_skb_verdict_peek();
1413 if (test__start_subtest("sockmap skb_verdict msg_f_peek with link"))
1414 test_sockmap_skb_verdict_peek_with_link();
1415 if (test__start_subtest("sockmap unconnected af_unix"))
1416 test_sockmap_unconnected_unix();
1417 if (test__start_subtest("sockmap one socket to many map entries"))
1418 test_sockmap_many_socket();
1419 if (test__start_subtest("sockmap one socket to many maps"))
1420 test_sockmap_many_maps();
1421 if (test__start_subtest("sockmap same socket replace"))
1422 test_sockmap_same_sock();
1423 if (test__start_subtest("sockmap sk_msg attach sockmap helpers with link"))
1424 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKMAP);
1425 if (test__start_subtest("sockhash sk_msg attach sockhash helpers with link"))
1426 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKHASH);
1427 if (test__start_subtest("sockmap skb_verdict vsock poll"))
1428 test_sockmap_skb_verdict_vsock_poll();
1429 if (test__start_subtest("sockmap vsock unconnected"))
1430 test_sockmap_vsock_unconnected();
1431 if (test__start_subtest("sockmap with zc"))
1432 test_sockmap_zc();
1433 if (test__start_subtest("sockmap recover"))
1434 test_sockmap_copied_seq(false);
1435 if (test__start_subtest("sockmap recover with strp"))
1436 test_sockmap_copied_seq(true);
1437 if (test__start_subtest("sockmap tcp multi channels"))
1438 test_sockmap_multi_channels(SOCK_STREAM);
1439 if (test__start_subtest("sockmap udp multi channels"))
1440 test_sockmap_multi_channels(SOCK_DGRAM);
1441}