projects/linux·blob master · refs

1993 lines · 47.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4#include <linux/bpf.h>
5#include <linux/btf_ids.h>
6#include <linux/filter.h>
7#include <linux/errno.h>
8#include <linux/file.h>
9#include <linux/net.h>
10#include <linux/workqueue.h>
11#include <linux/skmsg.h>
12#include <linux/list.h>
13#include <linux/jhash.h>
14#include <linux/sock_diag.h>
15#include <net/udp.h>
16
17struct bpf_stab {
18 struct bpf_map map;
19 struct sock **sks;
20 struct sk_psock_progs progs;
21 spinlock_t lock;
22};
23
24#define SOCK_CREATE_FLAG_MASK \
25 (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
26
27/* This mutex is used to
28 * - protect race between prog/link attach/detach and link prog update, and
29 * - protect race between releasing and accessing map in bpf_link.
30 * A single global mutex lock is used since it is expected contention is low.
31 */
32static DEFINE_MUTEX(sockmap_mutex);
33
34static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
35 struct bpf_prog *old, struct bpf_link *link,
36 u32 which);
37static struct sk_psock_progs *sock_map_progs(struct bpf_map *map);
38
39static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
40{
41 struct bpf_stab *stab;
42
43 if (attr->max_entries == 0 ||
44 attr->max_entries > INT_MAX ||
45 attr->key_size != 4 ||
46 (attr->value_size != sizeof(u32) &&
47 attr->value_size != sizeof(u64)) ||
48 attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
49 return ERR_PTR(-EINVAL);
50
51 stab = bpf_map_area_alloc(sizeof(*stab), NUMA_NO_NODE);
52 if (!stab)
53 return ERR_PTR(-ENOMEM);
54
55 bpf_map_init_from_attr(&stab->map, attr);
56 spin_lock_init(&stab->lock);
57
58 stab->sks = bpf_map_area_alloc((u64) stab->map.max_entries *
59 sizeof(struct sock *),
60 stab->map.numa_node);
61 if (!stab->sks) {
62 bpf_map_area_free(stab);
63 return ERR_PTR(-ENOMEM);
64 }
65
66 return &stab->map;
67}
68
69int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog)
70{
71 struct bpf_map *map;
72 int ret;
73
74 if (attr->attach_flags || attr->replace_bpf_fd)
75 return -EINVAL;
76
77 CLASS(fd, f)(attr->target_fd);
78 map = __bpf_map_get(f);
79 if (IS_ERR(map))
80 return PTR_ERR(map);
81 mutex_lock(&sockmap_mutex);
82 ret = sock_map_prog_update(map, prog, NULL, NULL, attr->attach_type);
83 mutex_unlock(&sockmap_mutex);
84 return ret;
85}
86
87int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
88{
89 struct bpf_prog *prog;
90 struct bpf_map *map;
91 int ret;
92
93 if (attr->attach_flags || attr->replace_bpf_fd)
94 return -EINVAL;
95
96 CLASS(fd, f)(attr->target_fd);
97 map = __bpf_map_get(f);
98 if (IS_ERR(map))
99 return PTR_ERR(map);
100
101 prog = bpf_prog_get(attr->attach_bpf_fd);
102 if (IS_ERR(prog))
103 return PTR_ERR(prog);
104
105 if (prog->type != ptype) {
106 ret = -EINVAL;
107 goto put_prog;
108 }
109
110 mutex_lock(&sockmap_mutex);
111 ret = sock_map_prog_update(map, NULL, prog, NULL, attr->attach_type);
112 mutex_unlock(&sockmap_mutex);
113put_prog:
114 bpf_prog_put(prog);
115 return ret;
116}
117
118static void sock_map_sk_acquire(struct sock *sk)
119 __acquires(&sk->sk_lock.slock)
120{
121 lock_sock(sk);
122 rcu_read_lock();
123}
124
125static void sock_map_sk_release(struct sock *sk)
126 __releases(&sk->sk_lock.slock)
127{
128 rcu_read_unlock();
129 release_sock(sk);
130}
131
132static void sock_map_add_link(struct sk_psock *psock,
133 struct sk_psock_link *link,
134 struct bpf_map *map, void *link_raw)
135{
136 link->link_raw = link_raw;
137 link->map = map;
138 spin_lock_bh(&psock->link_lock);
139 list_add_tail(&link->list, &psock->link);
140 spin_unlock_bh(&psock->link_lock);
141}
142
143static void sock_map_del_link(struct sock *sk,
144 struct sk_psock *psock, void *link_raw)
145{
146 bool strp_stop = false, verdict_stop = false;
147 struct sk_psock_link *link, *tmp;
148
149 spin_lock_bh(&psock->link_lock);
150 list_for_each_entry_safe(link, tmp, &psock->link, list) {
151 if (link->link_raw == link_raw) {
152 struct bpf_map *map = link->map;
153 struct sk_psock_progs *progs = sock_map_progs(map);
154
155 if (psock->saved_data_ready && progs->stream_parser)
156 strp_stop = true;
157 if (psock->saved_data_ready && progs->stream_verdict)
158 verdict_stop = true;
159 if (psock->saved_data_ready && progs->skb_verdict)
160 verdict_stop = true;
161 list_del(&link->list);
162 sk_psock_free_link(link);
163 break;
164 }
165 }
166 spin_unlock_bh(&psock->link_lock);
167 if (strp_stop || verdict_stop) {
168 write_lock_bh(&sk->sk_callback_lock);
169 if (strp_stop)
170 sk_psock_stop_strp(sk, psock);
171 if (verdict_stop)
172 sk_psock_stop_verdict(sk, psock);
173
174 if (psock->psock_update_sk_prot)
175 psock->psock_update_sk_prot(sk, psock, false);
176 write_unlock_bh(&sk->sk_callback_lock);
177 }
178}
179
180static void sock_map_unref(struct sock *sk, void *link_raw)
181{
182 struct sk_psock *psock = sk_psock(sk);
183
184 if (likely(psock)) {
185 sock_map_del_link(sk, psock, link_raw);
186 sk_psock_put(sk, psock);
187 }
188}
189
190static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)
191{
192 if (!sk->sk_prot->psock_update_sk_prot)
193 return -EINVAL;
194 psock->psock_update_sk_prot = sk->sk_prot->psock_update_sk_prot;
195 return sk->sk_prot->psock_update_sk_prot(sk, psock, false);
196}
197
198static struct sk_psock *sock_map_psock_get_checked(struct sock *sk)
199{
200 struct sk_psock *psock;
201
202 rcu_read_lock();
203 psock = sk_psock(sk);
204 if (psock) {
205 if (sk->sk_prot->close != sock_map_close) {
206 psock = ERR_PTR(-EBUSY);
207 goto out;
208 }
209
210 if (!refcount_inc_not_zero(&psock->refcnt))
211 psock = ERR_PTR(-EBUSY);
212 }
213out:
214 rcu_read_unlock();
215 return psock;
216}
217
218static int sock_map_link(struct bpf_map *map, struct sock *sk)
219{
220 struct sk_psock_progs *progs = sock_map_progs(map);
221 struct bpf_prog *stream_verdict = NULL;
222 struct bpf_prog *stream_parser = NULL;
223 struct bpf_prog *skb_verdict = NULL;
224 struct bpf_prog *msg_parser = NULL;
225 struct sk_psock *psock;
226 int ret;
227
228 stream_verdict = READ_ONCE(progs->stream_verdict);
229 if (stream_verdict) {
230 stream_verdict = bpf_prog_inc_not_zero(stream_verdict);
231 if (IS_ERR(stream_verdict))
232 return PTR_ERR(stream_verdict);
233 }
234
235 stream_parser = READ_ONCE(progs->stream_parser);
236 if (stream_parser) {
237 stream_parser = bpf_prog_inc_not_zero(stream_parser);
238 if (IS_ERR(stream_parser)) {
239 ret = PTR_ERR(stream_parser);
240 goto out_put_stream_verdict;
241 }
242 }
243
244 msg_parser = READ_ONCE(progs->msg_parser);
245 if (msg_parser) {
246 msg_parser = bpf_prog_inc_not_zero(msg_parser);
247 if (IS_ERR(msg_parser)) {
248 ret = PTR_ERR(msg_parser);
249 goto out_put_stream_parser;
250 }
251 }
252
253 skb_verdict = READ_ONCE(progs->skb_verdict);
254 if (skb_verdict) {
255 skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
256 if (IS_ERR(skb_verdict)) {
257 ret = PTR_ERR(skb_verdict);
258 goto out_put_msg_parser;
259 }
260 }
261
262 psock = sock_map_psock_get_checked(sk);
263 if (IS_ERR(psock)) {
264 ret = PTR_ERR(psock);
265 goto out_progs;
266 }
267
268 if (psock) {
269 if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||
270 (stream_parser && READ_ONCE(psock->progs.stream_parser)) ||
271 (skb_verdict && READ_ONCE(psock->progs.skb_verdict)) ||
272 (skb_verdict && READ_ONCE(psock->progs.stream_verdict)) ||
273 (stream_verdict && READ_ONCE(psock->progs.skb_verdict)) ||
274 (stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {
275 sk_psock_put(sk, psock);
276 ret = -EBUSY;
277 goto out_progs;
278 }
279 } else {
280 psock = sk_psock_init(sk, map->numa_node);
281 if (IS_ERR(psock)) {
282 ret = PTR_ERR(psock);
283 goto out_progs;
284 }
285 }
286
287 if (msg_parser)
288 psock_set_prog(&psock->progs.msg_parser, msg_parser);
289 if (stream_parser)
290 psock_set_prog(&psock->progs.stream_parser, stream_parser);
291 if (stream_verdict)
292 psock_set_prog(&psock->progs.stream_verdict, stream_verdict);
293 if (skb_verdict)
294 psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
295
296 /* msg_* and stream_* programs references tracked in psock after this
297 * point. Reference dec and cleanup will occur through psock destructor
298 */
299 ret = sock_map_init_proto(sk, psock);
300 if (ret < 0) {
301 sk_psock_put(sk, psock);
302 goto out;
303 }
304
305 write_lock_bh(&sk->sk_callback_lock);
306 if (stream_parser && stream_verdict && !psock->saved_data_ready) {
307 if (sk_is_tcp(sk))
308 ret = sk_psock_init_strp(sk, psock);
309 else
310 ret = -EOPNOTSUPP;
311 if (ret) {
312 write_unlock_bh(&sk->sk_callback_lock);
313 sk_psock_put(sk, psock);
314 goto out;
315 }
316 sk_psock_start_strp(sk, psock);
317 } else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {
318 sk_psock_start_verdict(sk,psock);
319 } else if (!stream_verdict && skb_verdict && !psock->saved_data_ready) {
320 sk_psock_start_verdict(sk, psock);
321 }
322 write_unlock_bh(&sk->sk_callback_lock);
323 return 0;
324out_progs:
325 if (skb_verdict)
326 bpf_prog_put(skb_verdict);
327out_put_msg_parser:
328 if (msg_parser)
329 bpf_prog_put(msg_parser);
330out_put_stream_parser:
331 if (stream_parser)
332 bpf_prog_put(stream_parser);
333out_put_stream_verdict:
334 if (stream_verdict)
335 bpf_prog_put(stream_verdict);
336out:
337 return ret;
338}
339
340static void sock_map_free(struct bpf_map *map)
341{
342 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
343 int i;
344
345 /* After the sync no updates or deletes will be in-flight so it
346 * is safe to walk map and remove entries without risking a race
347 * in EEXIST update case.
348 */
349 synchronize_rcu();
350 for (i = 0; i < stab->map.max_entries; i++) {
351 struct sock **psk = &stab->sks[i];
352 struct sock *sk;
353
354 sk = xchg(psk, NULL);
355 if (sk) {
356 sock_hold(sk);
357 lock_sock(sk);
358 rcu_read_lock();
359 sock_map_unref(sk, psk);
360 rcu_read_unlock();
361 release_sock(sk);
362 sock_put(sk);
363 }
364 }
365
366 /* wait for psock readers accessing its map link */
367 synchronize_rcu();
368
369 bpf_map_area_free(stab->sks);
370 bpf_map_area_free(stab);
371}
372
373static void sock_map_release_progs(struct bpf_map *map)
374{
375 psock_progs_drop(&container_of(map, struct bpf_stab, map)->progs);
376}
377
378static struct sock *__sock_map_lookup_elem(struct bpf_map *map, u32 key)
379{
380 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
381
382 WARN_ON_ONCE(!rcu_read_lock_held());
383
384 if (unlikely(key >= map->max_entries))
385 return NULL;
386 return READ_ONCE(stab->sks[key]);
387}
388
389static void *sock_map_lookup(struct bpf_map *map, void *key)
390{
391 struct sock *sk;
392
393 sk = __sock_map_lookup_elem(map, *(u32 *)key);
394 if (!sk)
395 return NULL;
396 if (sk_is_refcounted(sk))
397 sock_hold(sk);
398 return sk;
399}
400
401static void *sock_map_lookup_sys(struct bpf_map *map, void *key)
402{
403 struct sock *sk;
404
405 if (map->value_size != sizeof(u64))
406 return ERR_PTR(-ENOSPC);
407
408 sk = __sock_map_lookup_elem(map, *(u32 *)key);
409 if (!sk)
410 return ERR_PTR(-ENOENT);
411
412 __sock_gen_cookie(sk);
413 return &sk->sk_cookie;
414}
415
416static int __sock_map_delete(struct bpf_stab *stab, struct sock *sk_test,
417 struct sock **psk)
418{
419 struct sock *sk = NULL;
420 int err = 0;
421
422 spin_lock_bh(&stab->lock);
423 if (!sk_test || sk_test == *psk)
424 sk = xchg(psk, NULL);
425
426 if (likely(sk))
427 sock_map_unref(sk, psk);
428 else
429 err = -EINVAL;
430
431 spin_unlock_bh(&stab->lock);
432 return err;
433}
434
435static void sock_map_delete_from_link(struct bpf_map *map, struct sock *sk,
436 void *link_raw)
437{
438 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
439
440 __sock_map_delete(stab, sk, link_raw);
441}
442
443static long sock_map_delete_elem(struct bpf_map *map, void *key)
444{
445 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
446 u32 i = *(u32 *)key;
447 struct sock **psk;
448
449 if (unlikely(i >= map->max_entries))
450 return -EINVAL;
451
452 psk = &stab->sks[i];
453 return __sock_map_delete(stab, NULL, psk);
454}
455
456static int sock_map_get_next_key(struct bpf_map *map, void *key, void *next)
457{
458 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
459 u32 i = key ? *(u32 *)key : U32_MAX;
460 u32 *key_next = next;
461
462 if (i == stab->map.max_entries - 1)
463 return -ENOENT;
464 if (i >= stab->map.max_entries)
465 *key_next = 0;
466 else
467 *key_next = i + 1;
468 return 0;
469}
470
471static int sock_map_update_common(struct bpf_map *map, u32 idx,
472 struct sock *sk, u64 flags)
473{
474 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
475 struct sk_psock_link *link;
476 struct sk_psock *psock;
477 struct sock *osk;
478 int ret;
479
480 WARN_ON_ONCE(!rcu_read_lock_held());
481 if (unlikely(flags > BPF_EXIST))
482 return -EINVAL;
483 if (unlikely(idx >= map->max_entries))
484 return -E2BIG;
485
486 link = sk_psock_init_link();
487 if (!link)
488 return -ENOMEM;
489
490 ret = sock_map_link(map, sk);
491 if (ret < 0)
492 goto out_free;
493
494 psock = sk_psock(sk);
495 WARN_ON_ONCE(!psock);
496
497 spin_lock_bh(&stab->lock);
498 osk = stab->sks[idx];
499 if (osk && flags == BPF_NOEXIST) {
500 ret = -EEXIST;
501 goto out_unlock;
502 } else if (!osk && flags == BPF_EXIST) {
503 ret = -ENOENT;
504 goto out_unlock;
505 }
506
507 sock_map_add_link(psock, link, map, &stab->sks[idx]);
508 stab->sks[idx] = sk;
509 if (osk)
510 sock_map_unref(osk, &stab->sks[idx]);
511 spin_unlock_bh(&stab->lock);
512 return 0;
513out_unlock:
514 spin_unlock_bh(&stab->lock);
515 if (psock)
516 sk_psock_put(sk, psock);
517out_free:
518 sk_psock_free_link(link);
519 return ret;
520}
521
522static bool sock_map_op_okay(const struct bpf_sock_ops_kern *ops)
523{
524 return ops->op == BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB ||
525 ops->op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB ||
526 ops->op == BPF_SOCK_OPS_TCP_LISTEN_CB;
527}
528
529static bool sock_map_redirect_allowed(const struct sock *sk)
530{
531 if (sk_is_tcp(sk))
532 return sk->sk_state != TCP_LISTEN;
533 else
534 return READ_ONCE(sk->sk_state) == TCP_ESTABLISHED;
535}
536
537static bool sock_map_sk_is_suitable(const struct sock *sk)
538{
539 return !!sk->sk_prot->psock_update_sk_prot;
540}
541
542static bool sock_map_sk_state_allowed(const struct sock *sk)
543{
544 if (sk_is_tcp(sk))
545 return (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_LISTEN);
546 if (sk_is_udp(sk))
547 return sk_hashed(sk);
548 if (sk_is_stream_unix(sk))
549 return (1 << READ_ONCE(sk->sk_state)) & TCPF_ESTABLISHED;
550 if (sk_is_vsock(sk) &&
551 (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET))
552 return (1 << sk->sk_state) & TCPF_ESTABLISHED;
553 return true;
554}
555
556static int sock_hash_update_common(struct bpf_map *map, void *key,
557 struct sock *sk, u64 flags);
558
559int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
560 u64 flags)
561{
562 struct socket *sock;
563 struct sock *sk;
564 int ret;
565 u64 ufd;
566
567 if (map->value_size == sizeof(u64))
568 ufd = *(u64 *)value;
569 else
570 ufd = *(u32 *)value;
571 if (ufd > S32_MAX)
572 return -EINVAL;
573
574 sock = sockfd_lookup(ufd, &ret);
575 if (!sock)
576 return ret;
577 sk = sock->sk;
578 if (!sk) {
579 ret = -EINVAL;
580 goto out;
581 }
582 if (!sock_map_sk_is_suitable(sk)) {
583 ret = -EOPNOTSUPP;
584 goto out;
585 }
586
587 sock_map_sk_acquire(sk);
588 if (!sock_map_sk_state_allowed(sk))
589 ret = -EOPNOTSUPP;
590 else if (map->map_type == BPF_MAP_TYPE_SOCKMAP)
591 ret = sock_map_update_common(map, *(u32 *)key, sk, flags);
592 else
593 ret = sock_hash_update_common(map, key, sk, flags);
594 sock_map_sk_release(sk);
595out:
596 sockfd_put(sock);
597 return ret;
598}
599
600static long sock_map_update_elem(struct bpf_map *map, void *key,
601 void *value, u64 flags)
602{
603 struct sock *sk = (struct sock *)value;
604 int ret;
605
606 if (unlikely(!sk || !sk_fullsock(sk)))
607 return -EINVAL;
608
609 if (!sock_map_sk_is_suitable(sk))
610 return -EOPNOTSUPP;
611
612 local_bh_disable();
613 bh_lock_sock(sk);
614 if (!sock_map_sk_state_allowed(sk))
615 ret = -EOPNOTSUPP;
616 else if (map->map_type == BPF_MAP_TYPE_SOCKMAP)
617 ret = sock_map_update_common(map, *(u32 *)key, sk, flags);
618 else
619 ret = sock_hash_update_common(map, key, sk, flags);
620 bh_unlock_sock(sk);
621 local_bh_enable();
622 return ret;
623}
624
625BPF_CALL_4(bpf_sock_map_update, struct bpf_sock_ops_kern *, sops,
626 struct bpf_map *, map, void *, key, u64, flags)
627{
628 WARN_ON_ONCE(!rcu_read_lock_held());
629
630 if (likely(sock_map_sk_is_suitable(sops->sk) &&
631 sock_map_op_okay(sops)))
632 return sock_map_update_common(map, *(u32 *)key, sops->sk,
633 flags);
634 return -EOPNOTSUPP;
635}
636
637const struct bpf_func_proto bpf_sock_map_update_proto = {
638 .func = bpf_sock_map_update,
639 .gpl_only = false,
640 .pkt_access = true,
641 .ret_type = RET_INTEGER,
642 .arg1_type = ARG_PTR_TO_CTX,
643 .arg2_type = ARG_CONST_MAP_PTR,
644 .arg3_type = ARG_PTR_TO_MAP_KEY,
645 .arg4_type = ARG_ANYTHING,
646};
647
648BPF_CALL_4(bpf_sk_redirect_map, struct sk_buff *, skb,
649 struct bpf_map *, map, u32, key, u64, flags)
650{
651 struct sock *sk;
652
653 if (unlikely(flags & ~(BPF_F_INGRESS)))
654 return SK_DROP;
655
656 sk = __sock_map_lookup_elem(map, key);
657 if (unlikely(!sk || !sock_map_redirect_allowed(sk)))
658 return SK_DROP;
659 if ((flags & BPF_F_INGRESS) && sk_is_vsock(sk))
660 return SK_DROP;
661
662 skb_bpf_set_redir(skb, sk, flags & BPF_F_INGRESS);
663 return SK_PASS;
664}
665
666const struct bpf_func_proto bpf_sk_redirect_map_proto = {
667 .func = bpf_sk_redirect_map,
668 .gpl_only = false,
669 .ret_type = RET_INTEGER,
670 .arg1_type = ARG_PTR_TO_CTX,
671 .arg2_type = ARG_CONST_MAP_PTR,
672 .arg3_type = ARG_ANYTHING,
673 .arg4_type = ARG_ANYTHING,
674};
675
676BPF_CALL_4(bpf_msg_redirect_map, struct sk_msg *, msg,
677 struct bpf_map *, map, u32, key, u64, flags)
678{
679 struct sock *sk;
680
681 if (unlikely(flags & ~(BPF_F_INGRESS)))
682 return SK_DROP;
683
684 sk = __sock_map_lookup_elem(map, key);
685 if (unlikely(!sk || !sock_map_redirect_allowed(sk)))
686 return SK_DROP;
687 if (!(flags & BPF_F_INGRESS) && !sk_is_tcp(sk))
688 return SK_DROP;
689 if (sk_is_vsock(sk))
690 return SK_DROP;
691
692 msg->flags = flags;
693 msg->sk_redir = sk;
694 return SK_PASS;
695}
696
697const struct bpf_func_proto bpf_msg_redirect_map_proto = {
698 .func = bpf_msg_redirect_map,
699 .gpl_only = false,
700 .ret_type = RET_INTEGER,
701 .arg1_type = ARG_PTR_TO_CTX,
702 .arg2_type = ARG_CONST_MAP_PTR,
703 .arg3_type = ARG_ANYTHING,
704 .arg4_type = ARG_ANYTHING,
705};
706
707struct sock_map_seq_info {
708 struct bpf_map *map;
709 struct sock *sk;
710 u32 index;
711};
712
713struct bpf_iter__sockmap {
714 __bpf_md_ptr(struct bpf_iter_meta *, meta);
715 __bpf_md_ptr(struct bpf_map *, map);
716 __bpf_md_ptr(void *, key);
717 __bpf_md_ptr(struct sock *, sk);
718};
719
720DEFINE_BPF_ITER_FUNC(sockmap, struct bpf_iter_meta *meta,
721 struct bpf_map *map, void *key,
722 struct sock *sk)
723
724static void *sock_map_seq_lookup_elem(struct sock_map_seq_info *info)
725{
726 if (unlikely(info->index >= info->map->max_entries))
727 return NULL;
728
729 info->sk = __sock_map_lookup_elem(info->map, info->index);
730
731 /* can't return sk directly, since that might be NULL */
732 return info;
733}
734
735static void *sock_map_seq_start(struct seq_file *seq, loff_t *pos)
736 __acquires(rcu)
737{
738 struct sock_map_seq_info *info = seq->private;
739
740 if (*pos == 0)
741 ++*pos;
742
743 /* pairs with sock_map_seq_stop */
744 rcu_read_lock();
745 return sock_map_seq_lookup_elem(info);
746}
747
748static void *sock_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
749 __must_hold(rcu)
750{
751 struct sock_map_seq_info *info = seq->private;
752
753 ++*pos;
754 ++info->index;
755
756 return sock_map_seq_lookup_elem(info);
757}
758
759static int sock_map_seq_show(struct seq_file *seq, void *v)
760 __must_hold(rcu)
761{
762 struct sock_map_seq_info *info = seq->private;
763 struct bpf_iter__sockmap ctx = {};
764 struct bpf_iter_meta meta;
765 struct bpf_prog *prog;
766
767 meta.seq = seq;
768 prog = bpf_iter_get_info(&meta, !v);
769 if (!prog)
770 return 0;
771
772 ctx.meta = &meta;
773 ctx.map = info->map;
774 if (v) {
775 ctx.key = &info->index;
776 ctx.sk = info->sk;
777 }
778
779 return bpf_iter_run_prog(prog, &ctx);
780}
781
782static void sock_map_seq_stop(struct seq_file *seq, void *v)
783 __releases(rcu)
784{
785 if (!v)
786 (void)sock_map_seq_show(seq, NULL);
787
788 /* pairs with sock_map_seq_start */
789 rcu_read_unlock();
790}
791
792static const struct seq_operations sock_map_seq_ops = {
793 .start = sock_map_seq_start,
794 .next = sock_map_seq_next,
795 .stop = sock_map_seq_stop,
796 .show = sock_map_seq_show,
797};
798
799static int sock_map_init_seq_private(void *priv_data,
800 struct bpf_iter_aux_info *aux)
801{
802 struct sock_map_seq_info *info = priv_data;
803
804 bpf_map_inc_with_uref(aux->map);
805 info->map = aux->map;
806 return 0;
807}
808
809static void sock_map_fini_seq_private(void *priv_data)
810{
811 struct sock_map_seq_info *info = priv_data;
812
813 bpf_map_put_with_uref(info->map);
814}
815
816static u64 sock_map_mem_usage(const struct bpf_map *map)
817{
818 u64 usage = sizeof(struct bpf_stab);
819
820 usage += (u64)map->max_entries * sizeof(struct sock *);
821 return usage;
822}
823
824static const struct bpf_iter_seq_info sock_map_iter_seq_info = {
825 .seq_ops = &sock_map_seq_ops,
826 .init_seq_private = sock_map_init_seq_private,
827 .fini_seq_private = sock_map_fini_seq_private,
828 .seq_priv_size = sizeof(struct sock_map_seq_info),
829};
830
831BTF_ID_LIST_SINGLE(sock_map_btf_ids, struct, bpf_stab)
832const struct bpf_map_ops sock_map_ops = {
833 .map_meta_equal = bpf_map_meta_equal,
834 .map_alloc = sock_map_alloc,
835 .map_free = sock_map_free,
836 .map_get_next_key = sock_map_get_next_key,
837 .map_lookup_elem_sys_only = sock_map_lookup_sys,
838 .map_update_elem = sock_map_update_elem,
839 .map_delete_elem = sock_map_delete_elem,
840 .map_lookup_elem = sock_map_lookup,
841 .map_release_uref = sock_map_release_progs,
842 .map_check_btf = map_check_no_btf,
843 .map_mem_usage = sock_map_mem_usage,
844 .map_btf_id = &sock_map_btf_ids[0],
845 .iter_seq_info = &sock_map_iter_seq_info,
846};
847
848struct bpf_shtab_elem {
849 struct rcu_head rcu;
850 u32 hash;
851 struct sock *sk;
852 struct hlist_node node;
853 u8 key[];
854};
855
856struct bpf_shtab_bucket {
857 struct hlist_head head;
858 spinlock_t lock;
859};
860
861struct bpf_shtab {
862 struct bpf_map map;
863 struct bpf_shtab_bucket *buckets;
864 u32 buckets_num;
865 u32 elem_size;
866 struct sk_psock_progs progs;
867 atomic_t count;
868};
869
870static inline u32 sock_hash_bucket_hash(const void *key, u32 len)
871{
872 return jhash(key, len, 0);
873}
874
875static struct bpf_shtab_bucket *sock_hash_select_bucket(struct bpf_shtab *htab,
876 u32 hash)
877{
878 return &htab->buckets[hash & (htab->buckets_num - 1)];
879}
880
881static struct bpf_shtab_elem *
882sock_hash_lookup_elem_raw(struct hlist_head *head, u32 hash, void *key,
883 u32 key_size)
884{
885 struct bpf_shtab_elem *elem;
886
887 hlist_for_each_entry_rcu(elem, head, node) {
888 if (elem->hash == hash &&
889 !memcmp(&elem->key, key, key_size))
890 return elem;
891 }
892
893 return NULL;
894}
895
896static struct sock *__sock_hash_lookup_elem(struct bpf_map *map, void *key)
897{
898 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
899 u32 key_size = map->key_size, hash;
900 struct bpf_shtab_bucket *bucket;
901 struct bpf_shtab_elem *elem;
902
903 WARN_ON_ONCE(!rcu_read_lock_held());
904
905 hash = sock_hash_bucket_hash(key, key_size);
906 bucket = sock_hash_select_bucket(htab, hash);
907 elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
908
909 return elem ? elem->sk : NULL;
910}
911
912static void sock_hash_free_elem(struct bpf_shtab *htab,
913 struct bpf_shtab_elem *elem)
914{
915 atomic_dec(&htab->count);
916 kfree_rcu(elem, rcu);
917}
918
919static void sock_hash_delete_from_link(struct bpf_map *map, struct sock *sk,
920 void *link_raw)
921{
922 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
923 struct bpf_shtab_elem *elem_probe, *elem = link_raw;
924 struct bpf_shtab_bucket *bucket;
925
926 WARN_ON_ONCE(!rcu_read_lock_held());
927 bucket = sock_hash_select_bucket(htab, elem->hash);
928
929 /* elem may be deleted in parallel from the map, but access here
930 * is okay since it's going away only after RCU grace period.
931 * However, we need to check whether it's still present.
932 */
933 spin_lock_bh(&bucket->lock);
934 elem_probe = sock_hash_lookup_elem_raw(&bucket->head, elem->hash,
935 elem->key, map->key_size);
936 if (elem_probe && elem_probe == elem) {
937 hlist_del_rcu(&elem->node);
938 sock_map_unref(elem->sk, elem);
939 sock_hash_free_elem(htab, elem);
940 }
941 spin_unlock_bh(&bucket->lock);
942}
943
944static long sock_hash_delete_elem(struct bpf_map *map, void *key)
945{
946 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
947 u32 hash, key_size = map->key_size;
948 struct bpf_shtab_bucket *bucket;
949 struct bpf_shtab_elem *elem;
950 int ret = -ENOENT;
951
952 hash = sock_hash_bucket_hash(key, key_size);
953 bucket = sock_hash_select_bucket(htab, hash);
954
955 spin_lock_bh(&bucket->lock);
956 elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
957 if (elem) {
958 hlist_del_rcu(&elem->node);
959 sock_map_unref(elem->sk, elem);
960 sock_hash_free_elem(htab, elem);
961 ret = 0;
962 }
963 spin_unlock_bh(&bucket->lock);
964 return ret;
965}
966
967static struct bpf_shtab_elem *sock_hash_alloc_elem(struct bpf_shtab *htab,
968 void *key, u32 key_size,
969 u32 hash, struct sock *sk,
970 struct bpf_shtab_elem *old)
971{
972 struct bpf_shtab_elem *new;
973
974 if (atomic_inc_return(&htab->count) > htab->map.max_entries) {
975 if (!old) {
976 atomic_dec(&htab->count);
977 return ERR_PTR(-E2BIG);
978 }
979 }
980
981 new = bpf_map_kmalloc_node(&htab->map, htab->elem_size,
982 GFP_ATOMIC | __GFP_NOWARN,
983 htab->map.numa_node);
984 if (!new) {
985 atomic_dec(&htab->count);
986 return ERR_PTR(-ENOMEM);
987 }
988 memcpy(new->key, key, key_size);
989 new->sk = sk;
990 new->hash = hash;
991 return new;
992}
993
994static int sock_hash_update_common(struct bpf_map *map, void *key,
995 struct sock *sk, u64 flags)
996{
997 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
998 u32 key_size = map->key_size, hash;
999 struct bpf_shtab_elem *elem, *elem_new;
1000 struct bpf_shtab_bucket *bucket;
1001 struct sk_psock_link *link;
1002 struct sk_psock *psock;
1003 int ret;
1004
1005 WARN_ON_ONCE(!rcu_read_lock_held());
1006 if (unlikely(flags > BPF_EXIST))
1007 return -EINVAL;
1008
1009 link = sk_psock_init_link();
1010 if (!link)
1011 return -ENOMEM;
1012
1013 ret = sock_map_link(map, sk);
1014 if (ret < 0)
1015 goto out_free;
1016
1017 psock = sk_psock(sk);
1018 WARN_ON_ONCE(!psock);
1019
1020 hash = sock_hash_bucket_hash(key, key_size);
1021 bucket = sock_hash_select_bucket(htab, hash);
1022
1023 spin_lock_bh(&bucket->lock);
1024 elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
1025 if (elem && flags == BPF_NOEXIST) {
1026 ret = -EEXIST;
1027 goto out_unlock;
1028 } else if (!elem && flags == BPF_EXIST) {
1029 ret = -ENOENT;
1030 goto out_unlock;
1031 }
1032
1033 elem_new = sock_hash_alloc_elem(htab, key, key_size, hash, sk, elem);
1034 if (IS_ERR(elem_new)) {
1035 ret = PTR_ERR(elem_new);
1036 goto out_unlock;
1037 }
1038
1039 sock_map_add_link(psock, link, map, elem_new);
1040 /* Add new element to the head of the list, so that
1041 * concurrent search will find it before old elem.
1042 */
1043 hlist_add_head_rcu(&elem_new->node, &bucket->head);
1044 if (elem) {
1045 hlist_del_rcu(&elem->node);
1046 sock_map_unref(elem->sk, elem);
1047 sock_hash_free_elem(htab, elem);
1048 }
1049 spin_unlock_bh(&bucket->lock);
1050 return 0;
1051out_unlock:
1052 spin_unlock_bh(&bucket->lock);
1053 sk_psock_put(sk, psock);
1054out_free:
1055 sk_psock_free_link(link);
1056 return ret;
1057}
1058
1059static int sock_hash_get_next_key(struct bpf_map *map, void *key,
1060 void *key_next)
1061{
1062 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1063 struct bpf_shtab_elem *elem, *elem_next;
1064 u32 hash, key_size = map->key_size;
1065 struct hlist_head *head;
1066 int i = 0;
1067
1068 if (!key)
1069 goto find_first_elem;
1070 hash = sock_hash_bucket_hash(key, key_size);
1071 head = &sock_hash_select_bucket(htab, hash)->head;
1072 elem = sock_hash_lookup_elem_raw(head, hash, key, key_size);
1073 if (!elem)
1074 goto find_first_elem;
1075
1076 elem_next = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&elem->node)),
1077 struct bpf_shtab_elem, node);
1078 if (elem_next) {
1079 memcpy(key_next, elem_next->key, key_size);
1080 return 0;
1081 }
1082
1083 i = hash & (htab->buckets_num - 1);
1084 i++;
1085find_first_elem:
1086 for (; i < htab->buckets_num; i++) {
1087 head = &sock_hash_select_bucket(htab, i)->head;
1088 elem_next = hlist_entry_safe(rcu_dereference(hlist_first_rcu(head)),
1089 struct bpf_shtab_elem, node);
1090 if (elem_next) {
1091 memcpy(key_next, elem_next->key, key_size);
1092 return 0;
1093 }
1094 }
1095
1096 return -ENOENT;
1097}
1098
1099static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
1100{
1101 struct bpf_shtab *htab;
1102 int i, err;
1103
1104 if (attr->max_entries == 0 ||
1105 attr->key_size == 0 ||
1106 (attr->value_size != sizeof(u32) &&
1107 attr->value_size != sizeof(u64)) ||
1108 attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
1109 return ERR_PTR(-EINVAL);
1110 if (attr->key_size > MAX_BPF_STACK)
1111 return ERR_PTR(-E2BIG);
1112
1113 htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);
1114 if (!htab)
1115 return ERR_PTR(-ENOMEM);
1116
1117 bpf_map_init_from_attr(&htab->map, attr);
1118
1119 htab->buckets_num = roundup_pow_of_two(htab->map.max_entries);
1120 htab->elem_size = sizeof(struct bpf_shtab_elem) +
1121 round_up(htab->map.key_size, 8);
1122 if (htab->buckets_num == 0 ||
1123 htab->buckets_num > U32_MAX / sizeof(struct bpf_shtab_bucket)) {
1124 err = -EINVAL;
1125 goto free_htab;
1126 }
1127
1128 htab->buckets = bpf_map_area_alloc(htab->buckets_num *
1129 sizeof(struct bpf_shtab_bucket),
1130 htab->map.numa_node);
1131 if (!htab->buckets) {
1132 err = -ENOMEM;
1133 goto free_htab;
1134 }
1135
1136 for (i = 0; i < htab->buckets_num; i++) {
1137 INIT_HLIST_HEAD(&htab->buckets[i].head);
1138 spin_lock_init(&htab->buckets[i].lock);
1139 }
1140
1141 return &htab->map;
1142free_htab:
1143 bpf_map_area_free(htab);
1144 return ERR_PTR(err);
1145}
1146
1147static void sock_hash_free(struct bpf_map *map)
1148{
1149 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1150 struct bpf_shtab_bucket *bucket;
1151 struct hlist_head unlink_list;
1152 struct bpf_shtab_elem *elem;
1153 struct hlist_node *node;
1154 int i;
1155
1156 /* After the sync no updates or deletes will be in-flight so it
1157 * is safe to walk map and remove entries without risking a race
1158 * in EEXIST update case.
1159 */
1160 synchronize_rcu();
1161 for (i = 0; i < htab->buckets_num; i++) {
1162 bucket = sock_hash_select_bucket(htab, i);
1163
1164 /* We are racing with sock_hash_delete_from_link to
1165 * enter the spin-lock critical section. Every socket on
1166 * the list is still linked to sockhash. Since link
1167 * exists, psock exists and holds a ref to socket. That
1168 * lets us to grab a socket ref too.
1169 */
1170 spin_lock_bh(&bucket->lock);
1171 hlist_for_each_entry(elem, &bucket->head, node)
1172 sock_hold(elem->sk);
1173 hlist_move_list(&bucket->head, &unlink_list);
1174 spin_unlock_bh(&bucket->lock);
1175
1176 /* Process removed entries out of atomic context to
1177 * block for socket lock before deleting the psock's
1178 * link to sockhash.
1179 */
1180 hlist_for_each_entry_safe(elem, node, &unlink_list, node) {
1181 hlist_del(&elem->node);
1182 lock_sock(elem->sk);
1183 rcu_read_lock();
1184 sock_map_unref(elem->sk, elem);
1185 rcu_read_unlock();
1186 release_sock(elem->sk);
1187 sock_put(elem->sk);
1188 sock_hash_free_elem(htab, elem);
1189 }
1190 cond_resched();
1191 }
1192
1193 /* wait for psock readers accessing its map link */
1194 synchronize_rcu();
1195
1196 bpf_map_area_free(htab->buckets);
1197 bpf_map_area_free(htab);
1198}
1199
1200static void *sock_hash_lookup_sys(struct bpf_map *map, void *key)
1201{
1202 struct sock *sk;
1203
1204 if (map->value_size != sizeof(u64))
1205 return ERR_PTR(-ENOSPC);
1206
1207 sk = __sock_hash_lookup_elem(map, key);
1208 if (!sk)
1209 return ERR_PTR(-ENOENT);
1210
1211 __sock_gen_cookie(sk);
1212 return &sk->sk_cookie;
1213}
1214
1215static void *sock_hash_lookup(struct bpf_map *map, void *key)
1216{
1217 struct sock *sk;
1218
1219 sk = __sock_hash_lookup_elem(map, key);
1220 if (!sk)
1221 return NULL;
1222 if (sk_is_refcounted(sk))
1223 sock_hold(sk);
1224 return sk;
1225}
1226
1227static void sock_hash_release_progs(struct bpf_map *map)
1228{
1229 psock_progs_drop(&container_of(map, struct bpf_shtab, map)->progs);
1230}
1231
1232BPF_CALL_4(bpf_sock_hash_update, struct bpf_sock_ops_kern *, sops,
1233 struct bpf_map *, map, void *, key, u64, flags)
1234{
1235 WARN_ON_ONCE(!rcu_read_lock_held());
1236
1237 if (likely(sock_map_sk_is_suitable(sops->sk) &&
1238 sock_map_op_okay(sops)))
1239 return sock_hash_update_common(map, key, sops->sk, flags);
1240 return -EOPNOTSUPP;
1241}
1242
1243const struct bpf_func_proto bpf_sock_hash_update_proto = {
1244 .func = bpf_sock_hash_update,
1245 .gpl_only = false,
1246 .pkt_access = true,
1247 .ret_type = RET_INTEGER,
1248 .arg1_type = ARG_PTR_TO_CTX,
1249 .arg2_type = ARG_CONST_MAP_PTR,
1250 .arg3_type = ARG_PTR_TO_MAP_KEY,
1251 .arg4_type = ARG_ANYTHING,
1252};
1253
1254BPF_CALL_4(bpf_sk_redirect_hash, struct sk_buff *, skb,
1255 struct bpf_map *, map, void *, key, u64, flags)
1256{
1257 struct sock *sk;
1258
1259 if (unlikely(flags & ~(BPF_F_INGRESS)))
1260 return SK_DROP;
1261
1262 sk = __sock_hash_lookup_elem(map, key);
1263 if (unlikely(!sk || !sock_map_redirect_allowed(sk)))
1264 return SK_DROP;
1265 if ((flags & BPF_F_INGRESS) && sk_is_vsock(sk))
1266 return SK_DROP;
1267
1268 skb_bpf_set_redir(skb, sk, flags & BPF_F_INGRESS);
1269 return SK_PASS;
1270}
1271
1272const struct bpf_func_proto bpf_sk_redirect_hash_proto = {
1273 .func = bpf_sk_redirect_hash,
1274 .gpl_only = false,
1275 .ret_type = RET_INTEGER,
1276 .arg1_type = ARG_PTR_TO_CTX,
1277 .arg2_type = ARG_CONST_MAP_PTR,
1278 .arg3_type = ARG_PTR_TO_MAP_KEY,
1279 .arg4_type = ARG_ANYTHING,
1280};
1281
1282BPF_CALL_4(bpf_msg_redirect_hash, struct sk_msg *, msg,
1283 struct bpf_map *, map, void *, key, u64, flags)
1284{
1285 struct sock *sk;
1286
1287 if (unlikely(flags & ~(BPF_F_INGRESS)))
1288 return SK_DROP;
1289
1290 sk = __sock_hash_lookup_elem(map, key);
1291 if (unlikely(!sk || !sock_map_redirect_allowed(sk)))
1292 return SK_DROP;
1293 if (!(flags & BPF_F_INGRESS) && !sk_is_tcp(sk))
1294 return SK_DROP;
1295 if (sk_is_vsock(sk))
1296 return SK_DROP;
1297
1298 msg->flags = flags;
1299 msg->sk_redir = sk;
1300 return SK_PASS;
1301}
1302
1303const struct bpf_func_proto bpf_msg_redirect_hash_proto = {
1304 .func = bpf_msg_redirect_hash,
1305 .gpl_only = false,
1306 .ret_type = RET_INTEGER,
1307 .arg1_type = ARG_PTR_TO_CTX,
1308 .arg2_type = ARG_CONST_MAP_PTR,
1309 .arg3_type = ARG_PTR_TO_MAP_KEY,
1310 .arg4_type = ARG_ANYTHING,
1311};
1312
1313struct sock_hash_seq_info {
1314 struct bpf_map *map;
1315 struct bpf_shtab *htab;
1316 u32 bucket_id;
1317};
1318
1319static void *sock_hash_seq_find_next(struct sock_hash_seq_info *info,
1320 struct bpf_shtab_elem *prev_elem)
1321{
1322 const struct bpf_shtab *htab = info->htab;
1323 struct bpf_shtab_bucket *bucket;
1324 struct bpf_shtab_elem *elem;
1325 struct hlist_node *node;
1326
1327 /* try to find next elem in the same bucket */
1328 if (prev_elem) {
1329 node = rcu_dereference(hlist_next_rcu(&prev_elem->node));
1330 elem = hlist_entry_safe(node, struct bpf_shtab_elem, node);
1331 if (elem)
1332 return elem;
1333
1334 /* no more elements, continue in the next bucket */
1335 info->bucket_id++;
1336 }
1337
1338 for (; info->bucket_id < htab->buckets_num; info->bucket_id++) {
1339 bucket = &htab->buckets[info->bucket_id];
1340 node = rcu_dereference(hlist_first_rcu(&bucket->head));
1341 elem = hlist_entry_safe(node, struct bpf_shtab_elem, node);
1342 if (elem)
1343 return elem;
1344 }
1345
1346 return NULL;
1347}
1348
1349static void *sock_hash_seq_start(struct seq_file *seq, loff_t *pos)
1350 __acquires(rcu)
1351{
1352 struct sock_hash_seq_info *info = seq->private;
1353
1354 if (*pos == 0)
1355 ++*pos;
1356
1357 /* pairs with sock_hash_seq_stop */
1358 rcu_read_lock();
1359 return sock_hash_seq_find_next(info, NULL);
1360}
1361
1362static void *sock_hash_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1363 __must_hold(rcu)
1364{
1365 struct sock_hash_seq_info *info = seq->private;
1366
1367 ++*pos;
1368 return sock_hash_seq_find_next(info, v);
1369}
1370
1371static int sock_hash_seq_show(struct seq_file *seq, void *v)
1372 __must_hold(rcu)
1373{
1374 struct sock_hash_seq_info *info = seq->private;
1375 struct bpf_iter__sockmap ctx = {};
1376 struct bpf_shtab_elem *elem = v;
1377 struct bpf_iter_meta meta;
1378 struct bpf_prog *prog;
1379
1380 meta.seq = seq;
1381 prog = bpf_iter_get_info(&meta, !elem);
1382 if (!prog)
1383 return 0;
1384
1385 ctx.meta = &meta;
1386 ctx.map = info->map;
1387 if (elem) {
1388 ctx.key = elem->key;
1389 ctx.sk = elem->sk;
1390 }
1391
1392 return bpf_iter_run_prog(prog, &ctx);
1393}
1394
1395static void sock_hash_seq_stop(struct seq_file *seq, void *v)
1396 __releases(rcu)
1397{
1398 if (!v)
1399 (void)sock_hash_seq_show(seq, NULL);
1400
1401 /* pairs with sock_hash_seq_start */
1402 rcu_read_unlock();
1403}
1404
1405static const struct seq_operations sock_hash_seq_ops = {
1406 .start = sock_hash_seq_start,
1407 .next = sock_hash_seq_next,
1408 .stop = sock_hash_seq_stop,
1409 .show = sock_hash_seq_show,
1410};
1411
1412static int sock_hash_init_seq_private(void *priv_data,
1413 struct bpf_iter_aux_info *aux)
1414{
1415 struct sock_hash_seq_info *info = priv_data;
1416
1417 bpf_map_inc_with_uref(aux->map);
1418 info->map = aux->map;
1419 info->htab = container_of(aux->map, struct bpf_shtab, map);
1420 return 0;
1421}
1422
1423static void sock_hash_fini_seq_private(void *priv_data)
1424{
1425 struct sock_hash_seq_info *info = priv_data;
1426
1427 bpf_map_put_with_uref(info->map);
1428}
1429
1430static u64 sock_hash_mem_usage(const struct bpf_map *map)
1431{
1432 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1433 u64 usage = sizeof(*htab);
1434
1435 usage += htab->buckets_num * sizeof(struct bpf_shtab_bucket);
1436 usage += atomic_read(&htab->count) * (u64)htab->elem_size;
1437 return usage;
1438}
1439
1440static const struct bpf_iter_seq_info sock_hash_iter_seq_info = {
1441 .seq_ops = &sock_hash_seq_ops,
1442 .init_seq_private = sock_hash_init_seq_private,
1443 .fini_seq_private = sock_hash_fini_seq_private,
1444 .seq_priv_size = sizeof(struct sock_hash_seq_info),
1445};
1446
1447BTF_ID_LIST_SINGLE(sock_hash_map_btf_ids, struct, bpf_shtab)
1448const struct bpf_map_ops sock_hash_ops = {
1449 .map_meta_equal = bpf_map_meta_equal,
1450 .map_alloc = sock_hash_alloc,
1451 .map_free = sock_hash_free,
1452 .map_get_next_key = sock_hash_get_next_key,
1453 .map_update_elem = sock_map_update_elem,
1454 .map_delete_elem = sock_hash_delete_elem,
1455 .map_lookup_elem = sock_hash_lookup,
1456 .map_lookup_elem_sys_only = sock_hash_lookup_sys,
1457 .map_release_uref = sock_hash_release_progs,
1458 .map_check_btf = map_check_no_btf,
1459 .map_mem_usage = sock_hash_mem_usage,
1460 .map_btf_id = &sock_hash_map_btf_ids[0],
1461 .iter_seq_info = &sock_hash_iter_seq_info,
1462};
1463
1464static struct sk_psock_progs *sock_map_progs(struct bpf_map *map)
1465{
1466 switch (map->map_type) {
1467 case BPF_MAP_TYPE_SOCKMAP:
1468 return &container_of(map, struct bpf_stab, map)->progs;
1469 case BPF_MAP_TYPE_SOCKHASH:
1470 return &container_of(map, struct bpf_shtab, map)->progs;
1471 default:
1472 break;
1473 }
1474
1475 return NULL;
1476}
1477
1478static int sock_map_prog_link_lookup(struct bpf_map *map, struct bpf_prog ***pprog,
1479 struct bpf_link ***plink, u32 which)
1480{
1481 struct sk_psock_progs *progs = sock_map_progs(map);
1482 struct bpf_prog **cur_pprog;
1483 struct bpf_link **cur_plink;
1484
1485 if (!progs)
1486 return -EOPNOTSUPP;
1487
1488 switch (which) {
1489 case BPF_SK_MSG_VERDICT:
1490 cur_pprog = &progs->msg_parser;
1491 cur_plink = &progs->msg_parser_link;
1492 break;
1493#if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
1494 case BPF_SK_SKB_STREAM_PARSER:
1495 cur_pprog = &progs->stream_parser;
1496 cur_plink = &progs->stream_parser_link;
1497 break;
1498#endif
1499 case BPF_SK_SKB_STREAM_VERDICT:
1500 if (progs->skb_verdict)
1501 return -EBUSY;
1502 cur_pprog = &progs->stream_verdict;
1503 cur_plink = &progs->stream_verdict_link;
1504 break;
1505 case BPF_SK_SKB_VERDICT:
1506 if (progs->stream_verdict)
1507 return -EBUSY;
1508 cur_pprog = &progs->skb_verdict;
1509 cur_plink = &progs->skb_verdict_link;
1510 break;
1511 default:
1512 return -EOPNOTSUPP;
1513 }
1514
1515 *pprog = cur_pprog;
1516 if (plink)
1517 *plink = cur_plink;
1518 return 0;
1519}
1520
1521static int sock_map_prog_attach_check(enum bpf_attach_type attach_type,
1522 struct bpf_prog *prog)
1523{
1524 /* A stream parser must not modify the skb, only measure it. */
1525 if (prog && attach_type == BPF_SK_SKB_STREAM_PARSER &&
1526 prog->aux->changes_pkt_data)
1527 return -EINVAL;
1528
1529 return 0;
1530}
1531
1532/* Handle the following four cases:
1533 * prog_attach: prog != NULL, old == NULL, link == NULL
1534 * prog_detach: prog == NULL, old != NULL, link == NULL
1535 * link_attach: prog != NULL, old == NULL, link != NULL
1536 * link_detach: prog == NULL, old != NULL, link != NULL
1537 */
1538static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
1539 struct bpf_prog *old, struct bpf_link *link,
1540 u32 which)
1541{
1542 struct bpf_prog **pprog;
1543 struct bpf_link **plink;
1544 int ret;
1545
1546 ret = sock_map_prog_link_lookup(map, &pprog, &plink, which);
1547 if (ret)
1548 return ret;
1549
1550 ret = sock_map_prog_attach_check(which, prog);
1551 if (ret)
1552 return ret;
1553
1554 /* for prog_attach/prog_detach/link_attach, return error if a bpf_link
1555 * exists for that prog.
1556 */
1557 if ((!link || prog) && *plink)
1558 return -EBUSY;
1559
1560 if (old) {
1561 ret = psock_replace_prog(pprog, prog, old);
1562 if (!ret)
1563 *plink = NULL;
1564 } else {
1565 psock_set_prog(pprog, prog);
1566 if (link)
1567 *plink = link;
1568 }
1569
1570 return ret;
1571}
1572
1573int sock_map_bpf_prog_query(const union bpf_attr *attr,
1574 union bpf_attr __user *uattr)
1575{
1576 __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
1577 u32 prog_cnt = 0, flags = 0;
1578 struct bpf_prog **pprog;
1579 struct bpf_prog *prog;
1580 struct bpf_map *map;
1581 u32 id = 0;
1582 int ret;
1583
1584 if (attr->query.query_flags)
1585 return -EINVAL;
1586
1587 CLASS(fd, f)(attr->target_fd);
1588 map = __bpf_map_get(f);
1589 if (IS_ERR(map))
1590 return PTR_ERR(map);
1591
1592 rcu_read_lock();
1593
1594 ret = sock_map_prog_link_lookup(map, &pprog, NULL, attr->query.attach_type);
1595 if (ret)
1596 goto end;
1597
1598 prog = *pprog;
1599 prog_cnt = !prog ? 0 : 1;
1600
1601 if (!attr->query.prog_cnt || !prog_ids || !prog_cnt)
1602 goto end;
1603
1604 /* we do not hold the refcnt, the bpf prog may be released
1605 * asynchronously and the id would be set to 0.
1606 */
1607 id = data_race(prog->aux->id);
1608 if (id == 0)
1609 prog_cnt = 0;
1610
1611end:
1612 rcu_read_unlock();
1613
1614 if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)) ||
1615 (id != 0 && copy_to_user(prog_ids, &id, sizeof(u32))) ||
1616 copy_to_user(&uattr->query.prog_cnt, &prog_cnt, sizeof(prog_cnt)))
1617 ret = -EFAULT;
1618
1619 return ret;
1620}
1621
1622static void sock_map_unlink(struct sock *sk, struct sk_psock_link *link)
1623{
1624 switch (link->map->map_type) {
1625 case BPF_MAP_TYPE_SOCKMAP:
1626 return sock_map_delete_from_link(link->map, sk,
1627 link->link_raw);
1628 case BPF_MAP_TYPE_SOCKHASH:
1629 return sock_hash_delete_from_link(link->map, sk,
1630 link->link_raw);
1631 default:
1632 break;
1633 }
1634}
1635
1636static void sock_map_remove_links(struct sock *sk, struct sk_psock *psock)
1637{
1638 struct sk_psock_link *link;
1639
1640 while ((link = sk_psock_link_pop(psock))) {
1641 sock_map_unlink(sk, link);
1642 sk_psock_free_link(link);
1643 }
1644}
1645
1646void sock_map_unhash(struct sock *sk)
1647{
1648 void (*saved_unhash)(struct sock *sk);
1649 struct sk_psock *psock;
1650
1651retry:
1652 rcu_read_lock();
1653 psock = sk_psock(sk);
1654 if (unlikely(!psock)) {
1655 rcu_read_unlock();
1656 saved_unhash = READ_ONCE(sk->sk_prot)->unhash;
1657 if (unlikely(saved_unhash == sock_map_unhash))
1658 goto retry;
1659 } else {
1660 saved_unhash = psock->saved_unhash;
1661 sock_map_remove_links(sk, psock);
1662 rcu_read_unlock();
1663
1664 if (WARN_ON_ONCE(saved_unhash == sock_map_unhash))
1665 return;
1666 }
1667
1668 if (saved_unhash)
1669 saved_unhash(sk);
1670}
1671EXPORT_SYMBOL_GPL(sock_map_unhash);
1672
1673void sock_map_destroy(struct sock *sk)
1674{
1675 void (*saved_destroy)(struct sock *sk);
1676 struct sk_psock *psock;
1677
1678retry:
1679 rcu_read_lock();
1680 psock = sk_psock_get(sk);
1681 if (unlikely(!psock)) {
1682 rcu_read_unlock();
1683 saved_destroy = READ_ONCE(sk->sk_prot)->destroy;
1684 if (unlikely(saved_destroy == sock_map_destroy))
1685 goto retry;
1686 } else {
1687 saved_destroy = psock->saved_destroy;
1688 sock_map_remove_links(sk, psock);
1689 rcu_read_unlock();
1690 sk_psock_stop(psock);
1691 sk_psock_put(sk, psock);
1692
1693 if (WARN_ON_ONCE(saved_destroy == sock_map_destroy))
1694 return;
1695 }
1696
1697 if (saved_destroy)
1698 saved_destroy(sk);
1699}
1700EXPORT_SYMBOL_GPL(sock_map_destroy);
1701
1702void sock_map_close(struct sock *sk, long timeout)
1703{
1704 void (*saved_close)(struct sock *sk, long timeout);
1705 struct sk_psock *psock;
1706
1707retry:
1708 lock_sock(sk);
1709 rcu_read_lock();
1710 psock = sk_psock_get(sk);
1711 if (likely(psock)) {
1712 saved_close = psock->saved_close;
1713 sock_map_remove_links(sk, psock);
1714 rcu_read_unlock();
1715 sk_psock_stop(psock);
1716 release_sock(sk);
1717 cancel_delayed_work_sync(&psock->work);
1718 sk_psock_put(sk, psock);
1719
1720 /* Make sure we do not recurse. This is a bug.
1721 * Leak the socket instead of crashing on a stack overflow.
1722 */
1723 if (WARN_ON_ONCE(saved_close == sock_map_close))
1724 return;
1725 } else {
1726 saved_close = READ_ONCE(sk->sk_prot)->close;
1727 rcu_read_unlock();
1728 release_sock(sk);
1729
1730 if (unlikely(saved_close == sock_map_close))
1731 goto retry;
1732 }
1733
1734 saved_close(sk, timeout);
1735}
1736EXPORT_SYMBOL_GPL(sock_map_close);
1737
1738struct sockmap_link {
1739 struct bpf_link link;
1740 struct bpf_map *map;
1741};
1742
1743static void sock_map_link_release(struct bpf_link *link)
1744{
1745 struct sockmap_link *sockmap_link = container_of(link, struct sockmap_link, link);
1746
1747 mutex_lock(&sockmap_mutex);
1748 if (!sockmap_link->map)
1749 goto out;
1750
1751 WARN_ON_ONCE(sock_map_prog_update(sockmap_link->map, NULL, link->prog, link,
1752 link->attach_type));
1753
1754 bpf_map_put_with_uref(sockmap_link->map);
1755 sockmap_link->map = NULL;
1756out:
1757 mutex_unlock(&sockmap_mutex);
1758}
1759
1760static int sock_map_link_detach(struct bpf_link *link)
1761{
1762 sock_map_link_release(link);
1763 return 0;
1764}
1765
1766static void sock_map_link_dealloc(struct bpf_link *link)
1767{
1768 kfree(link);
1769}
1770
1771/* Handle the following two cases:
1772 * case 1: link != NULL, prog != NULL, old != NULL
1773 * case 2: link != NULL, prog != NULL, old == NULL
1774 */
1775static int sock_map_link_update_prog(struct bpf_link *link,
1776 struct bpf_prog *prog,
1777 struct bpf_prog *old)
1778{
1779 const struct sockmap_link *sockmap_link = container_of(link, struct sockmap_link, link);
1780 struct bpf_prog **pprog, *old_link_prog;
1781 struct bpf_link **plink;
1782 int ret = 0;
1783
1784 mutex_lock(&sockmap_mutex);
1785
1786 /* If old prog is not NULL, ensure old prog is the same as link->prog. */
1787 if (old && link->prog != old) {
1788 ret = -EPERM;
1789 goto out;
1790 }
1791 /* Ensure link->prog has the same type/attach_type as the new prog. */
1792 if (link->prog->type != prog->type ||
1793 link->prog->expected_attach_type != prog->expected_attach_type) {
1794 ret = -EINVAL;
1795 goto out;
1796 }
1797
1798 ret = sock_map_prog_attach_check(link->attach_type, prog);
1799 if (ret)
1800 goto out;
1801
1802 if (!sockmap_link->map) {
1803 ret = -ENOLINK;
1804 goto out;
1805 }
1806
1807 ret = sock_map_prog_link_lookup(sockmap_link->map, &pprog, &plink,
1808 link->attach_type);
1809 if (ret)
1810 goto out;
1811
1812 /* return error if the stored bpf_link does not match the incoming bpf_link. */
1813 if (link != *plink) {
1814 ret = -EBUSY;
1815 goto out;
1816 }
1817
1818 if (old) {
1819 ret = psock_replace_prog(pprog, prog, old);
1820 if (ret)
1821 goto out;
1822 } else {
1823 psock_set_prog(pprog, prog);
1824 }
1825
1826 bpf_prog_inc(prog);
1827 old_link_prog = xchg(&link->prog, prog);
1828 bpf_prog_put(old_link_prog);
1829
1830out:
1831 mutex_unlock(&sockmap_mutex);
1832 return ret;
1833}
1834
1835static u32 sock_map_link_get_map_id(const struct sockmap_link *sockmap_link)
1836{
1837 u32 map_id = 0;
1838
1839 mutex_lock(&sockmap_mutex);
1840 if (sockmap_link->map)
1841 map_id = sockmap_link->map->id;
1842 mutex_unlock(&sockmap_mutex);
1843 return map_id;
1844}
1845
1846static int sock_map_link_fill_info(const struct bpf_link *link,
1847 struct bpf_link_info *info)
1848{
1849 const struct sockmap_link *sockmap_link = container_of(link, struct sockmap_link, link);
1850 u32 map_id = sock_map_link_get_map_id(sockmap_link);
1851
1852 info->sockmap.map_id = map_id;
1853 info->sockmap.attach_type = link->attach_type;
1854 return 0;
1855}
1856
1857static void sock_map_link_show_fdinfo(const struct bpf_link *link,
1858 struct seq_file *seq)
1859{
1860 const struct sockmap_link *sockmap_link = container_of(link, struct sockmap_link, link);
1861 u32 map_id = sock_map_link_get_map_id(sockmap_link);
1862
1863 seq_printf(seq, "map_id:\t%u\n", map_id);
1864 seq_printf(seq, "attach_type:\t%u\n", link->attach_type);
1865}
1866
1867static const struct bpf_link_ops sock_map_link_ops = {
1868 .release = sock_map_link_release,
1869 .dealloc = sock_map_link_dealloc,
1870 .detach = sock_map_link_detach,
1871 .update_prog = sock_map_link_update_prog,
1872 .fill_link_info = sock_map_link_fill_info,
1873 .show_fdinfo = sock_map_link_show_fdinfo,
1874};
1875
1876int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
1877{
1878 struct bpf_link_primer link_primer;
1879 struct sockmap_link *sockmap_link;
1880 enum bpf_attach_type attach_type;
1881 struct bpf_map *map;
1882 int ret;
1883
1884 if (attr->link_create.flags)
1885 return -EINVAL;
1886
1887 map = bpf_map_get_with_uref(attr->link_create.target_fd);
1888 if (IS_ERR(map))
1889 return PTR_ERR(map);
1890 if (map->map_type != BPF_MAP_TYPE_SOCKMAP && map->map_type != BPF_MAP_TYPE_SOCKHASH) {
1891 ret = -EINVAL;
1892 goto out;
1893 }
1894
1895 sockmap_link = kzalloc_obj(*sockmap_link, GFP_USER);
1896 if (!sockmap_link) {
1897 ret = -ENOMEM;
1898 goto out;
1899 }
1900
1901 attach_type = attr->link_create.attach_type;
1902 bpf_link_init(&sockmap_link->link, BPF_LINK_TYPE_SOCKMAP, &sock_map_link_ops, prog,
1903 attach_type);
1904 sockmap_link->map = map;
1905
1906 ret = bpf_link_prime(&sockmap_link->link, &link_primer);
1907 if (ret) {
1908 kfree(sockmap_link);
1909 goto out;
1910 }
1911
1912 mutex_lock(&sockmap_mutex);
1913 ret = sock_map_prog_update(map, prog, NULL, &sockmap_link->link, attach_type);
1914 mutex_unlock(&sockmap_mutex);
1915 if (ret) {
1916 bpf_link_cleanup(&link_primer);
1917 goto out;
1918 }
1919
1920 /* Increase refcnt for the prog since when old prog is replaced with
1921 * psock_replace_prog() and psock_set_prog() its refcnt will be decreased.
1922 *
1923 * Actually, we do not need to increase refcnt for the prog since bpf_link
1924 * will hold a reference. But in order to have less complexity w.r.t.
1925 * replacing/setting prog, let us increase the refcnt to make things simpler.
1926 */
1927 bpf_prog_inc(prog);
1928
1929 return bpf_link_settle(&link_primer);
1930
1931out:
1932 bpf_map_put_with_uref(map);
1933 return ret;
1934}
1935
1936static int sock_map_iter_attach_target(struct bpf_prog *prog,
1937 union bpf_iter_link_info *linfo,
1938 struct bpf_iter_aux_info *aux)
1939{
1940 struct bpf_map *map;
1941 int err = -EINVAL;
1942
1943 if (!linfo->map.map_fd)
1944 return -EBADF;
1945
1946 map = bpf_map_get_with_uref(linfo->map.map_fd);
1947 if (IS_ERR(map))
1948 return PTR_ERR(map);
1949
1950 if (map->map_type != BPF_MAP_TYPE_SOCKMAP &&
1951 map->map_type != BPF_MAP_TYPE_SOCKHASH)
1952 goto put_map;
1953
1954 if (prog->aux->max_rdonly_access > map->key_size) {
1955 err = -EACCES;
1956 goto put_map;
1957 }
1958
1959 aux->map = map;
1960 return 0;
1961
1962put_map:
1963 bpf_map_put_with_uref(map);
1964 return err;
1965}
1966
1967static void sock_map_iter_detach_target(struct bpf_iter_aux_info *aux)
1968{
1969 bpf_map_put_with_uref(aux->map);
1970}
1971
1972static struct bpf_iter_reg sock_map_iter_reg = {
1973 .target = "sockmap",
1974 .attach_target = sock_map_iter_attach_target,
1975 .detach_target = sock_map_iter_detach_target,
1976 .show_fdinfo = bpf_iter_map_show_fdinfo,
1977 .fill_link_info = bpf_iter_map_fill_link_info,
1978 .ctx_arg_info_size = 2,
1979 .ctx_arg_info = {
1980 { offsetof(struct bpf_iter__sockmap, key),
1981 PTR_TO_BUF | PTR_MAYBE_NULL | MEM_RDONLY },
1982 { offsetof(struct bpf_iter__sockmap, sk),
1983 PTR_TO_BTF_ID_OR_NULL },
1984 },
1985};
1986
1987static int __init bpf_sockmap_iter_init(void)
1988{
1989 sock_map_iter_reg.ctx_arg_info[1].btf_id =
1990 btf_sock_ids[BTF_SOCK_TYPE_SOCK];
1991 return bpf_iter_reg_target(&sock_map_iter_reg);
1992}
1993late_initcall(bpf_sockmap_iter_init);