projects/linux·blob master · refs

1281 lines · 30.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4#include <linux/skmsg.h>
5#include <linux/skbuff.h>
6#include <linux/scatterlist.h>
7
8#include <net/sock.h>
9#include <net/tcp.h>
10#include <trace/events/sock.h>
11
12static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce)
13{
14 if (msg->sg.end > msg->sg.start &&
15 elem_first_coalesce < msg->sg.end)
16 return true;
17
18 if (msg->sg.end < msg->sg.start &&
19 (elem_first_coalesce > msg->sg.start ||
20 elem_first_coalesce < msg->sg.end))
21 return true;
22
23 return false;
24}
25
26int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
27 int elem_first_coalesce)
28{
29 struct page_frag *pfrag = sk_page_frag(sk);
30 u32 osize = msg->sg.size;
31 int ret = 0;
32
33 len -= msg->sg.size;
34 while (len > 0) {
35 struct scatterlist *sge;
36 u32 orig_offset;
37 int use, i;
38
39 if (!sk_page_frag_refill(sk, pfrag)) {
40 ret = -ENOMEM;
41 goto msg_trim;
42 }
43
44 orig_offset = pfrag->offset;
45 use = min_t(int, len, pfrag->size - orig_offset);
46 if (!sk_wmem_schedule(sk, use)) {
47 ret = -ENOMEM;
48 goto msg_trim;
49 }
50
51 i = msg->sg.end;
52 sk_msg_iter_var_prev(i);
53 sge = &msg->sg.data[i];
54
55 if (sk_msg_try_coalesce_ok(msg, elem_first_coalesce) &&
56 sg_page(sge) == pfrag->page &&
57 sge->offset + sge->length == orig_offset) {
58 sge->length += use;
59 } else {
60 if (sk_msg_full(msg)) {
61 ret = -ENOSPC;
62 break;
63 }
64
65 sge = &msg->sg.data[msg->sg.end];
66 sg_unmark_end(sge);
67 sg_set_page(sge, pfrag->page, use, orig_offset);
68 __clear_bit(msg->sg.end, msg->sg.copy);
69 get_page(pfrag->page);
70 sk_msg_iter_next(msg, end);
71 }
72
73 sk_mem_charge(sk, use);
74 msg->sg.size += use;
75 pfrag->offset += use;
76 len -= use;
77 }
78
79 return ret;
80
81msg_trim:
82 sk_msg_trim(sk, msg, osize);
83 return ret;
84}
85EXPORT_SYMBOL_GPL(sk_msg_alloc);
86
87int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
88 u32 off, u32 len)
89{
90 int i = src->sg.start;
91 struct scatterlist *sge = sk_msg_elem(src, i);
92 struct scatterlist *sgd = NULL;
93 u32 sge_len, sge_off;
94
95 while (off) {
96 if (sge->length > off)
97 break;
98 off -= sge->length;
99 sk_msg_iter_var_next(i);
100 if (i == src->sg.end && off)
101 return -ENOSPC;
102 sge = sk_msg_elem(src, i);
103 }
104
105 while (len) {
106 sge_len = sge->length - off;
107 if (sge_len > len)
108 sge_len = len;
109
110 if (dst->sg.end)
111 sgd = sk_msg_elem(dst, dst->sg.end - 1);
112
113 if (sgd &&
114 (sg_page(sge) == sg_page(sgd)) &&
115 (sg_virt(sge) + off == sg_virt(sgd) + sgd->length)) {
116 sgd->length += sge_len;
117 dst->sg.size += sge_len;
118 } else if (!sk_msg_full(dst)) {
119 sge_off = sge->offset + off;
120 sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off);
121 } else {
122 return -ENOSPC;
123 }
124
125 off = 0;
126 len -= sge_len;
127 sk_mem_charge(sk, sge_len);
128 sk_msg_iter_var_next(i);
129 if (i == src->sg.end && len)
130 return -ENOSPC;
131 sge = sk_msg_elem(src, i);
132 }
133
134 return 0;
135}
136EXPORT_SYMBOL_GPL(sk_msg_clone);
137
138void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes)
139{
140 int i = msg->sg.start;
141
142 do {
143 struct scatterlist *sge = sk_msg_elem(msg, i);
144
145 if (bytes < sge->length) {
146 sge->length -= bytes;
147 sge->offset += bytes;
148 sk_mem_uncharge(sk, bytes);
149 break;
150 }
151
152 sk_mem_uncharge(sk, sge->length);
153 bytes -= sge->length;
154 sge->length = 0;
155 sge->offset = 0;
156 sk_msg_iter_var_next(i);
157 } while (bytes && i != msg->sg.end);
158 msg->sg.start = i;
159}
160EXPORT_SYMBOL_GPL(sk_msg_return_zero);
161
162void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes)
163{
164 int i = msg->sg.start;
165
166 do {
167 struct scatterlist *sge = &msg->sg.data[i];
168 int uncharge = (bytes < sge->length) ? bytes : sge->length;
169
170 sk_mem_uncharge(sk, uncharge);
171 bytes -= uncharge;
172 sk_msg_iter_var_next(i);
173 } while (i != msg->sg.end);
174}
175EXPORT_SYMBOL_GPL(sk_msg_return);
176
177static int sk_msg_free_elem(struct sock *sk, struct sk_msg *msg, u32 i,
178 bool charge)
179{
180 struct scatterlist *sge = sk_msg_elem(msg, i);
181 u32 len = sge->length;
182
183 /* When the skb owns the memory we free it from consume_skb path. */
184 if (!msg->skb) {
185 if (charge)
186 sk_mem_uncharge(sk, len);
187 put_page(sg_page(sge));
188 }
189 __clear_bit(i, msg->sg.copy);
190 memset(sge, 0, sizeof(*sge));
191 return len;
192}
193
194static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i,
195 bool charge)
196{
197 struct scatterlist *sge = sk_msg_elem(msg, i);
198 int freed = 0;
199
200 while (msg->sg.size) {
201 msg->sg.size -= sge->length;
202 freed += sk_msg_free_elem(sk, msg, i, charge);
203 sk_msg_iter_var_next(i);
204 sk_msg_check_to_free(msg, i, msg->sg.size);
205 sge = sk_msg_elem(msg, i);
206 }
207 consume_skb(msg->skb);
208 sk_msg_init(msg);
209 return freed;
210}
211
212int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg)
213{
214 return __sk_msg_free(sk, msg, msg->sg.start, false);
215}
216EXPORT_SYMBOL_GPL(sk_msg_free_nocharge);
217
218int sk_msg_free(struct sock *sk, struct sk_msg *msg)
219{
220 return __sk_msg_free(sk, msg, msg->sg.start, true);
221}
222EXPORT_SYMBOL_GPL(sk_msg_free);
223
224static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg,
225 u32 bytes, bool charge)
226{
227 struct scatterlist *sge;
228 u32 i = msg->sg.start;
229
230 while (bytes) {
231 sge = sk_msg_elem(msg, i);
232 if (!sge->length)
233 break;
234 if (bytes < sge->length) {
235 if (charge)
236 sk_mem_uncharge(sk, bytes);
237 sge->length -= bytes;
238 sge->offset += bytes;
239 msg->sg.size -= bytes;
240 break;
241 }
242
243 msg->sg.size -= sge->length;
244 bytes -= sge->length;
245 sk_msg_free_elem(sk, msg, i, charge);
246 sk_msg_iter_var_next(i);
247 sk_msg_check_to_free(msg, i, bytes);
248 }
249 msg->sg.start = i;
250}
251
252void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes)
253{
254 __sk_msg_free_partial(sk, msg, bytes, true);
255}
256EXPORT_SYMBOL_GPL(sk_msg_free_partial);
257
258void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
259 u32 bytes)
260{
261 __sk_msg_free_partial(sk, msg, bytes, false);
262}
263
264void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len)
265{
266 int trim = msg->sg.size - len;
267 u32 i = msg->sg.end;
268
269 if (trim <= 0) {
270 WARN_ON(trim < 0);
271 return;
272 }
273
274 sk_msg_iter_var_prev(i);
275 msg->sg.size = len;
276 while (msg->sg.data[i].length &&
277 trim >= msg->sg.data[i].length) {
278 trim -= msg->sg.data[i].length;
279 sk_msg_free_elem(sk, msg, i, true);
280 sk_msg_iter_var_prev(i);
281 if (!trim)
282 goto out;
283 }
284
285 msg->sg.data[i].length -= trim;
286 sk_mem_uncharge(sk, trim);
287 /* Adjust copybreak if it falls into the trimmed part of last buf */
288 if (msg->sg.curr == i && msg->sg.copybreak > msg->sg.data[i].length)
289 msg->sg.copybreak = msg->sg.data[i].length;
290out:
291 sk_msg_iter_var_next(i);
292 msg->sg.end = i;
293
294 /* If we trim data a full sg elem before curr pointer update
295 * copybreak and current so that any future copy operations
296 * start at new copy location.
297 * However trimmed data that has not yet been used in a copy op
298 * does not require an update.
299 */
300 if (!msg->sg.size) {
301 msg->sg.curr = msg->sg.start;
302 msg->sg.copybreak = 0;
303 } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >=
304 sk_msg_iter_dist(msg->sg.start, msg->sg.end)) {
305 sk_msg_iter_var_prev(i);
306 msg->sg.curr = i;
307 msg->sg.copybreak = msg->sg.data[i].length;
308 }
309}
310EXPORT_SYMBOL_GPL(sk_msg_trim);
311
312int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
313 struct sk_msg *msg, u32 bytes)
314{
315 int i, maxpages, ret = 0, num_elems = sk_msg_elem_used(msg);
316 const int to_max_pages = MAX_MSG_FRAGS;
317 struct page *pages[MAX_MSG_FRAGS];
318 ssize_t orig, copied, use, offset;
319
320 orig = msg->sg.size;
321 while (bytes > 0) {
322 i = 0;
323 maxpages = to_max_pages - num_elems;
324 if (maxpages == 0) {
325 ret = -EFAULT;
326 goto out;
327 }
328
329 copied = iov_iter_get_pages2(from, pages, bytes, maxpages,
330 &offset);
331 if (copied <= 0) {
332 ret = -EFAULT;
333 goto out;
334 }
335
336 bytes -= copied;
337 msg->sg.size += copied;
338
339 while (copied) {
340 use = min_t(int, copied, PAGE_SIZE - offset);
341 sg_set_page(&msg->sg.data[msg->sg.end],
342 pages[i], use, offset);
343 sg_unmark_end(&msg->sg.data[msg->sg.end]);
344 sk_mem_charge(sk, use);
345
346 offset = 0;
347 copied -= use;
348 sk_msg_iter_next(msg, end);
349 num_elems++;
350 i++;
351 }
352 /* When zerocopy is mixed with sk_msg_*copy* operations we
353 * may have a copybreak set in this case clear and prefer
354 * zerocopy remainder when possible.
355 */
356 msg->sg.copybreak = 0;
357 msg->sg.curr = msg->sg.end;
358 }
359out:
360 /* Revert iov_iter updates, msg will need to use 'trim' later if it
361 * also needs to be cleared.
362 */
363 if (ret)
364 iov_iter_revert(from, msg->sg.size - orig);
365 return ret;
366}
367EXPORT_SYMBOL_GPL(sk_msg_zerocopy_from_iter);
368
369int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
370 struct sk_msg *msg, u32 bytes)
371{
372 int ret = -ENOSPC, i = msg->sg.curr;
373 u32 copy, buf_size, copied = 0;
374 struct scatterlist *sge;
375 void *to;
376
377 do {
378 sge = sk_msg_elem(msg, i);
379 /* This is possible if a trim operation shrunk the buffer */
380 if (msg->sg.copybreak >= sge->length) {
381 msg->sg.copybreak = 0;
382 sk_msg_iter_var_next(i);
383 if (i == msg->sg.end)
384 break;
385 sge = sk_msg_elem(msg, i);
386 }
387
388 buf_size = sge->length - msg->sg.copybreak;
389 copy = (buf_size > bytes) ? bytes : buf_size;
390 to = sg_virt(sge) + msg->sg.copybreak;
391 msg->sg.copybreak += copy;
392 if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY)
393 ret = copy_from_iter_nocache(to, copy, from);
394 else
395 ret = copy_from_iter(to, copy, from);
396 if (ret != copy) {
397 ret = -EFAULT;
398 goto out;
399 }
400 bytes -= copy;
401 copied += copy;
402 if (!bytes)
403 break;
404 msg->sg.copybreak = 0;
405 sk_msg_iter_var_next(i);
406 } while (i != msg->sg.end);
407out:
408 msg->sg.curr = i;
409 return (ret < 0) ? ret : copied;
410}
411EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);
412
413int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
414 int len, int flags, int *copied_from_self)
415{
416 struct iov_iter *iter = &msg->msg_iter;
417 int peek = flags & MSG_PEEK;
418 struct sk_msg *msg_rx;
419 int i, copied = 0;
420 bool from_self;
421
422 msg_rx = sk_psock_peek_msg(psock);
423 if (copied_from_self)
424 *copied_from_self = 0;
425
426 while (copied != len) {
427 struct scatterlist *sge;
428
429 if (unlikely(!msg_rx))
430 break;
431
432 from_self = msg_rx->sk == sk;
433 i = msg_rx->sg.start;
434 do {
435 struct page *page;
436 int copy;
437
438 sge = sk_msg_elem(msg_rx, i);
439 copy = sge->length;
440 page = sg_page(sge);
441 if (copied + copy > len)
442 copy = len - copied;
443 if (copy)
444 copy = copy_page_to_iter(page, sge->offset, copy, iter);
445 if (!copy) {
446 copied = copied ? copied : -EFAULT;
447 goto out;
448 }
449
450 copied += copy;
451 if (from_self && copied_from_self)
452 *copied_from_self += copy;
453
454 if (likely(!peek)) {
455 sge->offset += copy;
456 sge->length -= copy;
457 if (!msg_rx->skb) {
458 sk_mem_uncharge(sk, copy);
459 atomic_sub(copy, &sk->sk_rmem_alloc);
460 }
461 msg_rx->sg.size -= copy;
462 sk_psock_msg_len_add(psock, -copy);
463
464 if (!sge->length) {
465 sk_msg_iter_var_next(i);
466 if (!msg_rx->skb)
467 put_page(page);
468 }
469 } else {
470 /* Lets not optimize peek case if copy_page_to_iter
471 * didn't copy the entire length lets just break.
472 */
473 if (copy != sge->length)
474 goto out;
475 sk_msg_iter_var_next(i);
476 }
477
478 if (copied == len)
479 break;
480 } while ((i != msg_rx->sg.end) && !sg_is_last(sge));
481
482 if (unlikely(peek)) {
483 msg_rx = sk_psock_next_msg(psock, msg_rx);
484 if (!msg_rx)
485 break;
486 continue;
487 }
488
489 msg_rx->sg.start = i;
490 if (!sge->length && (i == msg_rx->sg.end || sg_is_last(sge))) {
491 msg_rx = sk_psock_dequeue_msg(psock);
492 kfree_sk_msg(msg_rx);
493 }
494 msg_rx = sk_psock_peek_msg(psock);
495 }
496out:
497 return copied;
498}
499
500/* Receive sk_msg from psock->ingress_msg to @msg. */
501int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
502 int len, int flags)
503{
504 return __sk_msg_recvmsg(sk, psock, msg, len, flags, NULL);
505}
506EXPORT_SYMBOL_GPL(sk_msg_recvmsg);
507
508bool sk_msg_is_readable(struct sock *sk)
509{
510 struct sk_psock *psock;
511 bool empty = true;
512
513 rcu_read_lock();
514 psock = sk_psock(sk);
515 if (likely(psock))
516 empty = list_empty(&psock->ingress_msg);
517 rcu_read_unlock();
518 return !empty;
519}
520EXPORT_SYMBOL_GPL(sk_msg_is_readable);
521
522static struct sk_msg *alloc_sk_msg(gfp_t gfp)
523{
524 struct sk_msg *msg;
525
526 msg = kzalloc_obj(*msg, gfp | __GFP_NOWARN);
527 if (unlikely(!msg))
528 return NULL;
529 sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);
530 return msg;
531}
532
533static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
534 struct sk_buff *skb)
535{
536 if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
537 return NULL;
538
539 if (!sk_rmem_schedule(sk, skb, skb->truesize))
540 return NULL;
541
542 return alloc_sk_msg(GFP_KERNEL);
543}
544
545static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb,
546 u32 off, u32 len,
547 struct sk_psock *psock,
548 struct sock *sk,
549 struct sk_msg *msg,
550 bool take_ref)
551{
552 int num_sge, copied;
553
554 /* skb_to_sgvec will fail when the total number of fragments in
555 * frag_list and frags exceeds MAX_MSG_FRAGS. For example, the
556 * caller may aggregate multiple skbs.
557 */
558 num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
559 if (num_sge < 0) {
560 /* skb linearize may fail with ENOMEM, but lets simply try again
561 * later if this happens. Under memory pressure we don't want to
562 * drop the skb. We need to linearize the skb so that the mapping
563 * in skb_to_sgvec can not error.
564 * Note that skb_linearize requires the skb not to be shared.
565 */
566 if (skb_linearize(skb))
567 return -EAGAIN;
568
569 num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
570 if (unlikely(num_sge < 0))
571 return num_sge;
572 }
573
574#if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
575 psock->ingress_bytes += len;
576#endif
577 copied = len;
578 msg->sg.start = 0;
579 msg->sg.size = copied;
580 msg->sg.end = num_sge;
581 msg->skb = take_ref ? skb_get(skb) : skb;
582
583 sk_psock_queue_msg(psock, msg);
584 sk_psock_data_ready(sk, psock);
585 return copied;
586}
587
588static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
589 u32 off, u32 len, bool take_ref);
590
591static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,
592 u32 off, u32 len)
593{
594 struct sock *sk = psock->sk;
595 struct sk_msg *msg;
596 int err;
597
598 /* If we are receiving on the same sock skb->sk is already assigned,
599 * skip memory accounting and owner transition seeing it already set
600 * correctly.
601 */
602 if (unlikely(skb->sk == sk))
603 return sk_psock_skb_ingress_self(psock, skb, off, len, true);
604 msg = sk_psock_create_ingress_msg(sk, skb);
605 if (!msg)
606 return -EAGAIN;
607
608 /* This will transition ownership of the data from the socket where
609 * the BPF program was run initiating the redirect to the socket
610 * we will eventually receive this data on. The data will be released
611 * from skb_consume found in __tcp_bpf_recvmsg() after its been copied
612 * into user buffers.
613 */
614 skb_set_owner_r(skb, sk);
615 err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, true);
616 if (err < 0)
617 kfree(msg);
618 return err;
619}
620
621/* Puts an skb on the ingress queue of the socket already assigned to the
622 * skb. In this case we do not need to check memory limits or skb_set_owner_r
623 * because the skb is already accounted for here.
624 */
625static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
626 u32 off, u32 len, bool take_ref)
627{
628 struct sk_msg *msg = alloc_sk_msg(GFP_ATOMIC);
629 struct sock *sk = psock->sk;
630 int err;
631
632 if (unlikely(!msg))
633 return -EAGAIN;
634 skb_set_owner_r(skb, sk);
635
636 /* This is used in tcp_bpf_recvmsg_parser() to determine whether the
637 * data originates from the socket's own protocol stack. No need to
638 * refcount sk because msg's lifetime is bound to sk via the ingress_msg.
639 */
640 msg->sk = sk;
641 err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, take_ref);
642 if (err < 0)
643 kfree(msg);
644 return err;
645}
646
647static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,
648 u32 off, u32 len, bool ingress)
649{
650 if (!ingress) {
651 if (!sock_writeable(psock->sk))
652 return -EAGAIN;
653 return skb_send_sock(psock->sk, skb, off, len);
654 }
655
656 return sk_psock_skb_ingress(psock, skb, off, len);
657}
658
659static void sk_psock_skb_state(struct sk_psock *psock,
660 struct sk_psock_work_state *state,
661 int len, int off)
662{
663 spin_lock_bh(&psock->ingress_lock);
664 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
665 state->len = len;
666 state->off = off;
667 }
668 spin_unlock_bh(&psock->ingress_lock);
669}
670
671static void sk_psock_backlog(struct work_struct *work)
672{
673 struct delayed_work *dwork = to_delayed_work(work);
674 struct sk_psock *psock = container_of(dwork, struct sk_psock, work);
675 struct sk_psock_work_state *state = &psock->work_state;
676 struct sk_buff *skb = NULL;
677 u32 len = 0, off = 0;
678 bool ingress;
679 int ret;
680
681 /* If sk is quickly removed from the map and then added back, the old
682 * psock should not be scheduled, because there are now two psocks
683 * pointing to the same sk.
684 */
685 if (!sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
686 return;
687
688 /* Increment the psock refcnt to synchronize with close(fd) path in
689 * sock_map_close(), ensuring we wait for backlog thread completion
690 * before sk_socket freed. If refcnt increment fails, it indicates
691 * sock_map_close() completed with sk_socket potentially already freed.
692 */
693 if (!sk_psock_get(psock->sk))
694 return;
695 mutex_lock(&psock->work_mutex);
696 while ((skb = skb_peek(&psock->ingress_skb))) {
697 len = skb->len;
698 off = 0;
699 if (skb_bpf_strparser(skb)) {
700 struct strp_msg *stm = strp_msg(skb);
701
702 off = stm->offset;
703 len = stm->full_len;
704 }
705
706 /* Resume processing from previous partial state */
707 if (unlikely(state->len)) {
708 len = state->len;
709 off = state->off;
710 }
711
712 ingress = skb_bpf_ingress(skb);
713 skb_bpf_redirect_clear(skb);
714 do {
715 ret = -EIO;
716 if (!sock_flag(psock->sk, SOCK_DEAD))
717 ret = sk_psock_handle_skb(psock, skb, off,
718 len, ingress);
719 if (ret <= 0) {
720 if (ret == -EAGAIN) {
721 sk_psock_skb_state(psock, state, len, off);
722 /* Restore redir info we cleared before */
723 skb_bpf_set_redir(skb, psock->sk, ingress);
724 /* Delay slightly to prioritize any
725 * other work that might be here.
726 */
727 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
728 schedule_delayed_work(&psock->work, 1);
729 goto end;
730 }
731 /* Hard errors break pipe and stop xmit. */
732 sk_psock_report_error(psock, ret ? -ret : EPIPE);
733 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
734 goto end;
735 }
736 off += ret;
737 len -= ret;
738 } while (len);
739
740 /* The entire skb sent, clear state */
741 sk_psock_skb_state(psock, state, 0, 0);
742 skb = skb_dequeue(&psock->ingress_skb);
743 kfree_skb(skb);
744 }
745end:
746 mutex_unlock(&psock->work_mutex);
747 sk_psock_put(psock->sk, psock);
748}
749
750struct sk_psock *sk_psock_init(struct sock *sk, int node)
751{
752 struct sk_psock *psock;
753 struct proto *prot;
754
755 write_lock_bh(&sk->sk_callback_lock);
756
757 if (sk_is_inet(sk) && inet_csk_has_ulp(sk)) {
758 psock = ERR_PTR(-EINVAL);
759 goto out;
760 }
761
762 if (sk->sk_user_data) {
763 psock = ERR_PTR(-EBUSY);
764 goto out;
765 }
766
767 psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node);
768 if (!psock) {
769 psock = ERR_PTR(-ENOMEM);
770 goto out;
771 }
772
773 prot = READ_ONCE(sk->sk_prot);
774 psock->sk = sk;
775 psock->eval = __SK_NONE;
776 psock->sk_proto = prot;
777 psock->saved_unhash = prot->unhash;
778 psock->saved_destroy = prot->destroy;
779 psock->saved_close = prot->close;
780 psock->saved_write_space = sk->sk_write_space;
781
782 INIT_LIST_HEAD(&psock->link);
783 spin_lock_init(&psock->link_lock);
784
785 INIT_DELAYED_WORK(&psock->work, sk_psock_backlog);
786 mutex_init(&psock->work_mutex);
787 INIT_LIST_HEAD(&psock->ingress_msg);
788 spin_lock_init(&psock->ingress_lock);
789 skb_queue_head_init(&psock->ingress_skb);
790
791 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);
792 refcount_set(&psock->refcnt, 1);
793
794 __rcu_assign_sk_user_data_with_flags(sk, psock,
795 SK_USER_DATA_NOCOPY |
796 SK_USER_DATA_PSOCK);
797 sock_hold(sk);
798
799out:
800 write_unlock_bh(&sk->sk_callback_lock);
801 return psock;
802}
803EXPORT_SYMBOL_GPL(sk_psock_init);
804
805struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)
806{
807 struct sk_psock_link *link;
808
809 spin_lock_bh(&psock->link_lock);
810 link = list_first_entry_or_null(&psock->link, struct sk_psock_link,
811 list);
812 if (link)
813 list_del(&link->list);
814 spin_unlock_bh(&psock->link_lock);
815 return link;
816}
817
818static void __sk_psock_purge_ingress_msg(struct sk_psock *psock)
819{
820 struct sk_msg *msg, *tmp;
821
822 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {
823 list_del(&msg->list);
824 if (!msg->skb)
825 atomic_sub(msg->sg.size, &psock->sk->sk_rmem_alloc);
826 sk_psock_msg_len_add(psock, -msg->sg.size);
827 sk_msg_free(psock->sk, msg);
828 kfree(msg);
829 }
830 WARN_ON_ONCE(psock->msg_tot_len);
831}
832
833static void __sk_psock_zap_ingress(struct sk_psock *psock)
834{
835 struct sk_buff *skb;
836
837 while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {
838 skb_bpf_redirect_clear(skb);
839 sock_drop(psock->sk, skb);
840 }
841 __sk_psock_purge_ingress_msg(psock);
842}
843
844static void sk_psock_link_destroy(struct sk_psock *psock)
845{
846 struct sk_psock_link *link, *tmp;
847
848 list_for_each_entry_safe(link, tmp, &psock->link, list) {
849 list_del(&link->list);
850 sk_psock_free_link(link);
851 }
852}
853
854void sk_psock_stop(struct sk_psock *psock)
855{
856 spin_lock_bh(&psock->ingress_lock);
857 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
858 sk_psock_cork_free(psock);
859 spin_unlock_bh(&psock->ingress_lock);
860}
861
862static void sk_psock_done_strp(struct sk_psock *psock);
863
864static void sk_psock_destroy(struct work_struct *work)
865{
866 struct sk_psock *psock = container_of(to_rcu_work(work),
867 struct sk_psock, rwork);
868 /* No sk_callback_lock since already detached. */
869
870 sk_psock_done_strp(psock);
871
872 cancel_delayed_work_sync(&psock->work);
873 __sk_psock_zap_ingress(psock);
874 mutex_destroy(&psock->work_mutex);
875
876 psock_progs_drop(&psock->progs);
877
878 sk_psock_link_destroy(psock);
879 sk_psock_cork_free(psock);
880
881 if (psock->sk_redir)
882 sock_put(psock->sk_redir);
883 if (psock->sk_pair)
884 sock_put(psock->sk_pair);
885 sock_put(psock->sk);
886 kfree(psock);
887}
888
889void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
890{
891 write_lock_bh(&sk->sk_callback_lock);
892 sk_psock_restore_proto(sk, psock);
893 rcu_assign_sk_user_data(sk, NULL);
894 if (psock->progs.stream_parser)
895 sk_psock_stop_strp(sk, psock);
896 else if (psock->progs.stream_verdict || psock->progs.skb_verdict)
897 sk_psock_stop_verdict(sk, psock);
898 write_unlock_bh(&sk->sk_callback_lock);
899
900 sk_psock_stop(psock);
901
902 INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);
903 queue_rcu_work(system_percpu_wq, &psock->rwork);
904}
905EXPORT_SYMBOL_GPL(sk_psock_drop);
906
907static int sk_psock_map_verd(int verdict, bool redir)
908{
909 switch (verdict) {
910 case SK_PASS:
911 return redir ? __SK_REDIRECT : __SK_PASS;
912 case SK_DROP:
913 default:
914 break;
915 }
916
917 return __SK_DROP;
918}
919
920int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
921 struct sk_msg *msg)
922{
923 struct bpf_prog *prog;
924 int ret;
925
926 rcu_read_lock();
927 prog = READ_ONCE(psock->progs.msg_parser);
928 if (unlikely(!prog)) {
929 ret = __SK_PASS;
930 goto out;
931 }
932
933 sk_msg_compute_data_pointers(msg);
934 msg->sk = sk;
935 ret = bpf_prog_run_pin_on_cpu(prog, msg);
936 msg->sk = NULL;
937 ret = sk_psock_map_verd(ret, msg->sk_redir);
938 psock->apply_bytes = msg->apply_bytes;
939 if (ret == __SK_REDIRECT) {
940 if (psock->sk_redir) {
941 sock_put(psock->sk_redir);
942 psock->sk_redir = NULL;
943 }
944 if (!msg->sk_redir) {
945 ret = __SK_DROP;
946 goto out;
947 }
948 psock->redir_ingress = sk_msg_to_ingress(msg);
949 psock->sk_redir = msg->sk_redir;
950 sock_hold(psock->sk_redir);
951 }
952out:
953 rcu_read_unlock();
954 return ret;
955}
956EXPORT_SYMBOL_GPL(sk_psock_msg_verdict);
957
958static int sk_psock_skb_redirect(struct sk_psock *from, struct sk_buff *skb)
959{
960 struct sk_psock *psock_other;
961 struct sock *sk_other;
962
963 sk_other = skb_bpf_redirect_fetch(skb);
964 /* This error is a buggy BPF program, it returned a redirect
965 * return code, but then didn't set a redirect interface.
966 */
967 if (unlikely(!sk_other)) {
968 skb_bpf_redirect_clear(skb);
969 sock_drop(from->sk, skb);
970 return -EIO;
971 }
972 psock_other = sk_psock(sk_other);
973 /* This error indicates the socket is being torn down or had another
974 * error that caused the pipe to break. We can't send a packet on
975 * a socket that is in this state so we drop the skb.
976 */
977 if (!psock_other || sock_flag(sk_other, SOCK_DEAD)) {
978 skb_bpf_redirect_clear(skb);
979 sock_drop(from->sk, skb);
980 return -EIO;
981 }
982 spin_lock_bh(&psock_other->ingress_lock);
983 if (!sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
984 spin_unlock_bh(&psock_other->ingress_lock);
985 skb_bpf_redirect_clear(skb);
986 sock_drop(from->sk, skb);
987 return -EIO;
988 }
989
990 skb_queue_tail(&psock_other->ingress_skb, skb);
991 schedule_delayed_work(&psock_other->work, 0);
992 spin_unlock_bh(&psock_other->ingress_lock);
993 return 0;
994}
995
996static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,
997 int verdict)
998{
999 struct sock *sk_other;
1000 int err = 0;
1001 u32 len, off;
1002
1003 if (verdict == __SK_REDIRECT && skb_bpf_ingress(skb) &&
1004 skb_bpf_redirect_fetch(skb) == psock->sk)
1005 verdict = __SK_PASS;
1006
1007 switch (verdict) {
1008 case __SK_PASS:
1009 err = -EIO;
1010 sk_other = psock->sk;
1011 if (sock_flag(sk_other, SOCK_DEAD) ||
1012 !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
1013 goto out_free;
1014
1015 skb_bpf_set_ingress(skb);
1016
1017 /* If the queue is empty then we can submit directly
1018 * into the msg queue. If its not empty we have to
1019 * queue work otherwise we may get OOO data. Otherwise,
1020 * if sk_psock_skb_ingress errors will be handled by
1021 * retrying later from workqueue.
1022 */
1023 if (skb_queue_empty(&psock->ingress_skb)) {
1024 len = skb->len;
1025 off = 0;
1026 if (skb_bpf_strparser(skb)) {
1027 struct strp_msg *stm = strp_msg(skb);
1028
1029 off = stm->offset;
1030 len = stm->full_len;
1031 }
1032 err = sk_psock_skb_ingress_self(psock, skb, off, len, false);
1033 }
1034 if (err < 0) {
1035 spin_lock_bh(&psock->ingress_lock);
1036 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
1037 skb_queue_tail(&psock->ingress_skb, skb);
1038 schedule_delayed_work(&psock->work, 0);
1039 err = 0;
1040 }
1041 spin_unlock_bh(&psock->ingress_lock);
1042 if (err < 0)
1043 goto out_free;
1044 }
1045 break;
1046 case __SK_REDIRECT:
1047 tcp_eat_skb(psock->sk, skb);
1048 err = sk_psock_skb_redirect(psock, skb);
1049 break;
1050 case __SK_DROP:
1051 default:
1052out_free:
1053 skb_bpf_redirect_clear(skb);
1054 tcp_eat_skb(psock->sk, skb);
1055 sock_drop(psock->sk, skb);
1056 }
1057
1058 return err;
1059}
1060
1061static void sk_psock_write_space(struct sock *sk)
1062{
1063 struct sk_psock *psock;
1064 void (*write_space)(struct sock *sk) = NULL;
1065
1066 rcu_read_lock();
1067 psock = sk_psock(sk);
1068 if (likely(psock)) {
1069 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
1070 schedule_delayed_work(&psock->work, 0);
1071 write_space = psock->saved_write_space;
1072 }
1073 rcu_read_unlock();
1074 if (write_space)
1075 write_space(sk);
1076}
1077
1078#if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
1079static void sk_psock_strp_read(struct strparser *strp, struct sk_buff *skb)
1080{
1081 struct sk_psock *psock;
1082 struct bpf_prog *prog;
1083 int ret = __SK_DROP;
1084 struct sock *sk;
1085
1086 rcu_read_lock();
1087 sk = strp->sk;
1088 psock = sk_psock(sk);
1089 if (unlikely(!psock)) {
1090 sock_drop(sk, skb);
1091 goto out;
1092 }
1093 prog = READ_ONCE(psock->progs.stream_verdict);
1094 if (likely(prog)) {
1095 skb->sk = sk;
1096 skb_dst_drop(skb);
1097 skb_bpf_redirect_clear(skb);
1098 ret = bpf_prog_run_pin_on_cpu(prog, skb);
1099 skb_bpf_set_strparser(skb);
1100 ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
1101 skb->sk = NULL;
1102 }
1103 sk_psock_verdict_apply(psock, skb, ret);
1104out:
1105 rcu_read_unlock();
1106}
1107
1108static int sk_psock_strp_read_done(struct strparser *strp, int err)
1109{
1110 return err;
1111}
1112
1113static int sk_psock_strp_parse(struct strparser *strp, struct sk_buff *skb)
1114{
1115 struct sk_psock *psock = container_of(strp, struct sk_psock, strp);
1116 struct bpf_prog *prog;
1117 int ret = skb->len;
1118
1119 rcu_read_lock();
1120 prog = READ_ONCE(psock->progs.stream_parser);
1121 if (likely(prog)) {
1122 skb->sk = psock->sk;
1123 ret = bpf_prog_run_pin_on_cpu(prog, skb);
1124 skb->sk = NULL;
1125 }
1126 rcu_read_unlock();
1127 return ret;
1128}
1129
1130/* Called with socket lock held. */
1131static void sk_psock_strp_data_ready(struct sock *sk)
1132{
1133 struct sk_psock *psock;
1134
1135 trace_sk_data_ready(sk);
1136
1137 rcu_read_lock();
1138 psock = sk_psock(sk);
1139 if (likely(psock)) {
1140 read_lock_bh(&sk->sk_callback_lock);
1141 strp_data_ready(&psock->strp);
1142 read_unlock_bh(&sk->sk_callback_lock);
1143 }
1144 rcu_read_unlock();
1145}
1146
1147int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
1148{
1149 int ret;
1150
1151 static const struct strp_callbacks cb = {
1152 .rcv_msg = sk_psock_strp_read,
1153 .read_sock_done = sk_psock_strp_read_done,
1154 .parse_msg = sk_psock_strp_parse,
1155 };
1156
1157 ret = strp_init(&psock->strp, sk, &cb);
1158 if (!ret)
1159 sk_psock_set_state(psock, SK_PSOCK_RX_STRP_ENABLED);
1160
1161 if (sk_is_tcp(sk)) {
1162 psock->strp.cb.read_sock = tcp_bpf_strp_read_sock;
1163 psock->copied_seq = tcp_sk(sk)->copied_seq;
1164 }
1165 return ret;
1166}
1167
1168void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
1169{
1170 if (psock->saved_data_ready)
1171 return;
1172
1173 psock->saved_data_ready = sk->sk_data_ready;
1174 WRITE_ONCE(sk->sk_data_ready, sk_psock_strp_data_ready);
1175 WRITE_ONCE(sk->sk_write_space, sk_psock_write_space);
1176}
1177
1178void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
1179{
1180 psock_set_prog(&psock->progs.stream_parser, NULL);
1181
1182 if (!psock->saved_data_ready)
1183 return;
1184
1185 WRITE_ONCE(sk->sk_data_ready, psock->saved_data_ready);
1186 WRITE_ONCE(psock->saved_data_ready, NULL);
1187 strp_stop(&psock->strp);
1188}
1189
1190static void sk_psock_done_strp(struct sk_psock *psock)
1191{
1192 /* Parser has been stopped */
1193 if (sk_psock_test_state(psock, SK_PSOCK_RX_STRP_ENABLED))
1194 strp_done(&psock->strp);
1195}
1196#else
1197static void sk_psock_done_strp(struct sk_psock *psock)
1198{
1199}
1200#endif /* CONFIG_BPF_STREAM_PARSER */
1201
1202static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb)
1203{
1204 struct sk_psock *psock;
1205 struct bpf_prog *prog;
1206 int ret = __SK_DROP;
1207 int len = skb->len;
1208
1209 rcu_read_lock();
1210 psock = sk_psock(sk);
1211 if (unlikely(!psock)) {
1212 len = 0;
1213 tcp_eat_skb(sk, skb);
1214 sock_drop(sk, skb);
1215 goto out;
1216 }
1217 prog = READ_ONCE(psock->progs.stream_verdict);
1218 if (!prog)
1219 prog = READ_ONCE(psock->progs.skb_verdict);
1220 if (likely(prog)) {
1221 skb_dst_drop(skb);
1222 skb_bpf_redirect_clear(skb);
1223 ret = bpf_prog_run_pin_on_cpu(prog, skb);
1224 ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
1225 }
1226 ret = sk_psock_verdict_apply(psock, skb, ret);
1227 if (ret < 0)
1228 len = ret;
1229out:
1230 rcu_read_unlock();
1231 return len;
1232}
1233
1234static void sk_psock_verdict_data_ready(struct sock *sk)
1235{
1236 const struct proto_ops *ops = NULL;
1237 struct sk_psock *psock;
1238 struct socket *sock;
1239 int copied;
1240
1241 trace_sk_data_ready(sk);
1242
1243 rcu_read_lock();
1244 sock = READ_ONCE(sk->sk_socket);
1245 if (likely(sock))
1246 ops = READ_ONCE(sock->ops);
1247 rcu_read_unlock();
1248 if (!ops || !ops->read_skb)
1249 return;
1250
1251 copied = ops->read_skb(sk, sk_psock_verdict_recv);
1252 if (copied >= 0) {
1253 rcu_read_lock();
1254 psock = sk_psock(sk);
1255 if (psock)
1256 sk_psock_data_ready(sk, psock);
1257 rcu_read_unlock();
1258 }
1259}
1260
1261void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
1262{
1263 if (psock->saved_data_ready)
1264 return;
1265
1266 psock->saved_data_ready = sk->sk_data_ready;
1267 WRITE_ONCE(sk->sk_data_ready, sk_psock_verdict_data_ready);
1268 WRITE_ONCE(sk->sk_write_space, sk_psock_write_space);
1269}
1270
1271void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
1272{
1273 psock_set_prog(&psock->progs.stream_verdict, NULL);
1274 psock_set_prog(&psock->progs.skb_verdict, NULL);
1275
1276 if (!psock->saved_data_ready)
1277 return;
1278
1279 WRITE_ONCE(sk->sk_data_ready, psock->saved_data_ready);
1280 psock->saved_data_ready = NULL;
1281}