projects/linux·blob master · refs

2230 lines · 62.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Multipath TCP
3 *
4 * Copyright (c) 2017 - 2019, Intel Corporation.
5 */
6
7#define pr_fmt(fmt) "MPTCP: " fmt
8
9#include <linux/kernel.h>
10#include <linux/module.h>
11#include <linux/netdevice.h>
12#include <crypto/sha2.h>
13#include <crypto/utils.h>
14#include <net/sock.h>
15#include <net/inet_common.h>
16#include <net/inet_hashtables.h>
17#include <net/protocol.h>
18#if IS_ENABLED(CONFIG_MPTCP_IPV6)
19#include <net/ip6_route.h>
20#include <net/transp_v6.h>
21#endif
22#include <net/mptcp.h>
23
24#include "protocol.h"
25#include "mib.h"
26
27#include <trace/events/mptcp.h>
28#include <trace/events/sock.h>
29
30static void mptcp_subflow_ops_undo_override(struct sock *ssk);
31
32static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,
33 enum linux_mptcp_mib_field field)
34{
35 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field);
36}
37
38static void subflow_req_destructor(struct request_sock *req)
39{
40 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
41
42 pr_debug("subflow_req=%p\n", subflow_req);
43
44 if (subflow_req->msk)
45 sock_put((struct sock *)subflow_req->msk);
46
47 mptcp_token_destroy_request(req);
48}
49
50static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2,
51 void *hmac)
52{
53 u8 msg[8];
54
55 put_unaligned_be32(nonce1, &msg[0]);
56 put_unaligned_be32(nonce2, &msg[4]);
57
58 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac);
59}
60
61static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk)
62{
63 return mptcp_is_fully_established((void *)msk) &&
64 ((mptcp_pm_is_userspace(msk) &&
65 mptcp_userspace_pm_active(msk)) ||
66 READ_ONCE(msk->pm.accept_subflow));
67}
68
69/* validate received token and create truncated hmac and nonce for SYN-ACK */
70static void subflow_req_create_thmac(struct mptcp_subflow_request_sock *subflow_req)
71{
72 struct mptcp_sock *msk = subflow_req->msk;
73 u8 hmac[SHA256_DIGEST_SIZE];
74
75 subflow_req->local_nonce = get_random_u32();
76
77 subflow_generate_hmac(READ_ONCE(msk->local_key),
78 READ_ONCE(msk->remote_key),
79 subflow_req->local_nonce,
80 subflow_req->remote_nonce, hmac);
81
82 subflow_req->thmac = get_unaligned_be64(hmac);
83}
84
85static struct mptcp_sock *subflow_token_join_request(struct request_sock *req)
86{
87 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
88 struct mptcp_sock *msk;
89 int local_id;
90
91 msk = mptcp_token_get_sock(sock_net(req_to_sk(req)), subflow_req->token);
92 if (!msk) {
93 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN);
94 return NULL;
95 }
96
97 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req);
98 if (local_id < 0) {
99 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPJOINNOIDFOUND);
100 sock_put((struct sock *)msk);
101 return NULL;
102 }
103 subflow_req->local_id = local_id;
104 subflow_req->request_bkup = mptcp_pm_is_backup(msk, (struct sock_common *)req);
105
106 return msk;
107}
108
109static void subflow_init_req(struct request_sock *req, const struct sock *sk_listener)
110{
111 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
112
113 subflow_req->mp_capable = 0;
114 subflow_req->mp_join = 0;
115 subflow_req->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk_listener));
116 subflow_req->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk_listener));
117 subflow_req->msk = NULL;
118 mptcp_token_init_request(req);
119}
120
121static bool subflow_use_different_sport(struct mptcp_sock *msk, const struct sock *sk)
122{
123 return inet_sk(sk)->inet_sport != inet_sk((struct sock *)msk)->inet_sport;
124}
125
126static void subflow_add_reset_reason(struct sk_buff *skb, u8 reason)
127{
128 struct mptcp_ext *mpext = skb_ext_add(skb, SKB_EXT_MPTCP);
129
130 if (mpext) {
131 memset(mpext, 0, sizeof(*mpext));
132 mpext->reset_reason = reason;
133 }
134}
135
136static int subflow_reset_req_endp(struct request_sock *req, struct sk_buff *skb)
137{
138 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEENDPATTEMPT);
139 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
140 return -EPERM;
141}
142
143/* Init mptcp request socket.
144 *
145 * Returns an error code if a JOIN has failed and a TCP reset
146 * should be sent.
147 */
148static int subflow_check_req(struct request_sock *req,
149 const struct sock *sk_listener,
150 struct sk_buff *skb)
151{
152 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
153 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
154 struct mptcp_options_received mp_opt;
155 bool opt_mp_capable, opt_mp_join;
156
157 pr_debug("subflow_req=%p, listener=%p\n", subflow_req, listener);
158
159#ifdef CONFIG_TCP_MD5SIG
160 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of
161 * TCP option space.
162 */
163 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) {
164 MPTCP_INC_STATS(sock_net(sk_listener), MPTCP_MIB_MD5SIGRESET);
165 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP);
166 return -EINVAL;
167 }
168#endif
169
170 mptcp_get_options(skb, &mp_opt);
171
172 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYN);
173 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYN);
174 if (opt_mp_capable) {
175 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE);
176
177 if (unlikely(listener->pm_listener))
178 return subflow_reset_req_endp(req, skb);
179 } else if (opt_mp_join) {
180 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX);
181
182 if (mp_opt.backup)
183 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNBACKUPRX);
184 } else if (unlikely(listener->pm_listener)) {
185 return subflow_reset_req_endp(req, skb);
186 }
187
188 if (opt_mp_capable && listener->request_mptcp) {
189 int err, retries = MPTCP_TOKEN_MAX_RETRIES;
190
191 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
192again:
193 do {
194 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key));
195 } while (subflow_req->local_key == 0);
196
197 if (unlikely(req->syncookie)) {
198 mptcp_crypto_key_sha(subflow_req->local_key,
199 &subflow_req->token,
200 &subflow_req->idsn);
201 if (mptcp_token_exists(subflow_req->token)) {
202 if (retries-- > 0)
203 goto again;
204 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT);
205 } else {
206 subflow_req->mp_capable = 1;
207 }
208 return 0;
209 }
210
211 err = mptcp_token_new_request(req);
212 if (err == 0)
213 subflow_req->mp_capable = 1;
214 else if (retries-- > 0)
215 goto again;
216 else
217 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT);
218
219 } else if (opt_mp_join && listener->request_mptcp) {
220 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
221 subflow_req->mp_join = 1;
222 subflow_req->backup = mp_opt.backup;
223 subflow_req->remote_id = mp_opt.join_id;
224 subflow_req->token = mp_opt.token;
225 subflow_req->remote_nonce = mp_opt.nonce;
226 subflow_req->msk = subflow_token_join_request(req);
227
228 /* Can't fall back to TCP in this case. */
229 if (!subflow_req->msk) {
230 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP);
231 return -EPERM;
232 }
233
234 if (subflow_use_different_sport(subflow_req->msk, sk_listener)) {
235 pr_debug("syn inet_sport=%d %d\n",
236 ntohs(inet_sk(sk_listener)->inet_sport),
237 ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport));
238 if (!mptcp_pm_announced_has_ssk(subflow_req->msk, sk_listener)) {
239 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX);
240 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
241 return -EPERM;
242 }
243 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTSYNRX);
244 }
245
246 subflow_req_create_thmac(subflow_req);
247
248 if (unlikely(req->syncookie)) {
249 if (!mptcp_can_accept_new_subflow(subflow_req->msk)) {
250 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINREJECTED);
251 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
252 return -EPERM;
253 }
254
255 subflow_init_req_cookie_join_save(subflow_req, skb);
256 }
257
258 pr_debug("token=%u, remote_nonce=%u msk=%p\n", subflow_req->token,
259 subflow_req->remote_nonce, subflow_req->msk);
260 }
261
262 return 0;
263}
264
265int mptcp_subflow_init_cookie_req(struct request_sock *req,
266 const struct sock *sk_listener,
267 struct sk_buff *skb)
268{
269 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
270 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
271 struct mptcp_options_received mp_opt;
272 bool opt_mp_capable, opt_mp_join;
273 int err;
274
275 subflow_init_req(req, sk_listener);
276 mptcp_get_options(skb, &mp_opt);
277
278 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_ACK);
279 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK);
280 if (opt_mp_capable && listener->request_mptcp) {
281 if (mp_opt.sndr_key == 0)
282 return -EINVAL;
283
284 subflow_req->local_key = mp_opt.rcvr_key;
285 err = mptcp_token_new_request(req);
286 if (err)
287 return err;
288
289 subflow_req->mp_capable = 1;
290 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
291 } else if (opt_mp_join && listener->request_mptcp) {
292 if (!mptcp_token_join_cookie_init_state(subflow_req, skb))
293 return -EINVAL;
294
295 subflow_req->mp_join = 1;
296 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
297 }
298
299 return 0;
300}
301EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req);
302
303static enum sk_rst_reason mptcp_get_rst_reason(const struct sk_buff *skb)
304{
305 const struct mptcp_ext *mpext = mptcp_get_ext(skb);
306
307 if (!mpext)
308 return SK_RST_REASON_NOT_SPECIFIED;
309
310 return sk_rst_convert_mptcp_reason(mpext->reset_reason);
311}
312
313static struct dst_entry *subflow_v4_route_req(const struct sock *sk,
314 struct sk_buff *skb,
315 struct flowi *fl,
316 struct request_sock *req,
317 u32 tw_isn)
318{
319 struct dst_entry *dst;
320 int err;
321
322 tcp_rsk(req)->is_mptcp = 1;
323 subflow_init_req(req, sk);
324
325 dst = tcp_request_sock_ipv4_ops.route_req(sk, skb, fl, req, tw_isn);
326 if (!dst)
327 return NULL;
328
329 err = subflow_check_req(req, sk, skb);
330 if (err == 0)
331 return dst;
332
333 dst_release(dst);
334 if (!req->syncookie)
335 tcp_request_sock_ops.send_reset(sk, skb,
336 mptcp_get_rst_reason(skb));
337 return NULL;
338}
339
340static void subflow_prep_synack(const struct sock *sk, struct request_sock *req,
341 struct tcp_fastopen_cookie *foc,
342 enum tcp_synack_type synack_type)
343{
344 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
345 struct inet_request_sock *ireq = inet_rsk(req);
346
347 /* clear tstamp_ok, as needed depending on cookie */
348 if (foc && foc->len > -1)
349 ireq->tstamp_ok = 0;
350
351 if (synack_type == TCP_SYNACK_FASTOPEN)
352 mptcp_fastopen_subflow_synack_set_params(subflow, req);
353}
354
355static int subflow_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
356 struct flowi *fl,
357 struct request_sock *req,
358 struct tcp_fastopen_cookie *foc,
359 enum tcp_synack_type synack_type,
360 struct sk_buff *syn_skb)
361{
362 subflow_prep_synack(sk, req, foc, synack_type);
363
364 return tcp_request_sock_ipv4_ops.send_synack(sk, dst, fl, req, foc,
365 synack_type, syn_skb);
366}
367
368#if IS_ENABLED(CONFIG_MPTCP_IPV6)
369static int subflow_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
370 struct flowi *fl,
371 struct request_sock *req,
372 struct tcp_fastopen_cookie *foc,
373 enum tcp_synack_type synack_type,
374 struct sk_buff *syn_skb)
375{
376 subflow_prep_synack(sk, req, foc, synack_type);
377
378 return tcp_request_sock_ipv6_ops.send_synack(sk, dst, fl, req, foc,
379 synack_type, syn_skb);
380}
381
382static struct dst_entry *subflow_v6_route_req(const struct sock *sk,
383 struct sk_buff *skb,
384 struct flowi *fl,
385 struct request_sock *req,
386 u32 tw_isn)
387{
388 struct dst_entry *dst;
389 int err;
390
391 tcp_rsk(req)->is_mptcp = 1;
392 subflow_init_req(req, sk);
393
394 dst = tcp_request_sock_ipv6_ops.route_req(sk, skb, fl, req, tw_isn);
395 if (!dst)
396 return NULL;
397
398 err = subflow_check_req(req, sk, skb);
399 if (err == 0)
400 return dst;
401
402 dst_release(dst);
403 if (!req->syncookie)
404 tcp6_request_sock_ops.send_reset(sk, skb,
405 mptcp_get_rst_reason(skb));
406 return NULL;
407}
408#endif
409
410/* validate received truncated hmac and create hmac for third ACK */
411static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow)
412{
413 u8 hmac[SHA256_DIGEST_SIZE];
414 u64 thmac;
415
416 subflow_generate_hmac(subflow->remote_key, subflow->local_key,
417 subflow->remote_nonce, subflow->local_nonce,
418 hmac);
419
420 thmac = get_unaligned_be64(hmac);
421 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n",
422 subflow, subflow->token, thmac, subflow->thmac);
423
424 return thmac == subflow->thmac;
425}
426
427void mptcp_subflow_reset(struct sock *ssk)
428{
429 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
430 struct sock *sk = subflow->conn;
431
432 /* mptcp_mp_fail_no_response() can reach here on an already closed
433 * socket
434 */
435 if (ssk->sk_state == TCP_CLOSE)
436 return;
437
438 /* must hold: tcp_done() could drop last reference on parent */
439 sock_hold(sk);
440
441 subflow->resetting = 1;
442
443 /* No need to delay the actual close for to-be discarded data. */
444 __skb_queue_purge(&ssk->sk_receive_queue);
445 mptcp_send_active_reset_reason(ssk);
446 tcp_done(ssk);
447 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags))
448 mptcp_schedule_work(sk);
449
450 sock_put(sk);
451}
452
453static bool subflow_use_different_dport(struct mptcp_sock *msk, const struct sock *sk)
454{
455 return inet_sk(sk)->inet_dport != inet_sk((struct sock *)msk)->inet_dport;
456}
457
458void __mptcp_sync_state(struct sock *sk, int state)
459{
460 struct mptcp_subflow_context *subflow;
461 struct mptcp_sock *msk = mptcp_sk(sk);
462 struct sock *ssk = msk->first;
463
464 subflow = mptcp_subflow_ctx(ssk);
465 __mptcp_propagate_sndbuf(sk, ssk);
466
467 if (sk->sk_state == TCP_SYN_SENT) {
468 /* subflow->idsn is always available is TCP_SYN_SENT state,
469 * even for the FASTOPEN scenarios
470 */
471 WRITE_ONCE(msk->write_seq, subflow->idsn + 1);
472 WRITE_ONCE(msk->snd_nxt, msk->write_seq);
473 mptcp_set_state(sk, state);
474 sk->sk_state_change(sk);
475 }
476}
477
478static void subflow_set_remote_key(struct mptcp_sock *msk,
479 struct mptcp_subflow_context *subflow,
480 const struct mptcp_options_received *mp_opt)
481{
482 /* active MPC subflow will reach here multiple times:
483 * at subflow_finish_connect() time and at 4th ack time
484 */
485 if (subflow->remote_key_valid)
486 return;
487
488 subflow->remote_key_valid = 1;
489 subflow->remote_key = mp_opt->sndr_key;
490 mptcp_crypto_key_sha(subflow->remote_key, NULL, &subflow->iasn);
491 subflow->iasn++;
492
493 /* for fallback's sake */
494 subflow->map_seq = subflow->iasn;
495
496 WRITE_ONCE(msk->remote_key, subflow->remote_key);
497 WRITE_ONCE(msk->ack_seq, subflow->iasn);
498 WRITE_ONCE(msk->can_ack, true);
499 atomic64_set(&msk->rcv_wnd_sent, subflow->iasn);
500}
501
502static void mptcp_propagate_state(struct sock *sk, struct sock *ssk,
503 struct mptcp_subflow_context *subflow,
504 const struct mptcp_options_received *mp_opt)
505{
506 struct mptcp_sock *msk = mptcp_sk(sk);
507
508 mptcp_data_lock(sk);
509 if (mp_opt) {
510 /* Options are available only in the non fallback cases
511 * avoid updating rx path fields otherwise
512 */
513 WRITE_ONCE(msk->snd_una, subflow->idsn + 1);
514 WRITE_ONCE(msk->wnd_end, subflow->idsn + 1 + tcp_sk(ssk)->snd_wnd);
515 subflow_set_remote_key(msk, subflow, mp_opt);
516 }
517
518 if (!sock_owned_by_user(sk)) {
519 __mptcp_sync_state(sk, ssk->sk_state);
520 } else {
521 msk->pending_state = ssk->sk_state;
522 __set_bit(MPTCP_SYNC_STATE, &msk->cb_flags);
523 }
524 mptcp_data_unlock(sk);
525}
526
527static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
528{
529 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
530 struct mptcp_options_received mp_opt;
531 struct sock *parent = subflow->conn;
532 struct mptcp_sock *msk;
533
534 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb);
535
536 /* be sure no special action on any packet other than syn-ack */
537 if (subflow->conn_finished)
538 return;
539
540 msk = mptcp_sk(parent);
541 subflow->rel_write_seq = 1;
542 subflow->conn_finished = 1;
543 subflow->ssn_offset = TCP_SKB_CB(skb)->seq;
544 pr_debug("subflow=%p synack seq=%x\n", subflow, subflow->ssn_offset);
545
546 mptcp_get_options(skb, &mp_opt);
547 if (subflow->request_mptcp) {
548 if (!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYNACK)) {
549 if (!mptcp_try_fallback(sk,
550 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK)) {
551 MPTCP_INC_STATS(sock_net(sk),
552 MPTCP_MIB_FALLBACKFAILED);
553 goto do_reset;
554 }
555
556 goto fallback;
557 }
558
559 if (mp_opt.suboptions & OPTION_MPTCP_CSUMREQD)
560 WRITE_ONCE(msk->csum_enabled, true);
561 if (mp_opt.deny_join_id0)
562 WRITE_ONCE(msk->pm.remote_deny_join_id0, true);
563 subflow->mp_capable = 1;
564 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVEACK);
565 mptcp_finish_connect(sk);
566 mptcp_active_enable(parent);
567 mptcp_propagate_state(parent, sk, subflow, &mp_opt);
568 } else if (subflow->request_join) {
569 u8 hmac[SHA256_DIGEST_SIZE];
570
571 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYNACK)) {
572 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPJOINSYNACKNOMPJOIN);
573 subflow->reset_reason = MPTCP_RST_EMPTCP;
574 goto do_reset;
575 }
576
577 subflow->backup = mp_opt.backup;
578 subflow->thmac = mp_opt.thmac;
579 subflow->remote_nonce = mp_opt.nonce;
580 WRITE_ONCE(subflow->remote_id, mp_opt.join_id);
581 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u backup=%d\n",
582 subflow, subflow->thmac, subflow->remote_nonce,
583 subflow->backup);
584
585 if (!subflow_thmac_valid(subflow)) {
586 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKMAC);
587 subflow->reset_reason = MPTCP_RST_EMPTCP;
588 goto do_reset;
589 }
590
591 if (!mptcp_finish_join(sk))
592 goto do_reset;
593
594 subflow_generate_hmac(subflow->local_key, subflow->remote_key,
595 subflow->local_nonce,
596 subflow->remote_nonce,
597 hmac);
598 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN);
599
600 subflow->mp_join = 1;
601 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX);
602
603 if (subflow->backup)
604 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKBACKUPRX);
605
606 if (subflow_use_different_dport(msk, sk)) {
607 pr_debug("synack inet_dport=%d %d\n",
608 ntohs(inet_sk(sk)->inet_dport),
609 ntohs(inet_sk(parent)->inet_dport));
610 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINPORTSYNACKRX);
611 }
612 } else if (mptcp_check_fallback(sk)) {
613 /* It looks like MPTCP is blocked, while TCP is not */
614 if (subflow->mpc_drop)
615 mptcp_active_disable(parent);
616fallback:
617 mptcp_propagate_state(parent, sk, subflow, NULL);
618 }
619 return;
620
621do_reset:
622 subflow->reset_transient = 0;
623 mptcp_subflow_reset(sk);
624}
625
626static void subflow_set_local_id(struct mptcp_subflow_context *subflow, int local_id)
627{
628 WARN_ON_ONCE(local_id < 0 || local_id > 255);
629 WRITE_ONCE(subflow->local_id, local_id);
630}
631
632static int subflow_chk_local_id(struct sock *sk)
633{
634 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
635 struct mptcp_sock *msk = mptcp_sk(subflow->conn);
636 int err;
637
638 if (likely(subflow->local_id >= 0))
639 return 0;
640
641 err = mptcp_pm_get_local_id(msk, (struct sock_common *)sk);
642 if (err < 0)
643 return err;
644
645 subflow_set_local_id(subflow, err);
646 subflow->request_bkup = mptcp_pm_is_backup(msk, (struct sock_common *)sk);
647
648 return 0;
649}
650
651static int subflow_rebuild_header(struct sock *sk)
652{
653 int err = subflow_chk_local_id(sk);
654
655 if (unlikely(err < 0))
656 return err;
657
658 return inet_sk_rebuild_header(sk);
659}
660
661#if IS_ENABLED(CONFIG_MPTCP_IPV6)
662static int subflow_v6_rebuild_header(struct sock *sk)
663{
664 int err = subflow_chk_local_id(sk);
665
666 if (unlikely(err < 0))
667 return err;
668
669 return inet6_sk_rebuild_header(sk);
670}
671#endif
672
673static struct request_sock_ops mptcp_subflow_v4_request_sock_ops __ro_after_init;
674static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops __ro_after_init;
675
676static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb)
677{
678 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
679
680 pr_debug("subflow=%p\n", subflow);
681
682 /* Never answer to SYNs sent to broadcast or multicast */
683 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
684 goto drop;
685
686 return tcp_conn_request(&mptcp_subflow_v4_request_sock_ops,
687 &subflow_request_sock_ipv4_ops,
688 sk, skb);
689drop:
690 tcp_listendrop(sk);
691 return 0;
692}
693
694static void subflow_v4_req_destructor(struct request_sock *req)
695{
696 subflow_req_destructor(req);
697 tcp_request_sock_ops.destructor(req);
698}
699
700#if IS_ENABLED(CONFIG_MPTCP_IPV6)
701static struct request_sock_ops mptcp_subflow_v6_request_sock_ops __ro_after_init;
702static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops __ro_after_init;
703static struct inet_connection_sock_af_ops subflow_v6_specific __ro_after_init;
704static struct inet_connection_sock_af_ops subflow_v6m_specific __ro_after_init;
705static struct proto tcpv6_prot_override __ro_after_init;
706
707static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb)
708{
709 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
710
711 pr_debug("subflow=%p\n", subflow);
712
713 if (skb->protocol == htons(ETH_P_IP))
714 return subflow_v4_conn_request(sk, skb);
715
716 if (!ipv6_unicast_destination(skb))
717 goto drop;
718
719 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
720 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
721 return 0;
722 }
723
724 return tcp_conn_request(&mptcp_subflow_v6_request_sock_ops,
725 &subflow_request_sock_ipv6_ops, sk, skb);
726
727drop:
728 tcp_listendrop(sk);
729 return 0; /* don't send reset */
730}
731
732static void subflow_v6_req_destructor(struct request_sock *req)
733{
734 subflow_req_destructor(req);
735 tcp6_request_sock_ops.destructor(req);
736}
737#endif
738
739struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *ops,
740 struct sock *sk_listener,
741 bool attach_listener)
742{
743 if (ops->family == AF_INET)
744 ops = &mptcp_subflow_v4_request_sock_ops;
745#if IS_ENABLED(CONFIG_MPTCP_IPV6)
746 else if (ops->family == AF_INET6)
747 ops = &mptcp_subflow_v6_request_sock_ops;
748#endif
749
750 return inet_reqsk_alloc(ops, sk_listener, attach_listener);
751}
752EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc);
753
754/* validate hmac received in third ACK */
755static bool subflow_hmac_valid(const struct mptcp_subflow_request_sock *subflow_req,
756 const struct mptcp_options_received *mp_opt)
757{
758 struct mptcp_sock *msk = subflow_req->msk;
759 u8 hmac[SHA256_DIGEST_SIZE];
760
761 subflow_generate_hmac(READ_ONCE(msk->remote_key),
762 READ_ONCE(msk->local_key),
763 subflow_req->remote_nonce,
764 subflow_req->local_nonce, hmac);
765
766 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN);
767}
768
769static void subflow_ulp_fallback(struct sock *sk,
770 struct mptcp_subflow_context *old_ctx)
771{
772 struct inet_connection_sock *icsk = inet_csk(sk);
773
774 mptcp_subflow_tcp_fallback(sk, old_ctx);
775 icsk->icsk_ulp_ops = NULL;
776 rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
777 tcp_sk(sk)->is_mptcp = 0;
778
779 mptcp_subflow_ops_undo_override(sk);
780}
781
782void mptcp_subflow_drop_ctx(struct sock *ssk)
783{
784 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk);
785
786 if (!ctx)
787 return;
788
789 list_del(&mptcp_subflow_ctx(ssk)->node);
790 if (inet_csk(ssk)->icsk_ulp_ops) {
791 subflow_ulp_fallback(ssk, ctx);
792 if (ctx->conn)
793 sock_put(ctx->conn);
794 }
795
796 kfree_rcu(ctx, rcu);
797}
798
799void __mptcp_subflow_fully_established(struct mptcp_sock *msk,
800 struct mptcp_subflow_context *subflow,
801 const struct mptcp_options_received *mp_opt)
802{
803 subflow_set_remote_key(msk, subflow, mp_opt);
804 WRITE_ONCE(subflow->fully_established, true);
805 WRITE_ONCE(msk->fully_established, true);
806}
807
808static struct sock *subflow_syn_recv_sock(const struct sock *sk,
809 struct sk_buff *skb,
810 struct request_sock *req,
811 struct dst_entry *dst,
812 struct request_sock *req_unhash,
813 bool *own_req,
814 void (*opt_child_init)(struct sock *newsk,
815 const struct sock *sk))
816{
817 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk);
818 struct mptcp_subflow_request_sock *subflow_req;
819 struct mptcp_options_received mp_opt;
820 bool fallback, fallback_is_fatal;
821 enum sk_rst_reason reason;
822 struct mptcp_sock *owner;
823 struct sock *child;
824
825 pr_debug("listener=%p, req=%p, conn=%p\n", listener, req, listener->conn);
826
827 /* After child creation we must look for MPC even when options
828 * are not parsed
829 */
830 mp_opt.suboptions = 0;
831
832 /* hopefully temporary handling for MP_JOIN+syncookie */
833 subflow_req = mptcp_subflow_rsk(req);
834 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join;
835 fallback = !tcp_rsk(req)->is_mptcp;
836 if (fallback)
837 goto create_child;
838
839 /* if the sk is MP_CAPABLE, we try to fetch the client key */
840 if (subflow_req->mp_capable) {
841 /* we can receive and accept an in-window, out-of-order pkt,
842 * which may not carry the MP_CAPABLE opt even on mptcp enabled
843 * paths: always try to extract the peer key, and fallback
844 * for packets missing it.
845 * Even OoO DSS packets coming legitly after dropped or
846 * reordered MPC will cause fallback, but we don't have other
847 * options.
848 */
849 mptcp_get_options(skb, &mp_opt);
850 if (!(mp_opt.suboptions &
851 (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_ACK)))
852 fallback = true;
853
854 } else if (subflow_req->mp_join) {
855 mptcp_get_options(skb, &mp_opt);
856 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK))
857 fallback = true;
858 }
859
860create_child:
861 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst,
862 req_unhash, own_req, opt_child_init);
863
864 if (child && *own_req) {
865 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child);
866
867 tcp_rsk(req)->drop_req = false;
868
869 /* we need to fallback on ctx allocation failure and on pre-reqs
870 * checking above. In the latter scenario we additionally need
871 * to reset the context to non MPTCP status.
872 */
873 if (!ctx || fallback) {
874 if (fallback_is_fatal) {
875 if (!ctx)
876 MPTCP_INC_STATS(sock_net(sk),
877 MPTCP_MIB_MPJOINACKNOCTX);
878 else
879 MPTCP_INC_STATS(sock_net(sk),
880 MPTCP_MIB_MPJOINACKNOMPJOIN);
881 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP);
882 goto dispose_child;
883 }
884 goto fallback;
885 }
886
887 /* ssk inherits options of listener sk */
888 ctx->setsockopt_seq = listener->setsockopt_seq;
889
890 if (ctx->mp_capable) {
891 ctx->conn = mptcp_sk_clone_init(listener->conn, &mp_opt, child, req);
892 if (!ctx->conn)
893 goto fallback;
894
895 ctx->subflow_id = 1;
896 owner = mptcp_sk(ctx->conn);
897
898 if (mp_opt.deny_join_id0)
899 WRITE_ONCE(owner->pm.remote_deny_join_id0, true);
900
901 mptcp_pm_new_connection(owner, child, 1);
902
903 /* with OoO packets we can reach here without ingress
904 * mpc option
905 */
906 if (mp_opt.suboptions & OPTION_MPTCP_MPC_ACK) {
907 mptcp_pm_fully_established(owner, child);
908 ctx->pm_notified = 1;
909 }
910 } else if (ctx->mp_join) {
911 owner = subflow_req->msk;
912 if (!owner) {
913 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
914 goto dispose_child;
915 }
916
917 if (!subflow_hmac_valid(subflow_req, &mp_opt)) {
918 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);
919 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP);
920 goto dispose_child;
921 }
922
923 if (!mptcp_can_accept_new_subflow(owner)) {
924 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINREJECTED);
925 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
926 goto dispose_child;
927 }
928
929 /* move the msk reference ownership to the subflow */
930 subflow_req->msk = NULL;
931 ctx->conn = (struct sock *)owner;
932
933 if (subflow_use_different_sport(owner, sk)) {
934 pr_debug("ack inet_sport=%d %d\n",
935 ntohs(inet_sk(sk)->inet_sport),
936 ntohs(inet_sk((struct sock *)owner)->inet_sport));
937 if (!mptcp_pm_announced_has_ssk(owner, sk)) {
938 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX);
939 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
940 goto dispose_child;
941 }
942 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTACKRX);
943 }
944
945 if (!mptcp_finish_join(child)) {
946 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(child);
947
948 subflow_add_reset_reason(skb, subflow->reset_reason);
949 goto dispose_child;
950 }
951
952 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX);
953 tcp_rsk(req)->drop_req = true;
954 }
955 }
956
957 /* check for expected invariant - should never trigger, just help
958 * catching earlier subtle bugs
959 */
960 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp &&
961 (!mptcp_subflow_ctx(child) ||
962 !mptcp_subflow_ctx(child)->conn));
963 return child;
964
965dispose_child:
966 mptcp_subflow_drop_ctx(child);
967 tcp_rsk(req)->drop_req = true;
968 inet_csk_prepare_for_destroy_sock(child);
969 tcp_done(child);
970 reason = mptcp_get_rst_reason(skb);
971 req->rsk_ops->send_reset(sk, skb, reason);
972
973 /* The last child reference will be released by the caller */
974 return child;
975
976fallback:
977 if (fallback)
978 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK);
979 mptcp_subflow_drop_ctx(child);
980 return child;
981}
982
983static struct inet_connection_sock_af_ops subflow_specific __ro_after_init;
984static struct proto tcp_prot_override __ro_after_init;
985
986enum mapping_status {
987 MAPPING_OK,
988 MAPPING_INVALID,
989 MAPPING_EMPTY,
990 MAPPING_DATA_FIN,
991 MAPPING_DUMMY,
992 MAPPING_BAD_CSUM,
993 MAPPING_NODSS
994};
995
996static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn)
997{
998 pr_debug("Bad mapping: ssn=%d map_seq=%d map_data_len=%d\n",
999 ssn, subflow->map_subflow_seq, subflow->map_data_len);
1000}
1001
1002static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb)
1003{
1004 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1005 unsigned int skb_consumed;
1006
1007 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq;
1008 if (unlikely(skb_consumed >= skb->len)) {
1009 DEBUG_NET_WARN_ON_ONCE(1);
1010 return true;
1011 }
1012
1013 return skb->len - skb_consumed <= subflow->map_data_len -
1014 mptcp_subflow_get_map_offset(subflow);
1015}
1016
1017static bool validate_mapping(struct sock *ssk, struct sk_buff *skb)
1018{
1019 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1020 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
1021
1022 if (unlikely(before(ssn, subflow->map_subflow_seq))) {
1023 /* Mapping covers data later in the subflow stream,
1024 * currently unsupported.
1025 */
1026 dbg_bad_map(subflow, ssn);
1027 return false;
1028 }
1029 if (unlikely(!before(ssn, subflow->map_subflow_seq +
1030 subflow->map_data_len))) {
1031 /* Mapping does covers past subflow data, invalid */
1032 dbg_bad_map(subflow, ssn);
1033 return false;
1034 }
1035 return true;
1036}
1037
1038static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *skb,
1039 bool csum_reqd)
1040{
1041 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1042 u32 offset, seq, delta;
1043 __sum16 csum;
1044 int len;
1045
1046 if (!csum_reqd)
1047 return MAPPING_OK;
1048
1049 /* mapping already validated on previous traversal */
1050 if (subflow->map_csum_len == subflow->map_data_len)
1051 return MAPPING_OK;
1052
1053 /* traverse the receive queue, ensuring it contains a full
1054 * DSS mapping and accumulating the related csum.
1055 * Preserve the accoumlate csum across multiple calls, to compute
1056 * the csum only once
1057 */
1058 delta = subflow->map_data_len - subflow->map_csum_len;
1059 for (;;) {
1060 seq = tcp_sk(ssk)->copied_seq + subflow->map_csum_len;
1061 offset = seq - TCP_SKB_CB(skb)->seq;
1062
1063 /* if the current skb has not been accounted yet, csum its contents
1064 * up to the amount covered by the current DSS
1065 */
1066 if (offset < skb->len) {
1067 __wsum csum;
1068
1069 len = min(skb->len - offset, delta);
1070 csum = skb_checksum(skb, offset, len, 0);
1071 subflow->map_data_csum = csum_block_add(subflow->map_data_csum, csum,
1072 subflow->map_csum_len);
1073
1074 delta -= len;
1075 subflow->map_csum_len += len;
1076 }
1077 if (delta == 0)
1078 break;
1079
1080 if (skb_queue_is_last(&ssk->sk_receive_queue, skb)) {
1081 /* if this subflow is closed, the partial mapping
1082 * will be never completed; flush the pending skbs, so
1083 * that subflow_sched_work_if_closed() can kick in
1084 */
1085 if (unlikely(ssk->sk_state == TCP_CLOSE))
1086 while ((skb = skb_peek(&ssk->sk_receive_queue)))
1087 sk_eat_skb(ssk, skb);
1088
1089 /* not enough data to validate the csum */
1090 return MAPPING_EMPTY;
1091 }
1092
1093 /* the DSS mapping for next skbs will be validated later,
1094 * when a get_mapping_status call will process such skb
1095 */
1096 skb = skb->next;
1097 }
1098
1099 /* note that 'map_data_len' accounts only for the carried data, does
1100 * not include the eventual seq increment due to the data fin,
1101 * while the pseudo header requires the original DSS data len,
1102 * including that
1103 */
1104 csum = __mptcp_make_csum(subflow->map_seq,
1105 subflow->map_subflow_seq,
1106 subflow->map_data_len + subflow->map_data_fin,
1107 subflow->map_data_csum);
1108 if (unlikely(csum)) {
1109 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR);
1110 return MAPPING_BAD_CSUM;
1111 }
1112
1113 subflow->valid_csum_seen = 1;
1114 return MAPPING_OK;
1115}
1116
1117static enum mapping_status get_mapping_status(struct sock *ssk,
1118 struct mptcp_sock *msk)
1119{
1120 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1121 bool csum_reqd = READ_ONCE(msk->csum_enabled);
1122 struct mptcp_ext *mpext;
1123 struct sk_buff *skb;
1124 u16 data_len;
1125 u64 map_seq;
1126
1127 skb = skb_peek(&ssk->sk_receive_queue);
1128 if (!skb)
1129 return MAPPING_EMPTY;
1130
1131 if (mptcp_check_fallback(ssk))
1132 return MAPPING_DUMMY;
1133
1134 mpext = mptcp_get_ext(skb);
1135 if (!mpext || !mpext->use_map) {
1136 if (!subflow->map_valid && !skb->len) {
1137 /* the TCP stack deliver 0 len FIN pkt to the receive
1138 * queue, that is the only 0len pkts ever expected here,
1139 * and we can admit no mapping only for 0 len pkts
1140 */
1141 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
1142 WARN_ONCE(1, "0len seq %d:%d flags %x",
1143 TCP_SKB_CB(skb)->seq,
1144 TCP_SKB_CB(skb)->end_seq,
1145 TCP_SKB_CB(skb)->tcp_flags);
1146 sk_eat_skb(ssk, skb);
1147 return MAPPING_EMPTY;
1148 }
1149
1150 /* If the required DSS has likely been dropped by a middlebox */
1151 if (!subflow->map_valid)
1152 return MAPPING_NODSS;
1153
1154 goto validate_seq;
1155 }
1156
1157 trace_get_mapping_status(mpext);
1158
1159 data_len = mpext->data_len;
1160 if (data_len == 0) {
1161 pr_debug("infinite mapping received\n");
1162 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX);
1163 return MAPPING_INVALID;
1164 }
1165
1166 if (mpext->data_fin == 1) {
1167 u64 data_fin_seq;
1168
1169 if (data_len == 1) {
1170 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq,
1171 mpext->dsn64);
1172 pr_debug("DATA_FIN with no payload seq=%llu\n", mpext->data_seq);
1173 if (subflow->map_valid) {
1174 /* A DATA_FIN might arrive in a DSS
1175 * option before the previous mapping
1176 * has been fully consumed. Continue
1177 * handling the existing mapping.
1178 */
1179 skb_ext_del(skb, SKB_EXT_MPTCP);
1180 return MAPPING_OK;
1181 }
1182
1183 if (updated)
1184 mptcp_schedule_work((struct sock *)msk);
1185
1186 return MAPPING_DATA_FIN;
1187 }
1188
1189 data_fin_seq = mpext->data_seq + data_len - 1;
1190
1191 /* If mpext->data_seq is a 32-bit value, data_fin_seq must also
1192 * be limited to 32 bits.
1193 */
1194 if (!mpext->dsn64)
1195 data_fin_seq &= GENMASK_ULL(31, 0);
1196
1197 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64);
1198 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d\n",
1199 data_fin_seq, mpext->dsn64);
1200
1201 /* Adjust for DATA_FIN using 1 byte of sequence space */
1202 data_len--;
1203 }
1204
1205 map_seq = mptcp_expand_seq(READ_ONCE(msk->ack_seq), mpext->data_seq, mpext->dsn64);
1206 WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64);
1207
1208 if (subflow->map_valid) {
1209 /* Allow replacing only with an identical map */
1210 if (subflow->map_seq == map_seq &&
1211 subflow->map_subflow_seq == mpext->subflow_seq &&
1212 subflow->map_data_len == data_len &&
1213 subflow->map_csum_reqd == mpext->csum_reqd) {
1214 skb_ext_del(skb, SKB_EXT_MPTCP);
1215 goto validate_csum;
1216 }
1217
1218 /* If this skb data are fully covered by the current mapping,
1219 * the new map would need caching, which is not supported
1220 */
1221 if (skb_is_fully_mapped(ssk, skb)) {
1222 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH);
1223 return MAPPING_INVALID;
1224 }
1225
1226 /* will validate the next map after consuming the current one */
1227 goto validate_csum;
1228 }
1229
1230 subflow->map_seq = map_seq;
1231 subflow->map_subflow_seq = mpext->subflow_seq;
1232 subflow->map_data_len = data_len;
1233 subflow->map_valid = 1;
1234 subflow->map_data_fin = mpext->data_fin;
1235 subflow->mpc_map = mpext->mpc_map;
1236 subflow->map_csum_reqd = mpext->csum_reqd;
1237 subflow->map_csum_len = 0;
1238 subflow->map_data_csum = csum_unfold(mpext->csum);
1239
1240 /* Cfr RFC 8684 Section 3.3.0 */
1241 if (unlikely(subflow->map_csum_reqd != csum_reqd))
1242 return MAPPING_INVALID;
1243
1244 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u csum=%d:%u\n",
1245 subflow->map_seq, subflow->map_subflow_seq,
1246 subflow->map_data_len, subflow->map_csum_reqd,
1247 subflow->map_data_csum);
1248
1249validate_seq:
1250 /* we revalidate valid mapping on new skb, because we must ensure
1251 * the current skb is completely covered by the available mapping
1252 */
1253 if (!validate_mapping(ssk, skb)) {
1254 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSTCPMISMATCH);
1255 return MAPPING_INVALID;
1256 }
1257
1258 skb_ext_del(skb, SKB_EXT_MPTCP);
1259
1260validate_csum:
1261 return validate_data_csum(ssk, skb, csum_reqd);
1262}
1263
1264static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb,
1265 u64 limit)
1266{
1267 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1268 bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
1269 struct tcp_sock *tp = tcp_sk(ssk);
1270 u32 offset, incr, avail_len;
1271
1272 offset = tp->copied_seq - TCP_SKB_CB(skb)->seq;
1273 if (WARN_ON_ONCE(offset > skb->len))
1274 goto out;
1275
1276 avail_len = skb->len - offset;
1277 incr = limit >= avail_len ? avail_len + fin : limit;
1278
1279 pr_debug("discarding=%d len=%d offset=%d seq=%d\n", incr, skb->len,
1280 offset, subflow->map_subflow_seq);
1281 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA);
1282 tcp_sk(ssk)->copied_seq += incr;
1283
1284out:
1285 if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq))
1286 sk_eat_skb(ssk, skb);
1287 if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len)
1288 subflow->map_valid = 0;
1289}
1290
1291static bool subflow_is_done(const struct sock *sk)
1292{
1293 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE;
1294}
1295
1296/* sched mptcp worker for subflow cleanup if no more data is pending */
1297static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk)
1298{
1299 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1300 struct sock *sk = (struct sock *)msk;
1301
1302 if (likely(ssk->sk_state != TCP_CLOSE &&
1303 (ssk->sk_state != TCP_CLOSE_WAIT ||
1304 inet_sk_state_load(sk) != TCP_ESTABLISHED)))
1305 return;
1306
1307 if (!skb_queue_empty(&ssk->sk_receive_queue))
1308 return;
1309
1310 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags))
1311 mptcp_schedule_work(sk);
1312
1313 /* when the fallback subflow closes the rx side, trigger a 'dummy'
1314 * ingress data fin, so that the msk state will follow along
1315 */
1316 if (__mptcp_check_fallback(msk) && subflow_is_done(ssk) &&
1317 msk->first == ssk &&
1318 mptcp_update_rcv_data_fin(msk, subflow->map_seq +
1319 subflow->map_data_len, true))
1320 mptcp_schedule_work(sk);
1321}
1322
1323static bool mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk)
1324{
1325 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1326 unsigned long fail_tout;
1327
1328 /* we are really failing, prevent any later subflow join */
1329 spin_lock_bh(&msk->fallback_lock);
1330 if (!msk->allow_infinite_fallback) {
1331 spin_unlock_bh(&msk->fallback_lock);
1332 return false;
1333 }
1334 msk->allow_subflows = false;
1335 spin_unlock_bh(&msk->fallback_lock);
1336
1337 /* graceful failure can happen only on the MPC subflow */
1338 if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first)))
1339 return false;
1340
1341 /* since the close timeout take precedence on the fail one,
1342 * no need to start the latter when the first is already set
1343 */
1344 if (sock_flag((struct sock *)msk, SOCK_DEAD))
1345 return true;
1346
1347 /* we don't need extreme accuracy here, use a zero fail_tout as special
1348 * value meaning no fail timeout at all;
1349 */
1350 fail_tout = jiffies + TCP_RTO_MAX;
1351 if (!fail_tout)
1352 fail_tout = 1;
1353 WRITE_ONCE(subflow->fail_tout, fail_tout);
1354 tcp_send_ack(ssk);
1355
1356 mptcp_reset_tout_timer(msk, subflow->fail_tout);
1357 return true;
1358}
1359
1360static bool subflow_check_data_avail(struct sock *ssk)
1361{
1362 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1363 enum mapping_status status;
1364 struct mptcp_sock *msk;
1365 struct sk_buff *skb;
1366
1367 if (!skb_peek(&ssk->sk_receive_queue))
1368 WRITE_ONCE(subflow->data_avail, false);
1369 if (subflow->data_avail)
1370 return true;
1371
1372 msk = mptcp_sk(subflow->conn);
1373 for (;;) {
1374 u64 ack_seq;
1375 u64 old_ack;
1376
1377 status = get_mapping_status(ssk, msk);
1378 trace_subflow_check_data_avail(status, skb_peek(&ssk->sk_receive_queue));
1379 if (unlikely(status == MAPPING_INVALID || status == MAPPING_DUMMY ||
1380 status == MAPPING_BAD_CSUM || status == MAPPING_NODSS))
1381 goto fallback;
1382
1383 if (status != MAPPING_OK)
1384 goto no_data;
1385
1386 skb = skb_peek(&ssk->sk_receive_queue);
1387 if (WARN_ON_ONCE(!skb))
1388 goto no_data;
1389
1390 if (unlikely(!READ_ONCE(msk->can_ack)))
1391 goto fallback;
1392
1393 old_ack = READ_ONCE(msk->ack_seq);
1394 ack_seq = mptcp_subflow_get_mapped_dsn(subflow);
1395 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx\n", old_ack,
1396 ack_seq);
1397 if (unlikely(before64(ack_seq, old_ack))) {
1398 mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq);
1399 continue;
1400 }
1401
1402 WRITE_ONCE(subflow->data_avail, true);
1403 break;
1404 }
1405 return true;
1406
1407no_data:
1408 subflow_sched_work_if_closed(msk, ssk);
1409 return false;
1410
1411fallback:
1412 if (!__mptcp_check_fallback(msk)) {
1413 /* RFC 8684 section 3.7. */
1414 if (status == MAPPING_BAD_CSUM &&
1415 (subflow->mp_join || subflow->valid_csum_seen)) {
1416 subflow->send_mp_fail = 1;
1417
1418 if (!mptcp_subflow_fail(msk, ssk)) {
1419 subflow->reset_transient = 0;
1420 subflow->reset_reason = MPTCP_RST_EMIDDLEBOX;
1421 goto reset;
1422 }
1423 WRITE_ONCE(subflow->data_avail, true);
1424 return true;
1425 }
1426
1427 if (!mptcp_try_fallback(ssk, MPTCP_MIB_DSSFALLBACK)) {
1428 /* fatal protocol error, close the socket.
1429 * subflow_error_report() will introduce the appropriate barriers
1430 */
1431 subflow->reset_transient = 0;
1432 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSRESET);
1433 subflow->reset_reason = status == MAPPING_NODSS ?
1434 MPTCP_RST_EMIDDLEBOX :
1435 MPTCP_RST_EMPTCP;
1436
1437reset:
1438 WRITE_ONCE(ssk->sk_err, EBADMSG);
1439 tcp_set_state(ssk, TCP_CLOSE);
1440 while ((skb = skb_peek(&ssk->sk_receive_queue)))
1441 sk_eat_skb(ssk, skb);
1442 mptcp_send_active_reset_reason(ssk);
1443 WRITE_ONCE(subflow->data_avail, false);
1444 return false;
1445 }
1446 }
1447
1448 skb = skb_peek(&ssk->sk_receive_queue);
1449 subflow->map_valid = 1;
1450 subflow->map_data_len = skb->len;
1451 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
1452 subflow->map_seq = __mptcp_expand_seq(subflow->map_seq,
1453 subflow->iasn +
1454 TCP_SKB_CB(skb)->seq -
1455 subflow->ssn_offset - 1);
1456 WRITE_ONCE(subflow->data_avail, true);
1457 return true;
1458}
1459
1460bool mptcp_subflow_data_available(struct sock *sk)
1461{
1462 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1463
1464 /* check if current mapping is still valid */
1465 if (subflow->map_valid &&
1466 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) {
1467 subflow->map_valid = 0;
1468 WRITE_ONCE(subflow->data_avail, false);
1469
1470 pr_debug("Done with mapping: seq=%u data_len=%u\n",
1471 subflow->map_subflow_seq,
1472 subflow->map_data_len);
1473 }
1474
1475 return subflow_check_data_avail(sk);
1476}
1477
1478/* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy,
1479 * not the ssk one.
1480 *
1481 * In mptcp, rwin is about the mptcp-level connection data.
1482 *
1483 * Data that is still on the ssk rx queue can thus be ignored,
1484 * as far as mptcp peer is concerned that data is still inflight.
1485 * DSS ACK is updated when skb is moved to the mptcp rx queue.
1486 */
1487void mptcp_space(const struct sock *ssk, int *space, int *full_space)
1488{
1489 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1490 const struct sock *sk = subflow->conn;
1491
1492 *space = __mptcp_space(sk);
1493 *full_space = mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
1494}
1495
1496static void subflow_error_report(struct sock *ssk)
1497{
1498 struct sock *sk = mptcp_subflow_ctx(ssk)->conn;
1499
1500 /* bail early if this is a no-op, so that we avoid introducing a
1501 * problematic lockdep dependency between TCP accept queue lock
1502 * and msk socket spinlock
1503 */
1504 if (!sk->sk_socket)
1505 return;
1506
1507 mptcp_data_lock(sk);
1508 if (!sock_owned_by_user(sk))
1509 __mptcp_error_report(sk);
1510 else
1511 __set_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->cb_flags);
1512 mptcp_data_unlock(sk);
1513}
1514
1515static void subflow_data_ready(struct sock *sk)
1516{
1517 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1518 u16 state = 1 << inet_sk_state_load(sk);
1519 struct sock *parent = subflow->conn;
1520 struct mptcp_sock *msk;
1521
1522 trace_sk_data_ready(sk);
1523
1524 msk = mptcp_sk(parent);
1525 if (state & TCPF_LISTEN) {
1526 /* MPJ subflow are removed from accept queue before reaching here,
1527 * avoid stray wakeups
1528 */
1529 if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue))
1530 return;
1531
1532 parent->sk_data_ready(parent);
1533 return;
1534 }
1535
1536 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable &&
1537 !subflow->mp_join && !(state & TCPF_CLOSE));
1538
1539 if (mptcp_subflow_data_available(sk)) {
1540 mptcp_data_ready(parent, sk);
1541
1542 /* subflow-level lowat test are not relevant.
1543 * respect the msk-level threshold eventually mandating an immediate ack
1544 */
1545 if (mptcp_data_avail(msk) < parent->sk_rcvlowat &&
1546 (tcp_sk(sk)->rcv_nxt - tcp_sk(sk)->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss)
1547 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
1548 } else if (unlikely(sk->sk_err)) {
1549 subflow_error_report(sk);
1550 }
1551}
1552
1553static void subflow_write_space(struct sock *ssk)
1554{
1555 struct sock *sk = mptcp_subflow_ctx(ssk)->conn;
1556
1557 mptcp_propagate_sndbuf(sk, ssk);
1558 mptcp_write_space(sk);
1559}
1560
1561static const struct inet_connection_sock_af_ops *
1562subflow_default_af_ops(struct sock *sk)
1563{
1564#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1565 if (sk->sk_family == AF_INET6)
1566 return &subflow_v6_specific;
1567#endif
1568 return &subflow_specific;
1569}
1570
1571#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1572void mptcpv6_handle_mapped(struct sock *sk, bool mapped)
1573{
1574 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1575 struct inet_connection_sock *icsk = inet_csk(sk);
1576 const struct inet_connection_sock_af_ops *target;
1577
1578 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk);
1579
1580 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d\n",
1581 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped);
1582
1583 if (likely(icsk->icsk_af_ops == target))
1584 return;
1585
1586 subflow->icsk_af_ops = icsk->icsk_af_ops;
1587 icsk->icsk_af_ops = target;
1588}
1589#endif
1590
1591void mptcp_info2sockaddr(const struct mptcp_addr_info *info,
1592 struct sockaddr_storage *addr,
1593 unsigned short family)
1594{
1595 memset(addr, 0, sizeof(*addr));
1596 addr->ss_family = family;
1597 if (addr->ss_family == AF_INET) {
1598 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr;
1599
1600 if (info->family == AF_INET)
1601 in_addr->sin_addr = info->addr;
1602#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1603 else if (ipv6_addr_v4mapped(&info->addr6))
1604 in_addr->sin_addr.s_addr = info->addr6.s6_addr32[3];
1605#endif
1606 in_addr->sin_port = info->port;
1607 }
1608#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1609 else if (addr->ss_family == AF_INET6) {
1610 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr;
1611
1612 if (info->family == AF_INET)
1613 ipv6_addr_set_v4mapped(info->addr.s_addr,
1614 &in6_addr->sin6_addr);
1615 else
1616 in6_addr->sin6_addr = info->addr6;
1617 in6_addr->sin6_port = info->port;
1618 }
1619#endif
1620}
1621
1622int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_pm_local *local,
1623 const struct mptcp_addr_info *remote)
1624{
1625 struct mptcp_sock *msk = mptcp_sk(sk);
1626 struct mptcp_subflow_context *subflow;
1627 int local_id = local->addr.id;
1628 struct sockaddr_storage addr;
1629 int remote_id = remote->id;
1630 int err = -ENOTCONN;
1631 struct socket *sf;
1632 struct sock *ssk;
1633 u32 remote_token;
1634 int addrlen;
1635
1636 /* The userspace PM sent the request too early? */
1637 if (!mptcp_is_fully_established(sk))
1638 goto err_out;
1639
1640 err = mptcp_subflow_create_socket(sk, local->addr.family, &sf);
1641 if (err) {
1642 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXCREATSKERR);
1643 pr_debug("msk=%p local=%d remote=%d create sock error: %d\n",
1644 msk, local_id, remote_id, err);
1645 goto err_out;
1646 }
1647
1648 ssk = sf->sk;
1649 subflow = mptcp_subflow_ctx(ssk);
1650 do {
1651 subflow->local_nonce = get_random_u32();
1652 } while (!subflow->local_nonce);
1653
1654 /* if 'IPADDRANY', the ID will be set later, after the routing */
1655 if (local->addr.family == AF_INET) {
1656 if (!local->addr.addr.s_addr)
1657 local_id = -1;
1658#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1659 } else if (sk->sk_family == AF_INET6) {
1660 if (ipv6_addr_any(&local->addr.addr6))
1661 local_id = -1;
1662#endif
1663 }
1664
1665 if (local_id >= 0)
1666 subflow_set_local_id(subflow, local_id);
1667
1668 subflow->remote_key_valid = 1;
1669 subflow->remote_key = READ_ONCE(msk->remote_key);
1670 subflow->local_key = READ_ONCE(msk->local_key);
1671 subflow->token = msk->token;
1672 mptcp_info2sockaddr(&local->addr, &addr, ssk->sk_family);
1673
1674 addrlen = sizeof(struct sockaddr_in);
1675#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1676 if (addr.ss_family == AF_INET6)
1677 addrlen = sizeof(struct sockaddr_in6);
1678#endif
1679 ssk->sk_bound_dev_if = local->ifindex;
1680 err = kernel_bind(sf, (struct sockaddr_unsized *)&addr, addrlen);
1681 if (err) {
1682 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXBINDERR);
1683 pr_debug("msk=%p local=%d remote=%d bind error: %d\n",
1684 msk, local_id, remote_id, err);
1685 goto failed;
1686 }
1687
1688 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL);
1689 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d\n", msk,
1690 remote_token, local_id, remote_id);
1691 subflow->remote_token = remote_token;
1692 WRITE_ONCE(subflow->remote_id, remote_id);
1693 subflow->request_join = 1;
1694 subflow->request_bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
1695 subflow->subflow_id = msk->subflow_id++;
1696 mptcp_info2sockaddr(remote, &addr, ssk->sk_family);
1697
1698 sock_hold(ssk);
1699 list_add_tail(&subflow->node, &msk->conn_list);
1700 err = kernel_connect(sf, (struct sockaddr_unsized *)&addr, addrlen, O_NONBLOCK);
1701 if (err && err != -EINPROGRESS) {
1702 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXCONNECTERR);
1703 pr_debug("msk=%p local=%d remote=%d connect error: %d\n",
1704 msk, local_id, remote_id, err);
1705 goto failed_unlink;
1706 }
1707
1708 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTX);
1709
1710 /* discard the subflow socket */
1711 mptcp_sock_graft(ssk, sk->sk_socket);
1712 iput(SOCK_INODE(sf));
1713 mptcp_stop_tout_timer(sk);
1714 return 0;
1715
1716failed_unlink:
1717 list_del(&subflow->node);
1718 sock_put(mptcp_subflow_tcp_sock(subflow));
1719
1720failed:
1721 subflow->disposable = 1;
1722 sock_release(sf);
1723
1724err_out:
1725 /* we account subflows before the creation, and this failures will not
1726 * be caught by sk_state_change()
1727 */
1728 mptcp_pm_close_subflow(msk);
1729 return err;
1730}
1731
1732void __mptcp_inherit_memcg(struct sock *sk, struct sock *ssk, gfp_t gfp)
1733{
1734 /* Only if the msk has been accepted already (and not orphaned).*/
1735 if (!mem_cgroup_sockets_enabled || !sk->sk_socket)
1736 return;
1737
1738 mem_cgroup_sk_inherit(sk, ssk);
1739 __sk_charge(ssk, gfp);
1740}
1741
1742void __mptcp_inherit_cgrp_data(struct sock *sk, struct sock *ssk)
1743{
1744#ifdef CONFIG_SOCK_CGROUP_DATA
1745 struct sock_cgroup_data *sk_cd = &sk->sk_cgrp_data,
1746 *ssk_cd = &ssk->sk_cgrp_data;
1747
1748 /* only the additional subflows created by kworkers have to be modified */
1749 if (cgroup_id(sock_cgroup_ptr(sk_cd)) !=
1750 cgroup_id(sock_cgroup_ptr(ssk_cd))) {
1751 cgroup_sk_free(ssk_cd);
1752 *ssk_cd = *sk_cd;
1753 cgroup_sk_clone(sk_cd);
1754 }
1755#endif /* CONFIG_SOCK_CGROUP_DATA */
1756}
1757
1758static void mptcp_attach_cgroup(struct sock *parent, struct sock *child)
1759{
1760 __mptcp_inherit_cgrp_data(parent, child);
1761 if (mem_cgroup_sockets_enabled)
1762 mem_cgroup_sk_inherit(parent, child);
1763}
1764
1765static void mptcp_subflow_ops_override(struct sock *ssk)
1766{
1767#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1768 if (ssk->sk_prot == &tcpv6_prot)
1769 ssk->sk_prot = &tcpv6_prot_override;
1770 else
1771#endif
1772 ssk->sk_prot = &tcp_prot_override;
1773}
1774
1775static void mptcp_subflow_ops_undo_override(struct sock *ssk)
1776{
1777#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1778 if (ssk->sk_prot == &tcpv6_prot_override)
1779 ssk->sk_prot = &tcpv6_prot;
1780 else
1781#endif
1782 ssk->sk_prot = &tcp_prot;
1783}
1784
1785int mptcp_subflow_create_socket(struct sock *sk, unsigned short family,
1786 struct socket **new_sock)
1787{
1788 struct mptcp_subflow_context *subflow;
1789 struct net *net = sock_net(sk);
1790 struct socket *sf;
1791 int err;
1792
1793 /* un-accepted server sockets can reach here - on bad configuration
1794 * bail early to avoid greater trouble later
1795 */
1796 if (unlikely(!sk->sk_socket))
1797 return -EINVAL;
1798
1799 err = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP, &sf);
1800 if (err)
1801 return err;
1802
1803 lock_sock_nested(sf->sk, SINGLE_DEPTH_NESTING);
1804
1805 err = security_mptcp_add_subflow(sk, sf->sk);
1806 if (err)
1807 goto err_free;
1808
1809 /* the newly created socket has to be in the same cgroup as its parent */
1810 mptcp_attach_cgroup(sk, sf->sk);
1811
1812 /* kernel sockets do not by default acquire net ref, but TCP timer
1813 * needs it.
1814 * Update ns_tracker to current stack trace and refcounted tracker.
1815 */
1816 sk_net_refcnt_upgrade(sf->sk);
1817 err = tcp_set_ulp(sf->sk, "mptcp");
1818 if (err)
1819 goto err_free;
1820
1821 mptcp_sockopt_sync_locked(mptcp_sk(sk), sf->sk);
1822 release_sock(sf->sk);
1823
1824 /* the newly created socket really belongs to the owning MPTCP
1825 * socket, even if for additional subflows the allocation is performed
1826 * by a kernel workqueue. Adjust inode references, so that the
1827 * procfs/diag interfaces really show this one belonging to the correct
1828 * user.
1829 */
1830 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino;
1831 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid;
1832 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid;
1833
1834 subflow = mptcp_subflow_ctx(sf->sk);
1835 pr_debug("subflow=%p\n", subflow);
1836
1837 *new_sock = sf;
1838 sock_hold(sk);
1839 subflow->conn = sk;
1840 mptcp_subflow_ops_override(sf->sk);
1841
1842 return 0;
1843
1844err_free:
1845 release_sock(sf->sk);
1846 sock_release(sf);
1847 return err;
1848}
1849
1850static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk,
1851 gfp_t priority)
1852{
1853 struct inet_connection_sock *icsk = inet_csk(sk);
1854 struct mptcp_subflow_context *ctx;
1855
1856 ctx = kzalloc_obj(*ctx, priority);
1857 if (!ctx)
1858 return NULL;
1859
1860 rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
1861 INIT_LIST_HEAD(&ctx->node);
1862 INIT_LIST_HEAD(&ctx->delegated_node);
1863
1864 pr_debug("subflow=%p\n", ctx);
1865
1866 ctx->tcp_sock = sk;
1867 WRITE_ONCE(ctx->local_id, -1);
1868
1869 return ctx;
1870}
1871
1872static void __subflow_state_change(struct sock *sk)
1873{
1874 struct socket_wq *wq;
1875
1876 rcu_read_lock();
1877 wq = rcu_dereference(sk->sk_wq);
1878 if (skwq_has_sleeper(wq))
1879 wake_up_interruptible_all(&wq->wait);
1880 rcu_read_unlock();
1881}
1882
1883static void subflow_state_change(struct sock *sk)
1884{
1885 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1886 struct sock *parent = subflow->conn;
1887
1888 __subflow_state_change(sk);
1889
1890 /* Rx queue processing is unneeded, error reporting will take place at
1891 * __mptcp_close_ssk() time and subflow reset can't happen in case of
1892 * fallback: subflow_sched_work_if_closed() would be a no-op.
1893 */
1894 if (subflow->resetting)
1895 return;
1896
1897 /* as recvmsg() does not acquire the subflow socket for ssk selection
1898 * a fin packet carrying a DSS can be unnoticed if we don't trigger
1899 * the data available machinery here.
1900 */
1901 if (mptcp_subflow_data_available(sk))
1902 mptcp_data_ready(parent, sk);
1903 else if (unlikely(sk->sk_err))
1904 subflow_error_report(sk);
1905
1906 subflow_sched_work_if_closed(mptcp_sk(parent), sk);
1907}
1908
1909void mptcp_subflow_queue_clean(struct sock *listener_sk, struct sock *listener_ssk)
1910{
1911 struct request_sock_queue *queue = &inet_csk(listener_ssk)->icsk_accept_queue;
1912 struct request_sock *req, *head, *tail;
1913 struct mptcp_subflow_context *subflow;
1914 struct sock *sk, *ssk;
1915
1916 /* Due to lock dependencies no relevant lock can be acquired under rskq_lock.
1917 * Splice the req list, so that accept() can not reach the pending ssk after
1918 * the listener socket is released below.
1919 */
1920 spin_lock_bh(&queue->rskq_lock);
1921 head = queue->rskq_accept_head;
1922 tail = queue->rskq_accept_tail;
1923 queue->rskq_accept_head = NULL;
1924 queue->rskq_accept_tail = NULL;
1925 spin_unlock_bh(&queue->rskq_lock);
1926
1927 if (!head)
1928 return;
1929
1930 /* can't acquire the msk socket lock under the subflow one,
1931 * or will cause ABBA deadlock
1932 */
1933 release_sock(listener_ssk);
1934
1935 for (req = head; req; req = req->dl_next) {
1936 ssk = req->sk;
1937 if (!sk_is_mptcp(ssk))
1938 continue;
1939
1940 subflow = mptcp_subflow_ctx(ssk);
1941 if (!subflow || !subflow->conn)
1942 continue;
1943
1944 sk = subflow->conn;
1945 sock_hold(sk);
1946
1947 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
1948 __mptcp_unaccepted_force_close(sk);
1949 release_sock(sk);
1950
1951 /* lockdep will report a false positive ABBA deadlock
1952 * between cancel_work_sync and the listener socket.
1953 * The involved locks belong to different sockets WRT
1954 * the existing AB chain.
1955 * Using a per socket key is problematic as key
1956 * deregistration requires process context and must be
1957 * performed at socket disposal time, in atomic
1958 * context.
1959 * Just tell lockdep to consider the listener socket
1960 * released here.
1961 */
1962 mutex_release(&listener_sk->sk_lock.dep_map, _RET_IP_);
1963 mptcp_cancel_work(sk);
1964 mutex_acquire(&listener_sk->sk_lock.dep_map, 0, 0, _RET_IP_);
1965
1966 sock_put(sk);
1967 }
1968
1969 /* we are still under the listener msk socket lock */
1970 lock_sock_nested(listener_ssk, SINGLE_DEPTH_NESTING);
1971
1972 /* restore the listener queue, to let the TCP code clean it up */
1973 spin_lock_bh(&queue->rskq_lock);
1974 WARN_ON_ONCE(queue->rskq_accept_head);
1975 queue->rskq_accept_head = head;
1976 queue->rskq_accept_tail = tail;
1977 spin_unlock_bh(&queue->rskq_lock);
1978}
1979
1980static int subflow_ulp_init(struct sock *sk)
1981{
1982 struct inet_connection_sock *icsk = inet_csk(sk);
1983 struct mptcp_subflow_context *ctx;
1984 struct tcp_sock *tp = tcp_sk(sk);
1985 int err = 0;
1986
1987 /* disallow attaching ULP to a socket unless it has been
1988 * created with sock_create_kern()
1989 */
1990 if (!sk->sk_kern_sock) {
1991 err = -EOPNOTSUPP;
1992 goto out;
1993 }
1994
1995 ctx = subflow_create_ctx(sk, GFP_KERNEL);
1996 if (!ctx) {
1997 err = -ENOMEM;
1998 goto out;
1999 }
2000
2001 pr_debug("subflow=%p, family=%d\n", ctx, sk->sk_family);
2002
2003 tp->is_mptcp = 1;
2004 ctx->icsk_af_ops = icsk->icsk_af_ops;
2005 icsk->icsk_af_ops = subflow_default_af_ops(sk);
2006 ctx->tcp_state_change = sk->sk_state_change;
2007 ctx->tcp_error_report = sk->sk_error_report;
2008
2009 WARN_ON_ONCE(sk->sk_data_ready != sock_def_readable);
2010 WARN_ON_ONCE(sk->sk_write_space != sk_stream_write_space);
2011
2012 sk->sk_data_ready = subflow_data_ready;
2013 sk->sk_write_space = subflow_write_space;
2014 sk->sk_state_change = subflow_state_change;
2015 sk->sk_error_report = subflow_error_report;
2016out:
2017 return err;
2018}
2019
2020static void subflow_ulp_release(struct sock *ssk)
2021{
2022 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk);
2023 bool release = true;
2024 struct sock *sk;
2025
2026 if (!ctx)
2027 return;
2028
2029 sk = ctx->conn;
2030 if (sk) {
2031 /* if the msk has been orphaned, keep the ctx
2032 * alive, will be freed by __mptcp_close_ssk(),
2033 * when the subflow is still unaccepted
2034 */
2035 release = ctx->disposable || list_empty(&ctx->node);
2036
2037 /* inet_child_forget() does not call sk_state_change(),
2038 * explicitly trigger the socket close machinery
2039 */
2040 if (!release && !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW,
2041 &mptcp_sk(sk)->flags))
2042 mptcp_schedule_work(sk);
2043 sock_put(sk);
2044 }
2045
2046 mptcp_subflow_ops_undo_override(ssk);
2047 if (release)
2048 kfree_rcu(ctx, rcu);
2049}
2050
2051static void subflow_ulp_clone(const struct request_sock *req,
2052 struct sock *newsk,
2053 const gfp_t priority)
2054{
2055 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
2056 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk);
2057 struct mptcp_subflow_context *new_ctx;
2058
2059 if (!tcp_rsk(req)->is_mptcp ||
2060 (!subflow_req->mp_capable && !subflow_req->mp_join)) {
2061 subflow_ulp_fallback(newsk, old_ctx);
2062 return;
2063 }
2064
2065 new_ctx = subflow_create_ctx(newsk, priority);
2066 if (!new_ctx) {
2067 subflow_ulp_fallback(newsk, old_ctx);
2068 return;
2069 }
2070
2071 new_ctx->conn_finished = 1;
2072 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops;
2073 new_ctx->tcp_state_change = old_ctx->tcp_state_change;
2074 new_ctx->tcp_error_report = old_ctx->tcp_error_report;
2075 new_ctx->rel_write_seq = 1;
2076
2077 if (subflow_req->mp_capable) {
2078 /* see comments in subflow_syn_recv_sock(), MPTCP connection
2079 * is fully established only after we receive the remote key
2080 */
2081 new_ctx->mp_capable = 1;
2082 new_ctx->local_key = subflow_req->local_key;
2083 new_ctx->token = subflow_req->token;
2084 new_ctx->ssn_offset = subflow_req->ssn_offset;
2085 new_ctx->idsn = subflow_req->idsn;
2086
2087 /* this is the first subflow, id is always 0 */
2088 subflow_set_local_id(new_ctx, 0);
2089 } else if (subflow_req->mp_join) {
2090 new_ctx->ssn_offset = subflow_req->ssn_offset;
2091 new_ctx->mp_join = 1;
2092 WRITE_ONCE(new_ctx->fully_established, true);
2093 new_ctx->remote_key_valid = 1;
2094 new_ctx->backup = subflow_req->backup;
2095 new_ctx->request_bkup = subflow_req->request_bkup;
2096 WRITE_ONCE(new_ctx->remote_id, subflow_req->remote_id);
2097 new_ctx->token = subflow_req->token;
2098
2099 /* the subflow req id is valid, fetched via subflow_check_req()
2100 * and subflow_token_join_request()
2101 */
2102 subflow_set_local_id(new_ctx, subflow_req->local_id);
2103 }
2104}
2105
2106static void tcp_release_cb_override(struct sock *ssk)
2107{
2108 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
2109 long status;
2110
2111 /* process and clear all the pending actions, but leave the subflow into
2112 * the napi queue. To respect locking, only the same CPU that originated
2113 * the action can touch the list. mptcp_napi_poll will take care of it.
2114 */
2115 status = set_mask_bits(&subflow->delegated_status, MPTCP_DELEGATE_ACTIONS_MASK, 0);
2116 if (status)
2117 mptcp_subflow_process_delegated(ssk, status);
2118
2119 tcp_release_cb(ssk);
2120}
2121
2122static int tcp_abort_override(struct sock *ssk, int err)
2123{
2124 /* closing a listener subflow requires a great deal of care.
2125 * keep it simple and just prevent such operation
2126 */
2127 if (inet_sk_state_load(ssk) == TCP_LISTEN)
2128 return -EINVAL;
2129
2130 return tcp_abort(ssk, err);
2131}
2132
2133static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = {
2134 .name = "mptcp",
2135 .owner = THIS_MODULE,
2136 .init = subflow_ulp_init,
2137 .release = subflow_ulp_release,
2138 .clone = subflow_ulp_clone,
2139};
2140
2141static int subflow_ops_init(struct request_sock_ops *subflow_ops)
2142{
2143 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock);
2144
2145 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name,
2146 subflow_ops->obj_size, 0,
2147 SLAB_ACCOUNT |
2148 SLAB_TYPESAFE_BY_RCU,
2149 NULL);
2150 if (!subflow_ops->slab)
2151 return -ENOMEM;
2152
2153 return 0;
2154}
2155
2156void __init mptcp_subflow_init(void)
2157{
2158 mptcp_subflow_v4_request_sock_ops = tcp_request_sock_ops;
2159 mptcp_subflow_v4_request_sock_ops.slab_name = "request_sock_subflow_v4";
2160 mptcp_subflow_v4_request_sock_ops.destructor = subflow_v4_req_destructor;
2161
2162 if (subflow_ops_init(&mptcp_subflow_v4_request_sock_ops) != 0)
2163 panic("MPTCP: failed to init subflow v4 request sock ops\n");
2164
2165 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops;
2166 subflow_request_sock_ipv4_ops.route_req = subflow_v4_route_req;
2167 subflow_request_sock_ipv4_ops.send_synack = subflow_v4_send_synack;
2168
2169 subflow_specific = ipv4_specific;
2170 subflow_specific.conn_request = subflow_v4_conn_request;
2171 subflow_specific.syn_recv_sock = subflow_syn_recv_sock;
2172 subflow_specific.sk_rx_dst_set = subflow_finish_connect;
2173 subflow_specific.rebuild_header = subflow_rebuild_header;
2174
2175 tcp_prot_override = tcp_prot;
2176 tcp_prot_override.release_cb = tcp_release_cb_override;
2177 tcp_prot_override.diag_destroy = tcp_abort_override;
2178#ifdef CONFIG_BPF_SYSCALL
2179 /* Disable sockmap processing for subflows */
2180 tcp_prot_override.psock_update_sk_prot = NULL;
2181#endif
2182
2183 mptcp_diag_subflow_init(&subflow_ulp_ops);
2184
2185 if (tcp_register_ulp(&subflow_ulp_ops) != 0)
2186 panic("MPTCP: failed to register subflows to ULP\n");
2187}
2188
2189#if IS_ENABLED(CONFIG_MPTCP_IPV6)
2190void __init mptcp_subflow_v6_init(void)
2191{
2192 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock
2193 * structures for v4 and v6 have the same size. It should not changed in
2194 * the future but better to make sure to be warned if it is no longer
2195 * the case.
2196 */
2197 BUILD_BUG_ON(sizeof(struct tcp_request_sock) != sizeof(struct tcp6_request_sock));
2198
2199 mptcp_subflow_v6_request_sock_ops = tcp6_request_sock_ops;
2200 mptcp_subflow_v6_request_sock_ops.slab_name = "request_sock_subflow_v6";
2201 mptcp_subflow_v6_request_sock_ops.destructor = subflow_v6_req_destructor;
2202
2203 if (subflow_ops_init(&mptcp_subflow_v6_request_sock_ops) != 0)
2204 panic("MPTCP: failed to init subflow v6 request sock ops\n");
2205
2206 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops;
2207 subflow_request_sock_ipv6_ops.route_req = subflow_v6_route_req;
2208 subflow_request_sock_ipv6_ops.send_synack = subflow_v6_send_synack;
2209
2210 subflow_v6_specific = ipv6_specific;
2211 subflow_v6_specific.conn_request = subflow_v6_conn_request;
2212 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock;
2213 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect;
2214 subflow_v6_specific.rebuild_header = subflow_v6_rebuild_header;
2215
2216 subflow_v6m_specific = subflow_v6_specific;
2217 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit;
2218 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len;
2219 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced;
2220 subflow_v6m_specific.rebuild_header = subflow_rebuild_header;
2221
2222 tcpv6_prot_override = tcpv6_prot;
2223 tcpv6_prot_override.release_cb = tcp_release_cb_override;
2224 tcpv6_prot_override.diag_destroy = tcp_abort_override;
2225#ifdef CONFIG_BPF_SYSCALL
2226 /* Disable sockmap processing for subflows */
2227 tcpv6_prot_override.psock_update_sk_prot = NULL;
2228#endif
2229}
2230#endif