projects/linux·blob master · refs

1663 lines · 43.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Multipath TCP
3 *
4 * Copyright (c) 2025, Matthieu Baerts.
5 */
6
7#define pr_fmt(fmt) "MPTCP: " fmt
8
9#include <net/netns/generic.h>
10
11#include "protocol.h"
12#include "mib.h"
13#include "mptcp_pm_gen.h"
14
15static int pm_nl_pernet_id;
16
17struct pm_nl_pernet {
18 /* protects pernet updates */
19 spinlock_t lock;
20 struct list_head endp_list;
21 u8 endpoints;
22 u8 endp_signal_max;
23 u8 endp_subflow_max;
24 u8 endp_laminar_max;
25 u8 endp_fullmesh_max;
26 u8 limit_add_addr_accepted;
27 u8 limit_extra_subflows;
28 u8 next_id;
29 DECLARE_BITMAP(id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
30};
31
32#define MPTCP_PM_ADDR_MAX 8
33#define MPTCP_PM_SUBFLOWS_MAX 64
34
35static struct pm_nl_pernet *pm_nl_get_pernet(const struct net *net)
36{
37 return net_generic(net, pm_nl_pernet_id);
38}
39
40static struct pm_nl_pernet *
41pm_nl_get_pernet_from_msk(const struct mptcp_sock *msk)
42{
43 return pm_nl_get_pernet(sock_net((struct sock *)msk));
44}
45
46static struct pm_nl_pernet *genl_info_pm_nl(struct genl_info *info)
47{
48 return pm_nl_get_pernet(genl_info_net(info));
49}
50
51u8 mptcp_pm_get_endp_signal_max(const struct mptcp_sock *msk)
52{
53 const struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
54
55 return READ_ONCE(pernet->endp_signal_max);
56}
57EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_signal_max);
58
59u8 mptcp_pm_get_endp_subflow_max(const struct mptcp_sock *msk)
60{
61 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
62
63 return READ_ONCE(pernet->endp_subflow_max);
64}
65EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_subflow_max);
66
67u8 mptcp_pm_get_endp_laminar_max(const struct mptcp_sock *msk)
68{
69 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
70
71 return READ_ONCE(pernet->endp_laminar_max);
72}
73EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_laminar_max);
74
75u8 mptcp_pm_get_endp_fullmesh_max(const struct mptcp_sock *msk)
76{
77 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
78
79 return READ_ONCE(pernet->endp_fullmesh_max);
80}
81EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_fullmesh_max);
82
83u8 mptcp_pm_get_limit_add_addr_accepted(const struct mptcp_sock *msk)
84{
85 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
86
87 return READ_ONCE(pernet->limit_add_addr_accepted);
88}
89EXPORT_SYMBOL_GPL(mptcp_pm_get_limit_add_addr_accepted);
90
91u8 mptcp_pm_get_limit_extra_subflows(const struct mptcp_sock *msk)
92{
93 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
94
95 return READ_ONCE(pernet->limit_extra_subflows);
96}
97EXPORT_SYMBOL_GPL(mptcp_pm_get_limit_extra_subflows);
98
99static bool has_subflow_daddr(const struct mptcp_sock *msk,
100 const struct mptcp_addr_info *daddr)
101{
102 struct mptcp_subflow_context *subflow;
103 struct mptcp_addr_info cur;
104
105 mptcp_for_each_subflow(msk, subflow) {
106 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
107
108 if (!((1 << inet_sk_state_load(ssk)) &
109 (TCPF_ESTABLISHED | TCPF_SYN_SENT | TCPF_SYN_RECV)))
110 continue;
111
112 mptcp_remote_address((struct sock_common *)ssk, &cur);
113 if (mptcp_addresses_equal(&cur, daddr, daddr->port))
114 return true;
115 }
116
117 return false;
118}
119
120static bool
121select_local_address(const struct pm_nl_pernet *pernet,
122 const struct mptcp_sock *msk,
123 struct mptcp_pm_local *new_local)
124{
125 struct mptcp_pm_addr_entry *entry;
126 bool found = false;
127
128 msk_owned_by_me(msk);
129
130 rcu_read_lock();
131 list_for_each_entry_rcu(entry, &pernet->endp_list, list) {
132 if (!(entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW))
133 continue;
134
135 if (!test_bit(entry->addr.id, msk->pm.id_avail_bitmap))
136 continue;
137
138 new_local->addr = entry->addr;
139 new_local->flags = entry->flags;
140 new_local->ifindex = entry->ifindex;
141 found = true;
142 break;
143 }
144 rcu_read_unlock();
145
146 return found;
147}
148
149static bool
150select_signal_address(struct pm_nl_pernet *pernet, const struct mptcp_sock *msk,
151 struct mptcp_pm_local *new_local)
152{
153 struct mptcp_pm_addr_entry *entry;
154 bool found = false;
155
156 rcu_read_lock();
157 /* do not keep any additional per socket state, just signal
158 * the address list in order.
159 * Note: removal from the local address list during the msk life-cycle
160 * can lead to additional addresses not being announced.
161 */
162 list_for_each_entry_rcu(entry, &pernet->endp_list, list) {
163 if (!test_bit(entry->addr.id, msk->pm.id_avail_bitmap))
164 continue;
165
166 if (!(entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL))
167 continue;
168
169 new_local->addr = entry->addr;
170 new_local->flags = entry->flags;
171 new_local->ifindex = entry->ifindex;
172 found = true;
173 break;
174 }
175 rcu_read_unlock();
176
177 return found;
178}
179
180static unsigned int
181fill_remote_addr(struct mptcp_sock *msk, struct mptcp_addr_info *local,
182 struct mptcp_addr_info *addrs)
183{
184 bool deny_id0 = READ_ONCE(msk->pm.remote_deny_join_id0);
185 struct mptcp_addr_info remote = { 0 };
186 struct sock *sk = (struct sock *)msk;
187
188 if (deny_id0)
189 return 0;
190
191 mptcp_remote_address((struct sock_common *)sk, &remote);
192
193 if (!mptcp_pm_addr_families_match(sk, local, &remote))
194 return 0;
195
196 msk->pm.extra_subflows++;
197 *addrs = remote;
198
199 return 1;
200}
201
202static unsigned int
203fill_remote_addresses_fullmesh(struct mptcp_sock *msk,
204 struct mptcp_addr_info *local,
205 struct mptcp_addr_info *addrs,
206 int addrs_size)
207{
208 u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
209 bool deny_id0 = READ_ONCE(msk->pm.remote_deny_join_id0);
210 DECLARE_BITMAP(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1);
211 struct sock *sk = (struct sock *)msk, *ssk;
212 struct mptcp_subflow_context *subflow;
213 int i = 0;
214
215 /* Forbid creation of new subflows matching existing ones, possibly
216 * already created by incoming ADD_ADDR
217 */
218 bitmap_zero(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1);
219 mptcp_for_each_subflow(msk, subflow)
220 if (READ_ONCE(subflow->local_id) == local->id)
221 __set_bit(subflow->remote_id, unavail_id);
222
223 mptcp_for_each_subflow(msk, subflow) {
224 ssk = mptcp_subflow_tcp_sock(subflow);
225 mptcp_remote_address((struct sock_common *)ssk, &addrs[i]);
226 addrs[i].id = READ_ONCE(subflow->remote_id);
227 if (deny_id0 && !addrs[i].id)
228 continue;
229
230 if (test_bit(addrs[i].id, unavail_id))
231 continue;
232
233 if (!mptcp_pm_addr_families_match(sk, local, &addrs[i]))
234 continue;
235
236 /* forbid creating multiple address towards this id */
237 __set_bit(addrs[i].id, unavail_id);
238 msk->pm.extra_subflows++;
239 i++;
240
241 if (msk->pm.extra_subflows >= limit_extra_subflows ||
242 i == addrs_size)
243 break;
244 }
245
246 return i;
247}
248
249/* Fill all the remote addresses into the array addrs[],
250 * and return the array size.
251 */
252static unsigned int
253fill_remote_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *local,
254 bool fullmesh, struct mptcp_addr_info *addrs,
255 int addrs_size)
256{
257 /* Non-fullmesh: fill in the single entry corresponding to the primary
258 * MPC subflow remote address, and return 1, corresponding to 1 entry.
259 */
260 if (!fullmesh)
261 return fill_remote_addr(msk, local, addrs);
262
263 /* Fullmesh endpoint: fill all possible remote addresses */
264 return fill_remote_addresses_fullmesh(msk, local, addrs, addrs_size);
265}
266
267static struct mptcp_pm_addr_entry *
268__lookup_addr_by_id(struct pm_nl_pernet *pernet, unsigned int id)
269{
270 struct mptcp_pm_addr_entry *entry;
271
272 list_for_each_entry_rcu(entry, &pernet->endp_list, list,
273 lockdep_is_held(&pernet->lock)) {
274 if (entry->addr.id == id)
275 return entry;
276 }
277 return NULL;
278}
279
280static struct mptcp_pm_addr_entry *
281__lookup_addr(struct pm_nl_pernet *pernet, const struct mptcp_addr_info *info)
282{
283 struct mptcp_pm_addr_entry *entry;
284
285 list_for_each_entry_rcu(entry, &pernet->endp_list, list,
286 lockdep_is_held(&pernet->lock)) {
287 if (mptcp_addresses_equal(&entry->addr, info, entry->addr.port))
288 return entry;
289 }
290 return NULL;
291}
292
293static u8 mptcp_endp_get_local_id(struct mptcp_sock *msk,
294 const struct mptcp_addr_info *addr)
295{
296 return msk->mpc_endpoint_id == addr->id ? 0 : addr->id;
297}
298
299/* Set mpc_endpoint_id, and send MP_PRIO for ID0 if needed */
300static void mptcp_mpc_endpoint_setup(struct mptcp_sock *msk)
301{
302 struct mptcp_subflow_context *subflow;
303 struct mptcp_pm_addr_entry *entry;
304 struct mptcp_addr_info mpc_addr;
305 struct pm_nl_pernet *pernet;
306 bool backup = false;
307
308 /* do lazy endpoint usage accounting for the MPC subflows */
309 if (likely(msk->pm.status & BIT(MPTCP_PM_MPC_ENDPOINT_ACCOUNTED)) ||
310 !msk->first)
311 return;
312
313 subflow = mptcp_subflow_ctx(msk->first);
314 pernet = pm_nl_get_pernet_from_msk(msk);
315
316 mptcp_local_address((struct sock_common *)msk->first, &mpc_addr);
317 rcu_read_lock();
318 entry = __lookup_addr(pernet, &mpc_addr);
319 if (entry) {
320 __clear_bit(entry->addr.id, msk->pm.id_avail_bitmap);
321 msk->mpc_endpoint_id = entry->addr.id;
322 backup = !!(entry->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
323 }
324 rcu_read_unlock();
325
326 /* Send MP_PRIO */
327 if (backup)
328 mptcp_pm_send_ack(msk, subflow, true, backup);
329
330 msk->pm.status |= BIT(MPTCP_PM_MPC_ENDPOINT_ACCOUNTED);
331}
332
333static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk)
334{
335 u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
336 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
337 u8 endp_subflow_max = mptcp_pm_get_endp_subflow_max(msk);
338 u8 endp_signal_max = mptcp_pm_get_endp_signal_max(msk);
339 struct sock *sk = (struct sock *)msk;
340 bool signal_and_subflow = false;
341 struct mptcp_pm_local local;
342
343 mptcp_mpc_endpoint_setup(msk);
344 if (!mptcp_is_fully_established(sk))
345 return;
346
347 pr_debug("local %d:%d signal %d:%d subflows %d:%d\n",
348 msk->pm.local_addr_used, endp_subflow_max,
349 msk->pm.add_addr_signaled, endp_signal_max,
350 msk->pm.extra_subflows, limit_extra_subflows);
351
352 /* check first for announce */
353 if (msk->pm.add_addr_signaled < endp_signal_max) {
354 u8 endp_id;
355
356 /* due to racing events on both ends we can reach here while
357 * previous add address is still running: if we invoke now
358 * mptcp_pm_announce_addr(), that will fail and the
359 * corresponding id will be marked as used.
360 * Instead let the PM machinery reschedule us when the
361 * current address announce will be completed.
362 */
363 if (msk->pm.addr_signal & BIT(MPTCP_ADD_ADDR_SIGNAL))
364 return;
365
366 if (!select_signal_address(pernet, msk, &local))
367 goto subflow;
368
369 /* Special case for ID0: set the correct ID */
370 endp_id = local.addr.id;
371 if (endp_id == msk->mpc_endpoint_id)
372 local.addr.id = 0;
373
374 /* If the alloc fails, we are on memory pressure, not worth
375 * continuing, and trying to create subflows.
376 */
377 if (!mptcp_pm_announced_alloc(msk, &local.addr))
378 return;
379
380 __clear_bit(endp_id, msk->pm.id_avail_bitmap);
381 msk->pm.add_addr_signaled++;
382
383 mptcp_pm_announce_addr(msk, &local.addr, false);
384 mptcp_pm_addr_send_ack(msk);
385
386 if (local.flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)
387 signal_and_subflow = true;
388 }
389
390subflow:
391 /* No need to try establishing subflows to remote id0 if not allowed */
392 if (mptcp_pm_add_addr_c_flag_case(msk))
393 goto exit;
394
395 /* check if should create a new subflow */
396 while (msk->pm.local_addr_used < endp_subflow_max &&
397 msk->pm.extra_subflows < limit_extra_subflows) {
398 struct mptcp_addr_info addrs[MPTCP_PM_ADDR_MAX];
399 bool fullmesh;
400 int i, nr;
401
402 if (signal_and_subflow)
403 signal_and_subflow = false;
404 else if (!select_local_address(pernet, msk, &local))
405 break;
406
407 fullmesh = !!(local.flags & MPTCP_PM_ADDR_FLAG_FULLMESH);
408
409 __clear_bit(local.addr.id, msk->pm.id_avail_bitmap);
410
411 /* Special case for ID0: set the correct ID */
412 if (local.addr.id == msk->mpc_endpoint_id)
413 local.addr.id = 0;
414 else /* local_addr_used is not decr for ID 0 */
415 msk->pm.local_addr_used++;
416
417 nr = fill_remote_addresses_vec(msk, &local.addr, fullmesh,
418 addrs, ARRAY_SIZE(addrs));
419 if (nr == 0)
420 continue;
421
422 spin_unlock_bh(&msk->pm.lock);
423 for (i = 0; i < nr; i++)
424 __mptcp_subflow_connect(sk, &local, &addrs[i]);
425 spin_lock_bh(&msk->pm.lock);
426 }
427
428exit:
429 /* If an endpoint has both the signal and subflow flags, but it is not
430 * possible to create subflows -- the 'while' loop body above never
431 * executed -- then still mark the endp as used, which is somehow the
432 * case. This avoids issues later when removing the endpoint and calling
433 * __mark_subflow_endp_available(), which expects the increment here.
434 */
435 if (signal_and_subflow && local.addr.id != msk->mpc_endpoint_id)
436 msk->pm.local_addr_used++;
437
438 mptcp_pm_nl_check_work_pending(msk);
439}
440
441static void mptcp_pm_nl_fully_established(struct mptcp_sock *msk)
442{
443 mptcp_pm_create_subflow_or_signal_addr(msk);
444}
445
446static void mptcp_pm_nl_subflow_established(struct mptcp_sock *msk)
447{
448 mptcp_pm_create_subflow_or_signal_addr(msk);
449}
450
451static unsigned int
452fill_local_addresses_vec_fullmesh(struct mptcp_sock *msk,
453 struct mptcp_addr_info *remote,
454 struct mptcp_pm_local *locals,
455 int locals_size,
456 bool c_flag_case)
457{
458 u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
459 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
460 struct sock *sk = (struct sock *)msk;
461 struct mptcp_pm_addr_entry *entry;
462 struct mptcp_pm_local *local;
463 int i = 0;
464
465 rcu_read_lock();
466 list_for_each_entry_rcu(entry, &pernet->endp_list, list) {
467 bool is_id0;
468
469 if (!(entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH))
470 continue;
471
472 if (!mptcp_pm_addr_families_match(sk, &entry->addr, remote))
473 continue;
474
475 local = &locals[i];
476 local->addr = entry->addr;
477 local->flags = entry->flags;
478 local->ifindex = entry->ifindex;
479
480 is_id0 = local->addr.id == msk->mpc_endpoint_id;
481
482 if (c_flag_case &&
483 (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)) {
484 __clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
485
486 if (!is_id0)
487 msk->pm.local_addr_used++;
488 }
489
490 /* Special case for ID0: set the correct ID */
491 if (is_id0)
492 local->addr.id = 0;
493
494 msk->pm.extra_subflows++;
495 i++;
496
497 if (msk->pm.extra_subflows >= limit_extra_subflows ||
498 i == locals_size)
499 break;
500 }
501 rcu_read_unlock();
502
503 return i;
504}
505
506static unsigned int
507fill_local_laminar_endp(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
508 struct mptcp_pm_local *locals)
509{
510 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
511 DECLARE_BITMAP(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1);
512 struct mptcp_subflow_context *subflow;
513 struct sock *sk = (struct sock *)msk;
514 struct mptcp_pm_addr_entry *entry;
515 struct mptcp_pm_local *local;
516 int found = 0;
517
518 /* Forbid creation of new subflows matching existing ones, possibly
519 * already created by 'subflow' endpoints
520 */
521 bitmap_zero(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1);
522 mptcp_for_each_subflow(msk, subflow) {
523 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
524
525 if ((1 << inet_sk_state_load(ssk)) &
526 (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSING |
527 TCPF_CLOSE))
528 continue;
529
530 __set_bit(subflow_get_local_id(subflow), unavail_id);
531 }
532
533 rcu_read_lock();
534 list_for_each_entry_rcu(entry, &pernet->endp_list, list) {
535 if (!(entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR))
536 continue;
537
538 if (!mptcp_pm_addr_families_match(sk, &entry->addr, remote))
539 continue;
540
541 if (test_bit(mptcp_endp_get_local_id(msk, &entry->addr),
542 unavail_id))
543 continue;
544
545 local = &locals[0];
546 local->addr = entry->addr;
547 local->flags = entry->flags;
548 local->ifindex = entry->ifindex;
549
550 if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) {
551 __clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
552
553 if (local->addr.id != msk->mpc_endpoint_id)
554 msk->pm.local_addr_used++;
555 }
556
557 msk->pm.extra_subflows++;
558 found = 1;
559 break;
560 }
561 rcu_read_unlock();
562
563 return found;
564}
565
566static unsigned int
567fill_local_addresses_vec_c_flag(struct mptcp_sock *msk,
568 struct mptcp_addr_info *remote,
569 struct mptcp_pm_local *locals,
570 int locals_size)
571{
572 u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
573 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
574 u8 endp_subflow_max = mptcp_pm_get_endp_subflow_max(msk);
575 struct sock *sk = (struct sock *)msk;
576 struct mptcp_pm_local *local;
577 int i = 0;
578
579 while (msk->pm.local_addr_used < endp_subflow_max) {
580 local = &locals[i];
581
582 if (!select_local_address(pernet, msk, local))
583 break;
584
585 __clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
586
587 if (!mptcp_pm_addr_families_match(sk, &local->addr, remote))
588 continue;
589
590 if (local->addr.id == msk->mpc_endpoint_id)
591 continue;
592
593 msk->pm.local_addr_used++;
594 msk->pm.extra_subflows++;
595 i++;
596
597 if (msk->pm.extra_subflows >= limit_extra_subflows ||
598 i == locals_size)
599 break;
600 }
601
602 return i;
603}
604
605static unsigned int
606fill_local_address_any(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
607 struct mptcp_pm_local *local)
608{
609 struct sock *sk = (struct sock *)msk;
610
611 memset(local, 0, sizeof(*local));
612 local->addr.family =
613#if IS_ENABLED(CONFIG_MPTCP_IPV6)
614 remote->family == AF_INET6 &&
615 ipv6_addr_v4mapped(&remote->addr6) ? AF_INET :
616#endif
617 remote->family;
618
619 if (!mptcp_pm_addr_families_match(sk, &local->addr, remote))
620 return 0;
621
622 msk->pm.extra_subflows++;
623
624 return 1;
625}
626
627/* Fill all the local addresses into the array addrs[],
628 * and return the array size.
629 */
630static unsigned int
631fill_local_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
632 struct mptcp_pm_local *locals, int locals_size)
633{
634 bool c_flag_case = remote->id && mptcp_pm_add_addr_c_flag_case(msk);
635
636 /* If there is at least one MPTCP endpoint with a fullmesh flag */
637 if (mptcp_pm_get_endp_fullmesh_max(msk))
638 return fill_local_addresses_vec_fullmesh(msk, remote, locals,
639 locals_size,
640 c_flag_case);
641
642 /* If there is at least one MPTCP endpoint with a laminar flag */
643 if (mptcp_pm_get_endp_laminar_max(msk))
644 return fill_local_laminar_endp(msk, remote, locals);
645
646 /* Special case: peer sets the C flag, accept one ADD_ADDR if default
647 * limits are used -- accepting no ADD_ADDR -- and use subflow endpoints
648 */
649 if (c_flag_case)
650 return fill_local_addresses_vec_c_flag(msk, remote, locals,
651 locals_size);
652
653 /* No special case: fill in the single 'IPADDRANY' local address */
654 return fill_local_address_any(msk, remote, &locals[0]);
655}
656
657static void mptcp_pm_nl_add_addr_received(struct mptcp_sock *msk)
658{
659 u8 limit_add_addr_accepted = mptcp_pm_get_limit_add_addr_accepted(msk);
660 u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
661 struct mptcp_pm_local locals[MPTCP_PM_ADDR_MAX];
662 struct sock *sk = (struct sock *)msk;
663 struct mptcp_addr_info remote;
664 bool sf_created = false;
665 int i, nr;
666
667 pr_debug("accepted %d:%d remote family %d\n",
668 msk->pm.add_addr_accepted, limit_add_addr_accepted,
669 msk->pm.remote.family);
670
671 remote = msk->pm.remote;
672 mptcp_pm_announce_addr(msk, &remote, true);
673 mptcp_pm_addr_send_ack(msk);
674 mptcp_mpc_endpoint_setup(msk);
675
676 if (has_subflow_daddr(msk, &remote))
677 return;
678
679 /* pick id 0 port, if none is provided the remote address */
680 if (!remote.port)
681 remote.port = sk->sk_dport;
682
683 /* connect to the specified remote address, using whatever
684 * local address the routing configuration will pick.
685 */
686 nr = fill_local_addresses_vec(msk, &remote, locals, ARRAY_SIZE(locals));
687 if (nr == 0)
688 return;
689
690 spin_unlock_bh(&msk->pm.lock);
691 for (i = 0; i < nr; i++)
692 if (__mptcp_subflow_connect(sk, &locals[i], &remote) == 0)
693 sf_created = true;
694 spin_lock_bh(&msk->pm.lock);
695
696 if (sf_created) {
697 /* add_addr_accepted is not decr for ID 0 */
698 if (remote.id)
699 msk->pm.add_addr_accepted++;
700 if (msk->pm.add_addr_accepted >= limit_add_addr_accepted ||
701 msk->pm.extra_subflows >= limit_extra_subflows)
702 WRITE_ONCE(msk->pm.accept_addr, false);
703 }
704}
705
706void mptcp_pm_nl_rm_addr(struct mptcp_sock *msk, u8 rm_id)
707{
708 if (rm_id && !WARN_ON_ONCE(msk->pm.add_addr_accepted == 0)) {
709 u8 limit_add_addr_accepted =
710 mptcp_pm_get_limit_add_addr_accepted(msk);
711
712 /* Note: if the subflow has been closed before, this
713 * add_addr_accepted counter will not be decremented.
714 */
715 if (--msk->pm.add_addr_accepted < limit_add_addr_accepted)
716 WRITE_ONCE(msk->pm.accept_addr, true);
717 }
718}
719
720static bool address_use_port(struct mptcp_pm_addr_entry *entry)
721{
722 return (entry->flags &
723 (MPTCP_PM_ADDR_FLAG_SIGNAL | MPTCP_PM_ADDR_FLAG_SUBFLOW)) ==
724 MPTCP_PM_ADDR_FLAG_SIGNAL;
725}
726
727/* caller must ensure the RCU grace period is already elapsed */
728static void __mptcp_pm_release_addr_entry(struct mptcp_pm_addr_entry *entry)
729{
730 if (entry->lsk)
731 sock_release(entry->lsk);
732 kfree(entry);
733}
734
735static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet,
736 struct mptcp_pm_addr_entry *entry,
737 bool replace)
738{
739 struct mptcp_pm_addr_entry *cur, *del_entry = NULL;
740 int ret = -EINVAL;
741 u8 addr_max;
742
743 spin_lock_bh(&pernet->lock);
744 /* to keep the code simple, don't do IDR-like allocation for address ID,
745 * just bail when we exceed limits
746 */
747 if (pernet->next_id == MPTCP_PM_MAX_ADDR_ID)
748 pernet->next_id = 1;
749 if (pernet->endpoints == MPTCP_PM_MAX_ADDR_ID) {
750 ret = -ERANGE;
751 goto out;
752 }
753 if (test_bit(entry->addr.id, pernet->id_bitmap)) {
754 ret = -EBUSY;
755 goto out;
756 }
757
758 /* do not insert duplicate address, differentiate on port only
759 * singled addresses
760 */
761 if (!address_use_port(entry))
762 entry->addr.port = 0;
763 list_for_each_entry(cur, &pernet->endp_list, list) {
764 if (mptcp_addresses_equal(&cur->addr, &entry->addr,
765 cur->addr.port || entry->addr.port)) {
766 /* allow replacing the exiting endpoint only if such
767 * endpoint is an implicit one and the user-space
768 * did not provide an endpoint id
769 */
770 if (!(cur->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT)) {
771 ret = -EEXIST;
772 goto out;
773 }
774 if (entry->addr.id)
775 goto out;
776
777 /* allow callers that only need to look up the local
778 * addr's id to skip replacement. This allows them to
779 * avoid calling synchronize_rcu in the packet recv
780 * path.
781 */
782 if (!replace) {
783 kfree(entry);
784 ret = cur->addr.id;
785 goto out;
786 }
787
788 pernet->endpoints--;
789 entry->addr.id = cur->addr.id;
790 list_del_rcu(&cur->list);
791 del_entry = cur;
792 break;
793 }
794 }
795
796 if (!entry->addr.id) {
797find_next:
798 entry->addr.id = find_next_zero_bit(pernet->id_bitmap,
799 MPTCP_PM_MAX_ADDR_ID + 1,
800 pernet->next_id);
801 if (!entry->addr.id && pernet->next_id != 1) {
802 pernet->next_id = 1;
803 goto find_next;
804 }
805 }
806
807 if (!entry->addr.id)
808 goto out;
809
810 __set_bit(entry->addr.id, pernet->id_bitmap);
811 if (entry->addr.id > pernet->next_id)
812 pernet->next_id = entry->addr.id;
813
814 if (entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL) {
815 addr_max = pernet->endp_signal_max;
816 WRITE_ONCE(pernet->endp_signal_max, addr_max + 1);
817 }
818 if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) {
819 addr_max = pernet->endp_subflow_max;
820 WRITE_ONCE(pernet->endp_subflow_max, addr_max + 1);
821 }
822 if (entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR) {
823 addr_max = pernet->endp_laminar_max;
824 WRITE_ONCE(pernet->endp_laminar_max, addr_max + 1);
825 }
826 if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) {
827 addr_max = pernet->endp_fullmesh_max;
828 WRITE_ONCE(pernet->endp_fullmesh_max, addr_max + 1);
829 }
830
831 pernet->endpoints++;
832 if (!entry->addr.port)
833 list_add_tail_rcu(&entry->list, &pernet->endp_list);
834 else
835 list_add_rcu(&entry->list, &pernet->endp_list);
836 ret = entry->addr.id;
837
838out:
839 spin_unlock_bh(&pernet->lock);
840
841 /* just replaced an existing entry, free it */
842 if (del_entry) {
843 synchronize_rcu();
844 __mptcp_pm_release_addr_entry(del_entry);
845 }
846 return ret;
847}
848
849static struct lock_class_key mptcp_slock_keys[2];
850static struct lock_class_key mptcp_keys[2];
851
852static int mptcp_pm_nl_create_listen_socket(struct sock *sk,
853 struct mptcp_pm_addr_entry *entry)
854{
855 bool is_ipv6 = entry->addr.family == AF_INET6;
856 int addrlen = sizeof(struct sockaddr_in);
857 struct sockaddr_storage addr;
858 struct sock *newsk, *ssk;
859 int backlog = 1024;
860 int err;
861
862 err = sock_create_kern(sock_net(sk), entry->addr.family,
863 SOCK_STREAM, IPPROTO_MPTCP, &entry->lsk);
864 if (err)
865 return err;
866
867 newsk = entry->lsk->sk;
868 if (!newsk)
869 return -EINVAL;
870
871 /* The subflow socket lock is acquired in a nested to the msk one
872 * in several places, even by the TCP stack, and this msk is a kernel
873 * socket: lockdep complains. Instead of propagating the _nested
874 * modifiers in several places, re-init the lock class for the msk
875 * socket to an mptcp specific one.
876 */
877 sock_lock_init_class_and_name(newsk,
878 is_ipv6 ? "mlock-AF_INET6" : "mlock-AF_INET",
879 &mptcp_slock_keys[is_ipv6],
880 is_ipv6 ? "msk_lock-AF_INET6" : "msk_lock-AF_INET",
881 &mptcp_keys[is_ipv6]);
882
883 lock_sock(newsk);
884 ssk = __mptcp_nmpc_sk(mptcp_sk(newsk));
885 release_sock(newsk);
886 if (IS_ERR(ssk))
887 return PTR_ERR(ssk);
888
889 mptcp_info2sockaddr(&entry->addr, &addr, entry->addr.family);
890#if IS_ENABLED(CONFIG_MPTCP_IPV6)
891 if (entry->addr.family == AF_INET6)
892 addrlen = sizeof(struct sockaddr_in6);
893#endif
894 if (ssk->sk_family == AF_INET)
895 err = inet_bind_sk(ssk, (struct sockaddr_unsized *)&addr, addrlen);
896#if IS_ENABLED(CONFIG_MPTCP_IPV6)
897 else if (ssk->sk_family == AF_INET6)
898 err = inet6_bind_sk(ssk, (struct sockaddr_unsized *)&addr, addrlen);
899#endif
900 if (err)
901 return err;
902
903 /* We don't use mptcp_set_state() here because it needs to be called
904 * under the msk socket lock. For the moment, that will not bring
905 * anything more than only calling inet_sk_state_store(), because the
906 * old status is known (TCP_CLOSE).
907 */
908 inet_sk_state_store(newsk, TCP_LISTEN);
909 lock_sock(ssk);
910 WRITE_ONCE(mptcp_subflow_ctx(ssk)->pm_listener, true);
911 err = __inet_listen_sk(ssk, backlog);
912 if (!err)
913 mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED);
914 release_sock(ssk);
915 return err;
916}
917
918int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk,
919 struct mptcp_pm_addr_entry *skc)
920{
921 struct mptcp_pm_addr_entry *entry;
922 struct pm_nl_pernet *pernet;
923 int ret;
924
925 pernet = pm_nl_get_pernet_from_msk(msk);
926
927 rcu_read_lock();
928 entry = __lookup_addr(pernet, &skc->addr);
929 ret = entry ? entry->addr.id : -1;
930 rcu_read_unlock();
931 if (ret >= 0)
932 return ret;
933
934 /* address not found, add to local list */
935 entry = kmemdup(skc, sizeof(*skc), GFP_ATOMIC);
936 if (!entry)
937 return -ENOMEM;
938
939 entry->addr.port = 0;
940 ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, false);
941 if (ret < 0)
942 kfree(entry);
943
944 return ret;
945}
946
947bool mptcp_pm_nl_is_backup(struct mptcp_sock *msk, struct mptcp_addr_info *skc)
948{
949 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
950 struct mptcp_pm_addr_entry *entry;
951 bool backup;
952
953 rcu_read_lock();
954 entry = __lookup_addr(pernet, skc);
955 backup = entry && !!(entry->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
956 rcu_read_unlock();
957
958 return backup;
959}
960
961static int mptcp_nl_add_subflow_or_signal_addr(struct net *net,
962 struct mptcp_addr_info *addr)
963{
964 struct mptcp_sock *msk;
965 long s_slot = 0, s_num = 0;
966
967 while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
968 struct sock *sk = (struct sock *)msk;
969 struct mptcp_addr_info mpc_addr;
970
971 if (!READ_ONCE(msk->fully_established) ||
972 mptcp_pm_is_userspace(msk))
973 goto next;
974
975 /* if the endp linked to the init sf is re-added with a != ID */
976 mptcp_local_address((struct sock_common *)msk, &mpc_addr);
977
978 lock_sock(sk);
979 spin_lock_bh(&msk->pm.lock);
980 if (mptcp_addresses_equal(addr, &mpc_addr, addr->port))
981 msk->mpc_endpoint_id = addr->id;
982 mptcp_pm_create_subflow_or_signal_addr(msk);
983 spin_unlock_bh(&msk->pm.lock);
984 release_sock(sk);
985
986next:
987 sock_put(sk);
988 cond_resched();
989 }
990
991 return 0;
992}
993
994/* Add an MPTCP endpoint */
995int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info)
996{
997 struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
998 struct mptcp_pm_addr_entry addr, *entry;
999 struct nlattr *attr;
1000 int ret;
1001
1002 if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ENDPOINT_ADDR))
1003 return -EINVAL;
1004
1005 attr = info->attrs[MPTCP_PM_ENDPOINT_ADDR];
1006 ret = mptcp_pm_parse_entry(attr, info, true, &addr);
1007 if (ret < 0)
1008 return ret;
1009
1010 if (addr.addr.port && !address_use_port(&addr)) {
1011 NL_SET_ERR_MSG_ATTR(info->extack, attr,
1012 "flags must have signal and not subflow when using port");
1013 return -EINVAL;
1014 }
1015
1016 if (addr.flags & MPTCP_PM_ADDR_FLAG_SIGNAL &&
1017 addr.flags & MPTCP_PM_ADDR_FLAG_FULLMESH) {
1018 NL_SET_ERR_MSG_ATTR(info->extack, attr,
1019 "flags mustn't have both signal and fullmesh");
1020 return -EINVAL;
1021 }
1022
1023 if (addr.flags & MPTCP_PM_ADDR_FLAG_IMPLICIT) {
1024 NL_SET_ERR_MSG_ATTR(info->extack, attr,
1025 "can't create IMPLICIT endpoint");
1026 return -EINVAL;
1027 }
1028
1029 entry = kmemdup(&addr, sizeof(addr), GFP_KERNEL_ACCOUNT);
1030 if (!entry) {
1031 GENL_SET_ERR_MSG(info, "can't allocate addr");
1032 return -ENOMEM;
1033 }
1034
1035 if (entry->addr.port) {
1036 ret = mptcp_pm_nl_create_listen_socket(skb->sk, entry);
1037 if (ret) {
1038 GENL_SET_ERR_MSG_FMT(info, "create listen socket error: %d", ret);
1039 goto out_free;
1040 }
1041 }
1042 ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true);
1043 if (ret < 0) {
1044 GENL_SET_ERR_MSG_FMT(info, "too many addresses or duplicate one: %d", ret);
1045 goto out_free;
1046 }
1047
1048 mptcp_nl_add_subflow_or_signal_addr(sock_net(skb->sk), &entry->addr);
1049 return 0;
1050
1051out_free:
1052 __mptcp_pm_release_addr_entry(entry);
1053 return ret;
1054}
1055
1056static void mptcp_pm_remove_announced(struct mptcp_sock *msk,
1057 const struct mptcp_addr_info *addr,
1058 bool force)
1059{
1060 struct mptcp_rm_list list = { .nr = 0 };
1061 bool announced;
1062
1063 list.ids[list.nr++] = mptcp_endp_get_local_id(msk, addr);
1064
1065 announced = mptcp_pm_announced_remove(msk, addr);
1066 if (announced || force) {
1067 spin_lock_bh(&msk->pm.lock);
1068 if (announced)
1069 msk->pm.add_addr_signaled--;
1070 mptcp_pm_remove_addr(msk, &list);
1071 spin_unlock_bh(&msk->pm.lock);
1072 }
1073}
1074
1075static void __mark_subflow_endp_available(struct mptcp_sock *msk, u8 id)
1076{
1077 /* If it was marked as used, and not ID 0, decrement local_addr_used */
1078 if (!__test_and_set_bit(id ? : msk->mpc_endpoint_id, msk->pm.id_avail_bitmap) &&
1079 id && !WARN_ON_ONCE(msk->pm.local_addr_used == 0))
1080 msk->pm.local_addr_used--;
1081}
1082
1083static int mptcp_nl_remove_subflow_and_signal_addr(struct net *net,
1084 const struct mptcp_pm_addr_entry *entry)
1085{
1086 const struct mptcp_addr_info *addr = &entry->addr;
1087 struct mptcp_rm_list list = { .nr = 1 };
1088 long s_slot = 0, s_num = 0;
1089 struct mptcp_sock *msk;
1090
1091 pr_debug("remove_id=%d\n", addr->id);
1092
1093 while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
1094 struct sock *sk = (struct sock *)msk;
1095 bool remove_subflow;
1096
1097 if (mptcp_pm_is_userspace(msk))
1098 goto next;
1099
1100 lock_sock(sk);
1101 remove_subflow = mptcp_pm_has_subflow_saddr(msk, addr);
1102 mptcp_pm_remove_announced(msk, addr, remove_subflow &&
1103 !(entry->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT));
1104
1105 list.ids[0] = mptcp_endp_get_local_id(msk, addr);
1106
1107 spin_lock_bh(&msk->pm.lock);
1108 if (remove_subflow)
1109 mptcp_pm_rm_subflow(msk, &list);
1110 if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)
1111 __mark_subflow_endp_available(msk, list.ids[0]);
1112 else /* mark endp ID as available, e.g. Signal or MPC endp */
1113 __set_bit(addr->id, msk->pm.id_avail_bitmap);
1114 spin_unlock_bh(&msk->pm.lock);
1115
1116 if (msk->mpc_endpoint_id == entry->addr.id)
1117 msk->mpc_endpoint_id = 0;
1118 release_sock(sk);
1119
1120next:
1121 sock_put(sk);
1122 cond_resched();
1123 }
1124
1125 return 0;
1126}
1127
1128static int mptcp_nl_remove_id_zero_address(struct net *net,
1129 struct mptcp_addr_info *addr)
1130{
1131 struct mptcp_rm_list list = { .nr = 0 };
1132 long s_slot = 0, s_num = 0;
1133 struct mptcp_sock *msk;
1134
1135 list.ids[list.nr++] = 0;
1136
1137 while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
1138 struct sock *sk = (struct sock *)msk;
1139 struct mptcp_addr_info msk_local;
1140 struct mptcp_addr_info anno_addr;
1141 bool announced;
1142
1143 if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk))
1144 goto next;
1145
1146 mptcp_local_address((struct sock_common *)msk, &msk_local);
1147 if (!mptcp_addresses_equal(&msk_local, addr, addr->port))
1148 goto next;
1149
1150 lock_sock(sk);
1151 /* Drop a possibly pending ADD_ADDR for this address. */
1152 anno_addr = msk_local;
1153 anno_addr.port = 0;
1154 announced = mptcp_pm_announced_remove(msk, &anno_addr);
1155 spin_lock_bh(&msk->pm.lock);
1156 if (announced)
1157 msk->pm.add_addr_signaled--;
1158 mptcp_pm_remove_addr(msk, &list);
1159 mptcp_pm_rm_subflow(msk, &list);
1160 __mark_subflow_endp_available(msk, 0);
1161 spin_unlock_bh(&msk->pm.lock);
1162 release_sock(sk);
1163
1164next:
1165 sock_put(sk);
1166 cond_resched();
1167 }
1168
1169 return 0;
1170}
1171
1172/* Remove an MPTCP endpoint */
1173int mptcp_pm_nl_del_addr_doit(struct sk_buff *skb, struct genl_info *info)
1174{
1175 struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1176 struct mptcp_pm_addr_entry addr, *entry;
1177 struct nlattr *attr;
1178 u8 addr_max;
1179 int ret;
1180
1181 if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ENDPOINT_ADDR))
1182 return -EINVAL;
1183
1184 attr = info->attrs[MPTCP_PM_ENDPOINT_ADDR];
1185 ret = mptcp_pm_parse_entry(attr, info, false, &addr);
1186 if (ret < 0)
1187 return ret;
1188
1189 /* the zero id address is special: the first address used by the msk
1190 * always gets such an id, so different subflows can have different zero
1191 * id addresses. Additionally zero id is not accounted for in id_bitmap.
1192 * Let's use an 'mptcp_rm_list' instead of the common remove code.
1193 */
1194 if (addr.addr.id == 0)
1195 return mptcp_nl_remove_id_zero_address(sock_net(skb->sk), &addr.addr);
1196
1197 spin_lock_bh(&pernet->lock);
1198 entry = __lookup_addr_by_id(pernet, addr.addr.id);
1199 if (!entry) {
1200 NL_SET_ERR_MSG_ATTR(info->extack, attr, "address not found");
1201 spin_unlock_bh(&pernet->lock);
1202 return -EINVAL;
1203 }
1204 if (entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL) {
1205 addr_max = pernet->endp_signal_max;
1206 WRITE_ONCE(pernet->endp_signal_max, addr_max - 1);
1207 }
1208 if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) {
1209 addr_max = pernet->endp_subflow_max;
1210 WRITE_ONCE(pernet->endp_subflow_max, addr_max - 1);
1211 }
1212 if (entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR) {
1213 addr_max = pernet->endp_laminar_max;
1214 WRITE_ONCE(pernet->endp_laminar_max, addr_max - 1);
1215 }
1216 if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) {
1217 addr_max = pernet->endp_fullmesh_max;
1218 WRITE_ONCE(pernet->endp_fullmesh_max, addr_max - 1);
1219 }
1220
1221 pernet->endpoints--;
1222 list_del_rcu(&entry->list);
1223 __clear_bit(entry->addr.id, pernet->id_bitmap);
1224 spin_unlock_bh(&pernet->lock);
1225
1226 mptcp_nl_remove_subflow_and_signal_addr(sock_net(skb->sk), entry);
1227 synchronize_rcu();
1228 __mptcp_pm_release_addr_entry(entry);
1229
1230 return ret;
1231}
1232
1233static void mptcp_pm_flush_addrs_and_subflows(struct mptcp_sock *msk,
1234 struct list_head *rm_list,
1235 struct mptcp_pm_addr_entry *entry)
1236{
1237 struct mptcp_rm_list alist, slist;
1238 bool more;
1239
1240again:
1241 alist.nr = 0;
1242 slist.nr = 0;
1243 more = false;
1244
1245 entry = list_prepare_entry(entry, rm_list, list);
1246 list_for_each_entry_continue(entry, rm_list, list) {
1247 if (mptcp_pm_has_subflow_saddr(msk, &entry->addr))
1248 slist.ids[slist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
1249
1250 if (mptcp_pm_announced_remove(msk, &entry->addr))
1251 alist.ids[alist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
1252
1253 if (slist.nr == MPTCP_RM_IDS_MAX ||
1254 alist.nr == MPTCP_RM_IDS_MAX) {
1255 more = !list_is_last(&entry->list, rm_list);
1256 break;
1257 }
1258 }
1259
1260 spin_lock_bh(&msk->pm.lock);
1261 if (alist.nr) {
1262 msk->pm.add_addr_signaled -= alist.nr;
1263 mptcp_pm_remove_addr(msk, &alist);
1264 }
1265 if (slist.nr)
1266 mptcp_pm_rm_subflow(msk, &slist);
1267 /* Reset counters: maybe some subflows have been removed before */
1268 if (!more) {
1269 bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
1270 msk->pm.local_addr_used = 0;
1271 }
1272 spin_unlock_bh(&msk->pm.lock);
1273
1274 if (more)
1275 goto again;
1276}
1277
1278static void mptcp_nl_flush_addrs_list(struct net *net,
1279 struct list_head *rm_list)
1280{
1281 long s_slot = 0, s_num = 0;
1282 struct mptcp_sock *msk;
1283
1284 if (list_empty(rm_list))
1285 return;
1286
1287 while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
1288 struct sock *sk = (struct sock *)msk;
1289
1290 if (!mptcp_pm_is_userspace(msk)) {
1291 lock_sock(sk);
1292 mptcp_pm_flush_addrs_and_subflows(msk, rm_list, NULL);
1293 release_sock(sk);
1294 }
1295
1296 sock_put(sk);
1297 cond_resched();
1298 }
1299}
1300
1301/* caller must ensure the RCU grace period is already elapsed */
1302static void __flush_addrs(struct list_head *list)
1303{
1304 while (!list_empty(list)) {
1305 struct mptcp_pm_addr_entry *cur;
1306
1307 cur = list_entry(list->next,
1308 struct mptcp_pm_addr_entry, list);
1309 list_del_rcu(&cur->list);
1310 __mptcp_pm_release_addr_entry(cur);
1311 }
1312}
1313
1314static void __reset_counters(struct pm_nl_pernet *pernet)
1315{
1316 WRITE_ONCE(pernet->endp_signal_max, 0);
1317 WRITE_ONCE(pernet->endp_subflow_max, 0);
1318 WRITE_ONCE(pernet->endp_laminar_max, 0);
1319 WRITE_ONCE(pernet->endp_fullmesh_max, 0);
1320 pernet->endpoints = 0;
1321}
1322
1323int mptcp_pm_nl_flush_addrs_doit(struct sk_buff *skb, struct genl_info *info)
1324{
1325 struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1326 struct list_head free_list;
1327
1328 spin_lock_bh(&pernet->lock);
1329 free_list = pernet->endp_list;
1330 INIT_LIST_HEAD_RCU(&pernet->endp_list);
1331 __reset_counters(pernet);
1332 pernet->next_id = 1;
1333 bitmap_zero(pernet->id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
1334 spin_unlock_bh(&pernet->lock);
1335
1336 if (free_list.next == &pernet->endp_list)
1337 return 0;
1338
1339 synchronize_rcu();
1340
1341 /* Adjust the pointers to free_list instead of pernet->endp_list */
1342 free_list.prev->next = &free_list;
1343 free_list.next->prev = &free_list;
1344
1345 mptcp_nl_flush_addrs_list(sock_net(skb->sk), &free_list);
1346 __flush_addrs(&free_list);
1347 return 0;
1348}
1349
1350int mptcp_pm_nl_get_addr(u8 id, struct mptcp_pm_addr_entry *addr,
1351 struct genl_info *info)
1352{
1353 struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1354 struct mptcp_pm_addr_entry *entry;
1355 int ret = -EINVAL;
1356
1357 rcu_read_lock();
1358 entry = __lookup_addr_by_id(pernet, id);
1359 if (entry) {
1360 *addr = *entry;
1361 ret = 0;
1362 }
1363 rcu_read_unlock();
1364
1365 return ret;
1366}
1367
1368int mptcp_pm_nl_dump_addr(struct sk_buff *msg,
1369 struct netlink_callback *cb)
1370{
1371 struct net *net = sock_net(msg->sk);
1372 struct mptcp_pm_addr_entry *entry;
1373 struct pm_nl_pernet *pernet;
1374 int id = cb->args[0];
1375 int i;
1376
1377 pernet = pm_nl_get_pernet(net);
1378
1379 rcu_read_lock();
1380 for (i = id; i < MPTCP_PM_MAX_ADDR_ID + 1; i++) {
1381 if (test_bit(i, pernet->id_bitmap)) {
1382 entry = __lookup_addr_by_id(pernet, i);
1383 if (!entry)
1384 break;
1385
1386 if (entry->addr.id <= id)
1387 continue;
1388
1389 if (mptcp_pm_genl_fill_addr(msg, cb, entry) < 0)
1390 break;
1391
1392 id = entry->addr.id;
1393 }
1394 }
1395 rcu_read_unlock();
1396
1397 cb->args[0] = id;
1398 return msg->len;
1399}
1400
1401static int parse_limit(struct genl_info *info, int id, unsigned int *limit)
1402{
1403 struct nlattr *attr = info->attrs[id];
1404
1405 if (!attr)
1406 return 0;
1407
1408 *limit = nla_get_u32(attr);
1409 if (*limit > MPTCP_PM_SUBFLOWS_MAX) {
1410 NL_SET_ERR_MSG_ATTR_FMT(info->extack, attr,
1411 "limit greater than maximum (%u)",
1412 MPTCP_PM_SUBFLOWS_MAX);
1413 return -EINVAL;
1414 }
1415 return 0;
1416}
1417
1418int mptcp_pm_nl_set_limits_doit(struct sk_buff *skb, struct genl_info *info)
1419{
1420 struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1421 unsigned int rcv_addrs, subflows;
1422 int ret;
1423
1424 spin_lock_bh(&pernet->lock);
1425 rcv_addrs = pernet->limit_add_addr_accepted;
1426 ret = parse_limit(info, MPTCP_PM_ATTR_RCV_ADD_ADDRS, &rcv_addrs);
1427 if (ret)
1428 goto unlock;
1429
1430 subflows = pernet->limit_extra_subflows;
1431 ret = parse_limit(info, MPTCP_PM_ATTR_SUBFLOWS, &subflows);
1432 if (ret)
1433 goto unlock;
1434
1435 WRITE_ONCE(pernet->limit_add_addr_accepted, rcv_addrs);
1436 WRITE_ONCE(pernet->limit_extra_subflows, subflows);
1437
1438unlock:
1439 spin_unlock_bh(&pernet->lock);
1440 return ret;
1441}
1442
1443int mptcp_pm_nl_get_limits_doit(struct sk_buff *skb, struct genl_info *info)
1444{
1445 struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1446 struct sk_buff *msg;
1447 void *reply;
1448
1449 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1450 if (!msg)
1451 return -ENOMEM;
1452
1453 reply = genlmsg_put_reply(msg, info, &mptcp_genl_family, 0,
1454 MPTCP_PM_CMD_GET_LIMITS);
1455 if (!reply)
1456 goto fail;
1457
1458 if (nla_put_u32(msg, MPTCP_PM_ATTR_RCV_ADD_ADDRS,
1459 READ_ONCE(pernet->limit_add_addr_accepted)))
1460 goto fail;
1461
1462 if (nla_put_u32(msg, MPTCP_PM_ATTR_SUBFLOWS,
1463 READ_ONCE(pernet->limit_extra_subflows)))
1464 goto fail;
1465
1466 genlmsg_end(msg, reply);
1467 return genlmsg_reply(msg, info);
1468
1469fail:
1470 GENL_SET_ERR_MSG(info, "not enough space in Netlink message");
1471 nlmsg_free(msg);
1472 return -EMSGSIZE;
1473}
1474
1475static void mptcp_pm_nl_fullmesh(struct mptcp_sock *msk,
1476 struct mptcp_addr_info *addr)
1477{
1478 struct mptcp_rm_list list = { .nr = 0 };
1479
1480 list.ids[list.nr++] = mptcp_endp_get_local_id(msk, addr);
1481
1482 spin_lock_bh(&msk->pm.lock);
1483 mptcp_pm_rm_subflow(msk, &list);
1484 __mark_subflow_endp_available(msk, list.ids[0]);
1485 mptcp_pm_create_subflow_or_signal_addr(msk);
1486 spin_unlock_bh(&msk->pm.lock);
1487}
1488
1489static void mptcp_pm_nl_set_flags_all(struct net *net,
1490 struct mptcp_pm_addr_entry *local,
1491 u8 changed)
1492{
1493 u8 is_subflow = !!(local->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW);
1494 u8 bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
1495 long s_slot = 0, s_num = 0;
1496 struct mptcp_sock *msk;
1497
1498 if (changed == MPTCP_PM_ADDR_FLAG_FULLMESH && !is_subflow)
1499 return;
1500
1501 while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
1502 struct sock *sk = (struct sock *)msk;
1503
1504 if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk))
1505 goto next;
1506
1507 lock_sock(sk);
1508 if (changed & MPTCP_PM_ADDR_FLAG_BACKUP)
1509 mptcp_pm_mp_prio_send_ack(msk, &local->addr, NULL, bkup);
1510 /* Subflows will only be recreated if the SUBFLOW flag is set */
1511 if (is_subflow && (changed & MPTCP_PM_ADDR_FLAG_FULLMESH))
1512 mptcp_pm_nl_fullmesh(msk, &local->addr);
1513 release_sock(sk);
1514
1515next:
1516 sock_put(sk);
1517 cond_resched();
1518 }
1519}
1520
1521int mptcp_pm_nl_set_flags(struct mptcp_pm_addr_entry *local,
1522 struct genl_info *info)
1523{
1524 struct nlattr *attr = info->attrs[MPTCP_PM_ATTR_ADDR];
1525 u8 changed, mask = MPTCP_PM_ADDR_FLAG_BACKUP |
1526 MPTCP_PM_ADDR_FLAG_FULLMESH;
1527 struct net *net = genl_info_net(info);
1528 struct mptcp_pm_addr_entry *entry;
1529 struct pm_nl_pernet *pernet;
1530 u8 lookup_by_id = 0;
1531
1532 pernet = pm_nl_get_pernet(net);
1533
1534 if (local->addr.family == AF_UNSPEC) {
1535 lookup_by_id = 1;
1536 if (!local->addr.id) {
1537 NL_SET_ERR_MSG_ATTR(info->extack, attr,
1538 "missing address ID");
1539 return -EOPNOTSUPP;
1540 }
1541 }
1542
1543 spin_lock_bh(&pernet->lock);
1544 entry = lookup_by_id ? __lookup_addr_by_id(pernet, local->addr.id) :
1545 __lookup_addr(pernet, &local->addr);
1546 if (!entry) {
1547 spin_unlock_bh(&pernet->lock);
1548 NL_SET_ERR_MSG_ATTR(info->extack, attr, "address not found");
1549 return -EINVAL;
1550 }
1551 if ((local->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) &&
1552 (entry->flags & (MPTCP_PM_ADDR_FLAG_SIGNAL |
1553 MPTCP_PM_ADDR_FLAG_IMPLICIT))) {
1554 spin_unlock_bh(&pernet->lock);
1555 NL_SET_ERR_MSG_ATTR(info->extack, attr, "invalid addr flags");
1556 return -EINVAL;
1557 }
1558
1559 changed = (local->flags ^ entry->flags) & mask;
1560 entry->flags = (entry->flags & ~mask) | (local->flags & mask);
1561 *local = *entry;
1562
1563 if (changed & MPTCP_PM_ADDR_FLAG_FULLMESH) {
1564 u8 addr_max = pernet->endp_fullmesh_max;
1565
1566 if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH)
1567 addr_max++;
1568 else
1569 addr_max--;
1570
1571 WRITE_ONCE(pernet->endp_fullmesh_max, addr_max);
1572 }
1573
1574 spin_unlock_bh(&pernet->lock);
1575
1576 mptcp_pm_nl_set_flags_all(net, local, changed);
1577 return 0;
1578}
1579
1580bool mptcp_pm_nl_check_work_pending(struct mptcp_sock *msk)
1581{
1582 struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
1583
1584 if (msk->pm.extra_subflows == mptcp_pm_get_limit_extra_subflows(msk) ||
1585 (find_next_and_bit(pernet->id_bitmap, msk->pm.id_avail_bitmap,
1586 MPTCP_PM_MAX_ADDR_ID + 1, 0) == MPTCP_PM_MAX_ADDR_ID + 1)) {
1587 WRITE_ONCE(msk->pm.work_pending, false);
1588 return false;
1589 }
1590 return true;
1591}
1592
1593/* Called under PM lock */
1594void __mptcp_pm_kernel_worker(struct mptcp_sock *msk)
1595{
1596 struct mptcp_pm_data *pm = &msk->pm;
1597
1598 if (pm->status & BIT(MPTCP_PM_ADD_ADDR_RECEIVED)) {
1599 pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_RECEIVED);
1600 mptcp_pm_nl_add_addr_received(msk);
1601 }
1602 if (pm->status & BIT(MPTCP_PM_ESTABLISHED)) {
1603 pm->status &= ~BIT(MPTCP_PM_ESTABLISHED);
1604 mptcp_pm_nl_fully_established(msk);
1605 }
1606 if (pm->status & BIT(MPTCP_PM_SUBFLOW_ESTABLISHED)) {
1607 pm->status &= ~BIT(MPTCP_PM_SUBFLOW_ESTABLISHED);
1608 mptcp_pm_nl_subflow_established(msk);
1609 }
1610}
1611
1612static int __net_init pm_nl_init_net(struct net *net)
1613{
1614 struct pm_nl_pernet *pernet = pm_nl_get_pernet(net);
1615
1616 INIT_LIST_HEAD_RCU(&pernet->endp_list);
1617
1618 /* Cit. 2 subflows ought to be enough for anybody. */
1619 pernet->limit_extra_subflows = 2;
1620 pernet->next_id = 1;
1621 spin_lock_init(&pernet->lock);
1622
1623 /* No need to initialize other pernet fields, the struct is zeroed at
1624 * allocation time.
1625 */
1626
1627 return 0;
1628}
1629
1630static void __net_exit pm_nl_exit_net(struct list_head *net_list)
1631{
1632 struct net *net;
1633
1634 list_for_each_entry(net, net_list, exit_list) {
1635 struct pm_nl_pernet *pernet = pm_nl_get_pernet(net);
1636
1637 /* net is removed from namespace list, can't race with
1638 * other modifiers, also netns core already waited for a
1639 * RCU grace period.
1640 */
1641 __flush_addrs(&pernet->endp_list);
1642 }
1643}
1644
1645static struct pernet_operations mptcp_pm_pernet_ops = {
1646 .init = pm_nl_init_net,
1647 .exit_batch = pm_nl_exit_net,
1648 .id = &pm_nl_pernet_id,
1649 .size = sizeof(struct pm_nl_pernet),
1650};
1651
1652struct mptcp_pm_ops mptcp_pm_kernel = {
1653 .name = "kernel",
1654 .owner = THIS_MODULE,
1655};
1656
1657void __init mptcp_pm_kernel_register(void)
1658{
1659 if (register_pernet_subsys(&mptcp_pm_pernet_ops) < 0)
1660 panic("Failed to register MPTCP PM pernet subsystem.\n");
1661
1662 mptcp_pm_register(&mptcp_pm_kernel);
1663}