projects/linux·blob master · refs

940 lines · 21.0 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * ip6_flowlabel.c IPv6 flowlabel manager.
4 *
5 * Authors: Alexey Kuznetsov, <[email protected]>
6 */
7
8#include <linux/capability.h>
9#include <linux/errno.h>
10#include <linux/types.h>
11#include <linux/socket.h>
12#include <linux/net.h>
13#include <linux/netdevice.h>
14#include <linux/in6.h>
15#include <linux/proc_fs.h>
16#include <linux/seq_file.h>
17#include <linux/slab.h>
18#include <linux/export.h>
19#include <linux/pid_namespace.h>
20#include <linux/jump_label_ratelimit.h>
21
22#include <net/net_namespace.h>
23#include <net/sock.h>
24
25#include <net/ipv6.h>
26#include <net/rawv6.h>
27#include <net/transp_v6.h>
28
29#include <linux/uaccess.h>
30
31#define FL_MIN_LINGER 6 /* Minimal linger. It is set to 6sec specified
32 in old IPv6 RFC. Well, it was reasonable value.
33 */
34#define FL_MAX_LINGER 150 /* Maximal linger timeout */
35
36/* FL hash table */
37
38#define FL_MAX_PER_SOCK 32
39#define FL_MAX_SIZE 8192
40#define FL_HASH_MASK 255
41#define FL_HASH(l) (ntohl(l)&FL_HASH_MASK)
42
43static int fl_size;
44static struct ip6_flowlabel __rcu *fl_ht[FL_HASH_MASK+1];
45
46static void ip6_fl_gc(struct timer_list *unused);
47static DEFINE_TIMER(ip6_fl_gc_timer, ip6_fl_gc);
48
49/* FL hash table lock: it protects only of GC */
50
51static DEFINE_SPINLOCK(ip6_fl_lock);
52
53/* Big socket sock */
54
55static DEFINE_SPINLOCK(ip6_sk_fl_lock);
56
57DEFINE_STATIC_KEY_DEFERRED_FALSE(ipv6_flowlabel_exclusive, HZ);
58EXPORT_SYMBOL(ipv6_flowlabel_exclusive);
59
60#define for_each_fl_rcu(hash, fl) \
61 for (fl = rcu_dereference(fl_ht[(hash)]); \
62 fl != NULL; \
63 fl = rcu_dereference(fl->next))
64#define for_each_fl_continue_rcu(fl) \
65 for (fl = rcu_dereference(fl->next); \
66 fl != NULL; \
67 fl = rcu_dereference(fl->next))
68
69#define for_each_sk_fl_rcu(sk, sfl) \
70 for (sfl = rcu_dereference(inet_sk(sk)->ipv6_fl_list); \
71 sfl != NULL; \
72 sfl = rcu_dereference(sfl->next))
73
74static inline struct ip6_flowlabel *__fl_lookup(struct net *net, __be32 label)
75{
76 struct ip6_flowlabel *fl;
77
78 for_each_fl_rcu(FL_HASH(label), fl) {
79 if (fl->label == label && net_eq(fl->fl_net, net))
80 return fl;
81 }
82 return NULL;
83}
84
85static struct ip6_flowlabel *fl_lookup(struct net *net, __be32 label)
86{
87 struct ip6_flowlabel *fl;
88
89 rcu_read_lock();
90 fl = __fl_lookup(net, label);
91 if (fl && !atomic_inc_not_zero(&fl->users))
92 fl = NULL;
93 rcu_read_unlock();
94 return fl;
95}
96
97static bool fl_shared_exclusive(struct ip6_flowlabel *fl)
98{
99 return fl->share == IPV6_FL_S_EXCL ||
100 fl->share == IPV6_FL_S_PROCESS ||
101 fl->share == IPV6_FL_S_USER;
102}
103
104static void fl_free_rcu(struct rcu_head *head)
105{
106 struct ip6_flowlabel *fl = container_of(head, struct ip6_flowlabel, rcu);
107
108 if (fl->share == IPV6_FL_S_PROCESS)
109 put_pid(fl->owner.pid);
110 kfree(fl->opt);
111 kfree(fl);
112}
113
114
115static void fl_free(struct ip6_flowlabel *fl)
116{
117 if (!fl)
118 return;
119
120 if (fl_shared_exclusive(fl) || fl->opt)
121 static_branch_slow_dec_deferred(&ipv6_flowlabel_exclusive);
122
123 call_rcu(&fl->rcu, fl_free_rcu);
124}
125
126static void fl_release(struct ip6_flowlabel *fl)
127{
128 spin_lock_bh(&ip6_fl_lock);
129
130 fl->lastuse = jiffies;
131 if (atomic_dec_and_test(&fl->users)) {
132 unsigned long ttd = fl->lastuse + fl->linger;
133 if (time_after(ttd, fl->expires))
134 fl->expires = ttd;
135 ttd = fl->expires;
136 if (!timer_pending(&ip6_fl_gc_timer) ||
137 time_after(ip6_fl_gc_timer.expires, ttd))
138 mod_timer(&ip6_fl_gc_timer, ttd);
139 }
140 spin_unlock_bh(&ip6_fl_lock);
141}
142
143static void ip6_fl_gc(struct timer_list *unused)
144{
145 int i;
146 unsigned long now = jiffies;
147 unsigned long sched = 0;
148
149 spin_lock(&ip6_fl_lock);
150
151 for (i = 0; i <= FL_HASH_MASK; i++) {
152 struct ip6_flowlabel *fl;
153 struct ip6_flowlabel __rcu **flp;
154
155 flp = &fl_ht[i];
156 while ((fl = rcu_dereference_protected(*flp,
157 lockdep_is_held(&ip6_fl_lock))) != NULL) {
158 if (atomic_read(&fl->users) == 0) {
159 unsigned long ttd = fl->lastuse + fl->linger;
160 if (time_after(ttd, fl->expires))
161 fl->expires = ttd;
162 ttd = fl->expires;
163 if (time_after_eq(now, ttd)) {
164 *flp = fl->next;
165 fl_size--;
166 fl->fl_net->ipv6.flowlabel_count--;
167 fl_free(fl);
168 continue;
169 }
170 if (!sched || time_before(ttd, sched))
171 sched = ttd;
172 }
173 flp = &fl->next;
174 }
175 }
176 if (!sched && fl_size)
177 sched = now + FL_MAX_LINGER;
178 if (sched) {
179 mod_timer(&ip6_fl_gc_timer, sched);
180 }
181 spin_unlock(&ip6_fl_lock);
182}
183
184static void __net_exit ip6_fl_purge(struct net *net)
185{
186 int i;
187
188 spin_lock_bh(&ip6_fl_lock);
189 for (i = 0; i <= FL_HASH_MASK; i++) {
190 struct ip6_flowlabel *fl;
191 struct ip6_flowlabel __rcu **flp;
192
193 flp = &fl_ht[i];
194 while ((fl = rcu_dereference_protected(*flp,
195 lockdep_is_held(&ip6_fl_lock))) != NULL) {
196 if (net_eq(fl->fl_net, net) &&
197 atomic_read(&fl->users) == 0) {
198 *flp = fl->next;
199 fl_free(fl);
200 fl_size--;
201 net->ipv6.flowlabel_count--;
202 continue;
203 }
204 flp = &fl->next;
205 }
206 }
207 spin_unlock_bh(&ip6_fl_lock);
208}
209
210static struct ip6_flowlabel *fl_intern(struct net *net,
211 struct ip6_flowlabel *fl, __be32 label)
212{
213 struct ip6_flowlabel *lfl;
214
215 lockdep_assert_held(&ip6_fl_lock);
216
217 fl->label = label & IPV6_FLOWLABEL_MASK;
218
219 if (label == 0) {
220 for (;;) {
221 fl->label = htonl(get_random_u32())&IPV6_FLOWLABEL_MASK;
222 if (fl->label) {
223 lfl = __fl_lookup(net, fl->label);
224 if (!lfl)
225 break;
226 }
227 }
228 } else {
229 /*
230 * we dropper the ip6_fl_lock, so this entry could reappear
231 * and we need to recheck with it.
232 *
233 * OTOH no need to search the active socket first, like it is
234 * done in ipv6_flowlabel_opt - sock is locked, so new entry
235 * with the same label can only appear on another sock
236 */
237 lfl = __fl_lookup(net, fl->label);
238 if (lfl) {
239 atomic_inc(&lfl->users);
240 return lfl;
241 }
242 }
243
244 fl->lastuse = jiffies;
245 fl->next = fl_ht[FL_HASH(fl->label)];
246 rcu_assign_pointer(fl_ht[FL_HASH(fl->label)], fl);
247 fl_size++;
248 net->ipv6.flowlabel_count++;
249 return NULL;
250}
251
252
253
254/* Socket flowlabel lists */
255
256struct ip6_flowlabel *__fl6_sock_lookup(struct sock *sk, __be32 label)
257{
258 struct ipv6_fl_socklist *sfl;
259
260 label &= IPV6_FLOWLABEL_MASK;
261
262 rcu_read_lock();
263 for_each_sk_fl_rcu(sk, sfl) {
264 struct ip6_flowlabel *fl = sfl->fl;
265
266 if (fl->label == label && atomic_inc_not_zero(&fl->users)) {
267 fl->lastuse = jiffies;
268 rcu_read_unlock();
269 return fl;
270 }
271 }
272 rcu_read_unlock();
273 return NULL;
274}
275EXPORT_SYMBOL_GPL(__fl6_sock_lookup);
276
277void fl6_free_socklist(struct sock *sk)
278{
279 struct inet_sock *inet = inet_sk(sk);
280 struct ipv6_fl_socklist *sfl;
281
282 if (!rcu_access_pointer(inet->ipv6_fl_list))
283 return;
284
285 spin_lock_bh(&ip6_sk_fl_lock);
286 while ((sfl = rcu_dereference_protected(inet->ipv6_fl_list,
287 lockdep_is_held(&ip6_sk_fl_lock))) != NULL) {
288 inet->ipv6_fl_list = sfl->next;
289 spin_unlock_bh(&ip6_sk_fl_lock);
290
291 fl_release(sfl->fl);
292 kfree_rcu(sfl, rcu);
293
294 spin_lock_bh(&ip6_sk_fl_lock);
295 }
296 spin_unlock_bh(&ip6_sk_fl_lock);
297}
298
299/* Service routines */
300
301
302/*
303 It is the only difficult place. flowlabel enforces equal headers
304 before and including routing header, however user may supply options
305 following rthdr.
306 */
307
308struct ipv6_txoptions *fl6_merge_options(struct ipv6_txoptions *opt_space,
309 struct ip6_flowlabel *fl,
310 struct ipv6_txoptions *fopt)
311{
312 struct ipv6_txoptions *fl_opt = fl->opt;
313
314 if (!fopt || fopt->opt_flen == 0)
315 return fl_opt;
316
317 if (fl_opt) {
318 opt_space->hopopt = fl_opt->hopopt;
319 opt_space->dst0opt = fl_opt->dst0opt;
320 opt_space->srcrt = fl_opt->srcrt;
321 opt_space->opt_nflen = fl_opt->opt_nflen;
322 } else {
323 if (fopt->opt_nflen == 0)
324 return fopt;
325 opt_space->hopopt = NULL;
326 opt_space->dst0opt = NULL;
327 opt_space->srcrt = NULL;
328 opt_space->opt_nflen = 0;
329 }
330 opt_space->dst1opt = fopt->dst1opt;
331 opt_space->opt_flen = fopt->opt_flen;
332 opt_space->tot_len = fopt->tot_len;
333 return opt_space;
334}
335EXPORT_SYMBOL_GPL(fl6_merge_options);
336
337static unsigned long check_linger(unsigned long ttl)
338{
339 if (ttl < FL_MIN_LINGER)
340 return FL_MIN_LINGER*HZ;
341 if (ttl > FL_MAX_LINGER && !capable(CAP_NET_ADMIN))
342 return 0;
343 return ttl*HZ;
344}
345
346static int fl6_renew(struct ip6_flowlabel *fl, unsigned long linger, unsigned long expires)
347{
348 linger = check_linger(linger);
349 if (!linger)
350 return -EPERM;
351 expires = check_linger(expires);
352 if (!expires)
353 return -EPERM;
354
355 spin_lock_bh(&ip6_fl_lock);
356 fl->lastuse = jiffies;
357 if (time_before(fl->linger, linger))
358 fl->linger = linger;
359 if (time_before(expires, fl->linger))
360 expires = fl->linger;
361 if (time_before(fl->expires, fl->lastuse + expires))
362 fl->expires = fl->lastuse + expires;
363 spin_unlock_bh(&ip6_fl_lock);
364
365 return 0;
366}
367
368static struct ip6_flowlabel *
369fl_create(struct net *net, struct sock *sk, struct in6_flowlabel_req *freq,
370 sockptr_t optval, int optlen, int *err_p)
371{
372 struct ip6_flowlabel *fl = NULL;
373 int olen;
374 int addr_type;
375 int err;
376
377 olen = optlen - CMSG_ALIGN(sizeof(*freq));
378 err = -EINVAL;
379 if (olen > 64 * 1024)
380 goto done;
381
382 err = -ENOMEM;
383 fl = kzalloc_obj(*fl);
384 if (!fl)
385 goto done;
386
387 if (olen > 0) {
388 struct msghdr msg;
389 struct flowi6 flowi6;
390 struct ipcm6_cookie ipc6;
391
392 err = -ENOMEM;
393 fl->opt = kmalloc(sizeof(*fl->opt) + olen, GFP_KERNEL);
394 if (!fl->opt)
395 goto done;
396
397 memset(fl->opt, 0, sizeof(*fl->opt));
398 fl->opt->tot_len = sizeof(*fl->opt) + olen;
399 err = -EFAULT;
400 if (copy_from_sockptr_offset(fl->opt + 1, optval,
401 CMSG_ALIGN(sizeof(*freq)), olen))
402 goto done;
403
404 msg.msg_controllen = olen;
405 msg.msg_control = (void *)(fl->opt+1);
406 memset(&flowi6, 0, sizeof(flowi6));
407
408 ipc6.opt = fl->opt;
409 err = ip6_datagram_send_ctl(net, sk, &msg, &flowi6, &ipc6);
410 if (err)
411 goto done;
412 err = -EINVAL;
413 if (fl->opt->opt_flen)
414 goto done;
415 if (fl->opt->opt_nflen == 0) {
416 kfree(fl->opt);
417 fl->opt = NULL;
418 }
419 }
420
421 fl->fl_net = net;
422 fl->expires = jiffies;
423 err = fl6_renew(fl, freq->flr_linger, freq->flr_expires);
424 if (err)
425 goto done;
426 fl->share = freq->flr_share;
427 addr_type = ipv6_addr_type(&freq->flr_dst);
428 if ((addr_type & IPV6_ADDR_MAPPED) ||
429 addr_type == IPV6_ADDR_ANY) {
430 err = -EINVAL;
431 goto done;
432 }
433 fl->dst = freq->flr_dst;
434 atomic_set(&fl->users, 1);
435 switch (fl->share) {
436 case IPV6_FL_S_EXCL:
437 case IPV6_FL_S_ANY:
438 break;
439 case IPV6_FL_S_PROCESS:
440 fl->owner.pid = get_task_pid(current, PIDTYPE_PID);
441 break;
442 case IPV6_FL_S_USER:
443 fl->owner.uid = current_euid();
444 break;
445 default:
446 err = -EINVAL;
447 goto done;
448 }
449 if (fl_shared_exclusive(fl) || fl->opt) {
450 WRITE_ONCE(sock_net(sk)->ipv6.flowlabel_has_excl, 1);
451 static_branch_deferred_inc(&ipv6_flowlabel_exclusive);
452 }
453 return fl;
454
455done:
456 if (fl) {
457 kfree(fl->opt);
458 kfree(fl);
459 }
460 *err_p = err;
461 return NULL;
462}
463
464static bool fl_sock_at_lease_limit(const struct sock *sk)
465{
466 const struct ipv6_fl_socklist *sfl;
467 int count = 0;
468
469 rcu_read_lock();
470 for_each_sk_fl_rcu(sk, sfl) {
471 if (++count >= FL_MAX_PER_SOCK)
472 break;
473 }
474 rcu_read_unlock();
475
476 return count >= FL_MAX_PER_SOCK;
477}
478
479static int mem_check(struct sock *sk)
480{
481 const int unpriv_total_limit = FL_MAX_SIZE - (FL_MAX_SIZE / 4);
482 const int unpriv_user_limit = unpriv_total_limit / 2;
483 struct net *net = sock_net(sk);
484 int room;
485 struct ipv6_fl_socklist *sfl;
486 int count = 0;
487
488 lockdep_assert_held(&ip6_fl_lock);
489
490 room = FL_MAX_SIZE - fl_size;
491
492 if (room > FL_MAX_SIZE - FL_MAX_PER_SOCK)
493 return 0;
494
495 rcu_read_lock();
496 for_each_sk_fl_rcu(sk, sfl)
497 count++;
498 rcu_read_unlock();
499
500 if (room <= 0 ||
501 ((count >= FL_MAX_PER_SOCK ||
502 (count > 0 && room < FL_MAX_SIZE / 2) ||
503 room < FL_MAX_SIZE / 4 ||
504 net->ipv6.flowlabel_count >= unpriv_user_limit) &&
505 !capable(CAP_NET_ADMIN)))
506 return -ENOBUFS;
507
508 return 0;
509}
510
511static inline void fl_link(struct sock *sk, struct ipv6_fl_socklist *sfl,
512 struct ip6_flowlabel *fl)
513{
514 struct inet_sock *inet = inet_sk(sk);
515
516 spin_lock_bh(&ip6_sk_fl_lock);
517 sfl->fl = fl;
518 sfl->next = inet->ipv6_fl_list;
519 rcu_assign_pointer(inet->ipv6_fl_list, sfl);
520 spin_unlock_bh(&ip6_sk_fl_lock);
521}
522
523int ipv6_flowlabel_opt_get(struct sock *sk, struct in6_flowlabel_req *freq,
524 int flags)
525{
526 struct ipv6_pinfo *np = inet6_sk(sk);
527 struct ipv6_fl_socklist *sfl;
528
529 if (flags & IPV6_FL_F_REMOTE) {
530 freq->flr_label = np->rcv_flowinfo & IPV6_FLOWLABEL_MASK;
531 return 0;
532 }
533
534 if (inet6_test_bit(REPFLOW, sk)) {
535 freq->flr_label = np->flow_label;
536 return 0;
537 }
538
539 rcu_read_lock();
540
541 for_each_sk_fl_rcu(sk, sfl) {
542 if (sfl->fl->label == (np->flow_label & IPV6_FLOWLABEL_MASK)) {
543 spin_lock_bh(&ip6_fl_lock);
544 freq->flr_label = sfl->fl->label;
545 freq->flr_dst = sfl->fl->dst;
546 freq->flr_share = sfl->fl->share;
547 freq->flr_expires = (sfl->fl->expires - jiffies) / HZ;
548 freq->flr_linger = sfl->fl->linger / HZ;
549
550 spin_unlock_bh(&ip6_fl_lock);
551 rcu_read_unlock();
552 return 0;
553 }
554 }
555 rcu_read_unlock();
556
557 return -ENOENT;
558}
559
560#define socklist_dereference(__sflp) \
561 rcu_dereference_protected(__sflp, lockdep_is_held(&ip6_sk_fl_lock))
562
563static int ipv6_flowlabel_put(struct sock *sk, struct in6_flowlabel_req *freq)
564{
565 struct ipv6_pinfo *np = inet6_sk(sk);
566 struct ipv6_fl_socklist __rcu **sflp;
567 struct ipv6_fl_socklist *sfl;
568
569 if (freq->flr_flags & IPV6_FL_F_REFLECT) {
570 if (sk->sk_protocol != IPPROTO_TCP)
571 return -ENOPROTOOPT;
572 if (!inet6_test_bit(REPFLOW, sk))
573 return -ESRCH;
574 np->flow_label = 0;
575 inet6_clear_bit(REPFLOW, sk);
576 return 0;
577 }
578
579 spin_lock_bh(&ip6_sk_fl_lock);
580 for (sflp = &inet_sk(sk)->ipv6_fl_list;
581 (sfl = socklist_dereference(*sflp)) != NULL;
582 sflp = &sfl->next) {
583 if (sfl->fl->label == freq->flr_label)
584 goto found;
585 }
586 spin_unlock_bh(&ip6_sk_fl_lock);
587 return -ESRCH;
588found:
589 if (freq->flr_label == (np->flow_label & IPV6_FLOWLABEL_MASK))
590 np->flow_label &= ~IPV6_FLOWLABEL_MASK;
591 *sflp = sfl->next;
592 spin_unlock_bh(&ip6_sk_fl_lock);
593 fl_release(sfl->fl);
594 kfree_rcu(sfl, rcu);
595 return 0;
596}
597
598static int ipv6_flowlabel_renew(struct sock *sk, struct in6_flowlabel_req *freq)
599{
600 struct net *net = sock_net(sk);
601 struct ipv6_fl_socklist *sfl;
602 int err;
603
604 rcu_read_lock();
605 for_each_sk_fl_rcu(sk, sfl) {
606 if (sfl->fl->label == freq->flr_label) {
607 err = fl6_renew(sfl->fl, freq->flr_linger,
608 freq->flr_expires);
609 rcu_read_unlock();
610 return err;
611 }
612 }
613 rcu_read_unlock();
614
615 if (freq->flr_share == IPV6_FL_S_NONE &&
616 ns_capable(net->user_ns, CAP_NET_ADMIN)) {
617 struct ip6_flowlabel *fl = fl_lookup(net, freq->flr_label);
618
619 if (fl) {
620 err = fl6_renew(fl, freq->flr_linger,
621 freq->flr_expires);
622 fl_release(fl);
623 return err;
624 }
625 }
626 return -ESRCH;
627}
628
629static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq,
630 sockptr_t optval, int optlen)
631{
632 struct ipv6_fl_socklist *sfl, *sfl1 = NULL;
633 struct ip6_flowlabel *fl, *fl1 = NULL;
634 struct net *net = sock_net(sk);
635 int err;
636
637 if (freq->flr_flags & IPV6_FL_F_REFLECT) {
638 if (READ_ONCE(net->ipv6.sysctl.flowlabel_consistency)) {
639 net_info_ratelimited("Can not set IPV6_FL_F_REFLECT if flowlabel_consistency sysctl is enable\n");
640 return -EPERM;
641 }
642
643 if (sk->sk_protocol != IPPROTO_TCP)
644 return -ENOPROTOOPT;
645 inet6_set_bit(REPFLOW, sk);
646 return 0;
647 }
648
649 if (freq->flr_label & ~IPV6_FLOWLABEL_MASK)
650 return -EINVAL;
651 if (READ_ONCE(net->ipv6.sysctl.flowlabel_state_ranges) &&
652 (freq->flr_label & IPV6_FLOWLABEL_STATELESS_FLAG))
653 return -ERANGE;
654
655 fl = fl_create(net, sk, freq, optval, optlen, &err);
656 if (!fl)
657 return err;
658
659 sfl1 = kmalloc_obj(*sfl1);
660
661 if (freq->flr_label) {
662 err = -EEXIST;
663 rcu_read_lock();
664 for_each_sk_fl_rcu(sk, sfl) {
665 if (sfl->fl->label == freq->flr_label) {
666 if (freq->flr_flags & IPV6_FL_F_EXCL) {
667 rcu_read_unlock();
668 goto done;
669 }
670 fl1 = sfl->fl;
671 if (!atomic_inc_not_zero(&fl1->users))
672 fl1 = NULL;
673 break;
674 }
675 }
676 rcu_read_unlock();
677
678 if (!fl1)
679 fl1 = fl_lookup(net, freq->flr_label);
680 if (fl1) {
681recheck:
682 err = -EEXIST;
683 if (freq->flr_flags&IPV6_FL_F_EXCL)
684 goto release;
685 err = -EPERM;
686 if (fl1->share == IPV6_FL_S_EXCL ||
687 fl1->share != fl->share ||
688 ((fl1->share == IPV6_FL_S_PROCESS) &&
689 (fl1->owner.pid != fl->owner.pid)) ||
690 ((fl1->share == IPV6_FL_S_USER) &&
691 !uid_eq(fl1->owner.uid, fl->owner.uid)))
692 goto release;
693
694 err = -ENOMEM;
695 if (!sfl1)
696 goto release;
697 err = -ENOBUFS;
698 if (fl_sock_at_lease_limit(sk) &&
699 !capable(CAP_NET_ADMIN))
700 goto release;
701 if (fl->linger > fl1->linger)
702 fl1->linger = fl->linger;
703 if ((long)(fl->expires - fl1->expires) > 0)
704 fl1->expires = fl->expires;
705 fl_link(sk, sfl1, fl1);
706 fl_free(fl);
707 return 0;
708
709release:
710 fl_release(fl1);
711 goto done;
712 }
713 }
714 err = -ENOENT;
715 if (!(freq->flr_flags & IPV6_FL_F_CREATE))
716 goto done;
717
718 err = -ENOMEM;
719 if (!sfl1)
720 goto done;
721
722 rcu_read_lock();
723 spin_lock_bh(&ip6_fl_lock);
724 err = mem_check(sk);
725 if (err == 0)
726 fl1 = fl_intern(net, fl, freq->flr_label);
727 else
728 fl1 = NULL;
729 spin_unlock_bh(&ip6_fl_lock);
730 rcu_read_unlock();
731
732 if (err != 0)
733 goto done;
734
735 if (fl1)
736 goto recheck;
737
738 if (!freq->flr_label) {
739 size_t offset = offsetof(struct in6_flowlabel_req, flr_label);
740
741 if (copy_to_sockptr_offset(optval, offset, &fl->label,
742 sizeof(fl->label))) {
743 /* Intentionally ignore fault. */
744 }
745 }
746
747 fl_link(sk, sfl1, fl);
748 return 0;
749done:
750 fl_free(fl);
751 kfree(sfl1);
752 return err;
753}
754
755int ipv6_flowlabel_opt(struct sock *sk, sockptr_t optval, int optlen)
756{
757 struct in6_flowlabel_req freq;
758
759 if (optlen < sizeof(freq))
760 return -EINVAL;
761 if (copy_from_sockptr(&freq, optval, sizeof(freq)))
762 return -EFAULT;
763
764 switch (freq.flr_action) {
765 case IPV6_FL_A_PUT:
766 return ipv6_flowlabel_put(sk, &freq);
767 case IPV6_FL_A_RENEW:
768 return ipv6_flowlabel_renew(sk, &freq);
769 case IPV6_FL_A_GET:
770 return ipv6_flowlabel_get(sk, &freq, optval, optlen);
771 default:
772 return -EINVAL;
773 }
774}
775
776#ifdef CONFIG_PROC_FS
777
778struct ip6fl_iter_state {
779 struct seq_net_private p;
780 struct pid_namespace *pid_ns;
781 int bucket;
782};
783
784#define ip6fl_seq_private(seq) ((struct ip6fl_iter_state *)(seq)->private)
785
786static struct ip6_flowlabel *ip6fl_get_first(struct seq_file *seq)
787{
788 struct ip6_flowlabel *fl = NULL;
789 struct ip6fl_iter_state *state = ip6fl_seq_private(seq);
790 struct net *net = seq_file_net(seq);
791
792 for (state->bucket = 0; state->bucket <= FL_HASH_MASK; ++state->bucket) {
793 for_each_fl_rcu(state->bucket, fl) {
794 if (net_eq(fl->fl_net, net))
795 goto out;
796 }
797 }
798 fl = NULL;
799out:
800 return fl;
801}
802
803static struct ip6_flowlabel *ip6fl_get_next(struct seq_file *seq, struct ip6_flowlabel *fl)
804{
805 struct ip6fl_iter_state *state = ip6fl_seq_private(seq);
806 struct net *net = seq_file_net(seq);
807
808 for_each_fl_continue_rcu(fl) {
809 if (net_eq(fl->fl_net, net))
810 goto out;
811 }
812
813try_again:
814 if (++state->bucket <= FL_HASH_MASK) {
815 for_each_fl_rcu(state->bucket, fl) {
816 if (net_eq(fl->fl_net, net))
817 goto out;
818 }
819 goto try_again;
820 }
821 fl = NULL;
822
823out:
824 return fl;
825}
826
827static struct ip6_flowlabel *ip6fl_get_idx(struct seq_file *seq, loff_t pos)
828{
829 struct ip6_flowlabel *fl = ip6fl_get_first(seq);
830 if (fl)
831 while (pos && (fl = ip6fl_get_next(seq, fl)) != NULL)
832 --pos;
833 return pos ? NULL : fl;
834}
835
836static void *ip6fl_seq_start(struct seq_file *seq, loff_t *pos)
837 __acquires(RCU)
838{
839 struct ip6fl_iter_state *state = ip6fl_seq_private(seq);
840
841 state->pid_ns = proc_pid_ns(file_inode(seq->file)->i_sb);
842
843 rcu_read_lock();
844 return *pos ? ip6fl_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
845}
846
847static void *ip6fl_seq_next(struct seq_file *seq, void *v, loff_t *pos)
848{
849 struct ip6_flowlabel *fl;
850
851 if (v == SEQ_START_TOKEN)
852 fl = ip6fl_get_first(seq);
853 else
854 fl = ip6fl_get_next(seq, v);
855 ++*pos;
856 return fl;
857}
858
859static void ip6fl_seq_stop(struct seq_file *seq, void *v)
860 __releases(RCU)
861{
862 rcu_read_unlock();
863}
864
865static int ip6fl_seq_show(struct seq_file *seq, void *v)
866{
867 struct ip6fl_iter_state *state = ip6fl_seq_private(seq);
868 if (v == SEQ_START_TOKEN) {
869 seq_puts(seq, "Label S Owner Users Linger Expires Dst Opt\n");
870 } else {
871 struct ip6_flowlabel *fl = v;
872 seq_printf(seq,
873 "%05X %-1d %-6d %-6d %-6ld %-8ld %pi6 %-4d\n",
874 (unsigned int)ntohl(fl->label),
875 fl->share,
876 ((fl->share == IPV6_FL_S_PROCESS) ?
877 pid_nr_ns(fl->owner.pid, state->pid_ns) :
878 ((fl->share == IPV6_FL_S_USER) ?
879 from_kuid_munged(seq_user_ns(seq), fl->owner.uid) :
880 0)),
881 atomic_read(&fl->users),
882 fl->linger/HZ,
883 (long)(fl->expires - jiffies)/HZ,
884 &fl->dst,
885 fl->opt ? fl->opt->opt_nflen : 0);
886 }
887 return 0;
888}
889
890static const struct seq_operations ip6fl_seq_ops = {
891 .start = ip6fl_seq_start,
892 .next = ip6fl_seq_next,
893 .stop = ip6fl_seq_stop,
894 .show = ip6fl_seq_show,
895};
896
897static int __net_init ip6_flowlabel_proc_init(struct net *net)
898{
899 if (!proc_create_net("ip6_flowlabel", 0444, net->proc_net,
900 &ip6fl_seq_ops, sizeof(struct ip6fl_iter_state)))
901 return -ENOMEM;
902 return 0;
903}
904
905static void __net_exit ip6_flowlabel_proc_fini(struct net *net)
906{
907 remove_proc_entry("ip6_flowlabel", net->proc_net);
908}
909#else
910static inline int ip6_flowlabel_proc_init(struct net *net)
911{
912 return 0;
913}
914static inline void ip6_flowlabel_proc_fini(struct net *net)
915{
916}
917#endif
918
919static void __net_exit ip6_flowlabel_net_exit(struct net *net)
920{
921 ip6_fl_purge(net);
922 ip6_flowlabel_proc_fini(net);
923}
924
925static struct pernet_operations ip6_flowlabel_net_ops = {
926 .init = ip6_flowlabel_proc_init,
927 .exit = ip6_flowlabel_net_exit,
928};
929
930int ip6_flowlabel_init(void)
931{
932 return register_pernet_subsys(&ip6_flowlabel_net_ops);
933}
934
935void ip6_flowlabel_cleanup(void)
936{
937 static_key_deferred_flush(&ipv6_flowlabel_exclusive);
938 timer_delete(&ip6_fl_gc_timer);
939 unregister_pernet_subsys(&ip6_flowlabel_net_ops);
940}