projects/linux·blob master · refs

2821 lines · 73.7 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Functions to manage eBPF programs attached to cgroups
4 *
5 * Copyright (c) 2016 Daniel Mack
6 */
7
8#include <linux/kernel.h>
9#include <linux/atomic.h>
10#include <linux/cgroup.h>
11#include <linux/filter.h>
12#include <linux/slab.h>
13#include <linux/sysctl.h>
14#include <linux/string.h>
15#include <linux/bpf.h>
16#include <linux/bpf-cgroup.h>
17#include <linux/bpf_lsm.h>
18#include <linux/bpf_verifier.h>
19#include <net/sock.h>
20#include <net/bpf_sk_storage.h>
21
22#include "../cgroup/cgroup-internal.h"
23
24DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE);
25EXPORT_SYMBOL(cgroup_bpf_enabled_key);
26
27/*
28 * cgroup bpf destruction makes heavy use of work items and there can be a lot
29 * of concurrent destructions. Use a separate workqueue so that cgroup bpf
30 * destruction work items don't end up filling up max_active of system_percpu_wq
31 * which may lead to deadlock.
32 */
33static struct workqueue_struct *cgroup_bpf_destroy_wq;
34
35static int __init cgroup_bpf_wq_init(void)
36{
37 cgroup_bpf_destroy_wq = alloc_workqueue("cgroup_bpf_destroy",
38 WQ_PERCPU, 1);
39 if (!cgroup_bpf_destroy_wq)
40 panic("Failed to alloc workqueue for cgroup bpf destroy.\n");
41 return 0;
42}
43core_initcall(cgroup_bpf_wq_init);
44
45static int cgroup_bpf_lifetime_notify(struct notifier_block *nb,
46 unsigned long action, void *data);
47
48static struct notifier_block cgroup_bpf_lifetime_nb = {
49 .notifier_call = cgroup_bpf_lifetime_notify,
50};
51
52void __init cgroup_bpf_lifetime_notifier_init(void)
53{
54 BUG_ON(blocking_notifier_chain_register(&cgroup_lifetime_notifier,
55 &cgroup_bpf_lifetime_nb));
56}
57
58#ifdef CONFIG_BPF_LSM
59struct cgroup_lsm_atype {
60 u32 attach_btf_id;
61 int refcnt;
62 bool returns_errno;
63};
64
65static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM];
66
67static bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype)
68{
69 if (atype >= CGROUP_LSM_START && atype <= CGROUP_LSM_END)
70 return READ_ONCE(cgroup_lsm_atype[atype - CGROUP_LSM_START].returns_errno);
71 return true;
72}
73#else
74static bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype)
75{
76 return true;
77}
78#endif
79
80/* __always_inline is necessary to prevent indirect call through run_prog
81 * function pointer.
82 */
83static __always_inline int
84bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,
85 enum cgroup_bpf_attach_type atype,
86 const void *ctx, bpf_prog_run_fn run_prog,
87 int retval, u32 *ret_flags)
88{
89 const struct bpf_prog_array_item *item;
90 const struct bpf_prog *prog;
91 const struct bpf_prog_array *array;
92 struct bpf_run_ctx *old_run_ctx;
93 struct bpf_cg_run_ctx run_ctx;
94 u32 func_ret;
95
96 run_ctx.retval = retval;
97 rcu_read_lock_dont_migrate();
98 array = rcu_dereference(cgrp->effective[atype]);
99 item = &array->items[0];
100 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
101 while ((prog = READ_ONCE(item->prog))) {
102 run_ctx.prog_item = item;
103 func_ret = run_prog(prog, ctx);
104 if (ret_flags) {
105 *(ret_flags) |= (func_ret >> 1);
106 func_ret &= 1;
107 }
108 if (!func_ret && cgroup_bpf_hook_returns_errno(atype) &&
109 !IS_ERR_VALUE((long)run_ctx.retval))
110 run_ctx.retval = -EPERM;
111 item++;
112 }
113 bpf_reset_run_ctx(old_run_ctx);
114 rcu_read_unlock_migrate();
115 return run_ctx.retval;
116}
117
118unsigned int __cgroup_bpf_run_lsm_sock(const void *ctx,
119 const struct bpf_insn *insn)
120{
121 const struct bpf_prog *shim_prog;
122 struct sock *sk;
123 struct cgroup *cgrp;
124 int ret = 0;
125 u64 *args;
126
127 args = (u64 *)ctx;
128 sk = (void *)(unsigned long)args[0];
129 /*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/
130 shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi));
131
132 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
133 if (likely(cgrp))
134 ret = bpf_prog_run_array_cg(&cgrp->bpf,
135 shim_prog->aux->cgroup_atype,
136 ctx, bpf_prog_run, 0, NULL);
137 return ret;
138}
139
140unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx,
141 const struct bpf_insn *insn)
142{
143 const struct bpf_prog *shim_prog;
144 struct socket *sock;
145 struct cgroup *cgrp;
146 int ret = 0;
147 u64 *args;
148
149 args = (u64 *)ctx;
150 sock = (void *)(unsigned long)args[0];
151 /*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/
152 shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi));
153
154 cgrp = sock_cgroup_ptr(&sock->sk->sk_cgrp_data);
155 if (likely(cgrp))
156 ret = bpf_prog_run_array_cg(&cgrp->bpf,
157 shim_prog->aux->cgroup_atype,
158 ctx, bpf_prog_run, 0, NULL);
159 return ret;
160}
161
162unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
163 const struct bpf_insn *insn)
164{
165 const struct bpf_prog *shim_prog;
166 struct cgroup *cgrp;
167 int ret = 0;
168
169 /*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/
170 shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi));
171
172 /* We rely on trampoline's __bpf_prog_enter_lsm_cgroup to grab RCU read lock. */
173 cgrp = task_dfl_cgroup(current);
174 if (likely(cgrp))
175 ret = bpf_prog_run_array_cg(&cgrp->bpf,
176 shim_prog->aux->cgroup_atype,
177 ctx, bpf_prog_run, 0, NULL);
178 return ret;
179}
180
181#ifdef CONFIG_BPF_LSM
182static enum cgroup_bpf_attach_type
183bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
184{
185 int i;
186
187 lockdep_assert_held(&cgroup_mutex);
188
189 if (attach_type != BPF_LSM_CGROUP)
190 return to_cgroup_bpf_attach_type(attach_type);
191
192 for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype); i++)
193 if (cgroup_lsm_atype[i].attach_btf_id == attach_btf_id)
194 return CGROUP_LSM_START + i;
195
196 for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype); i++)
197 if (cgroup_lsm_atype[i].attach_btf_id == 0)
198 return CGROUP_LSM_START + i;
199
200 return -E2BIG;
201
202}
203
204void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype)
205{
206 int i = cgroup_atype - CGROUP_LSM_START;
207
208 lockdep_assert_held(&cgroup_mutex);
209
210 if (!cgroup_lsm_atype[i].attach_btf_id) {
211 cgroup_lsm_atype[i].attach_btf_id = attach_btf_id;
212 WRITE_ONCE(cgroup_lsm_atype[i].returns_errno,
213 bpf_lsm_hook_returns_errno(attach_btf_id));
214 } else {
215 WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id != attach_btf_id);
216 }
217 cgroup_lsm_atype[i].refcnt++;
218}
219
220void bpf_cgroup_atype_put(int cgroup_atype)
221{
222 int i = cgroup_atype - CGROUP_LSM_START;
223
224 cgroup_lock();
225 if (--cgroup_lsm_atype[i].refcnt <= 0) {
226 WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, true);
227 cgroup_lsm_atype[i].attach_btf_id = 0;
228 }
229 WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0);
230 cgroup_unlock();
231}
232#else
233static enum cgroup_bpf_attach_type
234bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
235{
236 if (attach_type != BPF_LSM_CGROUP)
237 return to_cgroup_bpf_attach_type(attach_type);
238 return -EOPNOTSUPP;
239}
240#endif /* CONFIG_BPF_LSM */
241
242static void cgroup_bpf_offline(struct cgroup *cgrp)
243{
244 cgroup_get(cgrp);
245 percpu_ref_kill(&cgrp->bpf.refcnt);
246}
247
248static void bpf_cgroup_storages_free(struct bpf_cgroup_storage *storages[])
249{
250 enum bpf_cgroup_storage_type stype;
251
252 for_each_cgroup_storage_type(stype)
253 bpf_cgroup_storage_free(storages[stype]);
254}
255
256static int bpf_cgroup_storages_alloc(struct bpf_cgroup_storage *storages[],
257 struct bpf_cgroup_storage *new_storages[],
258 enum bpf_attach_type type,
259 struct bpf_prog *prog,
260 struct cgroup *cgrp)
261{
262 enum bpf_cgroup_storage_type stype;
263 struct bpf_cgroup_storage_key key;
264 struct bpf_map *map;
265
266 key.cgroup_inode_id = cgroup_id(cgrp);
267 key.attach_type = type;
268
269 for_each_cgroup_storage_type(stype) {
270 map = prog->aux->cgroup_storage[stype];
271 if (!map)
272 continue;
273
274 storages[stype] = cgroup_storage_lookup((void *)map, &key, false);
275 if (storages[stype])
276 continue;
277
278 storages[stype] = bpf_cgroup_storage_alloc(prog, stype);
279 if (IS_ERR(storages[stype])) {
280 bpf_cgroup_storages_free(new_storages);
281 return -ENOMEM;
282 }
283
284 new_storages[stype] = storages[stype];
285 }
286
287 return 0;
288}
289
290static void bpf_cgroup_storages_assign(struct bpf_cgroup_storage *dst[],
291 struct bpf_cgroup_storage *src[])
292{
293 enum bpf_cgroup_storage_type stype;
294
295 for_each_cgroup_storage_type(stype)
296 dst[stype] = src[stype];
297}
298
299static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[],
300 struct cgroup *cgrp,
301 enum bpf_attach_type attach_type)
302{
303 enum bpf_cgroup_storage_type stype;
304
305 for_each_cgroup_storage_type(stype)
306 bpf_cgroup_storage_link(storages[stype], cgrp, attach_type);
307}
308
309/* Called when bpf_cgroup_link is auto-detached from dying cgroup.
310 * It drops cgroup and bpf_prog refcounts, and marks bpf_link as defunct. It
311 * doesn't free link memory, which will eventually be done by bpf_link's
312 * release() callback, when its last FD is closed.
313 */
314static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link)
315{
316 cgroup_put(link->cgroup);
317 link->cgroup = NULL;
318}
319
320/**
321 * cgroup_bpf_release() - put references of all bpf programs and
322 * release all cgroup bpf data
323 * @work: work structure embedded into the cgroup to modify
324 */
325static void cgroup_bpf_release(struct work_struct *work)
326{
327 struct cgroup *p, *cgrp = container_of(work, struct cgroup,
328 bpf.release_work);
329 struct bpf_prog_array *old_array;
330 struct list_head *storages = &cgrp->bpf.storages;
331 struct bpf_cgroup_storage *storage, *stmp;
332
333 unsigned int atype;
334
335 cgroup_lock();
336
337 for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) {
338 struct hlist_head *progs = &cgrp->bpf.progs[atype];
339 struct bpf_prog_list *pl;
340 struct hlist_node *pltmp;
341
342 hlist_for_each_entry_safe(pl, pltmp, progs, node) {
343 hlist_del(&pl->node);
344 if (pl->prog) {
345 if (pl->prog->expected_attach_type == BPF_LSM_CGROUP)
346 bpf_trampoline_unlink_cgroup_shim(pl->prog);
347 bpf_prog_put(pl->prog);
348 }
349 if (pl->link) {
350 if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP)
351 bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog);
352 bpf_cgroup_link_auto_detach(pl->link);
353 }
354 kfree(pl);
355 static_branch_dec(&cgroup_bpf_enabled_key[atype]);
356 }
357 old_array = rcu_dereference_protected(
358 cgrp->bpf.effective[atype],
359 lockdep_is_held(&cgroup_mutex));
360 bpf_prog_array_free(old_array);
361 }
362
363 list_for_each_entry_safe(storage, stmp, storages, list_cg) {
364 bpf_cgroup_storage_unlink(storage);
365 bpf_cgroup_storage_free(storage);
366 }
367
368 cgroup_unlock();
369
370 for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
371 cgroup_bpf_put(p);
372
373 percpu_ref_exit(&cgrp->bpf.refcnt);
374 cgroup_put(cgrp);
375}
376
377/**
378 * cgroup_bpf_release_fn() - callback used to schedule releasing
379 * of bpf cgroup data
380 * @ref: percpu ref counter structure
381 */
382static void cgroup_bpf_release_fn(struct percpu_ref *ref)
383{
384 struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt);
385
386 INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);
387 queue_work(cgroup_bpf_destroy_wq, &cgrp->bpf.release_work);
388}
389
390/* Get underlying bpf_prog of bpf_prog_list entry, regardless if it's through
391 * link or direct prog.
392 */
393static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl)
394{
395 if (pl->prog)
396 return pl->prog;
397 if (pl->link)
398 return pl->link->link.prog;
399 return NULL;
400}
401
402/* count number of elements in the list.
403 * it's slow but the list cannot be long
404 */
405static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt)
406{
407 struct bpf_prog_list *pl;
408 u32 cnt = 0;
409
410 hlist_for_each_entry(pl, head, node) {
411 if (!prog_list_prog(pl))
412 continue;
413 if (preorder_cnt && (pl->flags & BPF_F_PREORDER))
414 (*preorder_cnt)++;
415 cnt++;
416 }
417 return cnt;
418}
419
420/* if parent has non-overridable prog attached,
421 * disallow attaching new programs to the descendent cgroup.
422 * if parent has overridable or multi-prog, allow attaching
423 */
424static bool hierarchy_allows_attach(struct cgroup *cgrp,
425 enum cgroup_bpf_attach_type atype)
426{
427 struct cgroup *p;
428
429 p = cgroup_parent(cgrp);
430 if (!p)
431 return true;
432 do {
433 u32 flags = p->bpf.flags[atype];
434 u32 cnt;
435
436 if (flags & BPF_F_ALLOW_MULTI)
437 return true;
438 cnt = prog_list_length(&p->bpf.progs[atype], NULL);
439 WARN_ON_ONCE(cnt > 1);
440 if (cnt == 1)
441 return !!(flags & BPF_F_ALLOW_OVERRIDE);
442 p = cgroup_parent(p);
443 } while (p);
444 return true;
445}
446
447/* compute a chain of effective programs for a given cgroup:
448 * start from the list of programs in this cgroup and add
449 * all parent programs.
450 * Note that parent's F_ALLOW_OVERRIDE-type program is yielding
451 * to programs in this cgroup
452 */
453static int compute_effective_progs(struct cgroup *cgrp,
454 enum cgroup_bpf_attach_type atype,
455 struct bpf_prog_array **array)
456{
457 struct bpf_prog_array_item *item;
458 struct bpf_prog_array *progs;
459 struct bpf_prog_list *pl;
460 struct cgroup *p = cgrp;
461 int i, j, cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart;
462
463 /* count number of effective programs by walking parents */
464 do {
465 if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
466 cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt);
467 p = cgroup_parent(p);
468 } while (p);
469
470 progs = bpf_prog_array_alloc(cnt, GFP_KERNEL);
471 if (!progs)
472 return -ENOMEM;
473
474 /* populate the array with effective progs */
475 cnt = 0;
476 p = cgrp;
477 fstart = preorder_cnt;
478 bstart = preorder_cnt - 1;
479 do {
480 if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
481 continue;
482
483 init_bstart = bstart;
484 hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {
485 if (!prog_list_prog(pl))
486 continue;
487
488 if (pl->flags & BPF_F_PREORDER) {
489 item = &progs->items[bstart];
490 bstart--;
491 } else {
492 item = &progs->items[fstart];
493 fstart++;
494 }
495 item->prog = prog_list_prog(pl);
496 bpf_cgroup_storages_assign(item->cgroup_storage,
497 pl->storage);
498 cnt++;
499 }
500
501 /* reverse pre-ordering progs at this cgroup level */
502 for (i = bstart + 1, j = init_bstart; i < j; i++, j--)
503 swap(progs->items[i], progs->items[j]);
504
505 } while ((p = cgroup_parent(p)));
506
507 *array = progs;
508 return 0;
509}
510
511static void activate_effective_progs(struct cgroup *cgrp,
512 enum cgroup_bpf_attach_type atype,
513 struct bpf_prog_array *old_array)
514{
515 old_array = rcu_replace_pointer(cgrp->bpf.effective[atype], old_array,
516 lockdep_is_held(&cgroup_mutex));
517 /* free prog array after grace period, since __cgroup_bpf_run_*()
518 * might be still walking the array
519 */
520 bpf_prog_array_free(old_array);
521}
522
523/**
524 * cgroup_bpf_inherit() - inherit effective programs from parent
525 * @cgrp: the cgroup to modify
526 */
527static int cgroup_bpf_inherit(struct cgroup *cgrp)
528{
529/* has to use marco instead of const int, since compiler thinks
530 * that array below is variable length
531 */
532#define NR ARRAY_SIZE(cgrp->bpf.effective)
533 struct bpf_prog_array *arrays[NR] = {};
534 struct cgroup *p;
535 int ret, i;
536
537 ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
538 GFP_KERNEL);
539 if (ret)
540 return ret;
541
542 for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
543 cgroup_bpf_get(p);
544
545 for (i = 0; i < NR; i++)
546 INIT_HLIST_HEAD(&cgrp->bpf.progs[i]);
547
548 INIT_LIST_HEAD(&cgrp->bpf.storages);
549
550 for (i = 0; i < NR; i++)
551 if (compute_effective_progs(cgrp, i, &arrays[i]))
552 goto cleanup;
553
554 for (i = 0; i < NR; i++)
555 activate_effective_progs(cgrp, i, arrays[i]);
556
557 return 0;
558cleanup:
559 for (i = 0; i < NR; i++)
560 bpf_prog_array_free(arrays[i]);
561
562 for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
563 cgroup_bpf_put(p);
564
565 percpu_ref_exit(&cgrp->bpf.refcnt);
566
567 return -ENOMEM;
568}
569
570static int cgroup_bpf_lifetime_notify(struct notifier_block *nb,
571 unsigned long action, void *data)
572{
573 struct cgroup *cgrp = data;
574 int ret = 0;
575
576 if (cgrp->root != &cgrp_dfl_root)
577 return NOTIFY_OK;
578
579 switch (action) {
580 case CGROUP_LIFETIME_ONLINE:
581 ret = cgroup_bpf_inherit(cgrp);
582 break;
583 case CGROUP_LIFETIME_OFFLINE:
584 cgroup_bpf_offline(cgrp);
585 break;
586 }
587
588 return notifier_from_errno(ret);
589}
590
591static int update_effective_progs(struct cgroup *cgrp,
592 enum cgroup_bpf_attach_type atype)
593{
594 struct cgroup_subsys_state *css;
595 int err;
596
597 /* allocate and recompute effective prog arrays */
598 css_for_each_descendant_pre(css, &cgrp->self) {
599 struct cgroup *desc = container_of(css, struct cgroup, self);
600
601 if (percpu_ref_is_zero(&desc->bpf.refcnt))
602 continue;
603
604 err = compute_effective_progs(desc, atype, &desc->bpf.inactive);
605 if (err)
606 goto cleanup;
607 }
608
609 /* all allocations were successful. Activate all prog arrays */
610 css_for_each_descendant_pre(css, &cgrp->self) {
611 struct cgroup *desc = container_of(css, struct cgroup, self);
612
613 if (percpu_ref_is_zero(&desc->bpf.refcnt)) {
614 if (unlikely(desc->bpf.inactive)) {
615 bpf_prog_array_free(desc->bpf.inactive);
616 desc->bpf.inactive = NULL;
617 }
618 continue;
619 }
620
621 activate_effective_progs(desc, atype, desc->bpf.inactive);
622 desc->bpf.inactive = NULL;
623 }
624
625 return 0;
626
627cleanup:
628 /* oom while computing effective. Free all computed effective arrays
629 * since they were not activated
630 */
631 css_for_each_descendant_pre(css, &cgrp->self) {
632 struct cgroup *desc = container_of(css, struct cgroup, self);
633
634 bpf_prog_array_free(desc->bpf.inactive);
635 desc->bpf.inactive = NULL;
636 }
637
638 return err;
639}
640
641#define BPF_CGROUP_MAX_PROGS 64
642
643static struct bpf_prog_list *find_attach_entry(struct hlist_head *progs,
644 struct bpf_prog *prog,
645 struct bpf_cgroup_link *link,
646 struct bpf_prog *replace_prog,
647 bool allow_multi)
648{
649 struct bpf_prog_list *pl;
650
651 /* single-attach case */
652 if (!allow_multi) {
653 if (hlist_empty(progs))
654 return NULL;
655 return hlist_entry(progs->first, typeof(*pl), node);
656 }
657
658 hlist_for_each_entry(pl, progs, node) {
659 if (prog && pl->prog == prog && prog != replace_prog)
660 /* disallow attaching the same prog twice */
661 return ERR_PTR(-EINVAL);
662 if (link && pl->link == link)
663 /* disallow attaching the same link twice */
664 return ERR_PTR(-EINVAL);
665 }
666
667 /* direct prog multi-attach w/ replacement case */
668 if (replace_prog) {
669 hlist_for_each_entry(pl, progs, node) {
670 if (pl->prog == replace_prog)
671 /* a match found */
672 return pl;
673 }
674 /* prog to replace not found for cgroup */
675 return ERR_PTR(-ENOENT);
676 }
677
678 return NULL;
679}
680
681static struct bpf_link *bpf_get_anchor_link(u32 flags, u32 id_or_fd)
682{
683 struct bpf_link *link = ERR_PTR(-EINVAL);
684
685 if (flags & BPF_F_ID)
686 link = bpf_link_by_id(id_or_fd);
687 else if (id_or_fd)
688 link = bpf_link_get_from_fd(id_or_fd);
689 return link;
690}
691
692static struct bpf_prog *bpf_get_anchor_prog(u32 flags, u32 id_or_fd)
693{
694 struct bpf_prog *prog = ERR_PTR(-EINVAL);
695
696 if (flags & BPF_F_ID)
697 prog = bpf_prog_by_id(id_or_fd);
698 else if (id_or_fd)
699 prog = bpf_prog_get(id_or_fd);
700 return prog;
701}
702
703static struct bpf_prog_list *get_prog_list(struct hlist_head *progs, struct bpf_prog *prog,
704 struct bpf_cgroup_link *link, u32 flags, u32 id_or_fd)
705{
706 bool is_link = flags & BPF_F_LINK, is_id = flags & BPF_F_ID;
707 struct bpf_prog_list *pltmp, *pl = ERR_PTR(-EINVAL);
708 bool preorder = flags & BPF_F_PREORDER;
709 struct bpf_link *anchor_link = NULL;
710 struct bpf_prog *anchor_prog = NULL;
711 bool is_before, is_after;
712
713 is_before = flags & BPF_F_BEFORE;
714 is_after = flags & BPF_F_AFTER;
715 if (is_link || is_id || id_or_fd) {
716 /* flags must have either BPF_F_BEFORE or BPF_F_AFTER */
717 if (is_before == is_after)
718 return ERR_PTR(-EINVAL);
719 if ((is_link && !link) || (!is_link && !prog))
720 return ERR_PTR(-EINVAL);
721 } else if (!hlist_empty(progs)) {
722 /* flags cannot have both BPF_F_BEFORE and BPF_F_AFTER */
723 if (is_before && is_after)
724 return ERR_PTR(-EINVAL);
725 }
726
727 if (is_link) {
728 anchor_link = bpf_get_anchor_link(flags, id_or_fd);
729 if (IS_ERR(anchor_link))
730 return ERR_CAST(anchor_link);
731 } else if (is_id || id_or_fd) {
732 anchor_prog = bpf_get_anchor_prog(flags, id_or_fd);
733 if (IS_ERR(anchor_prog))
734 return ERR_CAST(anchor_prog);
735 }
736
737 if (!anchor_prog && !anchor_link) {
738 /* if there is no anchor_prog/anchor_link, then BPF_F_PREORDER
739 * doesn't matter since either prepend or append to a combined
740 * list of progs will end up with correct result.
741 */
742 hlist_for_each_entry(pltmp, progs, node) {
743 if (is_before)
744 return pltmp;
745 if (pltmp->node.next)
746 continue;
747 return pltmp;
748 }
749 return NULL;
750 }
751
752 hlist_for_each_entry(pltmp, progs, node) {
753 if ((anchor_prog && anchor_prog == pltmp->prog) ||
754 (anchor_link && anchor_link == &pltmp->link->link)) {
755 if (!!(pltmp->flags & BPF_F_PREORDER) != preorder)
756 goto out;
757 pl = pltmp;
758 goto out;
759 }
760 }
761
762 pl = ERR_PTR(-ENOENT);
763out:
764 if (anchor_link)
765 bpf_link_put(anchor_link);
766 else
767 bpf_prog_put(anchor_prog);
768 return pl;
769}
770
771static int insert_pl_to_hlist(struct bpf_prog_list *pl, struct hlist_head *progs,
772 struct bpf_prog *prog, struct bpf_cgroup_link *link,
773 u32 flags, u32 id_or_fd)
774{
775 struct bpf_prog_list *pltmp;
776
777 pltmp = get_prog_list(progs, prog, link, flags, id_or_fd);
778 if (IS_ERR(pltmp))
779 return PTR_ERR(pltmp);
780
781 if (!pltmp)
782 hlist_add_head(&pl->node, progs);
783 else if (flags & BPF_F_BEFORE)
784 hlist_add_before(&pl->node, &pltmp->node);
785 else
786 hlist_add_behind(&pl->node, &pltmp->node);
787
788 return 0;
789}
790
791/**
792 * __cgroup_bpf_attach() - Attach the program or the link to a cgroup, and
793 * propagate the change to descendants
794 * @cgrp: The cgroup which descendants to traverse
795 * @prog: A program to attach
796 * @link: A link to attach
797 * @replace_prog: Previously attached program to replace if BPF_F_REPLACE is set
798 * @type: Type of attach operation
799 * @flags: Option flags
800 * @id_or_fd: Relative prog id or fd
801 * @revision: bpf_prog_list revision
802 *
803 * Exactly one of @prog or @link can be non-null.
804 * Must be called with cgroup_mutex held.
805 */
806static int __cgroup_bpf_attach(struct cgroup *cgrp,
807 struct bpf_prog *prog, struct bpf_prog *replace_prog,
808 struct bpf_cgroup_link *link,
809 enum bpf_attach_type type, u32 flags, u32 id_or_fd,
810 u64 revision)
811{
812 u32 saved_flags = (flags & (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI));
813 struct bpf_prog *old_prog = NULL;
814 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
815 struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
816 struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
817 struct bpf_prog *new_prog = prog ? : link->link.prog;
818 enum cgroup_bpf_attach_type atype;
819 u32 old_flags, old_pl_flags;
820 struct bpf_prog_list *pl;
821 struct hlist_head *progs;
822 int err;
823
824 if (((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI)) ||
825 ((flags & BPF_F_REPLACE) && !(flags & BPF_F_ALLOW_MULTI)))
826 /* invalid combination */
827 return -EINVAL;
828 if ((flags & BPF_F_REPLACE) && (flags & (BPF_F_BEFORE | BPF_F_AFTER)))
829 /* only either replace or insertion with before/after */
830 return -EINVAL;
831 if (link && (prog || replace_prog))
832 /* only either link or prog/replace_prog can be specified */
833 return -EINVAL;
834 if (!!replace_prog != !!(flags & BPF_F_REPLACE))
835 /* replace_prog implies BPF_F_REPLACE, and vice versa */
836 return -EINVAL;
837
838 atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id);
839 if (atype < 0)
840 return -EINVAL;
841 if (revision && revision != cgrp->bpf.revisions[atype])
842 return -ESTALE;
843
844 progs = &cgrp->bpf.progs[atype];
845
846 if (!hierarchy_allows_attach(cgrp, atype))
847 return -EPERM;
848
849 if (!hlist_empty(progs) && cgrp->bpf.flags[atype] != saved_flags)
850 /* Disallow attaching non-overridable on top
851 * of existing overridable in this cgroup.
852 * Disallow attaching multi-prog if overridable or none
853 */
854 return -EPERM;
855
856 if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS)
857 return -E2BIG;
858
859 pl = find_attach_entry(progs, prog, link, replace_prog,
860 flags & BPF_F_ALLOW_MULTI);
861 if (IS_ERR(pl))
862 return PTR_ERR(pl);
863
864 if (bpf_cgroup_storages_alloc(storage, new_storage, type,
865 prog ? : link->link.prog, cgrp))
866 return -ENOMEM;
867
868 if (pl) {
869 old_prog = pl->prog;
870 old_pl_flags = pl->flags;
871 bpf_cgroup_storages_assign(old_storage, pl->storage);
872 } else {
873 pl = kmalloc_obj(*pl);
874 if (!pl) {
875 bpf_cgroup_storages_free(new_storage);
876 return -ENOMEM;
877 }
878
879 err = insert_pl_to_hlist(pl, progs, prog, link, flags, id_or_fd);
880 if (err) {
881 kfree(pl);
882 bpf_cgroup_storages_free(new_storage);
883 return err;
884 }
885 }
886
887 pl->prog = prog;
888 pl->link = link;
889 pl->flags = flags;
890 bpf_cgroup_storages_assign(pl->storage, storage);
891 old_flags = cgrp->bpf.flags[atype];
892 cgrp->bpf.flags[atype] = saved_flags;
893
894 if (type == BPF_LSM_CGROUP) {
895 err = bpf_trampoline_link_cgroup_shim(new_prog, atype, type);
896 if (err)
897 goto cleanup;
898 }
899
900 err = update_effective_progs(cgrp, atype);
901 if (err)
902 goto cleanup_trampoline;
903
904 cgrp->bpf.revisions[atype] += 1;
905 if (old_prog) {
906 if (type == BPF_LSM_CGROUP)
907 bpf_trampoline_unlink_cgroup_shim(old_prog);
908 bpf_prog_put(old_prog);
909 } else {
910 static_branch_inc(&cgroup_bpf_enabled_key[atype]);
911 }
912 bpf_cgroup_storages_link(new_storage, cgrp, type);
913 return 0;
914
915cleanup_trampoline:
916 if (type == BPF_LSM_CGROUP)
917 bpf_trampoline_unlink_cgroup_shim(new_prog);
918
919cleanup:
920 if (old_prog) {
921 pl->prog = old_prog;
922 pl->link = NULL;
923 pl->flags = old_pl_flags;
924 bpf_cgroup_storages_assign(pl->storage, old_storage);
925 }
926 bpf_cgroup_storages_free(new_storage);
927 if (!old_prog) {
928 hlist_del(&pl->node);
929 kfree(pl);
930 }
931 cgrp->bpf.flags[atype] = old_flags;
932 return err;
933}
934
935static int cgroup_bpf_attach(struct cgroup *cgrp,
936 struct bpf_prog *prog, struct bpf_prog *replace_prog,
937 struct bpf_cgroup_link *link,
938 enum bpf_attach_type type,
939 u32 flags, u32 id_or_fd, u64 revision)
940{
941 int ret;
942
943 cgroup_lock();
944 ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags,
945 id_or_fd, revision);
946 cgroup_unlock();
947 return ret;
948}
949
950static int effective_prog_pos(struct cgroup *cgrp,
951 enum cgroup_bpf_attach_type atype,
952 struct bpf_prog_list *target_pl)
953{
954 int cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart, pos = -1;
955 struct bpf_prog_list *pl;
956 struct cgroup *p = cgrp;
957
958 /* count effective programs to find where the preorder region ends */
959 do {
960 if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
961 cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt);
962 p = cgroup_parent(p);
963 } while (p);
964
965 /* replay compute_effective_progs() placement and record target's slot */
966 cnt = 0;
967 p = cgrp;
968 fstart = preorder_cnt;
969 bstart = preorder_cnt - 1;
970 do {
971 if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
972 continue;
973
974 init_bstart = bstart;
975 hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {
976 if (!prog_list_prog(pl))
977 continue;
978
979 if (pl->flags & BPF_F_PREORDER) {
980 if (pl == target_pl)
981 pos = bstart;
982 bstart--;
983 } else {
984 if (pl == target_pl)
985 pos = fstart;
986 fstart++;
987 }
988 cnt++;
989 }
990
991 /* reverse pre-ordering progs at this cgroup level */
992 if (pos >= bstart + 1 && pos <= init_bstart)
993 pos = bstart + 1 + init_bstart - pos;
994 } while ((p = cgroup_parent(p)));
995
996 return pos;
997}
998
999/* Swap updated BPF program for given link in effective program arrays across
1000 * all descendant cgroups. This function is guaranteed to succeed.
1001 */
1002static void replace_effective_prog(struct cgroup *cgrp,
1003 enum cgroup_bpf_attach_type atype,
1004 struct bpf_prog_list *pl)
1005{
1006 struct bpf_prog_array_item *item;
1007 struct cgroup_subsys_state *css;
1008 struct bpf_prog_array *progs;
1009 int pos;
1010
1011 css_for_each_descendant_pre(css, &cgrp->self) {
1012 struct cgroup *desc = container_of(css, struct cgroup, self);
1013
1014 if (percpu_ref_is_zero(&desc->bpf.refcnt))
1015 continue;
1016
1017 pos = effective_prog_pos(desc, atype, pl);
1018 if (WARN_ON_ONCE(pos < 0))
1019 continue;
1020
1021 progs = rcu_dereference_protected(
1022 desc->bpf.effective[atype],
1023 lockdep_is_held(&cgroup_mutex));
1024 item = &progs->items[pos];
1025 WRITE_ONCE(item->prog, pl->link->link.prog);
1026 }
1027}
1028
1029static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog,
1030 struct bpf_prog *new_prog)
1031{
1032 enum bpf_cgroup_storage_type stype;
1033
1034 for_each_cgroup_storage_type(stype) {
1035 if (old_prog->aux->cgroup_storage[stype] !=
1036 new_prog->aux->cgroup_storage[stype])
1037 return false;
1038 }
1039
1040 return true;
1041}
1042
1043/**
1044 * __cgroup_bpf_replace() - Replace link's program and propagate the change
1045 * to descendants
1046 * @cgrp: The cgroup which descendants to traverse
1047 * @link: A link for which to replace BPF program
1048 * @new_prog: &struct bpf_prog for the target BPF program with its refcnt
1049 * incremented
1050 *
1051 * Must be called with cgroup_mutex held.
1052 */
1053static int __cgroup_bpf_replace(struct cgroup *cgrp,
1054 struct bpf_cgroup_link *link,
1055 struct bpf_prog *new_prog)
1056{
1057 enum cgroup_bpf_attach_type atype;
1058 struct bpf_prog *old_prog;
1059 struct bpf_prog_list *pl;
1060 struct hlist_head *progs;
1061 bool found = false;
1062
1063 atype = bpf_cgroup_atype_find(link->link.attach_type, new_prog->aux->attach_btf_id);
1064 if (atype < 0)
1065 return -EINVAL;
1066
1067 progs = &cgrp->bpf.progs[atype];
1068
1069 if (link->link.prog->type != new_prog->type)
1070 return -EINVAL;
1071
1072 hlist_for_each_entry(pl, progs, node) {
1073 if (pl->link == link) {
1074 found = true;
1075 break;
1076 }
1077 }
1078 if (!found)
1079 return -ENOENT;
1080
1081 if (!cgroup_bpf_storages_compatible(link->link.prog, new_prog))
1082 return -EINVAL;
1083
1084 cgrp->bpf.revisions[atype] += 1;
1085 old_prog = xchg(&link->link.prog, new_prog);
1086 replace_effective_prog(cgrp, atype, pl);
1087 bpf_prog_put(old_prog);
1088 return 0;
1089}
1090
1091static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog,
1092 struct bpf_prog *old_prog)
1093{
1094 struct bpf_cgroup_link *cg_link;
1095 int ret;
1096
1097 cg_link = container_of(link, struct bpf_cgroup_link, link);
1098
1099 cgroup_lock();
1100 /* link might have been auto-released by dying cgroup, so fail */
1101 if (!cg_link->cgroup) {
1102 ret = -ENOLINK;
1103 goto out_unlock;
1104 }
1105 if (old_prog && link->prog != old_prog) {
1106 ret = -EPERM;
1107 goto out_unlock;
1108 }
1109 ret = __cgroup_bpf_replace(cg_link->cgroup, cg_link, new_prog);
1110out_unlock:
1111 cgroup_unlock();
1112 return ret;
1113}
1114
1115static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs,
1116 struct bpf_prog *prog,
1117 struct bpf_cgroup_link *link,
1118 bool allow_multi)
1119{
1120 struct bpf_prog_list *pl;
1121
1122 if (!allow_multi) {
1123 if (hlist_empty(progs))
1124 /* report error when trying to detach and nothing is attached */
1125 return ERR_PTR(-ENOENT);
1126
1127 /* to maintain backward compatibility NONE and OVERRIDE cgroups
1128 * allow detaching with invalid FD (prog==NULL) in legacy mode
1129 */
1130 return hlist_entry(progs->first, typeof(*pl), node);
1131 }
1132
1133 if (!prog && !link)
1134 /* to detach MULTI prog the user has to specify valid FD
1135 * of the program or link to be detached
1136 */
1137 return ERR_PTR(-EINVAL);
1138
1139 /* find the prog or link and detach it */
1140 hlist_for_each_entry(pl, progs, node) {
1141 if (pl->prog == prog && pl->link == link)
1142 return pl;
1143 }
1144 return ERR_PTR(-ENOENT);
1145}
1146
1147/**
1148 * purge_effective_progs() - After compute_effective_progs fails to alloc new
1149 * cgrp->bpf.inactive table we can recover by
1150 * recomputing the array in place.
1151 *
1152 * @cgrp: The cgroup which descendants to travers
1153 * @pl: The prog_list entry being detached
1154 * @atype: Type of detach operation
1155 */
1156static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl,
1157 enum cgroup_bpf_attach_type atype)
1158{
1159 struct cgroup_subsys_state *css;
1160 struct bpf_prog_array *progs;
1161 int pos;
1162
1163 /* recompute effective prog array in place */
1164 css_for_each_descendant_pre(css, &cgrp->self) {
1165 struct cgroup *desc = container_of(css, struct cgroup, self);
1166
1167 if (percpu_ref_is_zero(&desc->bpf.refcnt))
1168 continue;
1169
1170 pos = effective_prog_pos(desc, atype, pl);
1171 /* no link or prog match, skip the cgroup of this layer */
1172 if (pos < 0)
1173 continue;
1174
1175 progs = rcu_dereference_protected(
1176 desc->bpf.effective[atype],
1177 lockdep_is_held(&cgroup_mutex));
1178
1179 /* Remove the program from the array */
1180 WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos),
1181 "Failed to purge a prog from array at index %d", pos);
1182 }
1183}
1184
1185/**
1186 * __cgroup_bpf_detach() - Detach the program or link from a cgroup, and
1187 * propagate the change to descendants
1188 * @cgrp: The cgroup which descendants to traverse
1189 * @prog: A program to detach or NULL
1190 * @link: A link to detach or NULL
1191 * @type: Type of detach operation
1192 * @revision: bpf_prog_list revision
1193 *
1194 * At most one of @prog or @link can be non-NULL.
1195 * Must be called with cgroup_mutex held.
1196 */
1197static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
1198 struct bpf_cgroup_link *link, enum bpf_attach_type type,
1199 u64 revision)
1200{
1201 enum cgroup_bpf_attach_type atype;
1202 struct bpf_prog *old_prog;
1203 struct bpf_prog_list *pl;
1204 struct hlist_head *progs;
1205 u32 attach_btf_id = 0;
1206 u32 flags;
1207
1208 if (prog)
1209 attach_btf_id = prog->aux->attach_btf_id;
1210 if (link)
1211 attach_btf_id = link->link.prog->aux->attach_btf_id;
1212
1213 atype = bpf_cgroup_atype_find(type, attach_btf_id);
1214 if (atype < 0)
1215 return -EINVAL;
1216
1217 if (revision && revision != cgrp->bpf.revisions[atype])
1218 return -ESTALE;
1219
1220 progs = &cgrp->bpf.progs[atype];
1221 flags = cgrp->bpf.flags[atype];
1222
1223 if (prog && link)
1224 /* only one of prog or link can be specified */
1225 return -EINVAL;
1226
1227 pl = find_detach_entry(progs, prog, link, flags & BPF_F_ALLOW_MULTI);
1228 if (IS_ERR(pl))
1229 return PTR_ERR(pl);
1230
1231 /* mark it deleted, so it's ignored while recomputing effective */
1232 old_prog = pl->prog;
1233 pl->prog = NULL;
1234 pl->link = NULL;
1235
1236 if (update_effective_progs(cgrp, atype)) {
1237 /* if update effective array failed replace the prog with a dummy prog*/
1238 pl->prog = old_prog;
1239 pl->link = link;
1240 purge_effective_progs(cgrp, pl, atype);
1241 }
1242
1243 /* now can actually delete it from this cgroup list */
1244 hlist_del(&pl->node);
1245 cgrp->bpf.revisions[atype] += 1;
1246
1247 kfree(pl);
1248 if (hlist_empty(progs))
1249 /* last program was detached, reset flags to zero */
1250 cgrp->bpf.flags[atype] = 0;
1251 if (old_prog) {
1252 if (type == BPF_LSM_CGROUP)
1253 bpf_trampoline_unlink_cgroup_shim(old_prog);
1254 bpf_prog_put(old_prog);
1255 }
1256 static_branch_dec(&cgroup_bpf_enabled_key[atype]);
1257 return 0;
1258}
1259
1260static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
1261 enum bpf_attach_type type, u64 revision)
1262{
1263 int ret;
1264
1265 cgroup_lock();
1266 ret = __cgroup_bpf_detach(cgrp, prog, NULL, type, revision);
1267 cgroup_unlock();
1268 return ret;
1269}
1270
1271/* Must be called with cgroup_mutex held to avoid races. */
1272static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
1273 union bpf_attr __user *uattr, u32 uattr_size)
1274{
1275 __u32 __user *prog_attach_flags = u64_to_user_ptr(attr->query.prog_attach_flags);
1276 bool effective_query = attr->query.query_flags & BPF_F_QUERY_EFFECTIVE;
1277 __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
1278 enum bpf_attach_type type = attr->query.attach_type;
1279 enum cgroup_bpf_attach_type from_atype, to_atype;
1280 enum cgroup_bpf_attach_type atype;
1281 struct bpf_prog_array *effective;
1282 int cnt, ret = 0, i;
1283 int total_cnt = 0;
1284 u64 revision = 0;
1285 u32 flags;
1286
1287 if (effective_query && prog_attach_flags)
1288 return -EINVAL;
1289
1290 if (type == BPF_LSM_CGROUP) {
1291 if (!effective_query && attr->query.prog_cnt &&
1292 prog_ids && !prog_attach_flags)
1293 return -EINVAL;
1294
1295 from_atype = CGROUP_LSM_START;
1296 to_atype = CGROUP_LSM_END;
1297 flags = 0;
1298 } else {
1299 from_atype = to_cgroup_bpf_attach_type(type);
1300 if (from_atype < 0)
1301 return -EINVAL;
1302 to_atype = from_atype;
1303 flags = cgrp->bpf.flags[from_atype];
1304 }
1305
1306 for (atype = from_atype; atype <= to_atype; atype++) {
1307 if (effective_query) {
1308 effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
1309 lockdep_is_held(&cgroup_mutex));
1310 total_cnt += bpf_prog_array_length(effective);
1311 } else {
1312 total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);
1313 }
1314 }
1315
1316 /* always output uattr->query.attach_flags as 0 during effective query */
1317 flags = effective_query ? 0 : flags;
1318 if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
1319 return -EFAULT;
1320 if (copy_to_user(&uattr->query.prog_cnt, &total_cnt, sizeof(total_cnt)))
1321 return -EFAULT;
1322 if (!effective_query && from_atype == to_atype)
1323 revision = cgrp->bpf.revisions[from_atype];
1324 if (uattr_size >= offsetofend(union bpf_attr, query.revision) &&
1325 copy_to_user(&uattr->query.revision, &revision, sizeof(revision)))
1326 return -EFAULT;
1327 if (attr->query.prog_cnt == 0 || !prog_ids || !total_cnt)
1328 /* return early if user requested only program count + flags */
1329 return 0;
1330
1331 if (attr->query.prog_cnt < total_cnt) {
1332 total_cnt = attr->query.prog_cnt;
1333 ret = -ENOSPC;
1334 }
1335
1336 for (atype = from_atype; atype <= to_atype && total_cnt; atype++) {
1337 if (effective_query) {
1338 effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
1339 lockdep_is_held(&cgroup_mutex));
1340 cnt = min_t(int, bpf_prog_array_length(effective), total_cnt);
1341 ret = bpf_prog_array_copy_to_user(effective, prog_ids, cnt);
1342 } else {
1343 struct hlist_head *progs;
1344 struct bpf_prog_list *pl;
1345 struct bpf_prog *prog;
1346 u32 id;
1347
1348 progs = &cgrp->bpf.progs[atype];
1349 cnt = min_t(int, prog_list_length(progs, NULL), total_cnt);
1350 i = 0;
1351 hlist_for_each_entry(pl, progs, node) {
1352 prog = prog_list_prog(pl);
1353 id = prog->aux->id;
1354 if (copy_to_user(prog_ids + i, &id, sizeof(id)))
1355 return -EFAULT;
1356 if (++i == cnt)
1357 break;
1358 }
1359
1360 if (prog_attach_flags) {
1361 flags = cgrp->bpf.flags[atype];
1362
1363 for (i = 0; i < cnt; i++)
1364 if (copy_to_user(prog_attach_flags + i,
1365 &flags, sizeof(flags)))
1366 return -EFAULT;
1367 prog_attach_flags += cnt;
1368 }
1369 }
1370
1371 prog_ids += cnt;
1372 total_cnt -= cnt;
1373 }
1374 return ret;
1375}
1376
1377static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
1378 union bpf_attr __user *uattr, u32 uattr_size)
1379{
1380 int ret;
1381
1382 cgroup_lock();
1383 ret = __cgroup_bpf_query(cgrp, attr, uattr, uattr_size);
1384 cgroup_unlock();
1385 return ret;
1386}
1387
1388int cgroup_bpf_prog_attach(const union bpf_attr *attr,
1389 enum bpf_prog_type ptype, struct bpf_prog *prog)
1390{
1391 struct bpf_prog *replace_prog = NULL;
1392 struct cgroup *cgrp;
1393 int ret;
1394
1395 cgrp = cgroup_get_from_fd(attr->target_fd);
1396 if (IS_ERR(cgrp))
1397 return PTR_ERR(cgrp);
1398
1399 if ((attr->attach_flags & BPF_F_ALLOW_MULTI) &&
1400 (attr->attach_flags & BPF_F_REPLACE)) {
1401 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, ptype);
1402 if (IS_ERR(replace_prog)) {
1403 cgroup_put(cgrp);
1404 return PTR_ERR(replace_prog);
1405 }
1406 }
1407
1408 ret = cgroup_bpf_attach(cgrp, prog, replace_prog, NULL,
1409 attr->attach_type, attr->attach_flags,
1410 attr->relative_fd, attr->expected_revision);
1411
1412 if (replace_prog)
1413 bpf_prog_put(replace_prog);
1414 cgroup_put(cgrp);
1415 return ret;
1416}
1417
1418int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
1419{
1420 struct bpf_prog *prog;
1421 struct cgroup *cgrp;
1422 int ret;
1423
1424 cgrp = cgroup_get_from_fd(attr->target_fd);
1425 if (IS_ERR(cgrp))
1426 return PTR_ERR(cgrp);
1427
1428 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
1429 if (IS_ERR(prog))
1430 prog = NULL;
1431
1432 ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, attr->expected_revision);
1433 if (prog)
1434 bpf_prog_put(prog);
1435
1436 cgroup_put(cgrp);
1437 return ret;
1438}
1439
1440static void bpf_cgroup_link_release(struct bpf_link *link)
1441{
1442 struct bpf_cgroup_link *cg_link =
1443 container_of(link, struct bpf_cgroup_link, link);
1444 struct cgroup *cg;
1445
1446 /* link might have been auto-detached by dying cgroup already,
1447 * in that case our work is done here
1448 */
1449 if (!cg_link->cgroup)
1450 return;
1451
1452 cgroup_lock();
1453
1454 /* re-check cgroup under lock again */
1455 if (!cg_link->cgroup) {
1456 cgroup_unlock();
1457 return;
1458 }
1459
1460 WARN_ON(__cgroup_bpf_detach(cg_link->cgroup, NULL, cg_link,
1461 link->attach_type, 0));
1462 if (link->attach_type == BPF_LSM_CGROUP)
1463 bpf_trampoline_unlink_cgroup_shim(cg_link->link.prog);
1464
1465 cg = cg_link->cgroup;
1466 cg_link->cgroup = NULL;
1467
1468 cgroup_unlock();
1469
1470 cgroup_put(cg);
1471}
1472
1473static void bpf_cgroup_link_dealloc(struct bpf_link *link)
1474{
1475 struct bpf_cgroup_link *cg_link =
1476 container_of(link, struct bpf_cgroup_link, link);
1477
1478 kfree(cg_link);
1479}
1480
1481static int bpf_cgroup_link_detach(struct bpf_link *link)
1482{
1483 bpf_cgroup_link_release(link);
1484
1485 return 0;
1486}
1487
1488static void bpf_cgroup_link_show_fdinfo(const struct bpf_link *link,
1489 struct seq_file *seq)
1490{
1491 struct bpf_cgroup_link *cg_link =
1492 container_of(link, struct bpf_cgroup_link, link);
1493 u64 cg_id = 0;
1494
1495 cgroup_lock();
1496 if (cg_link->cgroup)
1497 cg_id = cgroup_id(cg_link->cgroup);
1498 cgroup_unlock();
1499
1500 seq_printf(seq,
1501 "cgroup_id:\t%llu\n"
1502 "attach_type:\t%d\n",
1503 cg_id,
1504 link->attach_type);
1505}
1506
1507static int bpf_cgroup_link_fill_link_info(const struct bpf_link *link,
1508 struct bpf_link_info *info)
1509{
1510 struct bpf_cgroup_link *cg_link =
1511 container_of(link, struct bpf_cgroup_link, link);
1512 u64 cg_id = 0;
1513
1514 cgroup_lock();
1515 if (cg_link->cgroup)
1516 cg_id = cgroup_id(cg_link->cgroup);
1517 cgroup_unlock();
1518
1519 info->cgroup.cgroup_id = cg_id;
1520 info->cgroup.attach_type = link->attach_type;
1521 return 0;
1522}
1523
1524static const struct bpf_link_ops bpf_cgroup_link_lops = {
1525 .release = bpf_cgroup_link_release,
1526 .dealloc = bpf_cgroup_link_dealloc,
1527 .detach = bpf_cgroup_link_detach,
1528 .update_prog = cgroup_bpf_replace,
1529 .show_fdinfo = bpf_cgroup_link_show_fdinfo,
1530 .fill_link_info = bpf_cgroup_link_fill_link_info,
1531};
1532
1533#define BPF_F_LINK_ATTACH_MASK \
1534 (BPF_F_ID | \
1535 BPF_F_BEFORE | \
1536 BPF_F_AFTER | \
1537 BPF_F_PREORDER | \
1538 BPF_F_LINK)
1539
1540int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
1541{
1542 struct bpf_link_primer link_primer;
1543 struct bpf_cgroup_link *link;
1544 struct cgroup *cgrp;
1545 int err;
1546
1547 if (attr->link_create.flags & (~BPF_F_LINK_ATTACH_MASK))
1548 return -EINVAL;
1549
1550 cgrp = cgroup_get_from_fd(attr->link_create.target_fd);
1551 if (IS_ERR(cgrp))
1552 return PTR_ERR(cgrp);
1553
1554 link = kzalloc_obj(*link, GFP_USER);
1555 if (!link) {
1556 err = -ENOMEM;
1557 goto out_put_cgroup;
1558 }
1559 bpf_link_init(&link->link, BPF_LINK_TYPE_CGROUP, &bpf_cgroup_link_lops,
1560 prog, attr->link_create.attach_type);
1561 link->cgroup = cgrp;
1562
1563 err = bpf_link_prime(&link->link, &link_primer);
1564 if (err) {
1565 kfree(link);
1566 goto out_put_cgroup;
1567 }
1568
1569 err = cgroup_bpf_attach(cgrp, NULL, NULL, link,
1570 link->link.attach_type, BPF_F_ALLOW_MULTI | attr->link_create.flags,
1571 attr->link_create.cgroup.relative_fd,
1572 attr->link_create.cgroup.expected_revision);
1573 if (err) {
1574 bpf_link_cleanup(&link_primer);
1575 goto out_put_cgroup;
1576 }
1577
1578 return bpf_link_settle(&link_primer);
1579
1580out_put_cgroup:
1581 cgroup_put(cgrp);
1582 return err;
1583}
1584
1585int cgroup_bpf_prog_query(const union bpf_attr *attr,
1586 union bpf_attr __user *uattr, u32 uattr_size)
1587{
1588 struct cgroup *cgrp;
1589 int ret;
1590
1591 cgrp = cgroup_get_from_fd(attr->query.target_fd);
1592 if (IS_ERR(cgrp))
1593 return PTR_ERR(cgrp);
1594
1595 ret = cgroup_bpf_query(cgrp, attr, uattr, uattr_size);
1596
1597 cgroup_put(cgrp);
1598 return ret;
1599}
1600
1601/**
1602 * __cgroup_bpf_run_filter_skb() - Run a program for packet filtering
1603 * @sk: The socket sending or receiving traffic
1604 * @skb: The skb that is being sent or received
1605 * @atype: The type of program to be executed
1606 *
1607 * If no socket is passed, or the socket is not of type INET or INET6,
1608 * this function does nothing and returns 0.
1609 *
1610 * The program type passed in via @type must be suitable for network
1611 * filtering. No further check is performed to assert that.
1612 *
1613 * For egress packets, this function can return:
1614 * NET_XMIT_SUCCESS (0) - continue with packet output
1615 * NET_XMIT_DROP (1) - drop packet and notify TCP to call cwr
1616 * NET_XMIT_CN (2) - continue with packet output and notify TCP
1617 * to call cwr
1618 * -err - drop packet
1619 *
1620 * For ingress packets, this function will return -EPERM if any
1621 * attached program was found and if it returned != 1 during execution.
1622 * Otherwise 0 is returned.
1623 */
1624int __cgroup_bpf_run_filter_skb(struct sock *sk,
1625 struct sk_buff *skb,
1626 enum cgroup_bpf_attach_type atype)
1627{
1628 unsigned int offset = -skb_network_offset(skb);
1629 struct sock *save_sk;
1630 void *saved_data_end;
1631 struct cgroup *cgrp;
1632 int ret;
1633
1634 if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)
1635 return 0;
1636
1637 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1638 save_sk = skb->sk;
1639 skb->sk = sk;
1640 __skb_push(skb, offset);
1641
1642 /* compute pointers for the bpf prog */
1643 bpf_compute_and_save_data_end(skb, &saved_data_end);
1644
1645 if (atype == CGROUP_INET_EGRESS) {
1646 u32 flags = 0;
1647 bool cn;
1648
1649 ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, skb,
1650 __bpf_prog_run_save_cb, 0, &flags);
1651
1652 /* Return values of CGROUP EGRESS BPF programs are:
1653 * 0: drop packet
1654 * 1: keep packet
1655 * 2: drop packet and cn
1656 * 3: keep packet and cn
1657 *
1658 * The returned value is then converted to one of the NET_XMIT
1659 * or an error code that is then interpreted as drop packet
1660 * (and no cn):
1661 * 0: NET_XMIT_SUCCESS skb should be transmitted
1662 * 1: NET_XMIT_DROP skb should be dropped and cn
1663 * 2: NET_XMIT_CN skb should be transmitted and cn
1664 * 3: -err skb should be dropped
1665 */
1666
1667 cn = flags & BPF_RET_SET_CN;
1668 if (ret && !IS_ERR_VALUE((long)ret))
1669 ret = -EFAULT;
1670 if (!ret)
1671 ret = (cn ? NET_XMIT_CN : NET_XMIT_SUCCESS);
1672 else
1673 ret = (cn ? NET_XMIT_DROP : ret);
1674 } else {
1675 ret = bpf_prog_run_array_cg(&cgrp->bpf, atype,
1676 skb, __bpf_prog_run_save_cb, 0,
1677 NULL);
1678 if (ret && !IS_ERR_VALUE((long)ret))
1679 ret = -EFAULT;
1680 }
1681 bpf_restore_data_end(skb, saved_data_end);
1682 __skb_pull(skb, offset);
1683 skb->sk = save_sk;
1684
1685 return ret;
1686}
1687EXPORT_SYMBOL(__cgroup_bpf_run_filter_skb);
1688
1689/**
1690 * __cgroup_bpf_run_filter_sk() - Run a program on a sock
1691 * @sk: sock structure to manipulate
1692 * @atype: The type of program to be executed
1693 *
1694 * socket is passed is expected to be of type INET or INET6.
1695 *
1696 * The program type passed in via @type must be suitable for sock
1697 * filtering. No further check is performed to assert that.
1698 *
1699 * This function will return %-EPERM if any if an attached program was found
1700 * and if it returned != 1 during execution. In all other cases, 0 is returned.
1701 */
1702int __cgroup_bpf_run_filter_sk(struct sock *sk,
1703 enum cgroup_bpf_attach_type atype)
1704{
1705 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1706
1707 return bpf_prog_run_array_cg(&cgrp->bpf, atype, sk, bpf_prog_run, 0,
1708 NULL);
1709}
1710EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk);
1711
1712/**
1713 * __cgroup_bpf_run_filter_sock_addr() - Run a program on a sock and
1714 * provided by user sockaddr
1715 * @sk: sock struct that will use sockaddr
1716 * @uaddr: sockaddr struct provided by user
1717 * @uaddrlen: Pointer to the size of the sockaddr struct provided by user. It is
1718 * read-only for AF_INET[6] uaddr but can be modified for AF_UNIX
1719 * uaddr.
1720 * @atype: The type of program to be executed
1721 * @t_ctx: Pointer to attach type specific context
1722 * @flags: Pointer to u32 which contains higher bits of BPF program
1723 * return value (OR'ed together).
1724 *
1725 * socket is expected to be of type INET, INET6 or UNIX.
1726 *
1727 * This function will return %-EPERM if an attached program is found and
1728 * returned value != 1 during execution. In all other cases, 0 is returned.
1729 */
1730int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
1731 struct sockaddr_unsized *uaddr,
1732 int *uaddrlen,
1733 enum cgroup_bpf_attach_type atype,
1734 void *t_ctx,
1735 u32 *flags)
1736{
1737 struct bpf_sock_addr_kern ctx = {
1738 .sk = sk,
1739 .uaddr = uaddr,
1740 .t_ctx = t_ctx,
1741 };
1742 struct sockaddr_storage storage;
1743 struct cgroup *cgrp;
1744 int ret;
1745
1746 if (!sk_is_inet(sk) && !sk_is_unix(sk))
1747 return 0;
1748
1749 if (!ctx.uaddr) {
1750 memset(&storage, 0, sizeof(storage));
1751 ctx.uaddr = (struct sockaddr_unsized *)&storage;
1752 ctx.uaddrlen = 0;
1753 } else {
1754 ctx.uaddrlen = *uaddrlen;
1755 }
1756
1757 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1758 ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run,
1759 0, flags);
1760
1761 if (!ret && uaddr)
1762 *uaddrlen = ctx.uaddrlen;
1763
1764 return ret;
1765}
1766EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_addr);
1767
1768/**
1769 * __cgroup_bpf_run_filter_sock_ops() - Run a program on a sock
1770 * @sk: socket to get cgroup from
1771 * @sock_ops: bpf_sock_ops_kern struct to pass to program. Contains
1772 * sk with connection information (IP addresses, etc.) May not contain
1773 * cgroup info if it is a req sock.
1774 * @atype: The type of program to be executed
1775 *
1776 * socket passed is expected to be of type INET or INET6.
1777 *
1778 * The program type passed in via @type must be suitable for sock_ops
1779 * filtering. No further check is performed to assert that.
1780 *
1781 * This function will return %-EPERM if any if an attached program was found
1782 * and if it returned != 1 during execution. In all other cases, 0 is returned.
1783 */
1784int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
1785 struct bpf_sock_ops_kern *sock_ops,
1786 enum cgroup_bpf_attach_type atype)
1787{
1788 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1789
1790 return bpf_prog_run_array_cg(&cgrp->bpf, atype, sock_ops, bpf_prog_run,
1791 0, NULL);
1792}
1793EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops);
1794
1795int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
1796 short access, enum cgroup_bpf_attach_type atype)
1797{
1798 struct cgroup *cgrp;
1799 struct bpf_cgroup_dev_ctx ctx = {
1800 .access_type = (access << 16) | dev_type,
1801 .major = major,
1802 .minor = minor,
1803 };
1804 int ret;
1805
1806 rcu_read_lock();
1807 cgrp = task_dfl_cgroup(current);
1808 ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run, 0,
1809 NULL);
1810 rcu_read_unlock();
1811
1812 return ret;
1813}
1814
1815BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
1816{
1817 /* flags argument is not used now,
1818 * but provides an ability to extend the API.
1819 * verifier checks that its value is correct.
1820 */
1821 enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
1822 struct bpf_cgroup_storage *storage;
1823 struct bpf_cg_run_ctx *ctx;
1824 void *ptr;
1825
1826 /* get current cgroup storage from BPF run context */
1827 ctx = container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);
1828 storage = ctx->prog_item->cgroup_storage[stype];
1829
1830 if (stype == BPF_CGROUP_STORAGE_SHARED)
1831 ptr = &READ_ONCE(storage->buf)->data[0];
1832 else
1833 ptr = this_cpu_ptr(storage->percpu_buf);
1834
1835 return (unsigned long)ptr;
1836}
1837
1838const struct bpf_func_proto bpf_get_local_storage_proto = {
1839 .func = bpf_get_local_storage,
1840 .gpl_only = false,
1841 .ret_type = RET_PTR_TO_MAP_VALUE,
1842 .arg1_type = ARG_CONST_MAP_PTR,
1843 .arg2_type = ARG_ANYTHING,
1844};
1845
1846BPF_CALL_0(bpf_get_retval)
1847{
1848 struct bpf_cg_run_ctx *ctx =
1849 container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);
1850
1851 return ctx->retval;
1852}
1853
1854const struct bpf_func_proto bpf_get_retval_proto = {
1855 .func = bpf_get_retval,
1856 .gpl_only = false,
1857 .ret_type = RET_INTEGER,
1858};
1859
1860BPF_CALL_1(bpf_set_retval, int, retval)
1861{
1862 struct bpf_cg_run_ctx *ctx =
1863 container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);
1864
1865 ctx->retval = retval;
1866 return 0;
1867}
1868
1869const struct bpf_func_proto bpf_set_retval_proto = {
1870 .func = bpf_set_retval,
1871 .gpl_only = false,
1872 .ret_type = RET_INTEGER,
1873 .arg1_type = ARG_ANYTHING,
1874};
1875
1876static const struct bpf_func_proto *
1877cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1878{
1879 const struct bpf_func_proto *func_proto;
1880
1881 func_proto = cgroup_common_func_proto(func_id, prog);
1882 if (func_proto)
1883 return func_proto;
1884
1885 switch (func_id) {
1886 case BPF_FUNC_perf_event_output:
1887 return &bpf_event_output_data_proto;
1888 default:
1889 return bpf_base_func_proto(func_id, prog);
1890 }
1891}
1892
1893static bool cgroup_dev_is_valid_access(int off, int size,
1894 enum bpf_access_type type,
1895 const struct bpf_prog *prog,
1896 struct bpf_insn_access_aux *info)
1897{
1898 const int size_default = sizeof(__u32);
1899
1900 if (type == BPF_WRITE)
1901 return false;
1902
1903 if (off < 0 || off + size > sizeof(struct bpf_cgroup_dev_ctx))
1904 return false;
1905 /* The verifier guarantees that size > 0. */
1906 if (off % size != 0)
1907 return false;
1908
1909 switch (off) {
1910 case bpf_ctx_range(struct bpf_cgroup_dev_ctx, access_type):
1911 bpf_ctx_record_field_size(info, size_default);
1912 if (!bpf_ctx_narrow_access_ok(off, size, size_default))
1913 return false;
1914 break;
1915 default:
1916 if (size != size_default)
1917 return false;
1918 }
1919
1920 return true;
1921}
1922
1923const struct bpf_prog_ops cg_dev_prog_ops = {
1924};
1925
1926const struct bpf_verifier_ops cg_dev_verifier_ops = {
1927 .get_func_proto = cgroup_dev_func_proto,
1928 .is_valid_access = cgroup_dev_is_valid_access,
1929};
1930
1931/**
1932 * __cgroup_bpf_run_filter_sysctl - Run a program on sysctl
1933 *
1934 * @head: sysctl table header
1935 * @table: sysctl table
1936 * @write: sysctl is being read (= 0) or written (= 1)
1937 * @buf: pointer to buffer (in and out)
1938 * @pcount: value-result argument: value is size of buffer pointed to by @buf,
1939 * result is size of @new_buf if program set new value, initial value
1940 * otherwise
1941 * @ppos: value-result argument: value is position at which read from or write
1942 * to sysctl is happening, result is new position if program overrode it,
1943 * initial value otherwise
1944 * @atype: type of program to be executed
1945 *
1946 * Program is run when sysctl is being accessed, either read or written, and
1947 * can allow or deny such access.
1948 *
1949 * This function will return %-EPERM if an attached program is found and
1950 * returned value != 1 during execution. In all other cases 0 is returned.
1951 */
1952int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head,
1953 const struct ctl_table *table, int write,
1954 char **buf, size_t *pcount, loff_t *ppos,
1955 enum cgroup_bpf_attach_type atype)
1956{
1957 struct bpf_sysctl_kern ctx = {
1958 .head = head,
1959 .table = table,
1960 .write = write,
1961 .ppos = ppos,
1962 .cur_val = NULL,
1963 .cur_len = PAGE_SIZE,
1964 .new_val = NULL,
1965 .new_len = 0,
1966 .new_updated = 0,
1967 };
1968 struct cgroup *cgrp;
1969 loff_t pos = 0;
1970 int ret;
1971
1972 ctx.cur_val = kmalloc_track_caller(ctx.cur_len, GFP_KERNEL);
1973 if (!ctx.cur_val ||
1974 table->proc_handler(table, 0, ctx.cur_val, &ctx.cur_len, &pos)) {
1975 /* Let BPF program decide how to proceed. */
1976 ctx.cur_len = 0;
1977 }
1978
1979 if (write && *buf && *pcount) {
1980 /* BPF program should be able to override new value with a
1981 * buffer bigger than provided by user.
1982 */
1983 ctx.new_val = kmalloc_track_caller(PAGE_SIZE, GFP_KERNEL);
1984 ctx.new_len = min_t(size_t, PAGE_SIZE, *pcount);
1985 if (ctx.new_val) {
1986 memcpy(ctx.new_val, *buf, ctx.new_len);
1987 } else {
1988 /* Let BPF program decide how to proceed. */
1989 ctx.new_len = 0;
1990 }
1991 }
1992
1993 rcu_read_lock();
1994 cgrp = task_dfl_cgroup(current);
1995 ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run, 0,
1996 NULL);
1997 rcu_read_unlock();
1998
1999 kfree(ctx.cur_val);
2000
2001 if (!ret && ctx.new_updated) {
2002 kvfree(*buf);
2003 *buf = ctx.new_val;
2004 *pcount = ctx.new_len;
2005 } else {
2006 kfree(ctx.new_val);
2007 }
2008
2009 return ret;
2010}
2011
2012#ifdef CONFIG_NET
2013static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen,
2014 struct bpf_sockopt_buf *buf)
2015{
2016 if (unlikely(max_optlen < 0))
2017 return -EINVAL;
2018
2019 if (unlikely(max_optlen > PAGE_SIZE)) {
2020 /* We don't expose optvals that are greater than PAGE_SIZE
2021 * to the BPF program.
2022 */
2023 max_optlen = PAGE_SIZE;
2024 }
2025
2026 if (max_optlen <= sizeof(buf->data)) {
2027 /* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE
2028 * bytes avoid the cost of kzalloc.
2029 */
2030 ctx->optval = buf->data;
2031 ctx->optval_end = ctx->optval + max_optlen;
2032 return max_optlen;
2033 }
2034
2035 ctx->optval = kzalloc(max_optlen, GFP_USER);
2036 if (!ctx->optval)
2037 return -ENOMEM;
2038
2039 ctx->optval_end = ctx->optval + max_optlen;
2040
2041 return max_optlen;
2042}
2043
2044static void sockopt_free_buf(struct bpf_sockopt_kern *ctx,
2045 struct bpf_sockopt_buf *buf)
2046{
2047 if (ctx->optval == buf->data)
2048 return;
2049 kfree(ctx->optval);
2050}
2051
2052static bool sockopt_buf_allocated(struct bpf_sockopt_kern *ctx,
2053 struct bpf_sockopt_buf *buf)
2054{
2055 return ctx->optval != buf->data;
2056}
2057
2058int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
2059 int *optname, sockptr_t optval,
2060 int *optlen, char **kernel_optval)
2061{
2062 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
2063 struct bpf_sockopt_buf buf = {};
2064 struct bpf_sockopt_kern ctx = {
2065 .sk = sk,
2066 .level = *level,
2067 .optname = *optname,
2068 };
2069 int ret, max_optlen;
2070
2071 /* Allocate a bit more than the initial user buffer for
2072 * BPF program. The canonical use case is overriding
2073 * TCP_CONGESTION(nv) to TCP_CONGESTION(cubic).
2074 */
2075 max_optlen = max_t(int, 16, *optlen);
2076 max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
2077 if (max_optlen < 0)
2078 return max_optlen;
2079
2080 ctx.optlen = *optlen;
2081
2082 if (copy_from_sockptr(ctx.optval, optval,
2083 min(*optlen, max_optlen))) {
2084 ret = -EFAULT;
2085 goto out;
2086 }
2087
2088 lock_sock(sk);
2089 ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_SETSOCKOPT,
2090 &ctx, bpf_prog_run, 0, NULL);
2091 release_sock(sk);
2092
2093 if (ret)
2094 goto out;
2095
2096 if (ctx.optlen == -1) {
2097 /* optlen set to -1, bypass kernel */
2098 ret = 1;
2099 } else if (ctx.optlen > max_optlen || ctx.optlen < -1) {
2100 /* optlen is out of bounds */
2101 if (*optlen > PAGE_SIZE && ctx.optlen >= 0) {
2102 pr_info_once("bpf setsockopt: ignoring program buffer with optlen=%d (max_optlen=%d)\n",
2103 ctx.optlen, max_optlen);
2104 ret = 0;
2105 goto out;
2106 }
2107 ret = -EFAULT;
2108 } else {
2109 /* optlen within bounds, run kernel handler */
2110 ret = 0;
2111
2112 /* export any potential modifications */
2113 *level = ctx.level;
2114 *optname = ctx.optname;
2115
2116 /* optlen == 0 from BPF indicates that we should
2117 * use original userspace data.
2118 */
2119 if (ctx.optlen != 0) {
2120 *optlen = ctx.optlen;
2121 /* We've used bpf_sockopt_kern->buf as an intermediary
2122 * storage, but the BPF program indicates that we need
2123 * to pass this data to the kernel setsockopt handler.
2124 * No way to export on-stack buf, have to allocate a
2125 * new buffer.
2126 */
2127 if (!sockopt_buf_allocated(&ctx, &buf)) {
2128 void *p = kmalloc(ctx.optlen, GFP_USER);
2129
2130 if (!p) {
2131 ret = -ENOMEM;
2132 goto out;
2133 }
2134 memcpy(p, ctx.optval, ctx.optlen);
2135 *kernel_optval = p;
2136 } else {
2137 *kernel_optval = ctx.optval;
2138 }
2139 /* export and don't free sockopt buf */
2140 return 0;
2141 }
2142 }
2143
2144out:
2145 sockopt_free_buf(&ctx, &buf);
2146 return ret;
2147}
2148
2149int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
2150 int optname, sockptr_t optval,
2151 sockptr_t optlen, int max_optlen,
2152 int retval)
2153{
2154 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
2155 struct bpf_sockopt_buf buf = {};
2156 struct bpf_sockopt_kern ctx = {
2157 .sk = sk,
2158 .level = level,
2159 .optname = optname,
2160 .current_task = current,
2161 };
2162 int orig_optlen;
2163 int ret;
2164
2165 orig_optlen = max_optlen;
2166 ctx.optlen = max_optlen;
2167 max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
2168 if (max_optlen < 0)
2169 return max_optlen;
2170
2171 if (!retval) {
2172 /* If kernel getsockopt finished successfully,
2173 * copy whatever was returned to the user back
2174 * into our temporary buffer. Set optlen to the
2175 * one that kernel returned as well to let
2176 * BPF programs inspect the value.
2177 */
2178 if (copy_from_sockptr(&ctx.optlen, optlen,
2179 sizeof(ctx.optlen))) {
2180 ret = -EFAULT;
2181 goto out;
2182 }
2183
2184 if (ctx.optlen < 0) {
2185 ret = -EFAULT;
2186 goto out;
2187 }
2188 orig_optlen = ctx.optlen;
2189
2190 if (copy_from_sockptr(ctx.optval, optval,
2191 min(ctx.optlen, max_optlen))) {
2192 ret = -EFAULT;
2193 goto out;
2194 }
2195 }
2196
2197 lock_sock(sk);
2198 ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_GETSOCKOPT,
2199 &ctx, bpf_prog_run, retval, NULL);
2200 release_sock(sk);
2201
2202 if (ret < 0)
2203 goto out;
2204
2205 if (!sockptr_is_null(optval) &&
2206 (ctx.optlen > max_optlen || ctx.optlen < 0)) {
2207 if (orig_optlen > PAGE_SIZE && ctx.optlen >= 0) {
2208 pr_info_once("bpf getsockopt: ignoring program buffer with optlen=%d (max_optlen=%d)\n",
2209 ctx.optlen, max_optlen);
2210 ret = retval;
2211 goto out;
2212 }
2213 ret = -EFAULT;
2214 goto out;
2215 }
2216
2217 if (ctx.optlen != 0) {
2218 if (!sockptr_is_null(optval) &&
2219 copy_to_sockptr(optval, ctx.optval, ctx.optlen)) {
2220 ret = -EFAULT;
2221 goto out;
2222 }
2223 if (copy_to_sockptr(optlen, &ctx.optlen, sizeof(ctx.optlen))) {
2224 ret = -EFAULT;
2225 goto out;
2226 }
2227 }
2228
2229out:
2230 sockopt_free_buf(&ctx, &buf);
2231 return ret;
2232}
2233
2234int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
2235 int optname, void *optval,
2236 int *optlen, int retval)
2237{
2238 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
2239 struct bpf_sockopt_kern ctx = {
2240 .sk = sk,
2241 .level = level,
2242 .optname = optname,
2243 .optlen = *optlen,
2244 .optval = optval,
2245 .optval_end = optval + *optlen,
2246 .current_task = current,
2247 };
2248 int ret;
2249
2250 /* Note that __cgroup_bpf_run_filter_getsockopt doesn't copy
2251 * user data back into BPF buffer when reval != 0. This is
2252 * done as an optimization to avoid extra copy, assuming
2253 * kernel won't populate the data in case of an error.
2254 * Here we always pass the data and memset() should
2255 * be called if that data shouldn't be "exported".
2256 */
2257
2258 ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_GETSOCKOPT,
2259 &ctx, bpf_prog_run, retval, NULL);
2260 if (ret < 0)
2261 return ret;
2262
2263 if (ctx.optlen > *optlen || ctx.optlen < 0)
2264 return -EFAULT;
2265
2266 /* BPF programs can shrink the buffer, export the modifications.
2267 */
2268 if (ctx.optlen != 0)
2269 *optlen = ctx.optlen;
2270
2271 return ret;
2272}
2273#endif
2274
2275static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,
2276 size_t *lenp)
2277{
2278 ssize_t tmp_ret = 0, ret;
2279
2280 if (dir->header.parent) {
2281 tmp_ret = sysctl_cpy_dir(dir->header.parent, bufp, lenp);
2282 if (tmp_ret < 0)
2283 return tmp_ret;
2284 }
2285
2286 ret = strscpy(*bufp, dir->header.ctl_table[0].procname, *lenp);
2287 if (ret < 0)
2288 return ret;
2289 *bufp += ret;
2290 *lenp -= ret;
2291 ret += tmp_ret;
2292
2293 /* Avoid leading slash. */
2294 if (!ret)
2295 return ret;
2296
2297 tmp_ret = strscpy(*bufp, "/", *lenp);
2298 if (tmp_ret < 0)
2299 return tmp_ret;
2300 *bufp += tmp_ret;
2301 *lenp -= tmp_ret;
2302
2303 return ret + tmp_ret;
2304}
2305
2306BPF_CALL_4(bpf_sysctl_get_name, struct bpf_sysctl_kern *, ctx, char *, buf,
2307 size_t, buf_len, u64, flags)
2308{
2309 ssize_t tmp_ret = 0, ret;
2310
2311 if (!buf)
2312 return -EINVAL;
2313
2314 if (!(flags & BPF_F_SYSCTL_BASE_NAME)) {
2315 if (!ctx->head)
2316 return -EINVAL;
2317 tmp_ret = sysctl_cpy_dir(ctx->head->parent, &buf, &buf_len);
2318 if (tmp_ret < 0)
2319 return tmp_ret;
2320 }
2321
2322 ret = strscpy(buf, ctx->table->procname, buf_len);
2323
2324 return ret < 0 ? ret : tmp_ret + ret;
2325}
2326
2327static const struct bpf_func_proto bpf_sysctl_get_name_proto = {
2328 .func = bpf_sysctl_get_name,
2329 .gpl_only = false,
2330 .ret_type = RET_INTEGER,
2331 .arg1_type = ARG_PTR_TO_CTX,
2332 .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE,
2333 .arg3_type = ARG_MEM_SIZE,
2334 .arg4_type = ARG_ANYTHING,
2335};
2336
2337static int copy_sysctl_value(char *dst, size_t dst_len, char *src,
2338 size_t src_len)
2339{
2340 if (!dst)
2341 return -EINVAL;
2342
2343 if (!dst_len)
2344 return -E2BIG;
2345
2346 if (!src || !src_len) {
2347 memset(dst, 0, dst_len);
2348 return -EINVAL;
2349 }
2350
2351 memcpy(dst, src, min(dst_len, src_len));
2352
2353 if (dst_len > src_len) {
2354 memset(dst + src_len, '\0', dst_len - src_len);
2355 return src_len;
2356 }
2357
2358 dst[dst_len - 1] = '\0';
2359
2360 return -E2BIG;
2361}
2362
2363BPF_CALL_3(bpf_sysctl_get_current_value, struct bpf_sysctl_kern *, ctx,
2364 char *, buf, size_t, buf_len)
2365{
2366 return copy_sysctl_value(buf, buf_len, ctx->cur_val, ctx->cur_len);
2367}
2368
2369static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = {
2370 .func = bpf_sysctl_get_current_value,
2371 .gpl_only = false,
2372 .ret_type = RET_INTEGER,
2373 .arg1_type = ARG_PTR_TO_CTX,
2374 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
2375 .arg3_type = ARG_MEM_SIZE,
2376};
2377
2378BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf,
2379 size_t, buf_len)
2380{
2381 if (!ctx->write) {
2382 if (buf && buf_len)
2383 memset(buf, '\0', buf_len);
2384 return -EINVAL;
2385 }
2386 return copy_sysctl_value(buf, buf_len, ctx->new_val, ctx->new_len);
2387}
2388
2389static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = {
2390 .func = bpf_sysctl_get_new_value,
2391 .gpl_only = false,
2392 .ret_type = RET_INTEGER,
2393 .arg1_type = ARG_PTR_TO_CTX,
2394 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
2395 .arg3_type = ARG_MEM_SIZE,
2396};
2397
2398BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx,
2399 const char *, buf, size_t, buf_len)
2400{
2401 if (!ctx->write || !ctx->new_val || !ctx->new_len || !buf || !buf_len)
2402 return -EINVAL;
2403
2404 if (buf_len > PAGE_SIZE - 1)
2405 return -E2BIG;
2406
2407 memcpy(ctx->new_val, buf, buf_len);
2408 ((char *)ctx->new_val)[buf_len] = '\0';
2409 ctx->new_len = buf_len;
2410 ctx->new_updated = 1;
2411
2412 return 0;
2413}
2414
2415static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = {
2416 .func = bpf_sysctl_set_new_value,
2417 .gpl_only = false,
2418 .ret_type = RET_INTEGER,
2419 .arg1_type = ARG_PTR_TO_CTX,
2420 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
2421 .arg3_type = ARG_MEM_SIZE,
2422};
2423
2424static const struct bpf_func_proto *
2425sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
2426{
2427 const struct bpf_func_proto *func_proto;
2428
2429 func_proto = cgroup_common_func_proto(func_id, prog);
2430 if (func_proto)
2431 return func_proto;
2432
2433 switch (func_id) {
2434 case BPF_FUNC_sysctl_get_name:
2435 return &bpf_sysctl_get_name_proto;
2436 case BPF_FUNC_sysctl_get_current_value:
2437 return &bpf_sysctl_get_current_value_proto;
2438 case BPF_FUNC_sysctl_get_new_value:
2439 return &bpf_sysctl_get_new_value_proto;
2440 case BPF_FUNC_sysctl_set_new_value:
2441 return &bpf_sysctl_set_new_value_proto;
2442 case BPF_FUNC_ktime_get_coarse_ns:
2443 return &bpf_ktime_get_coarse_ns_proto;
2444 case BPF_FUNC_perf_event_output:
2445 return &bpf_event_output_data_proto;
2446 default:
2447 return bpf_base_func_proto(func_id, prog);
2448 }
2449}
2450
2451static bool sysctl_is_valid_access(int off, int size, enum bpf_access_type type,
2452 const struct bpf_prog *prog,
2453 struct bpf_insn_access_aux *info)
2454{
2455 const int size_default = sizeof(__u32);
2456
2457 if (off < 0 || off + size > sizeof(struct bpf_sysctl) || off % size)
2458 return false;
2459
2460 switch (off) {
2461 case bpf_ctx_range(struct bpf_sysctl, write):
2462 if (type != BPF_READ)
2463 return false;
2464 bpf_ctx_record_field_size(info, size_default);
2465 return bpf_ctx_narrow_access_ok(off, size, size_default);
2466 case bpf_ctx_range(struct bpf_sysctl, file_pos):
2467 if (type == BPF_READ) {
2468 bpf_ctx_record_field_size(info, size_default);
2469 return bpf_ctx_narrow_access_ok(off, size, size_default);
2470 } else {
2471 return size == size_default;
2472 }
2473 default:
2474 return false;
2475 }
2476}
2477
2478static u32 sysctl_convert_ctx_access(enum bpf_access_type type,
2479 const struct bpf_insn *si,
2480 struct bpf_insn *insn_buf,
2481 struct bpf_prog *prog, u32 *target_size)
2482{
2483 struct bpf_insn *insn = insn_buf;
2484 u32 read_size;
2485
2486 switch (si->off) {
2487 case offsetof(struct bpf_sysctl, write):
2488 *insn++ = BPF_LDX_MEM(
2489 BPF_SIZE(si->code), si->dst_reg, si->src_reg,
2490 bpf_target_off(struct bpf_sysctl_kern, write,
2491 sizeof_field(struct bpf_sysctl_kern,
2492 write),
2493 target_size));
2494 break;
2495 case offsetof(struct bpf_sysctl, file_pos):
2496 /* ppos is a pointer so it should be accessed via indirect
2497 * loads and stores. Also for stores additional temporary
2498 * register is used since neither src_reg nor dst_reg can be
2499 * overridden.
2500 */
2501 if (type == BPF_WRITE) {
2502 int treg = BPF_REG_9;
2503
2504 if (si->src_reg == treg || si->dst_reg == treg)
2505 --treg;
2506 if (si->src_reg == treg || si->dst_reg == treg)
2507 --treg;
2508 *insn++ = BPF_STX_MEM(
2509 BPF_DW, si->dst_reg, treg,
2510 offsetof(struct bpf_sysctl_kern, tmp_reg));
2511 *insn++ = BPF_LDX_MEM(
2512 BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos),
2513 treg, si->dst_reg,
2514 offsetof(struct bpf_sysctl_kern, ppos));
2515 *insn++ = BPF_RAW_INSN(
2516 BPF_CLASS(si->code) | BPF_MEM | BPF_SIZEOF(u32),
2517 treg, si->src_reg,
2518 bpf_ctx_narrow_access_offset(
2519 0, sizeof(u32), sizeof(loff_t)),
2520 si->imm);
2521 *insn++ = BPF_LDX_MEM(
2522 BPF_DW, treg, si->dst_reg,
2523 offsetof(struct bpf_sysctl_kern, tmp_reg));
2524 } else {
2525 *insn++ = BPF_LDX_MEM(
2526 BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos),
2527 si->dst_reg, si->src_reg,
2528 offsetof(struct bpf_sysctl_kern, ppos));
2529 read_size = bpf_size_to_bytes(BPF_SIZE(si->code));
2530 *insn++ = BPF_LDX_MEM(
2531 BPF_SIZE(si->code), si->dst_reg, si->dst_reg,
2532 bpf_ctx_narrow_access_offset(
2533 0, read_size, sizeof(loff_t)));
2534 }
2535 *target_size = sizeof(u32);
2536 break;
2537 }
2538
2539 return insn - insn_buf;
2540}
2541
2542const struct bpf_verifier_ops cg_sysctl_verifier_ops = {
2543 .get_func_proto = sysctl_func_proto,
2544 .is_valid_access = sysctl_is_valid_access,
2545 .convert_ctx_access = sysctl_convert_ctx_access,
2546};
2547
2548const struct bpf_prog_ops cg_sysctl_prog_ops = {
2549};
2550
2551#ifdef CONFIG_NET
2552BPF_CALL_1(bpf_get_netns_cookie_sockopt, struct bpf_sockopt_kern *, ctx)
2553{
2554 const struct net *net = ctx ? sock_net(ctx->sk) : &init_net;
2555
2556 return net->net_cookie;
2557}
2558
2559static const struct bpf_func_proto bpf_get_netns_cookie_sockopt_proto = {
2560 .func = bpf_get_netns_cookie_sockopt,
2561 .gpl_only = false,
2562 .ret_type = RET_INTEGER,
2563 .arg1_type = ARG_PTR_TO_CTX_OR_NULL,
2564};
2565#endif
2566
2567static const struct bpf_func_proto *
2568cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
2569{
2570 const struct bpf_func_proto *func_proto;
2571
2572 func_proto = cgroup_common_func_proto(func_id, prog);
2573 if (func_proto)
2574 return func_proto;
2575
2576 switch (func_id) {
2577#ifdef CONFIG_NET
2578 case BPF_FUNC_get_netns_cookie:
2579 return &bpf_get_netns_cookie_sockopt_proto;
2580 case BPF_FUNC_sk_storage_get:
2581 return &bpf_sk_storage_get_proto;
2582 case BPF_FUNC_sk_storage_delete:
2583 return &bpf_sk_storage_delete_proto;
2584 case BPF_FUNC_setsockopt:
2585 if (prog->expected_attach_type == BPF_CGROUP_SETSOCKOPT)
2586 return &bpf_sk_setsockopt_proto;
2587 return NULL;
2588 case BPF_FUNC_getsockopt:
2589 if (prog->expected_attach_type == BPF_CGROUP_SETSOCKOPT)
2590 return &bpf_sk_getsockopt_proto;
2591 return NULL;
2592#endif
2593#ifdef CONFIG_INET
2594 case BPF_FUNC_tcp_sock:
2595 return &bpf_tcp_sock_proto;
2596#endif
2597 case BPF_FUNC_perf_event_output:
2598 return &bpf_event_output_data_proto;
2599 default:
2600 return bpf_base_func_proto(func_id, prog);
2601 }
2602}
2603
2604static bool cg_sockopt_is_valid_access(int off, int size,
2605 enum bpf_access_type type,
2606 const struct bpf_prog *prog,
2607 struct bpf_insn_access_aux *info)
2608{
2609 const int size_default = sizeof(__u32);
2610
2611 if (off < 0 || off >= sizeof(struct bpf_sockopt))
2612 return false;
2613
2614 if (off % size != 0)
2615 return false;
2616
2617 if (type == BPF_WRITE) {
2618 switch (off) {
2619 case offsetof(struct bpf_sockopt, retval):
2620 if (size != size_default)
2621 return false;
2622 return prog->expected_attach_type ==
2623 BPF_CGROUP_GETSOCKOPT;
2624 case offsetof(struct bpf_sockopt, optname):
2625 fallthrough;
2626 case offsetof(struct bpf_sockopt, level):
2627 if (size != size_default)
2628 return false;
2629 return prog->expected_attach_type ==
2630 BPF_CGROUP_SETSOCKOPT;
2631 case offsetof(struct bpf_sockopt, optlen):
2632 return size == size_default;
2633 default:
2634 return false;
2635 }
2636 }
2637
2638 switch (off) {
2639 case bpf_ctx_range_ptr(struct bpf_sockopt, sk):
2640 if (size != sizeof(__u64))
2641 return false;
2642 info->reg_type = PTR_TO_SOCKET;
2643 break;
2644 case bpf_ctx_range_ptr(struct bpf_sockopt, optval):
2645 if (size != sizeof(__u64))
2646 return false;
2647 info->reg_type = PTR_TO_PACKET;
2648 break;
2649 case bpf_ctx_range_ptr(struct bpf_sockopt, optval_end):
2650 if (size != sizeof(__u64))
2651 return false;
2652 info->reg_type = PTR_TO_PACKET_END;
2653 break;
2654 case bpf_ctx_range(struct bpf_sockopt, retval):
2655 if (size != size_default)
2656 return false;
2657 return prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT;
2658 default:
2659 if (size != size_default)
2660 return false;
2661 break;
2662 }
2663 return true;
2664}
2665
2666#define CG_SOCKOPT_READ_FIELD(F) \
2667 BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, F), \
2668 si->dst_reg, si->src_reg, \
2669 offsetof(struct bpf_sockopt_kern, F))
2670
2671#define CG_SOCKOPT_WRITE_FIELD(F) \
2672 BPF_RAW_INSN((BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, F) | \
2673 BPF_MEM | BPF_CLASS(si->code)), \
2674 si->dst_reg, si->src_reg, \
2675 offsetof(struct bpf_sockopt_kern, F), \
2676 si->imm)
2677
2678static u32 cg_sockopt_convert_ctx_access(enum bpf_access_type type,
2679 const struct bpf_insn *si,
2680 struct bpf_insn *insn_buf,
2681 struct bpf_prog *prog,
2682 u32 *target_size)
2683{
2684 struct bpf_insn *insn = insn_buf;
2685
2686 switch (si->off) {
2687 case offsetof(struct bpf_sockopt, sk):
2688 *insn++ = CG_SOCKOPT_READ_FIELD(sk);
2689 break;
2690 case offsetof(struct bpf_sockopt, level):
2691 if (type == BPF_WRITE)
2692 *insn++ = CG_SOCKOPT_WRITE_FIELD(level);
2693 else
2694 *insn++ = CG_SOCKOPT_READ_FIELD(level);
2695 break;
2696 case offsetof(struct bpf_sockopt, optname):
2697 if (type == BPF_WRITE)
2698 *insn++ = CG_SOCKOPT_WRITE_FIELD(optname);
2699 else
2700 *insn++ = CG_SOCKOPT_READ_FIELD(optname);
2701 break;
2702 case offsetof(struct bpf_sockopt, optlen):
2703 if (type == BPF_WRITE)
2704 *insn++ = CG_SOCKOPT_WRITE_FIELD(optlen);
2705 else
2706 *insn++ = CG_SOCKOPT_READ_FIELD(optlen);
2707 break;
2708 case offsetof(struct bpf_sockopt, retval):
2709 BUILD_BUG_ON(offsetof(struct bpf_cg_run_ctx, run_ctx) != 0);
2710
2711 if (type == BPF_WRITE) {
2712 int treg = BPF_REG_9;
2713
2714 if (si->src_reg == treg || si->dst_reg == treg)
2715 --treg;
2716 if (si->src_reg == treg || si->dst_reg == treg)
2717 --treg;
2718 *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, treg,
2719 offsetof(struct bpf_sockopt_kern, tmp_reg));
2720 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, current_task),
2721 treg, si->dst_reg,
2722 offsetof(struct bpf_sockopt_kern, current_task));
2723 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct task_struct, bpf_ctx),
2724 treg, treg,
2725 offsetof(struct task_struct, bpf_ctx));
2726 *insn++ = BPF_RAW_INSN(BPF_CLASS(si->code) | BPF_MEM |
2727 BPF_FIELD_SIZEOF(struct bpf_cg_run_ctx, retval),
2728 treg, si->src_reg,
2729 offsetof(struct bpf_cg_run_ctx, retval),
2730 si->imm);
2731 *insn++ = BPF_LDX_MEM(BPF_DW, treg, si->dst_reg,
2732 offsetof(struct bpf_sockopt_kern, tmp_reg));
2733 } else {
2734 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, current_task),
2735 si->dst_reg, si->src_reg,
2736 offsetof(struct bpf_sockopt_kern, current_task));
2737 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct task_struct, bpf_ctx),
2738 si->dst_reg, si->dst_reg,
2739 offsetof(struct task_struct, bpf_ctx));
2740 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_cg_run_ctx, retval),
2741 si->dst_reg, si->dst_reg,
2742 offsetof(struct bpf_cg_run_ctx, retval));
2743 }
2744 break;
2745 case offsetof(struct bpf_sockopt, optval):
2746 *insn++ = CG_SOCKOPT_READ_FIELD(optval);
2747 break;
2748 case offsetof(struct bpf_sockopt, optval_end):
2749 *insn++ = CG_SOCKOPT_READ_FIELD(optval_end);
2750 break;
2751 }
2752
2753 return insn - insn_buf;
2754}
2755
2756static int cg_sockopt_get_prologue(struct bpf_insn *insn_buf,
2757 bool direct_write,
2758 const struct bpf_prog *prog)
2759{
2760 /* Nothing to do for sockopt argument. The data is kzalloc'ated.
2761 */
2762 return 0;
2763}
2764
2765const struct bpf_verifier_ops cg_sockopt_verifier_ops = {
2766 .get_func_proto = cg_sockopt_func_proto,
2767 .is_valid_access = cg_sockopt_is_valid_access,
2768 .convert_ctx_access = cg_sockopt_convert_ctx_access,
2769 .gen_prologue = cg_sockopt_get_prologue,
2770};
2771
2772const struct bpf_prog_ops cg_sockopt_prog_ops = {
2773};
2774
2775/* Common helpers for cgroup hooks. */
2776const struct bpf_func_proto *
2777cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
2778{
2779 switch (func_id) {
2780 case BPF_FUNC_get_local_storage:
2781 return &bpf_get_local_storage_proto;
2782 case BPF_FUNC_get_retval:
2783 switch (prog->expected_attach_type) {
2784 case BPF_CGROUP_INET_INGRESS:
2785 case BPF_CGROUP_INET_EGRESS:
2786 case BPF_CGROUP_SOCK_OPS:
2787 case BPF_CGROUP_UDP4_RECVMSG:
2788 case BPF_CGROUP_UDP6_RECVMSG:
2789 case BPF_CGROUP_UNIX_RECVMSG:
2790 case BPF_CGROUP_INET4_GETPEERNAME:
2791 case BPF_CGROUP_INET6_GETPEERNAME:
2792 case BPF_CGROUP_UNIX_GETPEERNAME:
2793 case BPF_CGROUP_INET4_GETSOCKNAME:
2794 case BPF_CGROUP_INET6_GETSOCKNAME:
2795 case BPF_CGROUP_UNIX_GETSOCKNAME:
2796 return NULL;
2797 default:
2798 return &bpf_get_retval_proto;
2799 }
2800 case BPF_FUNC_set_retval:
2801 switch (prog->expected_attach_type) {
2802 case BPF_CGROUP_INET_INGRESS:
2803 case BPF_CGROUP_INET_EGRESS:
2804 case BPF_CGROUP_SOCK_OPS:
2805 case BPF_CGROUP_UDP4_RECVMSG:
2806 case BPF_CGROUP_UDP6_RECVMSG:
2807 case BPF_CGROUP_UNIX_RECVMSG:
2808 case BPF_CGROUP_INET4_GETPEERNAME:
2809 case BPF_CGROUP_INET6_GETPEERNAME:
2810 case BPF_CGROUP_UNIX_GETPEERNAME:
2811 case BPF_CGROUP_INET4_GETSOCKNAME:
2812 case BPF_CGROUP_INET6_GETSOCKNAME:
2813 case BPF_CGROUP_UNIX_GETSOCKNAME:
2814 return NULL;
2815 default:
2816 return &bpf_set_retval_proto;
2817 }
2818 default:
2819 return NULL;
2820 }
2821}