projects/linux·blob master · refs

1872 lines · 48.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2019 Facebook */
3#include <linux/hash.h>
4#include <linux/bpf.h>
5#include <linux/filter.h>
6#include <linux/ftrace.h>
7#include <linux/rbtree_latch.h>
8#include <linux/perf_event.h>
9#include <linux/btf.h>
10#include <linux/rcupdate_trace.h>
11#include <linux/rcupdate_wait.h>
12#include <linux/static_call.h>
13#include <linux/bpf_verifier.h>
14#include <linux/bpf_lsm.h>
15#include <linux/delay.h>
16
17/* dummy _ops. The verifier will operate on target program's ops. */
18const struct bpf_verifier_ops bpf_extension_verifier_ops = {
19};
20const struct bpf_prog_ops bpf_extension_prog_ops = {
21};
22
23/* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */
24#define TRAMPOLINE_HASH_BITS 10
25#define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS)
26
27static struct hlist_head trampoline_key_table[TRAMPOLINE_TABLE_SIZE];
28static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE];
29
30/* serializes access to trampoline tables */
31static DEFINE_MUTEX(trampoline_mutex);
32
33/*
34 * Keep 32 trampoline locks (5 bits) in the pool so trampoline_lock_all()
35 * stays below MAX_LOCK_DEPTH. Each pool slot has a distinct lockdep
36 * class because trampoline_lock_all() takes all pool mutexes at once;
37 * otherwise lockdep would report recursive locking on same-class mutexes.
38 */
39#define TRAMPOLINE_LOCKS_BITS 5
40#define TRAMPOLINE_LOCKS_TABLE_SIZE (1 << TRAMPOLINE_LOCKS_BITS)
41
42static struct {
43 struct mutex mutex;
44 struct lock_class_key key;
45} trampoline_locks[TRAMPOLINE_LOCKS_TABLE_SIZE];
46
47static struct mutex *select_trampoline_lock(struct bpf_trampoline *tr)
48{
49 return &trampoline_locks[hash_ptr(tr, TRAMPOLINE_LOCKS_BITS)].mutex;
50}
51
52static void trampoline_lock(struct bpf_trampoline *tr)
53{
54 mutex_lock(select_trampoline_lock(tr));
55}
56
57static void trampoline_unlock(struct bpf_trampoline *tr)
58{
59 mutex_unlock(select_trampoline_lock(tr));
60}
61
62struct bpf_trampoline_ops {
63 int (*register_fentry)(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *data);
64 int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *data);
65 int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im,
66 bool lock_direct_mutex, void *data);
67};
68
69#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
70static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex,
71 const struct bpf_trampoline_ops *ops, void *data);
72static const struct bpf_trampoline_ops trampoline_ops;
73
74#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS
75static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip)
76{
77 struct hlist_head *head_ip;
78 struct bpf_trampoline *tr;
79
80 mutex_lock(&trampoline_mutex);
81 head_ip = &trampoline_ip_table[hash_64(ip, TRAMPOLINE_HASH_BITS)];
82 hlist_for_each_entry(tr, head_ip, hlist_ip) {
83 if (tr->ip == ip)
84 goto out;
85 }
86 tr = NULL;
87out:
88 mutex_unlock(&trampoline_mutex);
89 return tr;
90}
91#else
92static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip)
93{
94 return ops->private;
95}
96#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */
97
98static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip,
99 enum ftrace_ops_cmd cmd)
100{
101 struct bpf_trampoline *tr;
102 int ret = 0;
103
104 tr = direct_ops_ip_lookup(ops, ip);
105 if (!tr)
106 return -EINVAL;
107
108 if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) {
109 /* This is called inside register_ftrace_direct_multi(), so
110 * trampoline's mutex is already locked.
111 */
112 lockdep_assert_held_once(select_trampoline_lock(tr));
113
114 /* Instead of updating the trampoline here, we propagate
115 * -EAGAIN to register_ftrace_direct(). Then we can
116 * retry register_ftrace_direct() after updating the
117 * trampoline.
118 */
119 if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
120 !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) {
121 if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY))
122 return -EBUSY;
123
124 tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
125 return -EAGAIN;
126 }
127
128 return 0;
129 }
130
131 /* The normal locking order is
132 * select_trampoline_lock(tr) => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c)
133 *
134 * The following two commands are called from
135 *
136 * prepare_direct_functions_for_ipmodify
137 * cleanup_direct_functions_after_ipmodify
138 *
139 * In both cases, direct_mutex is already locked. Use
140 * mutex_trylock(select_trampoline_lock(tr)) to avoid deadlock in race condition
141 * (something else holds the same pool lock).
142 */
143 if (!mutex_trylock(select_trampoline_lock(tr))) {
144 /* sleep 1 ms to make sure whatever holding select_trampoline_lock(tr)
145 * makes some progress.
146 */
147 msleep(1);
148 return -EAGAIN;
149 }
150
151 switch (cmd) {
152 case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER:
153 tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
154
155 if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
156 !(tr->flags & BPF_TRAMP_F_ORIG_STACK))
157 ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */,
158 &trampoline_ops, NULL);
159 break;
160 case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER:
161 tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY;
162
163 if (tr->flags & BPF_TRAMP_F_ORIG_STACK)
164 ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */,
165 &trampoline_ops, NULL);
166 break;
167 default:
168 ret = -EINVAL;
169 break;
170 }
171
172 trampoline_unlock(tr);
173 return ret;
174}
175#endif
176
177bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
178{
179 enum bpf_attach_type eatype = prog->expected_attach_type;
180 enum bpf_prog_type ptype = prog->type;
181
182 switch (ptype) {
183 case BPF_PROG_TYPE_TRACING:
184 if (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||
185 eatype == BPF_MODIFY_RETURN || eatype == BPF_TRACE_FSESSION ||
186 eatype == BPF_TRACE_FENTRY_MULTI || eatype == BPF_TRACE_FEXIT_MULTI ||
187 eatype == BPF_TRACE_FSESSION_MULTI)
188 return true;
189 return false;
190 case BPF_PROG_TYPE_LSM:
191 return eatype == BPF_LSM_MAC;
192 default:
193 return false;
194 }
195}
196
197void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym)
198{
199 ksym->start = (unsigned long) data;
200 ksym->end = ksym->start + size;
201}
202
203void bpf_image_ksym_add(struct bpf_ksym *ksym)
204{
205 bpf_ksym_add(ksym);
206 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
207 PAGE_SIZE, false, ksym->name);
208}
209
210void bpf_image_ksym_del(struct bpf_ksym *ksym)
211{
212 bpf_ksym_del(ksym);
213 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
214 PAGE_SIZE, true, ksym->name);
215}
216
217#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
218#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS
219/*
220 * We have only single direct_ops which contains all the direct call
221 * sites and is the only global ftrace_ops for all trampolines.
222 *
223 * We use 'update_ftrace_direct_*' api for attachment.
224 */
225struct ftrace_ops direct_ops = {
226 .ops_func = bpf_tramp_ftrace_ops_func,
227};
228
229static int direct_ops_alloc(struct bpf_trampoline *tr)
230{
231 tr->fops = &direct_ops;
232 return 0;
233}
234
235static void direct_ops_free(struct bpf_trampoline *tr) { }
236
237static struct ftrace_hash *hash_from_ip(struct bpf_trampoline *tr, void *ptr)
238{
239 unsigned long ip, addr = (unsigned long) ptr;
240 struct ftrace_hash *hash;
241
242 ip = ftrace_location(tr->ip);
243 if (!ip)
244 return NULL;
245 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
246 if (!hash)
247 return NULL;
248 if (bpf_trampoline_use_jmp(tr->flags))
249 addr = ftrace_jmp_set(addr);
250 if (!add_ftrace_hash_entry_direct(hash, ip, addr)) {
251 free_ftrace_hash(hash);
252 return NULL;
253 }
254 return hash;
255}
256
257static int direct_ops_add(struct bpf_trampoline *tr, void *addr)
258{
259 struct ftrace_hash *hash = hash_from_ip(tr, addr);
260 int err;
261
262 if (!hash)
263 return -ENOMEM;
264 err = update_ftrace_direct_add(tr->fops, hash);
265 free_ftrace_hash(hash);
266 return err;
267}
268
269static int direct_ops_del(struct bpf_trampoline *tr, void *addr)
270{
271 struct ftrace_hash *hash = hash_from_ip(tr, addr);
272 int err;
273
274 if (!hash)
275 return -ENOMEM;
276 err = update_ftrace_direct_del(tr->fops, hash);
277 free_ftrace_hash(hash);
278 return err;
279}
280
281static int direct_ops_mod(struct bpf_trampoline *tr, void *addr, bool lock_direct_mutex)
282{
283 struct ftrace_hash *hash = hash_from_ip(tr, addr);
284 int err;
285
286 if (!hash)
287 return -ENOMEM;
288 err = update_ftrace_direct_mod(tr->fops, hash, lock_direct_mutex);
289 free_ftrace_hash(hash);
290 return err;
291}
292#else
293/*
294 * We allocate ftrace_ops object for each trampoline and it contains
295 * call site specific for that trampoline.
296 *
297 * We use *_ftrace_direct api for attachment.
298 */
299static int direct_ops_alloc(struct bpf_trampoline *tr)
300{
301 tr->fops = kzalloc_obj(struct ftrace_ops);
302 if (!tr->fops)
303 return -ENOMEM;
304 tr->fops->private = tr;
305 tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
306 return 0;
307}
308
309static void direct_ops_free(struct bpf_trampoline *tr)
310{
311 if (!tr->fops)
312 return;
313 ftrace_free_filter(tr->fops);
314 kfree(tr->fops);
315}
316
317static int direct_ops_add(struct bpf_trampoline *tr, void *ptr)
318{
319 unsigned long addr = (unsigned long) ptr;
320 struct ftrace_ops *ops = tr->fops;
321 int ret;
322
323 if (bpf_trampoline_use_jmp(tr->flags))
324 addr = ftrace_jmp_set(addr);
325
326 ret = ftrace_set_filter_ip(ops, tr->ip, 0, 1);
327 if (ret)
328 return ret;
329 return register_ftrace_direct(ops, addr);
330}
331
332static int direct_ops_del(struct bpf_trampoline *tr, void *addr)
333{
334 return unregister_ftrace_direct(tr->fops, (long)addr, false);
335}
336
337static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex)
338{
339 unsigned long addr = (unsigned long) ptr;
340 struct ftrace_ops *ops = tr->fops;
341
342 if (bpf_trampoline_use_jmp(tr->flags))
343 addr = ftrace_jmp_set(addr);
344 if (lock_direct_mutex)
345 return modify_ftrace_direct(ops, addr);
346 return modify_ftrace_direct_nolock(ops, addr);
347}
348#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */
349#else
350static void direct_ops_free(struct bpf_trampoline *tr) { }
351
352static int direct_ops_alloc(struct bpf_trampoline *tr)
353{
354 return 0;
355}
356
357static int direct_ops_add(struct bpf_trampoline *tr, void *addr)
358{
359 return -ENODEV;
360}
361
362static int direct_ops_del(struct bpf_trampoline *tr, void *addr)
363{
364 return -ENODEV;
365}
366
367static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex)
368{
369 return -ENODEV;
370}
371#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
372
373static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip)
374{
375 struct bpf_trampoline *tr;
376 struct hlist_head *head;
377 int i;
378
379 mutex_lock(&trampoline_mutex);
380 head = &trampoline_key_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
381 hlist_for_each_entry(tr, head, hlist_key) {
382 if (tr->key == key) {
383 refcount_inc(&tr->refcnt);
384 goto out;
385 }
386 }
387 tr = kzalloc_obj(*tr);
388 if (!tr)
389 goto out;
390 if (direct_ops_alloc(tr)) {
391 kfree(tr);
392 tr = NULL;
393 goto out;
394 }
395
396 tr->key = key;
397 tr->ip = ftrace_location(ip);
398 INIT_HLIST_NODE(&tr->hlist_key);
399 INIT_HLIST_NODE(&tr->hlist_ip);
400 hlist_add_head(&tr->hlist_key, head);
401 head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)];
402 hlist_add_head(&tr->hlist_ip, head);
403 refcount_set(&tr->refcnt, 1);
404 INIT_LIST_HEAD(&tr->images);
405 for (i = 0; i < BPF_TRAMP_MAX; i++)
406 INIT_HLIST_HEAD(&tr->progs_hlist[i]);
407out:
408 mutex_unlock(&trampoline_mutex);
409 return tr;
410}
411
412static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flags,
413 void *old_addr, void *new_addr)
414{
415 enum bpf_text_poke_type new_t = BPF_MOD_CALL, old_t = BPF_MOD_CALL;
416 void *ip = tr->func.addr;
417
418 if (!new_addr)
419 new_t = BPF_MOD_NOP;
420 else if (bpf_trampoline_use_jmp(tr->flags))
421 new_t = BPF_MOD_JUMP;
422
423 if (!old_addr)
424 old_t = BPF_MOD_NOP;
425 else if (bpf_trampoline_use_jmp(orig_flags))
426 old_t = BPF_MOD_JUMP;
427
428 return bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr);
429}
430
431static void bpf_tramp_image_put(struct bpf_tramp_image *im);
432
433static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, void *data __maybe_unused)
434{
435 void *old_addr = tr->cur_image->image;
436 int ret;
437
438 if (tr->func.ftrace_managed)
439 ret = direct_ops_del(tr, old_addr);
440 else
441 ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL);
442
443 if (ret)
444 return ret;
445
446 bpf_tramp_image_put(tr->cur_image);
447 tr->cur_image = NULL;
448 return 0;
449}
450
451static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im,
452 bool lock_direct_mutex, void *data __maybe_unused)
453{
454 void *old_addr = tr->cur_image->image;
455 void *new_addr = im->image;
456 int ret;
457
458 if (tr->func.ftrace_managed) {
459 ret = direct_ops_mod(tr, new_addr, lock_direct_mutex);
460 } else {
461 ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr,
462 new_addr);
463 }
464
465 if (ret)
466 return ret;
467
468 bpf_tramp_image_put(tr->cur_image);
469 tr->cur_image = im;
470 return 0;
471}
472
473/* first time registering */
474static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
475 void *data __maybe_unused)
476{
477 void *new_addr = im->image;
478 void *ip = tr->func.addr;
479 unsigned long faddr;
480 int ret;
481
482 faddr = ftrace_location((unsigned long)ip);
483 if (faddr) {
484 if (!tr->fops)
485 return -ENOTSUPP;
486 tr->func.ftrace_managed = true;
487 }
488
489 if (tr->func.ftrace_managed) {
490 ret = direct_ops_add(tr, new_addr);
491 } else {
492 ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr);
493 }
494
495 if (ret)
496 return ret;
497
498 tr->cur_image = im;
499 return 0;
500}
501
502static const struct bpf_trampoline_ops trampoline_ops = {
503 .register_fentry = register_fentry,
504 .unregister_fentry = unregister_fentry,
505 .modify_fentry = modify_fentry,
506};
507
508static struct bpf_tramp_nodes *
509bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)
510{
511 struct bpf_tramp_node *node, **nodes;
512 struct bpf_tramp_nodes *tnodes;
513 int kind;
514
515 *total = 0;
516 tnodes = kzalloc_objs(*tnodes, BPF_TRAMP_MAX);
517 if (!tnodes)
518 return ERR_PTR(-ENOMEM);
519
520 for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
521 tnodes[kind].nr_nodes = tr->progs_cnt[kind];
522 *total += tr->progs_cnt[kind];
523 nodes = tnodes[kind].nodes;
524
525 hlist_for_each_entry(node, &tr->progs_hlist[kind], tramp_hlist) {
526 *ip_arg |= node->link->prog->call_get_func_ip;
527 *nodes++ = node;
528 }
529 }
530 return tnodes;
531}
532
533/*
534 * The arena base against which save_args() converts the arguments marked
535 * with BTF_FMODEL_ARENA_ARG. Only the struct_ops indirect trampoline
536 * converts: it dispatches to a single prog whose arena is known at
537 * generation time. Return 0 when there is nothing to convert.
538 */
539u64 bpf_tramp_arena_base(const struct btf_func_model *m,
540 struct bpf_tramp_nodes *tnodes, u32 flags)
541{
542 const struct bpf_prog *prog;
543 int i;
544
545 if (!(flags & BPF_TRAMP_F_INDIRECT) ||
546 tnodes[BPF_TRAMP_FENTRY].nr_nodes != 1)
547 return 0;
548
549 for (i = 0; i < m->nr_args; i++)
550 if (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG)
551 break;
552 if (i == m->nr_args)
553 return 0;
554
555 /* Verification rejects an arena argument without an arena. */
556 prog = tnodes[BPF_TRAMP_FENTRY].nodes[0]->link->prog;
557 if (WARN_ON_ONCE(!prog->aux->arena))
558 return 0;
559
560 return bpf_arena_get_kern_vm_start(prog->aux->arena);
561}
562
563static void bpf_tramp_image_free(struct bpf_tramp_image *im)
564{
565 bpf_image_ksym_del(&im->ksym);
566 arch_free_bpf_trampoline(im->image, im->size);
567 bpf_jit_uncharge_modmem(im->size);
568 percpu_ref_exit(&im->pcref);
569 kfree(im->skips);
570 kfree_rcu(im, rcu);
571}
572
573static void __bpf_tramp_image_put_deferred(struct work_struct *work)
574{
575 struct bpf_tramp_image *im;
576 struct bpf_trampoline *tr;
577
578 im = container_of(work, struct bpf_tramp_image, work);
579 tr = im->tr;
580 trampoline_lock(tr);
581 list_del(&im->list);
582 trampoline_unlock(tr);
583 bpf_tramp_image_free(im);
584 bpf_trampoline_put(tr);
585}
586
587/* callback, fexit step 3 or fentry step 2 */
588static void __bpf_tramp_image_put_rcu(struct rcu_head *rcu)
589{
590 struct bpf_tramp_image *im;
591
592 im = container_of(rcu, struct bpf_tramp_image, rcu);
593 INIT_WORK(&im->work, __bpf_tramp_image_put_deferred);
594 schedule_work(&im->work);
595}
596
597/* callback, fexit step 2. Called after percpu_ref_kill confirms. */
598static void __bpf_tramp_image_release(struct percpu_ref *pcref)
599{
600 struct bpf_tramp_image *im;
601
602 im = container_of(pcref, struct bpf_tramp_image, pcref);
603 call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu);
604}
605
606/* callback, fexit or fentry step 1 */
607static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu)
608{
609 struct bpf_tramp_image *im;
610
611 im = container_of(rcu, struct bpf_tramp_image, rcu);
612 if (im->call_orig)
613 /* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */
614 percpu_ref_kill(&im->pcref);
615 else
616 /* the case of fentry trampoline */
617 call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu);
618}
619
620static void bpf_tramp_image_put(struct bpf_tramp_image *im)
621{
622 /* The trampoline image that calls original function is using:
623 * rcu_read_lock_trace to protect sleepable bpf progs
624 * rcu_read_lock to protect normal bpf progs
625 * percpu_ref to protect trampoline itself
626 * rcu tasks to protect trampoline asm not covered by percpu_ref
627 * (which are few asm insns before __bpf_tramp_enter and
628 * after __bpf_tramp_exit)
629 *
630 * The trampoline is unreachable before bpf_tramp_image_put().
631 *
632 * Progs are patched out of the image when they are detached, see
633 * bpf_trampoline_skip_prog(), so they can be freed even if a task is
634 * still in the image.
635 * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on
636 * first few asm instructions to execute and call into
637 * __bpf_tramp_enter->percpu_ref_get.
638 * Then use percpu_ref_kill to wait for the trampoline and the original
639 * function to finish.
640 * Then use call_rcu_tasks() to make sure few asm insns in
641 * the trampoline epilogue are done as well.
642 *
643 * In !PREEMPT case the task that got interrupted in the first asm
644 * insns won't go through an RCU quiescent state which the
645 * percpu_ref_kill will be waiting for. Hence the first
646 * call_rcu_tasks() is not necessary.
647 */
648 if (im->call_orig) {
649 if (IS_ENABLED(CONFIG_TASKS_RCU))
650 call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
651 else
652 percpu_ref_kill(&im->pcref);
653 return;
654 }
655
656 /* The trampoline without fexit and fmod_ret progs doesn't call original
657 * function and doesn't use percpu_ref.
658 * Use call_rcu_tasks_trace() to wait for sleepable progs to finish.
659 * Then use call_rcu_tasks() to wait for the rest of trampoline asm
660 * and normal progs.
661 */
662 call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
663}
664
665static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size, int nr_progs)
666{
667 struct bpf_tramp_image *im;
668 struct bpf_ksym *ksym;
669 void *image;
670 int err = -ENOMEM;
671
672 im = kzalloc_obj(*im);
673 if (!im)
674 goto out;
675
676 im->skips = kzalloc_objs(*im->skips, nr_progs);
677 if (!im->skips)
678 goto out_free_im;
679
680 err = bpf_jit_charge_modmem(size);
681 if (err)
682 goto out_free_im;
683 im->size = size;
684
685 err = -ENOMEM;
686 im->image = image = arch_alloc_bpf_trampoline(size);
687 if (!image)
688 goto out_uncharge;
689
690 err = percpu_ref_init(&im->pcref, __bpf_tramp_image_release, 0, GFP_KERNEL);
691 if (err)
692 goto out_free_image;
693
694 ksym = &im->ksym;
695 INIT_LIST_HEAD_RCU(&ksym->lnode);
696 snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key);
697 bpf_image_ksym_init(image, size, ksym);
698 bpf_image_ksym_add(ksym);
699 return im;
700
701out_free_image:
702 arch_free_bpf_trampoline(im->image, im->size);
703out_uncharge:
704 bpf_jit_uncharge_modmem(size);
705out_free_im:
706 kfree(im->skips);
707 kfree(im);
708out:
709 return ERR_PTR(err);
710}
711
712void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags)
713{
714 trampoline_lock(tr);
715 tr->flags |= flags;
716 trampoline_unlock(tr);
717}
718
719static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex,
720 const struct bpf_trampoline_ops *ops, void *data)
721{
722 struct bpf_tramp_image *im;
723 struct bpf_tramp_nodes *tnodes;
724 u32 orig_flags = tr->flags;
725 bool ip_arg = false;
726 int err, total, size;
727
728 tnodes = bpf_trampoline_get_progs(tr, &total, &ip_arg);
729 if (IS_ERR(tnodes))
730 return PTR_ERR(tnodes);
731
732 if (total == 0) {
733 err = ops->unregister_fentry(tr, orig_flags, data);
734 goto out;
735 }
736
737 /* clear all bits except SHARE_IPMODIFY and TAIL_CALL_CTX */
738 tr->flags &= (BPF_TRAMP_F_SHARE_IPMODIFY | BPF_TRAMP_F_TAIL_CALL_CTX);
739
740 if (tnodes[BPF_TRAMP_FEXIT].nr_nodes ||
741 tnodes[BPF_TRAMP_MODIFY_RETURN].nr_nodes) {
742 /* NOTE: BPF_TRAMP_F_RESTORE_REGS and BPF_TRAMP_F_SKIP_FRAME
743 * should not be set together.
744 */
745 tr->flags |= BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
746 } else {
747 tr->flags |= BPF_TRAMP_F_RESTORE_REGS;
748 }
749
750 if (ip_arg)
751 tr->flags |= BPF_TRAMP_F_IP_ARG;
752
753#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
754again:
755 if (tr->flags & BPF_TRAMP_F_CALL_ORIG) {
756 if (tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) {
757 /* The BPF_TRAMP_F_SKIP_FRAME can be cleared in the
758 * first try, reset it in the second try.
759 */
760 tr->flags |= BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SKIP_FRAME;
761 } else if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_JMP)) {
762 /* Use "jmp" instead of "call" for the trampoline
763 * in the origin call case, and we don't need to
764 * skip the frame.
765 */
766 tr->flags &= ~BPF_TRAMP_F_SKIP_FRAME;
767 }
768 }
769#endif
770
771 size = arch_bpf_trampoline_size(&tr->func.model, tr->flags,
772 tnodes, tr->func.addr);
773 if (size < 0) {
774 err = size;
775 goto out;
776 }
777
778 if (size > PAGE_SIZE) {
779 err = -E2BIG;
780 goto out;
781 }
782
783 im = bpf_tramp_image_alloc(tr->key, size, total);
784 if (IS_ERR(im)) {
785 err = PTR_ERR(im);
786 goto out;
787 }
788 im->call_orig = tr->flags & BPF_TRAMP_F_CALL_ORIG;
789
790 err = arch_prepare_bpf_trampoline(im, im->image, im->image + size,
791 &tr->func.model, tr->flags, tnodes,
792 tr->func.addr);
793 if (err < 0)
794 goto out_free;
795
796 err = arch_protect_bpf_trampoline(im->image, im->size);
797 if (err)
798 goto out_free;
799
800 if (tr->cur_image)
801 /* progs already running at this address */
802 err = ops->modify_fentry(tr, orig_flags, im, lock_direct_mutex, data);
803 else
804 /* first time registering */
805 err = ops->register_fentry(tr, im, data);
806
807#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
808 if (err == -EAGAIN) {
809 /* -EAGAIN from bpf_tramp_ftrace_ops_func. Now
810 * BPF_TRAMP_F_SHARE_IPMODIFY is set, we can generate the
811 * trampoline again, and retry register.
812 */
813 bpf_tramp_image_free(im);
814 goto again;
815 }
816#endif
817
818out_free:
819 if (err) {
820 bpf_tramp_image_free(im);
821 } else {
822 /* track the image until it is freed, for bpf_trampoline_skip_prog() */
823 refcount_inc(&tr->refcnt);
824 im->tr = tr;
825 list_add(&im->list, &tr->images);
826 }
827out:
828 /* If any error happens, restore previous flags */
829 if (err)
830 tr->flags = orig_flags;
831 kfree(tnodes);
832 return err;
833}
834
835static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
836{
837 switch (prog->expected_attach_type) {
838 case BPF_TRACE_FENTRY:
839 case BPF_TRACE_FENTRY_MULTI:
840 return BPF_TRAMP_FENTRY;
841 case BPF_MODIFY_RETURN:
842 return BPF_TRAMP_MODIFY_RETURN;
843 case BPF_TRACE_FEXIT:
844 case BPF_TRACE_FEXIT_MULTI:
845 return BPF_TRAMP_FEXIT;
846 case BPF_TRACE_FSESSION:
847 case BPF_TRACE_FSESSION_MULTI:
848 return BPF_TRAMP_FSESSION;
849 case BPF_LSM_MAC:
850 if (!prog->aux->attach_func_proto->type)
851 /* The function returns void, we cannot modify its
852 * return value.
853 */
854 return BPF_TRAMP_FEXIT;
855 else
856 return BPF_TRAMP_MODIFY_RETURN;
857 default:
858 return BPF_TRAMP_REPLACE;
859 }
860}
861
862static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog)
863{
864 struct bpf_prog_aux *aux = tgt_prog->aux;
865
866 guard(mutex)(&aux->ext_mutex);
867 if (aux->prog_array_member_cnt)
868 /* Program extensions can not extend target prog when the target
869 * prog has been updated to any prog_array map as tail callee.
870 * It's to prevent a potential infinite loop like:
871 * tgt prog entry -> tgt prog subprog -> freplace prog entry
872 * --tailcall-> tgt prog entry.
873 */
874 return -EBUSY;
875
876 aux->is_extended = true;
877 return 0;
878}
879
880static struct bpf_tramp_node *fsession_exit(struct bpf_tramp_node *node)
881{
882 if (node->link->type == BPF_LINK_TYPE_TRACING) {
883 struct bpf_tracing_link *link;
884
885 link = container_of(node->link, struct bpf_tracing_link, link.link);
886 return &link->fexit;
887 } else if (node->link->type == BPF_LINK_TYPE_TRACING_MULTI) {
888 struct bpf_tracing_multi_link *link;
889 struct bpf_tracing_multi_node *mnode;
890
891 link = container_of(node->link, struct bpf_tracing_multi_link, link);
892 mnode = container_of(node, struct bpf_tracing_multi_node, node);
893 return &link->fexits[mnode - link->nodes];
894 }
895 return NULL;
896}
897
898static int bpf_trampoline_add_prog(struct bpf_trampoline *tr,
899 struct bpf_tramp_node *node,
900 int cnt)
901{
902 enum bpf_tramp_prog_type kind;
903 struct bpf_tramp_node *node_existing, *fexit;
904 struct hlist_head *prog_list;
905
906 kind = bpf_attach_type_to_tramp(node->link->prog);
907 if (kind == BPF_TRAMP_FSESSION) {
908 prog_list = &tr->progs_hlist[BPF_TRAMP_FENTRY];
909 cnt++;
910 } else {
911 prog_list = &tr->progs_hlist[kind];
912 }
913 if (cnt >= BPF_MAX_TRAMP_LINKS)
914 return -E2BIG;
915 if (!hlist_unhashed(&node->tramp_hlist))
916 /* prog already linked */
917 return -EBUSY;
918 hlist_for_each_entry(node_existing, prog_list, tramp_hlist) {
919 if (node_existing->link->prog != node->link->prog)
920 continue;
921 /* prog already linked */
922 return -EBUSY;
923 }
924
925 hlist_add_head(&node->tramp_hlist, prog_list);
926 node->link->prog->aux->tramp_linked = true;
927 if (kind == BPF_TRAMP_FSESSION) {
928 tr->progs_cnt[BPF_TRAMP_FENTRY]++;
929 fexit = fsession_exit(node);
930 if (WARN_ON_ONCE(!fexit))
931 return -EINVAL;
932 hlist_add_head(&fexit->tramp_hlist, &tr->progs_hlist[BPF_TRAMP_FEXIT]);
933 tr->progs_cnt[BPF_TRAMP_FEXIT]++;
934 } else {
935 tr->progs_cnt[kind]++;
936 }
937 return 0;
938}
939
940/*
941 * Patch the nop in front of a prog call to a jump over it. A task can be
942 * preempted anywhere in the image, so archs that need several instructions for
943 * a jump of any range patch a single near branch here instead.
944 */
945int __weak arch_bpf_trampoline_skip(void *nop, void *target)
946{
947 return bpf_arch_text_poke(nop, BPF_MOD_NOP, BPF_MOD_JUMP, NULL, target);
948}
949
950/*
951 * prog was detached and can be freed, but tasks may still be running in images
952 * that call it, sleeping in an earlier prog for example. They can be in any
953 * image that is not freed yet, not only in cur_image, so patch all of them to
954 * jump over prog.
955 */
956static void bpf_trampoline_skip_prog(struct bpf_trampoline *tr, struct bpf_prog *prog)
957{
958 struct bpf_tramp_image *im;
959 int i, err;
960
961 list_for_each_entry(im, &tr->images, list) {
962 for (i = 0; i < im->nr_skips; i++) {
963 struct bpf_tramp_skip *skip = &im->skips[i];
964
965 if (skip->prog != prog)
966 continue;
967 err = arch_bpf_trampoline_skip(skip->nop, skip->target);
968 WARN_ON_ONCE(err);
969 /* not a nop anymore, and prog's address can be reused */
970 skip->prog = NULL;
971 }
972 }
973}
974
975static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr,
976 struct bpf_tramp_node *node)
977{
978 enum bpf_tramp_prog_type kind;
979 struct bpf_tramp_node *fexit;
980
981 kind = bpf_attach_type_to_tramp(node->link->prog);
982 if (kind == BPF_TRAMP_FSESSION) {
983 fexit = fsession_exit(node);
984 if (WARN_ON_ONCE(!fexit))
985 return;
986 hlist_del_init(&fexit->tramp_hlist);
987 tr->progs_cnt[BPF_TRAMP_FEXIT]--;
988 kind = BPF_TRAMP_FENTRY;
989 }
990 hlist_del_init(&node->tramp_hlist);
991 tr->progs_cnt[kind]--;
992 bpf_trampoline_skip_prog(tr, node->link->prog);
993}
994
995static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node,
996 struct bpf_trampoline *tr,
997 struct bpf_prog *tgt_prog,
998 const struct bpf_trampoline_ops *ops,
999 void *data)
1000{
1001 enum bpf_tramp_prog_type kind;
1002 int err = 0;
1003 int cnt = 0, i;
1004
1005 kind = bpf_attach_type_to_tramp(node->link->prog);
1006 /*
1007 * Arena ctx args are converted only by struct_ops indirect
1008 * trampolines. They must never be attached to a generic trampoline.
1009 */
1010 if (WARN_ON_ONCE(bpf_prog_has_arena_ctx_arg(node->link->prog)))
1011 return -ENOTSUPP;
1012
1013 if (tr->extension_prog)
1014 /* cannot attach fentry/fexit if extension prog is attached.
1015 * cannot overwrite extension prog either.
1016 */
1017 return -EBUSY;
1018
1019 for (i = 0; i < BPF_TRAMP_MAX; i++)
1020 cnt += tr->progs_cnt[i];
1021
1022 if (kind == BPF_TRAMP_REPLACE) {
1023 /* Cannot attach extension if fentry/fexit are in use. */
1024 if (cnt)
1025 return -EBUSY;
1026 err = bpf_freplace_check_tgt_prog(tgt_prog);
1027 if (err)
1028 return err;
1029 tr->extension_prog = node->link->prog;
1030 return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP,
1031 BPF_MOD_JUMP, NULL,
1032 node->link->prog->bpf_func);
1033 }
1034 err = bpf_trampoline_add_prog(tr, node, cnt);
1035 if (err)
1036 return err;
1037 err = bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data);
1038 if (err)
1039 bpf_trampoline_remove_prog(tr, node);
1040 return err;
1041}
1042
1043int bpf_trampoline_link_prog(struct bpf_tramp_node *node,
1044 struct bpf_trampoline *tr,
1045 struct bpf_prog *tgt_prog)
1046{
1047 int err;
1048
1049 trampoline_lock(tr);
1050 err = __bpf_trampoline_link_prog(node, tr, tgt_prog, &trampoline_ops, NULL);
1051 trampoline_unlock(tr);
1052 return err;
1053}
1054
1055static int __bpf_trampoline_unlink_prog(struct bpf_tramp_node *node,
1056 struct bpf_trampoline *tr,
1057 struct bpf_prog *tgt_prog,
1058 const struct bpf_trampoline_ops *ops,
1059 void *data)
1060{
1061 enum bpf_tramp_prog_type kind;
1062 int err;
1063
1064 kind = bpf_attach_type_to_tramp(node->link->prog);
1065 if (kind == BPF_TRAMP_REPLACE) {
1066 WARN_ON_ONCE(!tr->extension_prog);
1067 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,
1068 BPF_MOD_NOP,
1069 tr->extension_prog->bpf_func, NULL);
1070 tr->extension_prog = NULL;
1071 guard(mutex)(&tgt_prog->aux->ext_mutex);
1072 tgt_prog->aux->is_extended = false;
1073 return err;
1074 }
1075 bpf_trampoline_remove_prog(tr, node);
1076 return bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data);
1077}
1078
1079/* bpf_trampoline_unlink_prog() should never fail. */
1080int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node,
1081 struct bpf_trampoline *tr,
1082 struct bpf_prog *tgt_prog)
1083{
1084 int err;
1085
1086 trampoline_lock(tr);
1087 err = __bpf_trampoline_unlink_prog(node, tr, tgt_prog, &trampoline_ops, NULL);
1088 trampoline_unlock(tr);
1089 return err;
1090}
1091
1092#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
1093static void bpf_shim_tramp_link_release(struct bpf_link *link)
1094{
1095 struct bpf_shim_tramp_link *shim_link =
1096 container_of(link, struct bpf_shim_tramp_link, link.link);
1097 int err;
1098
1099 /* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */
1100 if (!shim_link->trampoline)
1101 return;
1102
1103 err = bpf_trampoline_unlink_prog(&shim_link->link.node, shim_link->trampoline, NULL);
1104 WARN_ONCE(err, "bpf_trampoline_unlink_prog failed: %d\n", err);
1105
1106 bpf_trampoline_put(shim_link->trampoline);
1107}
1108
1109static void bpf_shim_tramp_link_dealloc(struct bpf_link *link)
1110{
1111 struct bpf_shim_tramp_link *shim_link =
1112 container_of(link, struct bpf_shim_tramp_link, link.link);
1113
1114 kfree(shim_link);
1115}
1116
1117static const struct bpf_link_ops bpf_shim_tramp_link_lops = {
1118 .release = bpf_shim_tramp_link_release,
1119 .dealloc = bpf_shim_tramp_link_dealloc,
1120};
1121
1122static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,
1123 bpf_func_t bpf_func,
1124 int cgroup_atype,
1125 enum bpf_attach_type attach_type)
1126{
1127 struct bpf_shim_tramp_link *shim_link = NULL;
1128 struct bpf_prog *p;
1129
1130 shim_link = kzalloc_obj(*shim_link, GFP_USER);
1131 if (!shim_link)
1132 return NULL;
1133
1134 p = bpf_prog_alloc(1, 0);
1135 if (!p) {
1136 kfree(shim_link);
1137 return NULL;
1138 }
1139
1140 p->jited = false;
1141 p->bpf_func = bpf_func;
1142
1143 p->aux->cgroup_atype = cgroup_atype;
1144 p->aux->attach_func_proto = prog->aux->attach_func_proto;
1145 p->aux->attach_btf_id = prog->aux->attach_btf_id;
1146 p->aux->attach_btf = prog->aux->attach_btf;
1147 btf_get(p->aux->attach_btf);
1148 p->type = BPF_PROG_TYPE_LSM;
1149 p->expected_attach_type = BPF_LSM_MAC;
1150 bpf_prog_inc(p);
1151 bpf_tramp_link_init(&shim_link->link, BPF_LINK_TYPE_UNSPEC,
1152 &bpf_shim_tramp_link_lops, p, attach_type, 0);
1153 bpf_cgroup_atype_get(p->aux->attach_btf_id, cgroup_atype);
1154
1155 return shim_link;
1156}
1157
1158static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr,
1159 bpf_func_t bpf_func)
1160{
1161 struct bpf_tramp_node *node;
1162 int kind;
1163
1164 for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
1165 hlist_for_each_entry(node, &tr->progs_hlist[kind], tramp_hlist) {
1166 struct bpf_prog *p = node->link->prog;
1167
1168 if (p->bpf_func == bpf_func)
1169 return container_of(node, struct bpf_shim_tramp_link, link.node);
1170 }
1171 }
1172
1173 return NULL;
1174}
1175
1176int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1177 int cgroup_atype,
1178 enum bpf_attach_type attach_type)
1179{
1180 struct bpf_shim_tramp_link *shim_link = NULL;
1181 struct bpf_attach_target_info tgt_info = {};
1182 struct bpf_trampoline *tr;
1183 bpf_func_t bpf_func;
1184 u64 key;
1185 int err;
1186
1187 err = bpf_check_attach_target(NULL, prog, NULL,
1188 prog->aux->attach_btf_id,
1189 &tgt_info);
1190 if (err)
1191 return err;
1192
1193 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
1194 prog->aux->attach_btf_id);
1195
1196 bpf_lsm_find_cgroup_shim(prog, &bpf_func);
1197 tr = bpf_trampoline_get(key, &tgt_info);
1198 if (!tr)
1199 return -ENOMEM;
1200
1201 trampoline_lock(tr);
1202
1203 shim_link = cgroup_shim_find(tr, bpf_func);
1204 if (shim_link && !IS_ERR(bpf_link_inc_not_zero(&shim_link->link.link))) {
1205 /* Reusing existing shim attached by the other program. */
1206 trampoline_unlock(tr);
1207 bpf_trampoline_put(tr); /* bpf_trampoline_get above */
1208 return 0;
1209 }
1210
1211 /* Allocate and install new shim. */
1212
1213 shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype, attach_type);
1214 if (!shim_link) {
1215 err = -ENOMEM;
1216 goto err;
1217 }
1218
1219 err = __bpf_trampoline_link_prog(&shim_link->link.node, tr, NULL, &trampoline_ops, NULL);
1220 if (err)
1221 goto err;
1222
1223 shim_link->trampoline = tr;
1224 /* note, we're still holding tr refcnt from above */
1225
1226 trampoline_unlock(tr);
1227
1228 return 0;
1229err:
1230 trampoline_unlock(tr);
1231
1232 if (shim_link)
1233 bpf_link_put(&shim_link->link.link);
1234
1235 /* have to release tr while _not_ holding pool mutex for trampoline */
1236 bpf_trampoline_put(tr); /* bpf_trampoline_get above */
1237
1238 return err;
1239}
1240
1241void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
1242{
1243 struct bpf_shim_tramp_link *shim_link = NULL;
1244 struct bpf_trampoline *tr;
1245 bpf_func_t bpf_func;
1246 u64 key;
1247
1248 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
1249 prog->aux->attach_btf_id);
1250
1251 bpf_lsm_find_cgroup_shim(prog, &bpf_func);
1252 tr = bpf_trampoline_lookup(key, 0);
1253 if (WARN_ON_ONCE(!tr))
1254 return;
1255
1256 trampoline_lock(tr);
1257 shim_link = cgroup_shim_find(tr, bpf_func);
1258 trampoline_unlock(tr);
1259
1260 if (shim_link)
1261 bpf_link_put(&shim_link->link.link);
1262
1263 bpf_trampoline_put(tr); /* bpf_trampoline_lookup above */
1264}
1265#endif
1266
1267struct bpf_trampoline *bpf_trampoline_get(u64 key,
1268 struct bpf_attach_target_info *tgt_info)
1269{
1270 struct bpf_trampoline *tr;
1271
1272 tr = bpf_trampoline_lookup(key, tgt_info->tgt_addr);
1273 if (!tr)
1274 return NULL;
1275
1276 trampoline_lock(tr);
1277 if (tr->func.addr)
1278 goto out;
1279
1280 memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel));
1281 tr->func.addr = (void *)tgt_info->tgt_addr;
1282out:
1283 trampoline_unlock(tr);
1284 return tr;
1285}
1286
1287void bpf_trampoline_put(struct bpf_trampoline *tr)
1288{
1289 int i;
1290
1291 if (!tr)
1292 return;
1293 mutex_lock(&trampoline_mutex);
1294 if (!refcount_dec_and_test(&tr->refcnt))
1295 goto out;
1296
1297 for (i = 0; i < BPF_TRAMP_MAX; i++)
1298 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i])))
1299 goto out;
1300
1301 /*
1302 * All progs are detached and the last image has been freed, images
1303 * hold a reference on the trampoline until then.
1304 */
1305 hlist_del(&tr->hlist_key);
1306 hlist_del(&tr->hlist_ip);
1307 direct_ops_free(tr);
1308 kfree(tr);
1309out:
1310 mutex_unlock(&trampoline_mutex);
1311}
1312
1313#define NO_START_TIME 1
1314static __always_inline u64 notrace bpf_prog_start_time(void)
1315{
1316 u64 start = NO_START_TIME;
1317
1318 if (static_branch_unlikely(&bpf_stats_enabled_key)) {
1319 start = sched_clock();
1320 if (unlikely(!start))
1321 start = NO_START_TIME;
1322 }
1323 return start;
1324}
1325
1326/* The logic is similar to bpf_prog_run(), but with an explicit
1327 * rcu_read_lock() and migrate_disable() which are required
1328 * for the trampoline. The macro is split into
1329 * call __bpf_prog_enter
1330 * call prog->bpf_func
1331 * call __bpf_prog_exit
1332 *
1333 * __bpf_prog_enter returns:
1334 * 0 - skip execution of the bpf prog
1335 * 1 - execute bpf prog
1336 * [2..MAX_U64] - execute bpf prog and record execution time.
1337 * This is start time.
1338 */
1339static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
1340 __acquires(RCU)
1341{
1342 rcu_read_lock_dont_migrate();
1343
1344 run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
1345
1346 if (unlikely(!bpf_prog_get_recursion_context(prog))) {
1347 bpf_prog_inc_misses_counter(prog);
1348 if (prog->aux->recursion_detected)
1349 prog->aux->recursion_detected(prog);
1350 return 0;
1351 }
1352 return bpf_prog_start_time();
1353}
1354
1355static void notrace __update_prog_stats(struct bpf_prog *prog, u64 start)
1356{
1357 struct bpf_prog_stats *stats;
1358 unsigned long flags;
1359 u64 duration;
1360
1361 /*
1362 * static_key could be enabled in __bpf_prog_enter* and disabled in
1363 * __bpf_prog_exit*. And vice versa. Check that 'start' is valid.
1364 */
1365 if (start <= NO_START_TIME)
1366 return;
1367
1368 duration = sched_clock() - start;
1369 stats = this_cpu_ptr(prog->stats);
1370 flags = u64_stats_update_begin_irqsave(&stats->syncp);
1371 u64_stats_inc(&stats->cnt);
1372 u64_stats_add(&stats->nsecs, duration);
1373 u64_stats_update_end_irqrestore(&stats->syncp, flags);
1374}
1375
1376static __always_inline void notrace update_prog_stats(struct bpf_prog *prog,
1377 u64 start)
1378{
1379 if (static_branch_unlikely(&bpf_stats_enabled_key))
1380 __update_prog_stats(prog, start);
1381}
1382
1383static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,
1384 struct bpf_tramp_run_ctx *run_ctx)
1385 __releases(RCU)
1386{
1387 bpf_reset_run_ctx(run_ctx->saved_run_ctx);
1388
1389 update_prog_stats(prog, start);
1390 bpf_prog_put_recursion_context(prog);
1391 rcu_read_unlock_migrate();
1392}
1393
1394static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
1395 struct bpf_tramp_run_ctx *run_ctx)
1396 __acquires(RCU)
1397{
1398 /* Runtime stats are exported via actual BPF_LSM_CGROUP
1399 * programs, not the shims.
1400 */
1401 rcu_read_lock_dont_migrate();
1402
1403 run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
1404
1405 return NO_START_TIME;
1406}
1407
1408static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
1409 struct bpf_tramp_run_ctx *run_ctx)
1410 __releases(RCU)
1411{
1412 bpf_reset_run_ctx(run_ctx->saved_run_ctx);
1413
1414 rcu_read_unlock_migrate();
1415}
1416
1417u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
1418 struct bpf_tramp_run_ctx *run_ctx)
1419{
1420 rcu_read_lock_trace();
1421 migrate_disable();
1422 might_fault();
1423
1424 run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
1425
1426 if (unlikely(!bpf_prog_get_recursion_context(prog))) {
1427 bpf_prog_inc_misses_counter(prog);
1428 if (prog->aux->recursion_detected)
1429 prog->aux->recursion_detected(prog);
1430 return 0;
1431 }
1432 return bpf_prog_start_time();
1433}
1434
1435void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
1436 struct bpf_tramp_run_ctx *run_ctx)
1437{
1438 bpf_reset_run_ctx(run_ctx->saved_run_ctx);
1439
1440 update_prog_stats(prog, start);
1441 bpf_prog_put_recursion_context(prog);
1442 migrate_enable();
1443 rcu_read_unlock_trace();
1444}
1445
1446static u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog,
1447 struct bpf_tramp_run_ctx *run_ctx)
1448{
1449 rcu_read_lock_trace();
1450 migrate_disable();
1451 might_fault();
1452
1453 run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
1454
1455 return bpf_prog_start_time();
1456}
1457
1458static void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
1459 struct bpf_tramp_run_ctx *run_ctx)
1460{
1461 bpf_reset_run_ctx(run_ctx->saved_run_ctx);
1462
1463 update_prog_stats(prog, start);
1464 migrate_enable();
1465 rcu_read_unlock_trace();
1466}
1467
1468static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,
1469 struct bpf_tramp_run_ctx *run_ctx)
1470 __acquires(RCU)
1471{
1472 rcu_read_lock_dont_migrate();
1473
1474 run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
1475
1476 return bpf_prog_start_time();
1477}
1478
1479static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,
1480 struct bpf_tramp_run_ctx *run_ctx)
1481 __releases(RCU)
1482{
1483 bpf_reset_run_ctx(run_ctx->saved_run_ctx);
1484
1485 update_prog_stats(prog, start);
1486 rcu_read_unlock_migrate();
1487}
1488
1489void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr)
1490{
1491 percpu_ref_get(&tr->pcref);
1492}
1493
1494void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr)
1495{
1496 percpu_ref_put(&tr->pcref);
1497}
1498
1499bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog)
1500{
1501 bool sleepable = prog->sleepable;
1502
1503 if (bpf_prog_check_recur(prog))
1504 return sleepable ? __bpf_prog_enter_sleepable_recur :
1505 __bpf_prog_enter_recur;
1506
1507 if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&
1508 prog->expected_attach_type == BPF_LSM_CGROUP)
1509 return __bpf_prog_enter_lsm_cgroup;
1510
1511 return sleepable ? __bpf_prog_enter_sleepable : __bpf_prog_enter;
1512}
1513
1514bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog)
1515{
1516 bool sleepable = prog->sleepable;
1517
1518 if (bpf_prog_check_recur(prog))
1519 return sleepable ? __bpf_prog_exit_sleepable_recur :
1520 __bpf_prog_exit_recur;
1521
1522 if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&
1523 prog->expected_attach_type == BPF_LSM_CGROUP)
1524 return __bpf_prog_exit_lsm_cgroup;
1525
1526 return sleepable ? __bpf_prog_exit_sleepable : __bpf_prog_exit;
1527}
1528
1529int __weak
1530arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1531 const struct btf_func_model *m, u32 flags,
1532 struct bpf_tramp_nodes *tnodes,
1533 void *func_addr)
1534{
1535 return -ENOTSUPP;
1536}
1537
1538void * __weak arch_alloc_bpf_trampoline(unsigned int size)
1539{
1540 void *image;
1541
1542 if (WARN_ON_ONCE(size > PAGE_SIZE))
1543 return NULL;
1544 image = bpf_jit_alloc_exec(PAGE_SIZE);
1545 if (image)
1546 set_vm_flush_reset_perms(image);
1547 return image;
1548}
1549
1550void __weak arch_free_bpf_trampoline(void *image, unsigned int size)
1551{
1552 WARN_ON_ONCE(size > PAGE_SIZE);
1553 /* bpf_jit_free_exec doesn't need "size", but
1554 * bpf_prog_pack_free() needs it.
1555 */
1556 bpf_jit_free_exec(image);
1557}
1558
1559int __weak arch_protect_bpf_trampoline(void *image, unsigned int size)
1560{
1561 WARN_ON_ONCE(size > PAGE_SIZE);
1562 return set_memory_rox((long)image, 1);
1563}
1564
1565int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1566 struct bpf_tramp_nodes *tnodes, void *func_addr)
1567{
1568 return -ENOTSUPP;
1569}
1570
1571#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && \
1572 defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS) && \
1573 defined(CONFIG_BPF_SYSCALL)
1574
1575static void trampoline_lock_all(void)
1576{
1577 int i;
1578
1579 for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++)
1580 mutex_lock(&trampoline_locks[i].mutex);
1581}
1582
1583static void trampoline_unlock_all(void)
1584{
1585 int i;
1586
1587 for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++)
1588 mutex_unlock(&trampoline_locks[i].mutex);
1589}
1590
1591static void remove_tracing_multi_data(struct bpf_tracing_multi_data *data)
1592{
1593 ftrace_hash_remove(data->reg);
1594 ftrace_hash_remove(data->unreg);
1595 ftrace_hash_remove(data->modify);
1596}
1597
1598static void clear_tracing_multi_data(struct bpf_tracing_multi_data *data)
1599{
1600 remove_tracing_multi_data(data);
1601
1602 free_ftrace_hash(data->reg);
1603 free_ftrace_hash(data->unreg);
1604 free_ftrace_hash(data->modify);
1605}
1606
1607static int init_tracing_multi_data(struct bpf_tracing_multi_data *data)
1608{
1609 data->reg = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
1610 data->unreg = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
1611 data->modify = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
1612
1613 if (!data->reg || !data->unreg || !data->modify) {
1614 clear_tracing_multi_data(data);
1615 return -ENOMEM;
1616 }
1617 return 0;
1618}
1619
1620static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry *entry,
1621 unsigned long ip, unsigned long direct)
1622{
1623 entry->ip = ip;
1624 entry->direct = direct;
1625 add_ftrace_hash_entry(hash, entry);
1626}
1627
1628static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr)
1629{
1630 unsigned long addr = (unsigned long) im->image;
1631 unsigned long ip = ftrace_location(tr->ip);
1632 struct bpf_tracing_multi_data *data = ptr;
1633
1634 if (bpf_trampoline_use_jmp(tr->flags))
1635 addr = ftrace_jmp_set(addr);
1636
1637 tr->func.ftrace_managed = true;
1638 ftrace_hash_add(data->reg, data->entry, ip, addr);
1639 tr->cur_image = im;
1640 return 0;
1641}
1642
1643static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr)
1644{
1645 unsigned long addr = (unsigned long) tr->cur_image->image;
1646 unsigned long ip = ftrace_location(tr->ip);
1647 struct bpf_tracing_multi_data *data = ptr;
1648
1649 if (bpf_trampoline_use_jmp(tr->flags))
1650 addr = ftrace_jmp_set(addr);
1651
1652 ftrace_hash_add(data->unreg, data->entry, ip, addr);
1653 tr->cur_image = NULL;
1654 return 0;
1655}
1656
1657static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im,
1658 bool lock_direct_mutex, void *ptr)
1659{
1660 unsigned long addr = (unsigned long) im->image;
1661 unsigned long ip = ftrace_location(tr->ip);
1662 struct bpf_tracing_multi_data *data = ptr;
1663
1664 if (bpf_trampoline_use_jmp(tr->flags))
1665 addr = ftrace_jmp_set(addr);
1666
1667 ftrace_hash_add(data->modify, data->entry, ip, addr);
1668 tr->cur_image = im;
1669 return 0;
1670}
1671
1672static const struct bpf_trampoline_ops trampoline_multi_ops = {
1673 .register_fentry = register_fentry_multi,
1674 .unregister_fentry = unregister_fentry_multi,
1675 .modify_fentry = modify_fentry_multi,
1676};
1677
1678static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr)
1679{
1680 tr->multi_attach.old_image = tr->cur_image;
1681 tr->multi_attach.old_flags = tr->flags;
1682}
1683
1684static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr)
1685{
1686 /*
1687 * Only free old_image if it is no longer the active image.
1688 * When bpf_trampoline_update() fails before modify_fentry_multi()/
1689 * unregister_fentry_multi() is called, cur_image is unchanged
1690 * (cur_image == old_image) and ftrace still points to it. Freeing
1691 * it would cause a UAF when ftrace calls into the freed memory.
1692 * On success, cur_image is either a new image or NULL, so
1693 * old_image != cur_image means the image is stale.
1694 */
1695 if (tr->multi_attach.old_image &&
1696 tr->multi_attach.old_image != tr->cur_image)
1697 bpf_tramp_image_put(tr->multi_attach.old_image);
1698
1699 tr->multi_attach.old_image = NULL;
1700 tr->multi_attach.old_flags = 0;
1701}
1702
1703static void bpf_trampoline_multi_attach_rollback(struct bpf_trampoline *tr)
1704{
1705 if (tr->cur_image)
1706 bpf_tramp_image_put(tr->cur_image);
1707 tr->cur_image = tr->multi_attach.old_image;
1708 tr->flags = tr->multi_attach.old_flags;
1709
1710 tr->multi_attach.old_image = NULL;
1711 tr->multi_attach.old_flags = 0;
1712}
1713
1714#define for_each_mnode_cnt(mnode, link, cnt) \
1715 for (i = 0, mnode = &link->nodes[i]; i < cnt; i++, mnode = &link->nodes[i])
1716
1717#define for_each_mnode(mnode, link) \
1718 for_each_mnode_cnt(mnode, link, link->nodes_cnt)
1719
1720int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
1721 struct bpf_tracing_multi_link *link)
1722{
1723 struct bpf_tracing_multi_data *data = &link->data;
1724 struct bpf_attach_target_info tgt_info = {};
1725 struct btf *btf = prog->aux->attach_btf;
1726 struct bpf_tracing_multi_node *mnode;
1727 struct bpf_trampoline *tr;
1728 int i, err, rollback_cnt;
1729 u64 key;
1730
1731 for_each_mnode(mnode, link) {
1732 rollback_cnt = i;
1733
1734 err = bpf_check_attach_btf_id_multi(btf, prog, ids[i], &tgt_info);
1735 if (err)
1736 goto rollback_put;
1737
1738 key = bpf_trampoline_compute_key(NULL, btf, ids[i]);
1739
1740 tr = bpf_trampoline_get(key, &tgt_info);
1741 if (!tr) {
1742 err = -ENOMEM;
1743 goto rollback_put;
1744 }
1745
1746 mnode->trampoline = tr;
1747 mnode->node.link = &link->link;
1748 mnode->node.cookie = link->cookies ? link->cookies[i] : 0;
1749
1750 if (prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) {
1751 link->fexits[i].link = &link->link;
1752 link->fexits[i].cookie = link->cookies ? link->cookies[i] : 0;
1753 }
1754
1755 cond_resched();
1756 }
1757
1758 err = init_tracing_multi_data(data);
1759 if (err) {
1760 rollback_cnt = link->nodes_cnt;
1761 goto rollback_put;
1762 }
1763
1764 trampoline_lock_all();
1765
1766 for_each_mnode(mnode, link) {
1767 bpf_trampoline_multi_attach_init(mnode->trampoline);
1768
1769 data->entry = &mnode->entry;
1770 err = __bpf_trampoline_link_prog(&mnode->node, mnode->trampoline, NULL,
1771 &trampoline_multi_ops, data);
1772 if (err) {
1773 rollback_cnt = i;
1774 goto rollback_unlink;
1775 }
1776 }
1777
1778 rollback_cnt = link->nodes_cnt;
1779 if (ftrace_hash_count(data->reg)) {
1780 err = update_ftrace_direct_add(&direct_ops, data->reg);
1781 if (err)
1782 goto rollback_unlink;
1783 }
1784
1785 if (ftrace_hash_count(data->modify)) {
1786 err = update_ftrace_direct_mod(&direct_ops, data->modify, true);
1787 if (err) {
1788 if (ftrace_hash_count(data->reg))
1789 WARN_ON_ONCE(update_ftrace_direct_del(&direct_ops, data->reg));
1790 goto rollback_unlink;
1791 }
1792 }
1793
1794 for_each_mnode(mnode, link)
1795 bpf_trampoline_multi_attach_free(mnode->trampoline);
1796
1797 trampoline_unlock_all();
1798
1799 remove_tracing_multi_data(data);
1800 return 0;
1801
1802rollback_unlink:
1803 for_each_mnode_cnt(mnode, link, rollback_cnt) {
1804 bpf_trampoline_remove_prog(mnode->trampoline, &mnode->node);
1805 bpf_trampoline_multi_attach_rollback(mnode->trampoline);
1806 }
1807
1808 trampoline_unlock_all();
1809
1810 clear_tracing_multi_data(data);
1811 rollback_cnt = link->nodes_cnt;
1812
1813rollback_put:
1814 for_each_mnode_cnt(mnode, link, rollback_cnt)
1815 bpf_trampoline_put(mnode->trampoline);
1816
1817 return err;
1818}
1819
1820void bpf_trampoline_multi_detach(struct bpf_prog *prog,
1821 struct bpf_tracing_multi_link *link)
1822{
1823 struct bpf_tracing_multi_data *data = &link->data;
1824 struct bpf_tracing_multi_node *mnode;
1825 int i, err;
1826
1827 trampoline_lock_all();
1828
1829 for_each_mnode(mnode, link) {
1830 data->entry = &mnode->entry;
1831 bpf_trampoline_multi_attach_init(mnode->trampoline);
1832 err = __bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline, NULL,
1833 &trampoline_multi_ops, data);
1834 WARN_ONCE(err, "__bpf_trampoline_unlink_prog failed: %d\n", err);
1835 }
1836
1837 if (ftrace_hash_count(data->unreg))
1838 WARN_ON_ONCE(update_ftrace_direct_del(&direct_ops, data->unreg));
1839 if (ftrace_hash_count(data->modify))
1840 WARN_ON_ONCE(update_ftrace_direct_mod(&direct_ops, data->modify, true));
1841
1842 for_each_mnode(mnode, link)
1843 bpf_trampoline_multi_attach_free(mnode->trampoline);
1844
1845 trampoline_unlock_all();
1846
1847 for_each_mnode(mnode, link)
1848 bpf_trampoline_put(mnode->trampoline);
1849
1850 clear_tracing_multi_data(data);
1851}
1852
1853#undef for_each_mnode_cnt
1854#undef for_each_mnode
1855
1856#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS &&
1857 CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS &&
1858 CONFIG_BPF_SYSCALL */
1859
1860static int __init init_trampolines(void)
1861{
1862 int i;
1863
1864 for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)
1865 INIT_HLIST_HEAD(&trampoline_key_table[i]);
1866 for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)
1867 INIT_HLIST_HEAD(&trampoline_ip_table[i]);
1868 for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++)
1869 __mutex_init(&trampoline_locks[i].mutex, "trampoline_lock", &trampoline_locks[i].key);
1870 return 0;
1871}
1872late_initcall(init_trampolines);