projects/linux·blob uaccess-cleanups · refs

1597 lines · 39.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Fprobe-based tracing events
4 * Copyright (C) 2022 Google LLC.
5 */
6#define pr_fmt(fmt) "trace_fprobe: " fmt
7
8#include <linux/fprobe.h>
9#include <linux/list.h>
10#include <linux/module.h>
11#include <linux/mutex.h>
12#include <linux/rculist.h>
13#include <linux/security.h>
14#include <linux/tracepoint.h>
15#include <linux/uaccess.h>
16
17#include <asm/ptrace.h>
18
19#include "trace_dynevent.h"
20#include "trace_probe.h"
21#include "trace_probe_kernel.h"
22#include "trace_probe_tmpl.h"
23
24#define FPROBE_EVENT_SYSTEM "fprobes"
25#define TRACEPOINT_EVENT_SYSTEM "tracepoints"
26#define RETHOOK_MAXACTIVE_MAX 4096
27
28static int trace_fprobe_create(const char *raw_command);
29static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);
30static int trace_fprobe_release(struct dyn_event *ev);
31static bool trace_fprobe_is_busy(struct dyn_event *ev);
32static bool trace_fprobe_match(const char *system, const char *event,
33 int argc, const char **argv, struct dyn_event *ev);
34
35static struct dyn_event_operations trace_fprobe_ops = {
36 .create = trace_fprobe_create,
37 .show = trace_fprobe_show,
38 .is_busy = trace_fprobe_is_busy,
39 .free = trace_fprobe_release,
40 .match = trace_fprobe_match,
41};
42
43/* List of tracepoint_user */
44static LIST_HEAD(tracepoint_user_list);
45static DEFINE_MUTEX(tracepoint_user_mutex);
46
47/* While living tracepoint_user, @tpoint can be NULL and @refcount != 0. */
48struct tracepoint_user {
49 struct list_head list;
50 const char *name;
51 struct tracepoint *tpoint;
52 unsigned int refcount;
53};
54
55/* NOTE: you must lock tracepoint_user_mutex. */
56#define for_each_tracepoint_user(tuser) \
57 list_for_each_entry(tuser, &tracepoint_user_list, list)
58
59static int tracepoint_user_register(struct tracepoint_user *tuser)
60{
61 struct tracepoint *tpoint = tuser->tpoint;
62
63 if (!tpoint)
64 return 0;
65
66 return tracepoint_probe_register_prio_may_exist(tpoint,
67 tpoint->probestub, NULL, 0);
68}
69
70static void tracepoint_user_unregister(struct tracepoint_user *tuser)
71{
72 if (!tuser->tpoint)
73 return;
74
75 WARN_ON_ONCE(tracepoint_probe_unregister(tuser->tpoint, tuser->tpoint->probestub, NULL));
76 tuser->tpoint = NULL;
77}
78
79static unsigned long tracepoint_user_ip(struct tracepoint_user *tuser)
80{
81 if (!tuser->tpoint)
82 return 0UL;
83
84 return (unsigned long)tuser->tpoint->probestub;
85}
86
87static void __tracepoint_user_free(struct tracepoint_user *tuser)
88{
89 if (!tuser)
90 return;
91 kfree(tuser->name);
92 kfree(tuser);
93}
94
95DEFINE_FREE(tuser_free, struct tracepoint_user *, __tracepoint_user_free(_T))
96
97static struct tracepoint_user *__tracepoint_user_init(const char *name, struct tracepoint *tpoint)
98{
99 struct tracepoint_user *tuser __free(tuser_free) = NULL;
100 int ret;
101
102 tuser = kzalloc_obj(*tuser);
103 if (!tuser)
104 return NULL;
105 tuser->name = kstrdup(name, GFP_KERNEL);
106 if (!tuser->name)
107 return NULL;
108
109 /* Register tracepoint if it is loaded. */
110 if (tpoint) {
111 tuser->tpoint = tpoint;
112 ret = tracepoint_user_register(tuser);
113 if (ret)
114 return ERR_PTR(ret);
115 }
116
117 tuser->refcount = 1;
118 INIT_LIST_HEAD(&tuser->list);
119 list_add(&tuser->list, &tracepoint_user_list);
120
121 return_ptr(tuser);
122}
123
124static struct tracepoint *find_tracepoint(const char *tp_name,
125 struct module **tp_mod);
126
127/*
128 * Get tracepoint_user if exist, or allocate new one and register it.
129 * If tracepoint is on a module, get its refcounter too.
130 * This returns errno or NULL (not loaded yet) or tracepoint_user.
131 */
132static struct tracepoint_user *tracepoint_user_find_get(const char *name, struct module **pmod)
133{
134 struct module *mod __free(module_put) = NULL;
135 struct tracepoint_user *tuser;
136 struct tracepoint *tpoint;
137
138 if (!name || !pmod)
139 return ERR_PTR(-EINVAL);
140
141 /* Get and lock the module which has tracepoint. */
142 tpoint = find_tracepoint(name, &mod);
143
144 guard(mutex)(&tracepoint_user_mutex);
145 /* Search existing tracepoint_user */
146 for_each_tracepoint_user(tuser) {
147 if (!strcmp(tuser->name, name)) {
148 tuser->refcount++;
149 *pmod = no_free_ptr(mod);
150 return tuser;
151 }
152 }
153
154 /* The corresponding tracepoint_user is not found. */
155 tuser = __tracepoint_user_init(name, tpoint);
156 if (!IS_ERR_OR_NULL(tuser))
157 *pmod = no_free_ptr(mod);
158
159 return tuser;
160}
161
162static void tracepoint_user_put(struct tracepoint_user *tuser)
163{
164 scoped_guard(mutex, &tracepoint_user_mutex) {
165 if (--tuser->refcount > 0)
166 return;
167
168 list_del(&tuser->list);
169 tracepoint_user_unregister(tuser);
170 }
171
172 __tracepoint_user_free(tuser);
173}
174
175DEFINE_FREE(tuser_put, struct tracepoint_user *,
176 if (!IS_ERR_OR_NULL(_T))
177 tracepoint_user_put(_T))
178
179/*
180 * Fprobe event core functions
181 */
182
183/*
184 * @tprobe is true for tracepoint probe.
185 * @tuser can be NULL if the trace_fprobe is disabled or the tracepoint is not
186 * loaded with a module. If @tuser != NULL, this trace_fprobe is enabled.
187 */
188struct trace_fprobe {
189 struct dyn_event devent;
190 struct fprobe fp;
191 const char *symbol;
192 bool tprobe;
193 struct tracepoint_user *tuser;
194 struct trace_probe tp;
195};
196
197static bool is_trace_fprobe(struct dyn_event *ev)
198{
199 return ev->ops == &trace_fprobe_ops;
200}
201
202static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev)
203{
204 return container_of(ev, struct trace_fprobe, devent);
205}
206
207/**
208 * for_each_trace_fprobe - iterate over the trace_fprobe list
209 * @pos: the struct trace_fprobe * for each entry
210 * @dpos: the struct dyn_event * to use as a loop cursor
211 */
212#define for_each_trace_fprobe(pos, dpos) \
213 for_each_dyn_event(dpos) \
214 if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos)))
215
216static bool trace_fprobe_is_return(struct trace_fprobe *tf)
217{
218 return tf->fp.exit_handler != NULL;
219}
220
221static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf)
222{
223 return tf->tprobe;
224}
225
226static const char *trace_fprobe_symbol(struct trace_fprobe *tf)
227{
228 return tf->symbol ? tf->symbol : "unknown";
229}
230
231static bool trace_fprobe_is_busy(struct dyn_event *ev)
232{
233 struct trace_fprobe *tf = to_trace_fprobe(ev);
234
235 return trace_probe_is_enabled(&tf->tp);
236}
237
238static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,
239 int argc, const char **argv)
240{
241 if (!argc)
242 return true;
243
244 if (strcmp(trace_fprobe_symbol(tf), argv[0]))
245 return false;
246 argc--; argv++;
247
248 return trace_probe_match_command_args(&tf->tp, argc, argv);
249}
250
251static bool trace_fprobe_match(const char *system, const char *event,
252 int argc, const char **argv, struct dyn_event *ev)
253{
254 struct trace_fprobe *tf = to_trace_fprobe(ev);
255
256 if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event))
257 return false;
258
259 if (system && strcmp(trace_probe_group_name(&tf->tp), system))
260 return false;
261
262 return trace_fprobe_match_command_head(tf, argc, argv);
263}
264
265static bool trace_fprobe_is_registered(struct trace_fprobe *tf)
266{
267 return fprobe_is_registered(&tf->fp);
268}
269
270/*
271 * Note that we don't verify the fetch_insn code, since it does not come
272 * from user space.
273 */
274static int
275process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
276 void *dest, void *base)
277{
278 struct ftrace_regs *fregs = rec;
279 unsigned long val;
280 int ret;
281
282retry:
283 /* 1st stage: get value from context */
284 switch (code->op) {
285 case FETCH_OP_STACK:
286 val = ftrace_regs_get_kernel_stack_nth(fregs, code->param);
287 break;
288 case FETCH_OP_STACKP:
289 val = ftrace_regs_get_stack_pointer(fregs);
290 break;
291 case FETCH_OP_RETVAL:
292 val = ftrace_regs_get_return_value(fregs);
293 break;
294#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
295 case FETCH_OP_ARG:
296 val = ftrace_regs_get_argument(fregs, code->param);
297 break;
298 case FETCH_OP_EDATA:
299 val = *(unsigned long *)((unsigned long)edata + code->offset);
300 break;
301#endif
302 case FETCH_NOP_SYMBOL: /* Ignore a place holder */
303 code++;
304 goto retry;
305 default:
306 ret = process_common_fetch_insn(code, &val);
307 if (ret < 0)
308 return ret;
309 }
310 code++;
311
312 return process_fetch_insn_bottom(code, val, dest, base);
313}
314NOKPROBE_SYMBOL(process_fetch_insn)
315
316/* function entry handler */
317static nokprobe_inline void
318__fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
319 struct ftrace_regs *fregs,
320 struct trace_event_file *trace_file)
321{
322 struct fentry_trace_entry_head *entry;
323 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
324 struct trace_event_buffer fbuffer;
325 int dsize;
326
327 if (WARN_ON_ONCE(call != trace_file->event_call))
328 return;
329
330 if (trace_trigger_soft_disabled(trace_file))
331 return;
332
333 dsize = __get_data_size(&tf->tp, fregs, NULL);
334
335 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
336 sizeof(*entry) + tf->tp.size + dsize);
337 if (!entry)
338 return;
339
340 fbuffer.regs = ftrace_get_regs(fregs);
341 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
342 entry->ip = entry_ip;
343 store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize);
344
345 trace_event_buffer_commit(&fbuffer);
346}
347
348static void
349fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
350 struct ftrace_regs *fregs)
351{
352 struct event_file_link *link;
353
354 trace_probe_for_each_link_rcu(link, &tf->tp)
355 __fentry_trace_func(tf, entry_ip, fregs, link->file);
356}
357NOKPROBE_SYMBOL(fentry_trace_func);
358
359static nokprobe_inline
360void store_fprobe_entry_data(void *edata, struct trace_probe *tp, struct ftrace_regs *fregs)
361{
362 struct probe_entry_arg *earg = tp->entry_arg;
363 unsigned long val = 0;
364 int i;
365
366 if (!earg)
367 return;
368
369 for (i = 0; i < earg->size; i++) {
370 struct fetch_insn *code = &earg->code[i];
371
372 switch (code->op) {
373 case FETCH_OP_ARG:
374 val = ftrace_regs_get_argument(fregs, code->param);
375 break;
376 case FETCH_OP_ST_EDATA:
377 *(unsigned long *)((unsigned long)edata + code->offset) = val;
378 break;
379 case FETCH_OP_END:
380 goto end;
381 default:
382 break;
383 }
384 }
385end:
386 return;
387}
388
389/* function exit handler */
390static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip,
391 unsigned long ret_ip, struct ftrace_regs *fregs,
392 void *entry_data)
393{
394 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
395
396 if (tf->tp.entry_arg)
397 store_fprobe_entry_data(entry_data, &tf->tp, fregs);
398
399 return 0;
400}
401NOKPROBE_SYMBOL(trace_fprobe_entry_handler)
402
403static nokprobe_inline void
404__fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
405 unsigned long ret_ip, struct ftrace_regs *fregs,
406 void *entry_data, struct trace_event_file *trace_file)
407{
408 struct fexit_trace_entry_head *entry;
409 struct trace_event_buffer fbuffer;
410 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
411 int dsize;
412
413 if (WARN_ON_ONCE(call != trace_file->event_call))
414 return;
415
416 if (trace_trigger_soft_disabled(trace_file))
417 return;
418
419 dsize = __get_data_size(&tf->tp, fregs, entry_data);
420
421 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
422 sizeof(*entry) + tf->tp.size + dsize);
423 if (!entry)
424 return;
425
426 fbuffer.regs = ftrace_get_regs(fregs);
427 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
428 entry->func = entry_ip;
429 entry->ret_ip = ret_ip;
430 store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize);
431
432 trace_event_buffer_commit(&fbuffer);
433}
434
435static void
436fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
437 unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data)
438{
439 struct event_file_link *link;
440
441 trace_probe_for_each_link_rcu(link, &tf->tp)
442 __fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data, link->file);
443}
444NOKPROBE_SYMBOL(fexit_trace_func);
445
446#ifdef CONFIG_PERF_EVENTS
447
448static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
449 struct ftrace_regs *fregs)
450{
451 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
452 struct fentry_trace_entry_head *entry;
453 struct hlist_head *head;
454 int size, __size, dsize;
455 struct pt_regs *regs;
456 int rctx;
457
458 head = this_cpu_ptr(call->perf_events);
459 if (hlist_empty(head))
460 return 0;
461
462 dsize = __get_data_size(&tf->tp, fregs, NULL);
463 __size = sizeof(*entry) + tf->tp.size + dsize;
464 size = ALIGN(__size + sizeof(u32), sizeof(u64));
465 size -= sizeof(u32);
466
467 entry = perf_trace_buf_alloc(size, &regs, &rctx);
468 if (!entry)
469 return 0;
470
471 regs = ftrace_fill_perf_regs(fregs, regs);
472
473 entry->ip = entry_ip;
474 store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize);
475 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
476 head, NULL);
477 return 0;
478}
479NOKPROBE_SYMBOL(fentry_perf_func);
480
481static void
482fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
483 unsigned long ret_ip, struct ftrace_regs *fregs,
484 void *entry_data)
485{
486 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
487 struct fexit_trace_entry_head *entry;
488 struct hlist_head *head;
489 int size, __size, dsize;
490 struct pt_regs *regs;
491 int rctx;
492
493 head = this_cpu_ptr(call->perf_events);
494 if (hlist_empty(head))
495 return;
496
497 dsize = __get_data_size(&tf->tp, fregs, entry_data);
498 __size = sizeof(*entry) + tf->tp.size + dsize;
499 size = ALIGN(__size + sizeof(u32), sizeof(u64));
500 size -= sizeof(u32);
501
502 entry = perf_trace_buf_alloc(size, &regs, &rctx);
503 if (!entry)
504 return;
505
506 regs = ftrace_fill_perf_regs(fregs, regs);
507
508 entry->func = entry_ip;
509 entry->ret_ip = ret_ip;
510 store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize);
511 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
512 head, NULL);
513}
514NOKPROBE_SYMBOL(fexit_perf_func);
515#endif /* CONFIG_PERF_EVENTS */
516
517static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip,
518 unsigned long ret_ip, struct ftrace_regs *fregs,
519 void *entry_data)
520{
521 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
522 unsigned int flags = trace_probe_load_flag(&tf->tp);
523 int ret = 0;
524
525 if (flags & TP_FLAG_TRACE)
526 fentry_trace_func(tf, entry_ip, fregs);
527
528#ifdef CONFIG_PERF_EVENTS
529 if (flags & TP_FLAG_PROFILE)
530 ret = fentry_perf_func(tf, entry_ip, fregs);
531#endif
532 return ret;
533}
534NOKPROBE_SYMBOL(fentry_dispatcher);
535
536static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip,
537 unsigned long ret_ip, struct ftrace_regs *fregs,
538 void *entry_data)
539{
540 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
541 unsigned int flags = trace_probe_load_flag(&tf->tp);
542
543 if (flags & TP_FLAG_TRACE)
544 fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data);
545#ifdef CONFIG_PERF_EVENTS
546 if (flags & TP_FLAG_PROFILE)
547 fexit_perf_func(tf, entry_ip, ret_ip, fregs, entry_data);
548#endif
549}
550NOKPROBE_SYMBOL(fexit_dispatcher);
551
552static void free_trace_fprobe(struct trace_fprobe *tf)
553{
554 if (tf) {
555 trace_probe_cleanup(&tf->tp);
556 if (tf->tuser)
557 tracepoint_user_put(tf->tuser);
558 kfree(tf->symbol);
559 kfree(tf);
560 }
561}
562
563/* Since alloc_trace_fprobe() can return error, check the pointer is ERR too. */
564DEFINE_FREE(free_trace_fprobe, struct trace_fprobe *, if (!IS_ERR_OR_NULL(_T)) free_trace_fprobe(_T))
565
566/*
567 * Allocate new trace_probe and initialize it (including fprobe).
568 */
569static struct trace_fprobe *alloc_trace_fprobe(const char *group,
570 const char *event,
571 const char *symbol,
572 int nargs, bool is_return,
573 bool is_tracepoint)
574{
575 struct trace_fprobe *tf __free(free_trace_fprobe) = NULL;
576 int ret = -ENOMEM;
577
578 tf = kzalloc_flex(*tf, tp.args, nargs);
579 if (!tf)
580 return ERR_PTR(ret);
581
582 tf->symbol = kstrdup(symbol, GFP_KERNEL);
583 if (!tf->symbol)
584 return ERR_PTR(-ENOMEM);
585
586 if (is_return)
587 tf->fp.exit_handler = fexit_dispatcher;
588 else
589 tf->fp.entry_handler = fentry_dispatcher;
590
591 tf->tprobe = is_tracepoint;
592
593 ret = trace_probe_init(&tf->tp, event, group, false, nargs);
594 if (ret < 0)
595 return ERR_PTR(ret);
596
597 dyn_event_init(&tf->devent, &trace_fprobe_ops);
598 return_ptr(tf);
599}
600
601static struct trace_fprobe *find_trace_fprobe(const char *event,
602 const char *group)
603{
604 struct dyn_event *pos;
605 struct trace_fprobe *tf;
606
607 for_each_trace_fprobe(tf, pos)
608 if (strcmp(trace_probe_name(&tf->tp), event) == 0 &&
609 strcmp(trace_probe_group_name(&tf->tp), group) == 0)
610 return tf;
611 return NULL;
612}
613
614/* Event entry printers */
615static enum print_line_t
616print_fentry_event(struct trace_iterator *iter, int flags,
617 struct trace_event *event)
618{
619 struct fentry_trace_entry_head *field;
620 struct trace_seq *s = &iter->seq;
621 struct trace_probe *tp;
622
623 field = (struct fentry_trace_entry_head *)iter->ent;
624 tp = trace_probe_primary_from_call(
625 container_of(event, struct trace_event_call, event));
626 if (WARN_ON_ONCE(!tp))
627 goto out;
628
629 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
630
631 if (!seq_print_ip_sym_offset(s, field->ip, flags))
632 goto out;
633
634 trace_seq_putc(s, ')');
635
636 if (trace_probe_print_args(s, tp->args, tp->nr_args,
637 (u8 *)&field[1], field) < 0)
638 goto out;
639
640 trace_seq_putc(s, '\n');
641 out:
642 return trace_handle_return(s);
643}
644
645static enum print_line_t
646print_fexit_event(struct trace_iterator *iter, int flags,
647 struct trace_event *event)
648{
649 struct fexit_trace_entry_head *field;
650 struct trace_seq *s = &iter->seq;
651 struct trace_probe *tp;
652
653 field = (struct fexit_trace_entry_head *)iter->ent;
654 tp = trace_probe_primary_from_call(
655 container_of(event, struct trace_event_call, event));
656 if (WARN_ON_ONCE(!tp))
657 goto out;
658
659 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
660
661 if (!seq_print_ip_sym_offset(s, field->ret_ip, flags))
662 goto out;
663
664 trace_seq_puts(s, " <- ");
665
666 if (!seq_print_ip_sym_no_offset(s, field->func, flags))
667 goto out;
668
669 trace_seq_putc(s, ')');
670
671 if (trace_probe_print_args(s, tp->args, tp->nr_args,
672 (u8 *)&field[1], field) < 0)
673 goto out;
674
675 trace_seq_putc(s, '\n');
676
677 out:
678 return trace_handle_return(s);
679}
680
681static int fentry_event_define_fields(struct trace_event_call *event_call)
682{
683 int ret;
684 struct fentry_trace_entry_head field;
685 struct trace_probe *tp;
686
687 tp = trace_probe_primary_from_call(event_call);
688 if (WARN_ON_ONCE(!tp))
689 return -ENOENT;
690
691 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
692
693 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
694}
695
696static int fexit_event_define_fields(struct trace_event_call *event_call)
697{
698 int ret;
699 struct fexit_trace_entry_head field;
700 struct trace_probe *tp;
701
702 tp = trace_probe_primary_from_call(event_call);
703 if (WARN_ON_ONCE(!tp))
704 return -ENOENT;
705
706 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
707 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
708
709 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
710}
711
712static struct trace_event_functions fentry_funcs = {
713 .trace = print_fentry_event
714};
715
716static struct trace_event_functions fexit_funcs = {
717 .trace = print_fexit_event
718};
719
720static struct trace_event_fields fentry_fields_array[] = {
721 { .type = TRACE_FUNCTION_TYPE,
722 .define_fields = fentry_event_define_fields },
723 {}
724};
725
726static struct trace_event_fields fexit_fields_array[] = {
727 { .type = TRACE_FUNCTION_TYPE,
728 .define_fields = fexit_event_define_fields },
729 {}
730};
731
732static int fprobe_register(struct trace_event_call *event,
733 enum trace_reg type, void *data);
734
735static inline void init_trace_event_call(struct trace_fprobe *tf)
736{
737 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
738
739 if (trace_fprobe_is_return(tf)) {
740 call->event.funcs = &fexit_funcs;
741 call->class->fields_array = fexit_fields_array;
742 } else {
743 call->event.funcs = &fentry_funcs;
744 call->class->fields_array = fentry_fields_array;
745 }
746
747 call->flags = TRACE_EVENT_FL_FPROBE;
748 call->class->reg = fprobe_register;
749}
750
751static int register_fprobe_event(struct trace_fprobe *tf)
752{
753 init_trace_event_call(tf);
754
755 return trace_probe_register_event_call(&tf->tp);
756}
757
758static int unregister_fprobe_event(struct trace_fprobe *tf)
759{
760 return trace_probe_unregister_event_call(&tf->tp);
761}
762
763static int __register_tracepoint_fprobe(struct trace_fprobe *tf)
764{
765 struct tracepoint_user *tuser __free(tuser_put) = NULL;
766 struct module *mod __free(module_put) = NULL;
767 unsigned long ip;
768 int ret;
769
770 if (WARN_ON_ONCE(tf->tuser))
771 return -EINVAL;
772
773 /* If the tracepoint is in a module, it must be locked in this function. */
774 tuser = tracepoint_user_find_get(tf->symbol, &mod);
775 /* This tracepoint is not loaded yet */
776 if (IS_ERR(tuser))
777 return PTR_ERR(tuser);
778 if (!tuser)
779 return -ENOMEM;
780
781 /* Register fprobe only if the tracepoint is loaded. */
782 if (tuser->tpoint) {
783 ip = tracepoint_user_ip(tuser);
784 if (WARN_ON_ONCE(!ip))
785 return -ENOENT;
786
787 ret = register_fprobe_ips(&tf->fp, &ip, 1);
788 if (ret < 0)
789 return ret;
790 }
791
792 tf->tuser = no_free_ptr(tuser);
793 return 0;
794}
795
796/* Returns an error if the target function is not available, or 0 */
797static int trace_fprobe_verify_target(struct trace_fprobe *tf)
798{
799 int ret;
800
801 /* Tracepoint should have a stub function. */
802 if (trace_fprobe_is_tracepoint(tf))
803 return 0;
804
805 /*
806 * Note: since we don't lock the module, even if this succeeded,
807 * register_fprobe() later can fail.
808 */
809 ret = fprobe_count_ips_from_filter(tf->symbol, NULL);
810 return (ret < 0) ? ret : 0;
811}
812
813/* Internal register function - just handle fprobe and flags */
814static int __register_trace_fprobe(struct trace_fprobe *tf)
815{
816 int i, ret;
817
818 /* Should we need new LOCKDOWN flag for fprobe? */
819 ret = security_locked_down(LOCKDOWN_KPROBES);
820 if (ret)
821 return ret;
822
823 if (trace_fprobe_is_registered(tf))
824 return -EINVAL;
825
826 for (i = 0; i < tf->tp.nr_args; i++) {
827 ret = traceprobe_update_arg(&tf->tp.args[i]);
828 if (ret)
829 return ret;
830 }
831
832 tf->fp.flags &= ~FPROBE_FL_DISABLED;
833
834 if (trace_fprobe_is_tracepoint(tf))
835 return __register_tracepoint_fprobe(tf);
836
837 /* TODO: handle filter, nofilter or symbol list */
838 return register_fprobe(&tf->fp, tf->symbol, NULL);
839}
840
841/* Internal unregister function - just handle fprobe and flags */
842static void __unregister_trace_fprobe(struct trace_fprobe *tf)
843{
844 if (trace_fprobe_is_registered(tf))
845 unregister_fprobe(&tf->fp);
846 if (tf->tuser) {
847 tracepoint_user_put(tf->tuser);
848 tf->tuser = NULL;
849 }
850}
851
852/* TODO: make this trace_*probe common function */
853/* Unregister a trace_probe and probe_event */
854static int unregister_trace_fprobe(struct trace_fprobe *tf)
855{
856 /* If other probes are on the event, just unregister fprobe */
857 if (trace_probe_has_sibling(&tf->tp))
858 goto unreg;
859
860 /* Enabled event can not be unregistered */
861 if (trace_probe_is_enabled(&tf->tp))
862 return -EBUSY;
863
864 /* If there's a reference to the dynamic event */
865 if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp)))
866 return -EBUSY;
867
868 /* Will fail if probe is being used by ftrace or perf */
869 if (unregister_fprobe_event(tf))
870 return -EBUSY;
871
872unreg:
873 __unregister_trace_fprobe(tf);
874 dyn_event_remove(&tf->devent);
875 trace_probe_unlink(&tf->tp);
876
877 return 0;
878}
879
880static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig,
881 struct trace_fprobe *comp)
882{
883 struct trace_probe_event *tpe = orig->tp.event;
884 int i;
885
886 list_for_each_entry(orig, &tpe->probes, tp.list) {
887 if (strcmp(trace_fprobe_symbol(orig),
888 trace_fprobe_symbol(comp)))
889 continue;
890
891 /*
892 * trace_probe_compare_arg_type() ensured that nr_args and
893 * each argument name and type are same. Let's compare comm.
894 */
895 for (i = 0; i < orig->tp.nr_args; i++) {
896 if (strcmp(orig->tp.args[i].comm,
897 comp->tp.args[i].comm))
898 break;
899 }
900
901 if (i == orig->tp.nr_args)
902 return true;
903 }
904
905 return false;
906}
907
908static int append_trace_fprobe_event(struct trace_fprobe *tf, struct trace_fprobe *to)
909{
910 int ret;
911
912 if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) ||
913 trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) {
914 trace_probe_log_set_index(0);
915 trace_probe_log_err(0, DIFF_PROBE_TYPE);
916 return -EEXIST;
917 }
918 ret = trace_probe_compare_arg_type(&tf->tp, &to->tp);
919 if (ret) {
920 /* Note that argument starts index = 2 */
921 trace_probe_log_set_index(ret + 1);
922 trace_probe_log_err(0, DIFF_ARG_TYPE);
923 return -EEXIST;
924 }
925 if (trace_fprobe_has_same_fprobe(to, tf)) {
926 trace_probe_log_set_index(0);
927 trace_probe_log_err(0, SAME_PROBE);
928 return -EEXIST;
929 }
930
931 /* Append to existing event */
932 ret = trace_probe_append(&tf->tp, &to->tp);
933 if (ret)
934 return ret;
935
936 ret = trace_fprobe_verify_target(tf);
937 if (ret)
938 trace_probe_unlink(&tf->tp);
939 else
940 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
941
942 return ret;
943}
944
945/* Register a trace_probe and probe_event, and check the fprobe is available. */
946static int register_trace_fprobe_event(struct trace_fprobe *tf)
947{
948 struct trace_fprobe *old_tf;
949 int ret;
950
951 guard(mutex)(&event_mutex);
952
953 old_tf = find_trace_fprobe(trace_probe_name(&tf->tp),
954 trace_probe_group_name(&tf->tp));
955 if (old_tf)
956 return append_trace_fprobe_event(tf, old_tf);
957
958 /* Register new event */
959 ret = register_fprobe_event(tf);
960 if (ret) {
961 if (ret == -EEXIST) {
962 trace_probe_log_set_index(0);
963 trace_probe_log_err(0, EVENT_EXIST);
964 } else
965 pr_warn("Failed to register probe event(%d)\n", ret);
966 return ret;
967 }
968
969 /* Verify fprobe is sane. */
970 ret = trace_fprobe_verify_target(tf);
971 if (ret < 0)
972 unregister_fprobe_event(tf);
973 else
974 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
975
976 return ret;
977}
978
979struct __find_tracepoint_cb_data {
980 const char *tp_name;
981 struct tracepoint *tpoint;
982 struct module *mod;
983};
984
985static void __find_tracepoint_module_cb(struct tracepoint *tp, struct module *mod, void *priv)
986{
987 struct __find_tracepoint_cb_data *data = priv;
988
989 if (!data->tpoint && !strcmp(data->tp_name, tp->name)) {
990 /* If module is not specified, try getting module refcount. */
991 if (!data->mod && mod) {
992 /* If failed to get refcount, ignore this tracepoint. */
993 if (!try_module_get(mod))
994 return;
995
996 data->mod = mod;
997 }
998 data->tpoint = tp;
999 }
1000}
1001
1002static void __find_tracepoint_cb(struct tracepoint *tp, void *priv)
1003{
1004 struct __find_tracepoint_cb_data *data = priv;
1005
1006 if (!data->tpoint && !strcmp(data->tp_name, tp->name))
1007 data->tpoint = tp;
1008}
1009
1010/*
1011 * Find a tracepoint from kernel and module. If the tracepoint is on the module,
1012 * the module's refcount is incremented and returned as *@tp_mod. Thus, if it is
1013 * not NULL, caller must call module_put(*tp_mod) after used the tracepoint.
1014 */
1015static struct tracepoint *find_tracepoint(const char *tp_name,
1016 struct module **tp_mod)
1017{
1018 struct __find_tracepoint_cb_data data = {
1019 .tp_name = tp_name,
1020 .mod = NULL,
1021 };
1022
1023 for_each_kernel_tracepoint(__find_tracepoint_cb, &data);
1024
1025 if (!data.tpoint && IS_ENABLED(CONFIG_MODULES)) {
1026 for_each_module_tracepoint(__find_tracepoint_module_cb, &data);
1027 *tp_mod = data.mod;
1028 }
1029
1030 return data.tpoint;
1031}
1032
1033#ifdef CONFIG_MODULES
1034/*
1035 * Find a tracepoint from specified module. In this case, this does not get the
1036 * module's refcount. The caller must ensure the module is not freed.
1037 */
1038static struct tracepoint *find_tracepoint_in_module(struct module *mod,
1039 const char *tp_name)
1040{
1041 struct __find_tracepoint_cb_data data = {
1042 .tp_name = tp_name,
1043 .mod = mod,
1044 };
1045
1046 for_each_tracepoint_in_module(mod, __find_tracepoint_module_cb, &data);
1047 return data.tpoint;
1048}
1049
1050/* These are CONFIG_MODULES=y specific functions. */
1051static bool tracepoint_user_within_module(struct tracepoint_user *tuser,
1052 struct module *mod)
1053{
1054 return within_module(tracepoint_user_ip(tuser), mod);
1055}
1056
1057static int tracepoint_user_register_again(struct tracepoint_user *tuser,
1058 struct tracepoint *tpoint)
1059{
1060 tuser->tpoint = tpoint;
1061 return tracepoint_user_register(tuser);
1062}
1063
1064static void tracepoint_user_unregister_clear(struct tracepoint_user *tuser)
1065{
1066 tracepoint_user_unregister(tuser);
1067 tuser->tpoint = NULL;
1068}
1069
1070/* module callback for tracepoint_user */
1071static int __tracepoint_probe_module_cb(struct notifier_block *self,
1072 unsigned long val, void *data)
1073{
1074 struct tp_module *tp_mod = data;
1075 struct tracepoint_user *tuser;
1076 struct tracepoint *tpoint;
1077
1078 if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING)
1079 return NOTIFY_DONE;
1080
1081 mutex_lock(&tracepoint_user_mutex);
1082 for_each_tracepoint_user(tuser) {
1083 if (val == MODULE_STATE_COMING) {
1084 /* This is not a tracepoint in this module. Skip it. */
1085 tpoint = find_tracepoint_in_module(tp_mod->mod, tuser->name);
1086 if (!tpoint)
1087 continue;
1088 WARN_ON_ONCE(tracepoint_user_register_again(tuser, tpoint));
1089 } else if (val == MODULE_STATE_GOING &&
1090 tracepoint_user_within_module(tuser, tp_mod->mod)) {
1091 /* Unregister all tracepoint_user in this module. */
1092 tracepoint_user_unregister_clear(tuser);
1093 }
1094 }
1095 mutex_unlock(&tracepoint_user_mutex);
1096
1097 return NOTIFY_DONE;
1098}
1099
1100static struct notifier_block tracepoint_module_nb = {
1101 .notifier_call = __tracepoint_probe_module_cb,
1102};
1103
1104/* module callback for tprobe events */
1105static int __tprobe_event_module_cb(struct notifier_block *self,
1106 unsigned long val, void *data)
1107{
1108 struct trace_fprobe *tf;
1109 struct dyn_event *pos;
1110 struct module *mod = data;
1111
1112 if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING)
1113 return NOTIFY_DONE;
1114
1115 mutex_lock(&event_mutex);
1116 for_each_trace_fprobe(tf, pos) {
1117 /* Skip fprobe and disabled tprobe events. */
1118 if (!trace_fprobe_is_tracepoint(tf) || !tf->tuser)
1119 continue;
1120
1121 /* Before this notification, tracepoint notifier has already done. */
1122 if (val == MODULE_STATE_COMING &&
1123 tracepoint_user_within_module(tf->tuser, mod)) {
1124 unsigned long ip = tracepoint_user_ip(tf->tuser);
1125
1126 WARN_ON_ONCE(register_fprobe_ips(&tf->fp, &ip, 1));
1127 } else if (val == MODULE_STATE_GOING &&
1128 /*
1129 * tracepoint_user_within_module() does not work here because
1130 * tracepoint_user is already unregistered and cleared tpoint.
1131 * Instead, checking whether the fprobe is registered but
1132 * tpoint is cleared(unregistered). Such unbalance probes
1133 * must be adjusted anyway.
1134 */
1135 trace_fprobe_is_registered(tf) &&
1136 !tf->tuser->tpoint) {
1137 unregister_fprobe(&tf->fp);
1138 }
1139 }
1140 mutex_unlock(&event_mutex);
1141
1142 return NOTIFY_DONE;
1143}
1144
1145/* NOTE: this must be called after tracepoint callback */
1146static struct notifier_block tprobe_event_module_nb = {
1147 .notifier_call = __tprobe_event_module_cb,
1148 /* Make sure this is later than tracepoint module notifier. */
1149 .priority = -10,
1150};
1151#endif /* CONFIG_MODULES */
1152
1153static int parse_symbol_and_return(int argc, const char *argv[],
1154 char **symbol, bool *is_return,
1155 bool is_tracepoint)
1156{
1157 char *tmp = strchr(argv[1], '%');
1158 int i;
1159
1160 if (tmp) {
1161 int len = tmp - argv[1];
1162
1163 if (!is_tracepoint && !strcmp(tmp, "%return")) {
1164 *is_return = true;
1165 } else {
1166 trace_probe_log_err(len, BAD_ADDR_SUFFIX);
1167 return -EINVAL;
1168 }
1169 *symbol = kmemdup_nul(argv[1], len, GFP_KERNEL);
1170 } else
1171 *symbol = kstrdup(argv[1], GFP_KERNEL);
1172 if (!*symbol)
1173 return -ENOMEM;
1174
1175 if (*is_return)
1176 return 0;
1177
1178 if (is_tracepoint) {
1179 tmp = *symbol;
1180 while (*tmp && (isalnum(*tmp) || *tmp == '_'))
1181 tmp++;
1182 if (*tmp) {
1183 /* find a wrong character. */
1184 trace_probe_log_err(tmp - *symbol, BAD_TP_NAME);
1185 kfree(*symbol);
1186 *symbol = NULL;
1187 return -EINVAL;
1188 }
1189 }
1190
1191 /* If there is $retval, this should be a return fprobe. */
1192 for (i = 2; i < argc; i++) {
1193 tmp = strstr(argv[i], "$retval");
1194 if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') {
1195 if (is_tracepoint) {
1196 trace_probe_log_set_index(i);
1197 trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE);
1198 kfree(*symbol);
1199 *symbol = NULL;
1200 return -EINVAL;
1201 }
1202 *is_return = true;
1203 break;
1204 }
1205 }
1206 return 0;
1207}
1208
1209static int trace_fprobe_create_internal(int argc, const char *argv[],
1210 struct traceprobe_parse_context *ctx)
1211{
1212 /*
1213 * Argument syntax:
1214 * - Add fentry probe:
1215 * f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS]
1216 * - Add fexit probe:
1217 * f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS]
1218 * - Add tracepoint probe:
1219 * t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS]
1220 *
1221 * Fetch args:
1222 * $retval : fetch return value
1223 * $stack : fetch stack address
1224 * $stackN : fetch Nth entry of stack (N:0-)
1225 * $argN : fetch Nth argument (N:1-)
1226 * $comm : fetch current task comm
1227 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
1228 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
1229 * Dereferencing memory fetch:
1230 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
1231 * Alias name of args:
1232 * NAME=FETCHARG : set NAME as alias of FETCHARG.
1233 * Type of args:
1234 * FETCHARG:TYPE : use TYPE instead of unsigned long.
1235 */
1236 struct trace_fprobe *tf __free(free_trace_fprobe) = NULL;
1237 const char *event = NULL, *group = FPROBE_EVENT_SYSTEM;
1238 struct module *mod __free(module_put) = NULL;
1239 const char **new_argv __free(kfree) = NULL;
1240 char *symbol __free(kfree) = NULL;
1241 char *ebuf __free(kfree) = NULL;
1242 char *gbuf __free(kfree) = NULL;
1243 char *sbuf __free(kfree) = NULL;
1244 char *abuf __free(kfree) = NULL;
1245 char *dbuf __free(kfree) = NULL;
1246 int i, new_argc = 0, ret = 0;
1247 bool is_tracepoint = false;
1248 bool is_return = false;
1249
1250 if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2)
1251 return -ECANCELED;
1252
1253 if (argv[0][0] == 't') {
1254 is_tracepoint = true;
1255 group = TRACEPOINT_EVENT_SYSTEM;
1256 }
1257
1258 if (argv[0][1] != '\0') {
1259 if (argv[0][1] != ':') {
1260 trace_probe_log_set_index(0);
1261 trace_probe_log_err(1, BAD_MAXACT);
1262 return -EINVAL;
1263 }
1264 event = &argv[0][2];
1265 }
1266
1267 trace_probe_log_set_index(1);
1268
1269 /* a symbol(or tracepoint) must be specified */
1270 ret = parse_symbol_and_return(argc, argv, &symbol, &is_return, is_tracepoint);
1271 if (ret < 0)
1272 return -EINVAL;
1273
1274 trace_probe_log_set_index(0);
1275 if (event) {
1276 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
1277 if (!gbuf)
1278 return -ENOMEM;
1279 ret = traceprobe_parse_event_name(&event, &group, gbuf,
1280 event - argv[0]);
1281 if (ret)
1282 return -EINVAL;
1283 }
1284
1285 if (!event) {
1286 ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
1287 if (!ebuf)
1288 return -ENOMEM;
1289 /* Make a new event name */
1290 if (is_tracepoint)
1291 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%s%s",
1292 isdigit(*symbol) ? "_" : "", symbol);
1293 else
1294 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%s__%s", symbol,
1295 is_return ? "exit" : "entry");
1296 sanitize_event_name(ebuf);
1297 event = ebuf;
1298 }
1299
1300 if (is_return)
1301 ctx->flags |= TPARG_FL_RETURN;
1302 else
1303 ctx->flags |= TPARG_FL_FENTRY;
1304
1305 ctx->funcname = NULL;
1306 if (is_tracepoint) {
1307 /* Get tracepoint and lock its module until the end of the registration. */
1308 struct tracepoint *tpoint;
1309
1310 ctx->flags |= TPARG_FL_TPOINT;
1311 mod = NULL;
1312 tpoint = find_tracepoint(symbol, &mod);
1313 if (tpoint) {
1314 sbuf = kmalloc(KSYM_NAME_LEN, GFP_KERNEL);
1315 if (!sbuf)
1316 return -ENOMEM;
1317 ctx->funcname = kallsyms_lookup((unsigned long)tpoint->probestub,
1318 NULL, NULL, NULL, sbuf);
1319 }
1320 }
1321 if (!ctx->funcname)
1322 ctx->funcname = symbol;
1323
1324 abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL);
1325 if (!abuf)
1326 return -ENOMEM;
1327 argc -= 2; argv += 2;
1328 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc,
1329 abuf, MAX_BTF_ARGS_LEN, ctx);
1330 if (IS_ERR(new_argv))
1331 return PTR_ERR(new_argv);
1332 if (new_argv) {
1333 argc = new_argc;
1334 argv = new_argv;
1335 }
1336 if (argc > MAX_TRACE_ARGS) {
1337 trace_probe_log_set_index(2);
1338 trace_probe_log_err(0, TOO_MANY_ARGS);
1339 return -E2BIG;
1340 }
1341
1342 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf);
1343 if (ret)
1344 return ret;
1345
1346 /* setup a probe */
1347 tf = alloc_trace_fprobe(group, event, symbol, argc, is_return, is_tracepoint);
1348 if (IS_ERR(tf)) {
1349 ret = PTR_ERR(tf);
1350 /* This must return -ENOMEM, else there is a bug */
1351 WARN_ON_ONCE(ret != -ENOMEM);
1352 return ret;
1353 }
1354
1355 /* parse arguments */
1356 for (i = 0; i < argc; i++) {
1357 trace_probe_log_set_index(i + 2);
1358 ctx->offset = 0;
1359 ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], ctx);
1360 if (ret)
1361 return ret; /* This can be -ENOMEM */
1362 }
1363
1364 if (is_return && tf->tp.entry_arg) {
1365 tf->fp.entry_handler = trace_fprobe_entry_handler;
1366 tf->fp.entry_data_size = traceprobe_get_entry_data_size(&tf->tp);
1367 if (ALIGN(tf->fp.entry_data_size, sizeof(long)) > MAX_FPROBE_DATA_SIZE) {
1368 trace_probe_log_set_index(2);
1369 trace_probe_log_err(0, TOO_MANY_EARGS);
1370 return -E2BIG;
1371 }
1372 }
1373
1374 ret = traceprobe_set_print_fmt(&tf->tp,
1375 is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL);
1376 if (ret < 0)
1377 return ret;
1378
1379 ret = register_trace_fprobe_event(tf);
1380 if (ret) {
1381 trace_probe_log_set_index(1);
1382 if (ret == -EILSEQ)
1383 trace_probe_log_err(0, BAD_INSN_BNDRY);
1384 else if (ret == -ENOENT)
1385 trace_probe_log_err(0, BAD_PROBE_ADDR);
1386 else if (ret != -ENOMEM && ret != -EEXIST)
1387 trace_probe_log_err(0, FAIL_REG_PROBE);
1388 return -EINVAL;
1389 }
1390
1391 /* 'tf' is successfully registered. To avoid freeing, assign NULL. */
1392 tf = NULL;
1393
1394 return 0;
1395}
1396
1397static int trace_fprobe_create_cb(int argc, const char *argv[])
1398{
1399 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
1400 int ret;
1401
1402 ctx = kzalloc_obj(*ctx);
1403 if (!ctx)
1404 return -ENOMEM;
1405
1406 ctx->flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE;
1407
1408 trace_probe_log_init("trace_fprobe", argc, argv);
1409 ret = trace_fprobe_create_internal(argc, argv, ctx);
1410 trace_probe_log_clear();
1411 return ret;
1412}
1413
1414static int trace_fprobe_create(const char *raw_command)
1415{
1416 return trace_probe_create(raw_command, trace_fprobe_create_cb);
1417}
1418
1419static int trace_fprobe_release(struct dyn_event *ev)
1420{
1421 struct trace_fprobe *tf = to_trace_fprobe(ev);
1422 int ret = unregister_trace_fprobe(tf);
1423
1424 if (!ret)
1425 free_trace_fprobe(tf);
1426 return ret;
1427}
1428
1429static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev)
1430{
1431 struct trace_fprobe *tf = to_trace_fprobe(ev);
1432 int i;
1433
1434 if (trace_fprobe_is_tracepoint(tf))
1435 seq_putc(m, 't');
1436 else
1437 seq_putc(m, 'f');
1438 seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp),
1439 trace_probe_name(&tf->tp));
1440
1441 seq_printf(m, " %s%s", trace_fprobe_symbol(tf),
1442 trace_fprobe_is_return(tf) ? "%return" : "");
1443
1444 for (i = 0; i < tf->tp.nr_args; i++)
1445 seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);
1446 seq_putc(m, '\n');
1447
1448 trace_probe_dump_args(m, &tf->tp);
1449
1450 return 0;
1451}
1452
1453/*
1454 * Enable trace_probe
1455 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
1456 */
1457static int enable_trace_fprobe(struct trace_event_call *call,
1458 struct trace_event_file *file)
1459{
1460 struct trace_probe *tp;
1461 struct trace_fprobe *tf;
1462 bool enabled;
1463 int ret = 0;
1464
1465 tp = trace_probe_primary_from_call(call);
1466 if (WARN_ON_ONCE(!tp))
1467 return -ENODEV;
1468 enabled = trace_probe_is_enabled(tp);
1469
1470 /* This also changes "enabled" state */
1471 if (file) {
1472 ret = trace_probe_add_file(tp, file);
1473 if (ret)
1474 return ret;
1475 } else
1476 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
1477
1478 if (!enabled) {
1479 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
1480 ret = __register_trace_fprobe(tf);
1481 if (ret < 0)
1482 goto err;
1483 }
1484 }
1485
1486 return 0;
1487
1488err:
1489 /* Failed to enable one of them. Roll back all */
1490 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list)
1491 __unregister_trace_fprobe(tf);
1492 if (file)
1493 trace_probe_remove_file(tp, file);
1494 else
1495 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1496 return ret;
1497}
1498
1499/*
1500 * Disable trace_probe
1501 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
1502 */
1503static int disable_trace_fprobe(struct trace_event_call *call,
1504 struct trace_event_file *file)
1505{
1506 struct trace_fprobe *tf;
1507 struct trace_probe *tp;
1508
1509 tp = trace_probe_primary_from_call(call);
1510 if (WARN_ON_ONCE(!tp))
1511 return -ENODEV;
1512
1513 if (file) {
1514 if (!trace_probe_get_file_link(tp, file))
1515 return -ENOENT;
1516 if (!trace_probe_has_single_file(tp))
1517 goto out;
1518 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
1519 } else
1520 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1521
1522 if (!trace_probe_is_enabled(tp)) {
1523 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
1524 unregister_fprobe(&tf->fp);
1525 if (tf->tuser) {
1526 tracepoint_user_put(tf->tuser);
1527 tf->tuser = NULL;
1528 }
1529 }
1530 }
1531
1532 out:
1533 if (file)
1534 /*
1535 * Synchronization is done in below function. For perf event,
1536 * file == NULL and perf_trace_event_unreg() calls
1537 * tracepoint_synchronize_unregister() to ensure synchronize
1538 * event. We don't need to care about it.
1539 */
1540 trace_probe_remove_file(tp, file);
1541
1542 return 0;
1543}
1544
1545/*
1546 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1547 */
1548static int fprobe_register(struct trace_event_call *event,
1549 enum trace_reg type, void *data)
1550{
1551 struct trace_event_file *file = data;
1552
1553 switch (type) {
1554 case TRACE_REG_REGISTER:
1555 return enable_trace_fprobe(event, file);
1556 case TRACE_REG_UNREGISTER:
1557 return disable_trace_fprobe(event, file);
1558
1559#ifdef CONFIG_PERF_EVENTS
1560 case TRACE_REG_PERF_REGISTER:
1561 return enable_trace_fprobe(event, NULL);
1562 case TRACE_REG_PERF_UNREGISTER:
1563 return disable_trace_fprobe(event, NULL);
1564 case TRACE_REG_PERF_OPEN:
1565 case TRACE_REG_PERF_CLOSE:
1566 case TRACE_REG_PERF_ADD:
1567 case TRACE_REG_PERF_DEL:
1568 return 0;
1569#endif
1570 }
1571 return 0;
1572}
1573
1574/*
1575 * Register dynevent at core_initcall. This allows kernel to setup fprobe
1576 * events in postcore_initcall without tracefs.
1577 */
1578static __init int init_fprobe_trace_early(void)
1579{
1580 int ret;
1581
1582 ret = dyn_event_register(&trace_fprobe_ops);
1583 if (ret)
1584 return ret;
1585
1586#ifdef CONFIG_MODULES
1587 ret = register_tracepoint_module_notifier(&tracepoint_module_nb);
1588 if (ret)
1589 return ret;
1590 ret = register_module_notifier(&tprobe_event_module_nb);
1591 if (ret)
1592 return ret;
1593#endif
1594
1595 return 0;
1596}
1597core_initcall(init_fprobe_trace_early);