projects/linux·blob master · refs

2224 lines · 53.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Kprobes-based tracing events
4 *
5 * Created by Masami Hiramatsu <[email protected]>
6 *
7 */
8#define pr_fmt(fmt) "trace_kprobe: " fmt
9
10#include <linux/bpf-cgroup.h>
11#include <linux/cleanup.h>
12#include <linux/error-injection.h>
13#include <linux/module.h>
14#include <linux/rculist.h>
15#include <linux/security.h>
16#include <linux/uaccess.h>
17
18#include <asm/setup.h> /* for COMMAND_LINE_SIZE */
19
20#include "trace_dynevent.h"
21#include "trace_kprobe_selftest.h"
22#include "trace_probe.h"
23#include "trace_probe_kernel.h"
24#include "trace_probe_tmpl.h"
25
26#define KPROBE_EVENT_SYSTEM "kprobes"
27#define KRETPROBE_MAXACTIVE_MAX 4096
28
29/* Kprobe early definition from command line */
30static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata;
31
32static int __init set_kprobe_boot_events(char *str)
33{
34 trace_append_boot_param(kprobe_boot_events_buf, str, ';',
35 COMMAND_LINE_SIZE);
36 disable_tracing_selftest("running kprobe events");
37
38 return 1;
39}
40__setup("kprobe_event=", set_kprobe_boot_events);
41
42static int trace_kprobe_create(const char *raw_command);
43static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev);
44static int trace_kprobe_release(struct dyn_event *ev);
45static bool trace_kprobe_is_busy(struct dyn_event *ev);
46static bool trace_kprobe_match(const char *system, const char *event,
47 int argc, const char **argv, struct dyn_event *ev);
48
49static struct dyn_event_operations trace_kprobe_ops = {
50 .create = trace_kprobe_create,
51 .show = trace_kprobe_show,
52 .is_busy = trace_kprobe_is_busy,
53 .free = trace_kprobe_release,
54 .match = trace_kprobe_match,
55};
56
57/*
58 * Kprobe event core functions
59 */
60struct trace_kprobe {
61 struct dyn_event devent;
62 struct kretprobe rp; /* Use rp.kp for kprobe use */
63 unsigned long __percpu *nhit;
64 const char *symbol; /* symbol name */
65 struct trace_probe tp;
66};
67
68static bool is_trace_kprobe(struct dyn_event *ev)
69{
70 return ev->ops == &trace_kprobe_ops;
71}
72
73static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev)
74{
75 return container_of(ev, struct trace_kprobe, devent);
76}
77
78/**
79 * for_each_trace_kprobe - iterate over the trace_kprobe list
80 * @pos: the struct trace_kprobe * for each entry
81 * @dpos: the struct dyn_event * to use as a loop cursor
82 */
83#define for_each_trace_kprobe(pos, dpos) \
84 for_each_dyn_event(dpos) \
85 if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos)))
86#define trace_kprobe_list_empty() list_empty(&dyn_event_list)
87
88static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
89{
90 return tk->rp.handler != NULL;
91}
92
93static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
94{
95 return tk->symbol ? tk->symbol : "unknown";
96}
97
98static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
99{
100 return tk->rp.kp.offset;
101}
102
103static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
104{
105 return kprobe_gone(&tk->rp.kp);
106}
107
108static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
109 struct module *mod)
110{
111 int len = strlen(module_name(mod));
112 const char *name = trace_kprobe_symbol(tk);
113
114 return strncmp(module_name(mod), name, len) == 0 && name[len] == ':';
115}
116
117#ifdef CONFIG_MODULES
118static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
119{
120 char *p;
121 bool ret;
122
123 if (!tk->symbol)
124 return false;
125 p = strchr(tk->symbol, ':');
126 if (!p)
127 return true;
128 *p = '\0';
129 scoped_guard(rcu)
130 ret = !!find_module(tk->symbol);
131 *p = ':';
132
133 return ret;
134}
135#else
136static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
137{
138 return false;
139}
140#endif
141
142static bool trace_kprobe_is_busy(struct dyn_event *ev)
143{
144 struct trace_kprobe *tk = to_trace_kprobe(ev);
145
146 return trace_probe_is_enabled(&tk->tp);
147}
148
149static bool trace_kprobe_match_command_head(struct trace_kprobe *tk,
150 int argc, const char **argv)
151{
152 char buf[32];
153 int len;
154
155 if (!argc)
156 return true;
157
158 if (!tk->symbol) {
159 snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr);
160 if (strcmp(buf, argv[0]))
161 return false;
162 } else if (tk->rp.kp.offset) {
163 len = strlen(trace_kprobe_symbol(tk));
164 if (strncmp(trace_kprobe_symbol(tk), argv[0], len) ||
165 argv[0][len] != '+')
166 return false;
167
168 snprintf(buf, sizeof(buf), "%u", tk->rp.kp.offset);
169 if (strcmp(buf, &argv[0][len + 1]))
170 return false;
171 } else if (strcmp(trace_kprobe_symbol(tk), argv[0]))
172 return false;
173
174 argc--; argv++;
175
176 return trace_probe_match_command_args(&tk->tp, argc, argv);
177}
178
179static bool trace_kprobe_match(const char *system, const char *event,
180 int argc, const char **argv, struct dyn_event *ev)
181{
182 struct trace_kprobe *tk = to_trace_kprobe(ev);
183
184 return (event[0] == '\0' ||
185 strcmp(trace_probe_name(&tk->tp), event) == 0) &&
186 (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) &&
187 trace_kprobe_match_command_head(tk, argc, argv);
188}
189
190static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
191{
192 unsigned long nhit = 0;
193 int cpu;
194
195 for_each_possible_cpu(cpu)
196 nhit += *per_cpu_ptr(tk->nhit, cpu);
197
198 return nhit;
199}
200
201static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk)
202{
203 return !(list_empty(&tk->rp.kp.list) &&
204 hlist_unhashed(&tk->rp.kp.hlist));
205}
206
207/* Return 0 if it fails to find the symbol address */
208static nokprobe_inline
209unsigned long trace_kprobe_address(struct trace_kprobe *tk)
210{
211 unsigned long addr;
212
213 if (tk->symbol) {
214 addr = (unsigned long)
215 kallsyms_lookup_name(trace_kprobe_symbol(tk));
216 if (addr)
217 addr += tk->rp.kp.offset;
218 } else {
219 addr = (unsigned long)tk->rp.kp.addr;
220 }
221 return addr;
222}
223
224static nokprobe_inline struct trace_kprobe *
225trace_kprobe_primary_from_call(struct trace_event_call *call)
226{
227 struct trace_probe *tp;
228
229 tp = trace_probe_primary_from_call(call);
230 if (WARN_ON_ONCE(!tp))
231 return NULL;
232
233 return container_of(tp, struct trace_kprobe, tp);
234}
235
236bool trace_kprobe_on_func_entry(struct trace_event_call *call)
237{
238 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
239
240 return tk ? (kprobe_on_func_entry(tk->rp.kp.addr,
241 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,
242 tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false;
243}
244
245bool trace_kprobe_error_injectable(struct trace_event_call *call)
246{
247 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
248
249 return tk ? within_error_injection_list(trace_kprobe_address(tk)) :
250 false;
251}
252
253static int register_kprobe_event(struct trace_kprobe *tk);
254static int unregister_kprobe_event(struct trace_kprobe *tk);
255
256static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
257static int kretprobe_dispatcher(struct kretprobe_instance *ri,
258 struct pt_regs *regs);
259
260static void free_trace_kprobe(struct trace_kprobe *tk)
261{
262 if (tk) {
263 trace_probe_cleanup(&tk->tp);
264 kfree(tk->symbol);
265 free_percpu(tk->nhit);
266 kfree(tk);
267 }
268}
269
270DEFINE_FREE(free_trace_kprobe, struct trace_kprobe *,
271 if (!IS_ERR_OR_NULL(_T)) free_trace_kprobe(_T))
272
273/*
274 * Allocate new trace_probe and initialize it (including kprobes).
275 */
276static struct trace_kprobe *alloc_trace_kprobe(const char *group,
277 const char *event,
278 void *addr,
279 const char *symbol,
280 unsigned long offs,
281 int maxactive,
282 int nargs, bool is_return)
283{
284 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL;
285 int ret = -ENOMEM;
286
287 tk = kzalloc_flex(*tk, tp.args, nargs);
288 if (!tk)
289 return ERR_PTR(ret);
290
291 tk->nhit = alloc_percpu(unsigned long);
292 if (!tk->nhit)
293 return ERR_PTR(ret);
294
295 if (symbol) {
296 tk->symbol = kstrdup(symbol, GFP_KERNEL);
297 if (!tk->symbol)
298 return ERR_PTR(ret);
299 tk->rp.kp.symbol_name = tk->symbol;
300 tk->rp.kp.offset = offs;
301 } else
302 tk->rp.kp.addr = addr;
303
304 if (is_return)
305 tk->rp.handler = kretprobe_dispatcher;
306 else
307 tk->rp.kp.pre_handler = kprobe_dispatcher;
308
309 tk->rp.maxactive = maxactive;
310 INIT_HLIST_NODE(&tk->rp.kp.hlist);
311 INIT_LIST_HEAD(&tk->rp.kp.list);
312
313 ret = trace_probe_init(&tk->tp, event, group, false, nargs);
314 if (ret < 0)
315 return ERR_PTR(ret);
316
317 dyn_event_init(&tk->devent, &trace_kprobe_ops);
318 return_ptr(tk);
319}
320
321static struct trace_kprobe *find_trace_kprobe(const char *event,
322 const char *group)
323{
324 struct dyn_event *pos;
325 struct trace_kprobe *tk;
326
327 for_each_trace_kprobe(tk, pos)
328 if (strcmp(trace_probe_name(&tk->tp), event) == 0 &&
329 strcmp(trace_probe_group_name(&tk->tp), group) == 0)
330 return tk;
331 return NULL;
332}
333
334static inline int __enable_trace_kprobe(struct trace_kprobe *tk)
335{
336 int ret = 0;
337
338 if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) {
339 if (trace_kprobe_is_return(tk))
340 ret = enable_kretprobe(&tk->rp);
341 else
342 ret = enable_kprobe(&tk->rp.kp);
343 }
344
345 return ret;
346}
347
348static void __disable_trace_kprobe(struct trace_probe *tp)
349{
350 struct trace_kprobe *tk;
351
352 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {
353 if (!trace_kprobe_is_registered(tk))
354 continue;
355 if (trace_kprobe_is_return(tk))
356 disable_kretprobe(&tk->rp);
357 else
358 disable_kprobe(&tk->rp.kp);
359 }
360}
361
362/*
363 * Enable trace_probe
364 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
365 */
366static int enable_trace_kprobe(struct trace_event_call *call,
367 struct trace_event_file *file)
368{
369 struct trace_probe *tp;
370 struct trace_kprobe *tk;
371 bool enabled;
372 int ret = 0;
373
374 tp = trace_probe_primary_from_call(call);
375 if (WARN_ON_ONCE(!tp))
376 return -ENODEV;
377 enabled = trace_probe_is_enabled(tp);
378
379 /* This also changes "enabled" state */
380 if (file) {
381 ret = trace_probe_add_file(tp, file);
382 if (ret)
383 return ret;
384 } else
385 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
386
387 if (enabled)
388 return 0;
389
390 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {
391 if (trace_kprobe_has_gone(tk))
392 continue;
393 ret = __enable_trace_kprobe(tk);
394 if (ret)
395 break;
396 enabled = true;
397 }
398
399 if (ret) {
400 /* Failed to enable one of them. Roll back all */
401 if (enabled)
402 __disable_trace_kprobe(tp);
403 if (file)
404 trace_probe_remove_file(tp, file);
405 else
406 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
407 }
408
409 return ret;
410}
411
412/*
413 * Disable trace_probe
414 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
415 */
416static int disable_trace_kprobe(struct trace_event_call *call,
417 struct trace_event_file *file)
418{
419 struct trace_probe *tp;
420
421 tp = trace_probe_primary_from_call(call);
422 if (WARN_ON_ONCE(!tp))
423 return -ENODEV;
424
425 if (file) {
426 if (!trace_probe_get_file_link(tp, file))
427 return -ENOENT;
428 if (!trace_probe_has_single_file(tp))
429 goto out;
430 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
431 } else
432 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
433
434 if (!trace_probe_is_enabled(tp))
435 __disable_trace_kprobe(tp);
436
437 out:
438 if (file)
439 /*
440 * Synchronization is done in below function. For perf event,
441 * file == NULL and perf_trace_event_unreg() calls
442 * tracepoint_synchronize_unregister() to ensure synchronize
443 * event. We don't need to care about it.
444 */
445 trace_probe_remove_file(tp, file);
446
447 return 0;
448}
449
450#if defined(CONFIG_DYNAMIC_FTRACE) && \
451 !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE)
452static bool __within_notrace_func(unsigned long addr)
453{
454 unsigned long offset, size;
455
456 if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset))
457 return false;
458
459 /* Get the entry address of the target function */
460 addr -= offset;
461
462 /*
463 * Since ftrace_location_range() does inclusive range check, we need
464 * to subtract 1 byte from the end address.
465 */
466 return !ftrace_location_range(addr, addr + size - 1);
467}
468
469static bool within_notrace_func(struct trace_kprobe *tk)
470{
471 unsigned long addr = trace_kprobe_address(tk);
472 char symname[KSYM_NAME_LEN], *p;
473
474 if (!__within_notrace_func(addr))
475 return false;
476
477 /* Check if the address is on a suffixed-symbol */
478 if (!lookup_symbol_name(addr, symname)) {
479 p = strchr(symname, '.');
480 if (!p)
481 return true;
482 *p = '\0';
483 addr = (unsigned long)kprobe_lookup_name(symname, 0);
484 if (addr)
485 return __within_notrace_func(addr);
486 }
487
488 return true;
489}
490#else
491#define within_notrace_func(tk) (false)
492#endif
493
494/* Internal register function - just handle k*probes and flags */
495static int __register_trace_kprobe(struct trace_kprobe *tk)
496{
497 int i, ret;
498
499 ret = security_locked_down(LOCKDOWN_KPROBES);
500 if (ret)
501 return ret;
502
503 if (trace_kprobe_is_registered(tk))
504 return -EINVAL;
505
506 if (within_notrace_func(tk)) {
507 pr_warn("Could not probe notrace function %ps\n",
508 (void *)trace_kprobe_address(tk));
509 return -EINVAL;
510 }
511
512 for (i = 0; i < tk->tp.nr_args; i++) {
513 ret = traceprobe_update_arg(&tk->tp.args[i]);
514 if (ret)
515 return ret;
516 }
517
518 /* Set/clear disabled flag according to tp->flag */
519 if (trace_probe_is_enabled(&tk->tp))
520 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
521 else
522 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
523
524 if (trace_kprobe_is_return(tk))
525 ret = register_kretprobe(&tk->rp);
526 else
527 ret = register_kprobe(&tk->rp.kp);
528
529 return ret;
530}
531
532/* Internal unregister function - just handle k*probes and flags */
533static void __unregister_trace_kprobe(struct trace_kprobe *tk)
534{
535 if (trace_kprobe_is_registered(tk)) {
536 if (trace_kprobe_is_return(tk))
537 unregister_kretprobe(&tk->rp);
538 else
539 unregister_kprobe(&tk->rp.kp);
540 /* Cleanup kprobe for reuse and mark it unregistered */
541 INIT_HLIST_NODE(&tk->rp.kp.hlist);
542 INIT_LIST_HEAD(&tk->rp.kp.list);
543 if (tk->rp.kp.symbol_name)
544 tk->rp.kp.addr = NULL;
545 }
546}
547
548/* Unregister a trace_probe and probe_event */
549static int unregister_trace_kprobe(struct trace_kprobe *tk)
550{
551 /* If other probes are on the event, just unregister kprobe */
552 if (trace_probe_has_sibling(&tk->tp))
553 goto unreg;
554
555 /* Enabled event can not be unregistered */
556 if (trace_probe_is_enabled(&tk->tp))
557 return -EBUSY;
558
559 /* If there's a reference to the dynamic event */
560 if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp)))
561 return -EBUSY;
562
563 /* Will fail if probe is being used by ftrace or perf */
564 if (unregister_kprobe_event(tk))
565 return -EBUSY;
566
567unreg:
568 __unregister_trace_kprobe(tk);
569 dyn_event_remove(&tk->devent);
570 trace_probe_unlink(&tk->tp);
571
572 return 0;
573}
574
575static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig,
576 struct trace_kprobe *comp)
577{
578 struct trace_probe_event *tpe = orig->tp.event;
579 int i;
580
581 list_for_each_entry(orig, &tpe->probes, tp.list) {
582 if (strcmp(trace_kprobe_symbol(orig),
583 trace_kprobe_symbol(comp)) ||
584 trace_kprobe_offset(orig) != trace_kprobe_offset(comp))
585 continue;
586
587 /*
588 * trace_probe_compare_arg_type() ensured that nr_args and
589 * each argument name and type are same. Let's compare comm.
590 */
591 for (i = 0; i < orig->tp.nr_args; i++) {
592 if (strcmp(orig->tp.args[i].comm,
593 comp->tp.args[i].comm))
594 break;
595 }
596
597 if (i == orig->tp.nr_args)
598 return true;
599 }
600
601 return false;
602}
603
604static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to)
605{
606 int ret;
607
608 ret = trace_probe_compare_arg_type(&tk->tp, &to->tp);
609 if (ret) {
610 /* Note that argument starts index = 2 */
611 trace_probe_log_set_index(ret + 1);
612 trace_probe_log_err(0, DIFF_ARG_TYPE);
613 return -EEXIST;
614 }
615 if (trace_kprobe_has_same_kprobe(to, tk)) {
616 trace_probe_log_set_index(0);
617 trace_probe_log_err(0, SAME_PROBE);
618 return -EEXIST;
619 }
620
621 /* Append to existing event */
622 ret = trace_probe_append(&tk->tp, &to->tp);
623 if (ret)
624 return ret;
625
626 /* Register k*probe */
627 ret = __register_trace_kprobe(tk);
628 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
629 pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
630 ret = 0;
631 }
632
633 if (ret)
634 trace_probe_unlink(&tk->tp);
635 else
636 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));
637
638 return ret;
639}
640
641/* Register a trace_probe and probe_event */
642static int register_trace_kprobe(struct trace_kprobe *tk)
643{
644 struct trace_kprobe *old_tk;
645 int ret;
646
647 guard(mutex)(&event_mutex);
648
649 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp),
650 trace_probe_group_name(&tk->tp));
651 if (old_tk) {
652 if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) {
653 trace_probe_log_set_index(0);
654 trace_probe_log_err(0, DIFF_PROBE_TYPE);
655 return -EEXIST;
656 }
657 return append_trace_kprobe(tk, old_tk);
658 }
659
660 /* Register new event */
661 ret = register_kprobe_event(tk);
662 if (ret) {
663 if (ret == -EEXIST) {
664 trace_probe_log_set_index(0);
665 trace_probe_log_err(0, EVENT_EXIST);
666 } else
667 pr_warn("Failed to register probe event(%d)\n", ret);
668 return ret;
669 }
670
671 /* Register k*probe */
672 ret = __register_trace_kprobe(tk);
673 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
674 pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
675 ret = 0;
676 }
677
678 if (ret < 0)
679 unregister_kprobe_event(tk);
680 else
681 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));
682
683 return ret;
684}
685
686#ifdef CONFIG_MODULES
687static int validate_module_probe_symbol(const char *modname, const char *symbol);
688
689static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk)
690{
691 const char *p;
692 int ret = 0;
693
694 p = strchr(trace_kprobe_symbol(tk), ':');
695 if (p)
696 ret = validate_module_probe_symbol(module_name(mod), p + 1);
697 if (!ret)
698 ret = __register_trace_kprobe(tk);
699 return ret;
700}
701
702/* Module notifier call back, checking event on the module */
703static int trace_kprobe_module_callback(struct notifier_block *nb,
704 unsigned long val, void *data)
705{
706 struct module *mod = data;
707 struct dyn_event *pos;
708 struct trace_kprobe *tk;
709 int ret;
710
711 if (val != MODULE_STATE_COMING)
712 return NOTIFY_DONE;
713
714 /* Update probes on coming module */
715 guard(mutex)(&event_mutex);
716 for_each_trace_kprobe(tk, pos) {
717 if (trace_kprobe_within_module(tk, mod)) {
718 /* Don't need to check busy - this should have gone. */
719 __unregister_trace_kprobe(tk);
720 ret = register_module_trace_kprobe(mod, tk);
721 if (ret)
722 pr_warn("Failed to re-register probe %s on %s: %d\n",
723 trace_probe_name(&tk->tp),
724 module_name(mod), ret);
725 }
726 }
727
728 return NOTIFY_DONE;
729}
730
731static struct notifier_block trace_kprobe_module_nb = {
732 .notifier_call = trace_kprobe_module_callback,
733 .priority = 2 /* Invoked after kprobe and jump_label module callback */
734};
735static int trace_kprobe_register_module_notifier(void)
736{
737 return register_module_notifier(&trace_kprobe_module_nb);
738}
739#else
740static int trace_kprobe_register_module_notifier(void)
741{
742 return 0;
743}
744#endif /* CONFIG_MODULES */
745
746static int count_symbols(void *data, unsigned long unused)
747{
748 unsigned int *count = data;
749
750 (*count)++;
751
752 return 0;
753}
754
755struct sym_count_ctx {
756 unsigned int count;
757 const char *name;
758};
759
760static int count_mod_symbols(void *data, const char *name, unsigned long unused)
761{
762 struct sym_count_ctx *ctx = data;
763
764 if (strcmp(name, ctx->name) == 0)
765 ctx->count++;
766
767 return 0;
768}
769
770static unsigned int number_of_same_symbols(const char *mod, const char *func_name)
771{
772 struct sym_count_ctx ctx = { .count = 0, .name = func_name };
773
774 if (!mod)
775 kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count);
776
777 /*
778 * If the symbol is found in vmlinux, use vmlinux resolution only.
779 * This prevents module symbols from shadowing vmlinux symbols
780 * and causing -EADDRNOTAVAIL for unqualified kprobe targets.
781 */
782 if (!mod && ctx.count > 0)
783 return ctx.count;
784
785 module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx);
786
787 return ctx.count;
788}
789
790static int validate_module_probe_symbol(const char *modname, const char *symbol)
791{
792 unsigned int count = number_of_same_symbols(modname, symbol);
793
794 if (count > 1) {
795 /*
796 * Users should use ADDR to remove the ambiguity of
797 * using KSYM only.
798 */
799 return -EADDRNOTAVAIL;
800 } else if (count == 0) {
801 /*
802 * We can return ENOENT earlier than when register the
803 * kprobe.
804 */
805 return -ENOENT;
806 }
807 return 0;
808}
809
810#ifdef CONFIG_MODULES
811/* Return NULL if the module is not loaded or under unloading. */
812static struct module *try_module_get_by_name(const char *name)
813{
814 struct module *mod;
815
816 guard(rcu)();
817 mod = find_module(name);
818 if (mod && !try_module_get(mod))
819 mod = NULL;
820 return mod;
821}
822#else
823#define try_module_get_by_name(name) (NULL)
824#endif
825
826static int validate_probe_symbol(char *symbol)
827{
828 struct module *mod = NULL;
829 char *modname = NULL, *p;
830 int ret = 0;
831
832 p = strchr(symbol, ':');
833 if (p) {
834 modname = symbol;
835 symbol = p + 1;
836 *p = '\0';
837 mod = try_module_get_by_name(modname);
838 if (!mod)
839 goto out;
840 }
841
842 ret = validate_module_probe_symbol(modname, symbol);
843out:
844 if (p)
845 *p = ':';
846 if (mod)
847 module_put(mod);
848 return ret;
849}
850
851static int trace_kprobe_entry_handler(struct kretprobe_instance *ri,
852 struct pt_regs *regs);
853
854static int trace_kprobe_create_internal(int argc, const char *argv[],
855 struct traceprobe_parse_context *ctx)
856{
857 /*
858 * Argument syntax:
859 * - Add kprobe:
860 * p[:[GRP/][EVENT]] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
861 * - Add kretprobe:
862 * r[MAXACTIVE][:[GRP/][EVENT]] [MOD:]KSYM[+0] [FETCHARGS]
863 * Or
864 * p[:[GRP/][EVENT]] [MOD:]KSYM[+0]%return [FETCHARGS]
865 *
866 * Fetch args:
867 * $retval : fetch return value
868 * $stack : fetch stack address
869 * $stackN : fetch Nth of stack (N:0-)
870 * $comm : fetch current task comm
871 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
872 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
873 * %REG : fetch register REG
874 * Dereferencing memory fetch:
875 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
876 * Alias name of args:
877 * NAME=FETCHARG : set NAME as alias of FETCHARG.
878 * Type of args:
879 * FETCHARG:TYPE : use TYPE instead of unsigned long.
880 */
881 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL;
882 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM;
883 const char **new_argv __free(kfree) = NULL;
884 int i, len, new_argc = 0, ret = 0;
885 char *symbol __free(kfree) = NULL;
886 char *ebuf __free(kfree) = NULL;
887 char *gbuf __free(kfree) = NULL;
888 char *abuf __free(kfree) = NULL;
889 char *dbuf __free(kfree) = NULL;
890 enum probe_print_type ptype;
891 bool is_return = false;
892 int maxactive = 0;
893 void *addr = NULL;
894 char *tmp = NULL;
895 long offset = 0;
896
897 switch (argv[0][0]) {
898 case 'r':
899 is_return = true;
900 break;
901 case 'p':
902 break;
903 default:
904 return -ECANCELED;
905 }
906 if (argc < 2)
907 return -ECANCELED;
908
909 event = strchr(&argv[0][1], ':');
910 if (event)
911 event++;
912
913 if (isdigit(argv[0][1])) {
914 char *buf __free(kfree) = NULL;
915
916 if (!is_return) {
917 trace_probe_log_err(1, BAD_MAXACT_TYPE);
918 return -EINVAL;
919 }
920 if (event)
921 len = event - &argv[0][1] - 1;
922 else
923 len = strlen(&argv[0][1]);
924 if (len > MAX_EVENT_NAME_LEN - 1) {
925 trace_probe_log_err(1, BAD_MAXACT);
926 return -EINVAL;
927 }
928 buf = kmemdup(&argv[0][1], len + 1, GFP_KERNEL);
929 if (!buf)
930 return -ENOMEM;
931 buf[len] = '\0';
932 ret = kstrtouint(buf, 0, &maxactive);
933 if (ret || !maxactive) {
934 trace_probe_log_err(1, BAD_MAXACT);
935 return -EINVAL;
936 }
937 /* kretprobes instances are iterated over via a list. The
938 * maximum should stay reasonable.
939 */
940 if (maxactive > KRETPROBE_MAXACTIVE_MAX) {
941 trace_probe_log_err(1, MAXACT_TOO_BIG);
942 return -EINVAL;
943 }
944 }
945
946 /* try to parse an address. if that fails, try to read the
947 * input as a symbol. */
948 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
949 trace_probe_log_set_index(1);
950 /* Check whether uprobe event specified */
951 if (strchr(argv[1], '/') && strchr(argv[1], ':'))
952 return -ECANCELED;
953
954 /* a symbol specified */
955 symbol = kstrdup(argv[1], GFP_KERNEL);
956 if (!symbol)
957 return -ENOMEM;
958
959 tmp = strchr(symbol, '%');
960 if (tmp) {
961 if (!strcmp(tmp, "%return")) {
962 *tmp = '\0';
963 is_return = true;
964 } else {
965 trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX);
966 return -EINVAL;
967 }
968 }
969
970 /* TODO: support .init module functions */
971 ret = traceprobe_split_symbol_offset(symbol, &offset);
972 if (ret || offset < 0 || offset > UINT_MAX) {
973 trace_probe_log_err(0, BAD_PROBE_ADDR);
974 return -EINVAL;
975 }
976 ret = validate_probe_symbol(symbol);
977 if (ret) {
978 if (ret == -EADDRNOTAVAIL)
979 trace_probe_log_err(0, NON_UNIQ_SYMBOL);
980 else
981 trace_probe_log_err(0, BAD_PROBE_ADDR);
982 return -EINVAL;
983 }
984 if (is_return)
985 ctx->flags |= TPARG_FL_RETURN;
986 ret = kprobe_on_func_entry(NULL, symbol, offset);
987 if (ret == 0 && !is_return)
988 ctx->flags |= TPARG_FL_FENTRY;
989 /* Defer the ENOENT case until register kprobe */
990 if (ret == -EINVAL && is_return) {
991 trace_probe_log_err(0, BAD_RETPROBE);
992 return -EINVAL;
993 }
994 }
995
996 trace_probe_log_set_index(0);
997 if (event) {
998 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
999 if (!gbuf)
1000 return -ENOMEM;
1001 ret = traceprobe_parse_event_name(&event, &group, gbuf,
1002 event - argv[0]);
1003 if (ret)
1004 return ret;
1005 }
1006
1007 if (!event) {
1008 /* Make a new event name */
1009 ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
1010 if (!ebuf)
1011 return -ENOMEM;
1012 if (symbol)
1013 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
1014 is_return ? 'r' : 'p', symbol, offset);
1015 else
1016 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_0x%p",
1017 is_return ? 'r' : 'p', addr);
1018 sanitize_event_name(ebuf);
1019 event = ebuf;
1020 }
1021
1022 abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL);
1023 if (!abuf)
1024 return -ENOMEM;
1025 argc -= 2; argv += 2;
1026 ctx->funcname = symbol;
1027 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc,
1028 abuf, MAX_BTF_ARGS_LEN, ctx);
1029 if (IS_ERR(new_argv)) {
1030 ret = PTR_ERR(new_argv);
1031 new_argv = NULL;
1032 return ret;
1033 }
1034 if (new_argv) {
1035 argc = new_argc;
1036 argv = new_argv;
1037 }
1038 if (argc > MAX_TRACE_ARGS) {
1039 trace_probe_log_set_index(2);
1040 trace_probe_log_err(0, TOO_MANY_ARGS);
1041 return -E2BIG;
1042 }
1043
1044 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf);
1045 if (ret)
1046 return ret;
1047
1048 /* setup a probe */
1049 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
1050 argc, is_return);
1051 if (IS_ERR(tk)) {
1052 ret = PTR_ERR(tk);
1053 /* This must return -ENOMEM, else there is a bug */
1054 WARN_ON_ONCE(ret != -ENOMEM);
1055 return ret; /* We know tk is not allocated */
1056 }
1057
1058 /* parse arguments */
1059 for (i = 0; i < argc; i++) {
1060 trace_probe_log_set_index(i + 2);
1061 ctx->offset = 0;
1062 ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], ctx);
1063 if (ret)
1064 return ret; /* This can be -ENOMEM */
1065 }
1066 /* entry handler for kretprobe */
1067 if (is_return && tk->tp.entry_arg) {
1068 tk->rp.entry_handler = trace_kprobe_entry_handler;
1069 tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp);
1070 }
1071
1072 ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1073 ret = traceprobe_set_print_fmt(&tk->tp, ptype);
1074 if (ret < 0)
1075 return ret;
1076
1077 ret = register_trace_kprobe(tk);
1078 if (ret) {
1079 trace_probe_log_set_index(1);
1080 if (ret == -EILSEQ)
1081 trace_probe_log_err(0, BAD_INSN_BNDRY);
1082 else if (ret == -ENOENT)
1083 trace_probe_log_err(0, BAD_PROBE_ADDR);
1084 else if (ret != -ENOMEM && ret != -EEXIST)
1085 trace_probe_log_err(0, FAIL_REG_PROBE);
1086 return ret;
1087 }
1088 /*
1089 * Here, 'tk' has been registered to the list successfully,
1090 * so we don't need to free it.
1091 */
1092 tk = NULL;
1093
1094 return 0;
1095}
1096
1097static int trace_kprobe_create_cb(int argc, const char *argv[])
1098{
1099 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
1100 int ret;
1101
1102 ctx = kzalloc_obj(*ctx);
1103 if (!ctx)
1104 return -ENOMEM;
1105 ctx->flags = TPARG_FL_KERNEL;
1106
1107 trace_probe_log_init("trace_kprobe", argc, argv);
1108
1109 ret = trace_kprobe_create_internal(argc, argv, ctx);
1110
1111 trace_probe_log_clear();
1112 return ret;
1113}
1114
1115static int trace_kprobe_create(const char *raw_command)
1116{
1117 return trace_probe_create(raw_command, trace_kprobe_create_cb);
1118}
1119
1120static int create_or_delete_trace_kprobe(const char *raw_command)
1121{
1122 int ret;
1123
1124 if (raw_command[0] == '-')
1125 return dyn_event_release(raw_command, &trace_kprobe_ops);
1126
1127 ret = dyn_event_create(raw_command, &trace_kprobe_ops);
1128 return ret == -ECANCELED ? -EINVAL : ret;
1129}
1130
1131static int trace_kprobe_run_command(struct dynevent_cmd *cmd)
1132{
1133 return create_or_delete_trace_kprobe(cmd->seq.buffer);
1134}
1135
1136/**
1137 * kprobe_event_cmd_init - Initialize a kprobe event command object
1138 * @cmd: A pointer to the dynevent_cmd struct representing the new event
1139 * @buf: A pointer to the buffer used to build the command
1140 * @maxlen: The length of the buffer passed in @buf
1141 *
1142 * Initialize a synthetic event command object. Use this before
1143 * calling any of the other kprobe_event functions.
1144 */
1145void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen)
1146{
1147 dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE,
1148 trace_kprobe_run_command);
1149}
1150EXPORT_SYMBOL_GPL(kprobe_event_cmd_init);
1151
1152/**
1153 * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list
1154 * @cmd: A pointer to the dynevent_cmd struct representing the new event
1155 * @kretprobe: Is this a return probe?
1156 * @name: The name of the kprobe event
1157 * @loc: The location of the kprobe event
1158 * @...: Variable number of arg (pairs), one pair for each field
1159 *
1160 * NOTE: Users normally won't want to call this function directly, but
1161 * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically
1162 * adds a NULL to the end of the arg list. If this function is used
1163 * directly, make sure the last arg in the variable arg list is NULL.
1164 *
1165 * Generate a kprobe event command to be executed by
1166 * kprobe_event_gen_cmd_end(). This function can be used to generate the
1167 * complete command or only the first part of it; in the latter case,
1168 * kprobe_event_add_fields() can be used to add more fields following this.
1169 *
1170 * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This
1171 * returns -EINVAL if @loc == NULL.
1172 *
1173 * Return: 0 if successful, error otherwise.
1174 */
1175int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe,
1176 const char *name, const char *loc, ...)
1177{
1178 char buf[MAX_EVENT_NAME_LEN];
1179 struct dynevent_arg arg;
1180 va_list args;
1181 int ret;
1182
1183 if (cmd->type != DYNEVENT_TYPE_KPROBE)
1184 return -EINVAL;
1185
1186 if (!loc)
1187 return -EINVAL;
1188
1189 if (kretprobe)
1190 snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name);
1191 else
1192 snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name);
1193
1194 ret = dynevent_str_add(cmd, buf);
1195 if (ret)
1196 return ret;
1197
1198 dynevent_arg_init(&arg, 0);
1199 arg.str = loc;
1200 ret = dynevent_arg_add(cmd, &arg, NULL);
1201 if (ret)
1202 return ret;
1203
1204 va_start(args, loc);
1205 for (;;) {
1206 const char *field;
1207
1208 field = va_arg(args, const char *);
1209 if (!field)
1210 break;
1211
1212 if (++cmd->n_fields > MAX_TRACE_ARGS) {
1213 ret = -EINVAL;
1214 break;
1215 }
1216
1217 arg.str = field;
1218 ret = dynevent_arg_add(cmd, &arg, NULL);
1219 if (ret)
1220 break;
1221 }
1222 va_end(args);
1223
1224 return ret;
1225}
1226EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start);
1227
1228/**
1229 * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list
1230 * @cmd: A pointer to the dynevent_cmd struct representing the new event
1231 * @...: Variable number of arg (pairs), one pair for each field
1232 *
1233 * NOTE: Users normally won't want to call this function directly, but
1234 * rather use the kprobe_event_add_fields() wrapper, which
1235 * automatically adds a NULL to the end of the arg list. If this
1236 * function is used directly, make sure the last arg in the variable
1237 * arg list is NULL.
1238 *
1239 * Add probe fields to an existing kprobe command using a variable
1240 * list of args. Fields are added in the same order they're listed.
1241 *
1242 * Return: 0 if successful, error otherwise.
1243 */
1244int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...)
1245{
1246 struct dynevent_arg arg;
1247 va_list args;
1248 int ret = 0;
1249
1250 if (cmd->type != DYNEVENT_TYPE_KPROBE)
1251 return -EINVAL;
1252
1253 dynevent_arg_init(&arg, 0);
1254
1255 va_start(args, cmd);
1256 for (;;) {
1257 const char *field;
1258
1259 field = va_arg(args, const char *);
1260 if (!field)
1261 break;
1262
1263 if (++cmd->n_fields > MAX_TRACE_ARGS) {
1264 ret = -EINVAL;
1265 break;
1266 }
1267
1268 arg.str = field;
1269 ret = dynevent_arg_add(cmd, &arg, NULL);
1270 if (ret)
1271 break;
1272 }
1273 va_end(args);
1274
1275 return ret;
1276}
1277EXPORT_SYMBOL_GPL(__kprobe_event_add_fields);
1278
1279/**
1280 * kprobe_event_delete - Delete a kprobe event
1281 * @name: The name of the kprobe event to delete
1282 *
1283 * Delete a kprobe event with the give @name from kernel code rather
1284 * than directly from the command line.
1285 *
1286 * Return: 0 if successful, error otherwise.
1287 */
1288int kprobe_event_delete(const char *name)
1289{
1290 char buf[MAX_EVENT_NAME_LEN];
1291
1292 snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name);
1293
1294 return create_or_delete_trace_kprobe(buf);
1295}
1296EXPORT_SYMBOL_GPL(kprobe_event_delete);
1297
1298static int trace_kprobe_release(struct dyn_event *ev)
1299{
1300 struct trace_kprobe *tk = to_trace_kprobe(ev);
1301 int ret = unregister_trace_kprobe(tk);
1302
1303 if (!ret)
1304 free_trace_kprobe(tk);
1305 return ret;
1306}
1307
1308static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev)
1309{
1310 struct trace_kprobe *tk = to_trace_kprobe(ev);
1311 int i;
1312
1313 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
1314 if (trace_kprobe_is_return(tk) && tk->rp.maxactive)
1315 seq_printf(m, "%d", tk->rp.maxactive);
1316 seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp),
1317 trace_probe_name(&tk->tp));
1318
1319 if (!tk->symbol)
1320 seq_printf(m, " 0x%p", tk->rp.kp.addr);
1321 else if (tk->rp.kp.offset)
1322 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
1323 tk->rp.kp.offset);
1324 else
1325 seq_printf(m, " %s", trace_kprobe_symbol(tk));
1326
1327 for (i = 0; i < tk->tp.nr_args; i++)
1328 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
1329 seq_putc(m, '\n');
1330
1331 trace_probe_dump_args(m, &tk->tp);
1332
1333 return 0;
1334}
1335
1336static int probes_seq_show(struct seq_file *m, void *v)
1337{
1338 struct dyn_event *ev = v;
1339
1340 if (!is_trace_kprobe(ev))
1341 return 0;
1342
1343 return trace_kprobe_show(m, ev);
1344}
1345
1346static const struct seq_operations probes_seq_op = {
1347 .start = dyn_event_seq_start,
1348 .next = dyn_event_seq_next,
1349 .stop = dyn_event_seq_stop,
1350 .show = probes_seq_show
1351};
1352
1353static int probes_open(struct inode *inode, struct file *file)
1354{
1355 int ret;
1356
1357 ret = security_locked_down(LOCKDOWN_TRACEFS);
1358 if (ret)
1359 return ret;
1360
1361 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1362 ret = dyn_events_release_all(&trace_kprobe_ops);
1363 if (ret < 0)
1364 return ret;
1365 }
1366
1367 return seq_open(file, &probes_seq_op);
1368}
1369
1370static ssize_t probes_write(struct file *file, const char __user *buffer,
1371 size_t count, loff_t *ppos)
1372{
1373 return trace_parse_run_command(file, buffer, count, ppos,
1374 create_or_delete_trace_kprobe);
1375}
1376
1377static const struct file_operations kprobe_events_ops = {
1378 .owner = THIS_MODULE,
1379 .open = probes_open,
1380 .read = seq_read,
1381 .llseek = seq_lseek,
1382 .release = seq_release,
1383 .write = probes_write,
1384};
1385
1386static unsigned long trace_kprobe_missed(struct trace_kprobe *tk)
1387{
1388 return trace_kprobe_is_return(tk) ?
1389 tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed;
1390}
1391
1392/* Probes profiling interfaces */
1393static int probes_profile_seq_show(struct seq_file *m, void *v)
1394{
1395 struct dyn_event *ev = v;
1396 struct trace_kprobe *tk;
1397 unsigned long nmissed;
1398
1399 if (!is_trace_kprobe(ev))
1400 return 0;
1401
1402 tk = to_trace_kprobe(ev);
1403 nmissed = trace_kprobe_missed(tk);
1404 seq_printf(m, " %-44s %15lu %15lu\n",
1405 trace_probe_name(&tk->tp),
1406 trace_kprobe_nhit(tk),
1407 nmissed);
1408
1409 return 0;
1410}
1411
1412static const struct seq_operations profile_seq_op = {
1413 .start = dyn_event_seq_start,
1414 .next = dyn_event_seq_next,
1415 .stop = dyn_event_seq_stop,
1416 .show = probes_profile_seq_show
1417};
1418
1419static int profile_open(struct inode *inode, struct file *file)
1420{
1421 int ret;
1422
1423 ret = security_locked_down(LOCKDOWN_TRACEFS);
1424 if (ret)
1425 return ret;
1426
1427 return seq_open(file, &profile_seq_op);
1428}
1429
1430static const struct file_operations kprobe_profile_ops = {
1431 .owner = THIS_MODULE,
1432 .open = profile_open,
1433 .read = seq_read,
1434 .llseek = seq_lseek,
1435 .release = seq_release,
1436};
1437
1438/* Note that we don't verify it, since the code does not come from user space */
1439static int
1440process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
1441 void *dest, void *base)
1442{
1443 struct pt_regs *regs = rec;
1444 unsigned long val;
1445 int ret;
1446
1447retry:
1448 /* 1st stage: get value from context */
1449 switch (code->op) {
1450 case FETCH_OP_REG:
1451 val = regs_get_register(regs, code->param);
1452 break;
1453 case FETCH_OP_STACK:
1454 val = regs_get_kernel_stack_nth(regs, code->param);
1455 break;
1456 case FETCH_OP_STACKP:
1457 val = kernel_stack_pointer(regs);
1458 break;
1459 case FETCH_OP_RETVAL:
1460 val = regs_return_value(regs);
1461 break;
1462#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
1463 case FETCH_OP_ARG:
1464 val = regs_get_kernel_argument(regs, code->param);
1465 break;
1466 case FETCH_OP_EDATA:
1467 val = *(unsigned long *)((unsigned long)edata + code->offset);
1468 break;
1469#endif
1470 case FETCH_NOP_SYMBOL: /* Ignore a place holder */
1471 code++;
1472 goto retry;
1473 default:
1474 ret = process_common_fetch_insn(code, &val);
1475 if (ret < 0)
1476 return ret;
1477 }
1478 code++;
1479
1480 return process_fetch_insn_bottom(code, val, dest, base);
1481}
1482NOKPROBE_SYMBOL(process_fetch_insn)
1483
1484/* Kprobe handler */
1485static nokprobe_inline void
1486__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
1487 struct trace_event_file *trace_file)
1488{
1489 struct kprobe_trace_entry_head *entry;
1490 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1491 struct trace_event_buffer fbuffer;
1492 int dsize;
1493
1494 WARN_ON(call != trace_file->event_call);
1495
1496 if (trace_trigger_soft_disabled(trace_file))
1497 return;
1498
1499 dsize = __get_data_size(&tk->tp, regs, NULL);
1500
1501 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
1502 sizeof(*entry) + tk->tp.size + dsize);
1503 if (!entry)
1504 return;
1505
1506 fbuffer.regs = regs;
1507 entry->ip = (unsigned long)tk->rp.kp.addr;
1508 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize);
1509
1510 trace_event_buffer_commit(&fbuffer);
1511}
1512
1513static void
1514kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
1515{
1516 struct event_file_link *link;
1517
1518 trace_probe_for_each_link_rcu(link, &tk->tp)
1519 __kprobe_trace_func(tk, regs, link->file);
1520}
1521NOKPROBE_SYMBOL(kprobe_trace_func);
1522
1523/* Kretprobe handler */
1524
1525static int trace_kprobe_entry_handler(struct kretprobe_instance *ri,
1526 struct pt_regs *regs)
1527{
1528 struct kretprobe *rp = get_kretprobe(ri);
1529 struct trace_kprobe *tk;
1530
1531 /*
1532 * There is a small chance that get_kretprobe(ri) returns NULL when
1533 * the kretprobe is unregister on another CPU between kretprobe's
1534 * trampoline_handler and this function.
1535 */
1536 if (unlikely(!rp))
1537 return -ENOENT;
1538
1539 tk = container_of(rp, struct trace_kprobe, rp);
1540
1541 /* store argument values into ri->data as entry data */
1542 if (tk->tp.entry_arg)
1543 store_trace_entry_data(ri->data, &tk->tp, regs);
1544
1545 return 0;
1546}
1547
1548
1549static nokprobe_inline void
1550__kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1551 struct pt_regs *regs,
1552 struct trace_event_file *trace_file)
1553{
1554 struct kretprobe_trace_entry_head *entry;
1555 struct trace_event_buffer fbuffer;
1556 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1557 int dsize;
1558
1559 WARN_ON(call != trace_file->event_call);
1560
1561 if (trace_trigger_soft_disabled(trace_file))
1562 return;
1563
1564 dsize = __get_data_size(&tk->tp, regs, ri->data);
1565
1566 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
1567 sizeof(*entry) + tk->tp.size + dsize);
1568 if (!entry)
1569 return;
1570
1571 fbuffer.regs = regs;
1572 entry->func = (unsigned long)tk->rp.kp.addr;
1573 entry->ret_ip = get_kretprobe_retaddr(ri);
1574 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize);
1575
1576 trace_event_buffer_commit(&fbuffer);
1577}
1578
1579static void
1580kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1581 struct pt_regs *regs)
1582{
1583 struct event_file_link *link;
1584
1585 trace_probe_for_each_link_rcu(link, &tk->tp)
1586 __kretprobe_trace_func(tk, ri, regs, link->file);
1587}
1588NOKPROBE_SYMBOL(kretprobe_trace_func);
1589
1590/* Event entry printers */
1591static enum print_line_t
1592print_kprobe_event(struct trace_iterator *iter, int flags,
1593 struct trace_event *event)
1594{
1595 struct kprobe_trace_entry_head *field;
1596 struct trace_seq *s = &iter->seq;
1597 struct trace_probe *tp;
1598
1599 field = (struct kprobe_trace_entry_head *)iter->ent;
1600 tp = trace_probe_primary_from_call(
1601 container_of(event, struct trace_event_call, event));
1602 if (WARN_ON_ONCE(!tp))
1603 goto out;
1604
1605 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
1606
1607 if (!seq_print_ip_sym_offset(s, field->ip, flags))
1608 goto out;
1609
1610 trace_seq_putc(s, ')');
1611
1612 if (trace_probe_print_args(s, tp->args, tp->nr_args,
1613 (u8 *)&field[1], field) < 0)
1614 goto out;
1615
1616 trace_seq_putc(s, '\n');
1617 out:
1618 return trace_handle_return(s);
1619}
1620
1621static enum print_line_t
1622print_kretprobe_event(struct trace_iterator *iter, int flags,
1623 struct trace_event *event)
1624{
1625 struct kretprobe_trace_entry_head *field;
1626 struct trace_seq *s = &iter->seq;
1627 struct trace_probe *tp;
1628
1629 field = (struct kretprobe_trace_entry_head *)iter->ent;
1630 tp = trace_probe_primary_from_call(
1631 container_of(event, struct trace_event_call, event));
1632 if (WARN_ON_ONCE(!tp))
1633 goto out;
1634
1635 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
1636
1637 if (!seq_print_ip_sym_offset(s, field->ret_ip, flags))
1638 goto out;
1639
1640 trace_seq_puts(s, " <- ");
1641
1642 if (!seq_print_ip_sym_no_offset(s, field->func, flags))
1643 goto out;
1644
1645 trace_seq_putc(s, ')');
1646
1647 if (trace_probe_print_args(s, tp->args, tp->nr_args,
1648 (u8 *)&field[1], field) < 0)
1649 goto out;
1650
1651 trace_seq_putc(s, '\n');
1652
1653 out:
1654 return trace_handle_return(s);
1655}
1656
1657
1658static int kprobe_event_define_fields(struct trace_event_call *event_call)
1659{
1660 int ret;
1661 struct kprobe_trace_entry_head field;
1662 struct trace_probe *tp;
1663
1664 tp = trace_probe_primary_from_call(event_call);
1665 if (WARN_ON_ONCE(!tp))
1666 return -ENOENT;
1667
1668 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1669
1670 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1671}
1672
1673static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1674{
1675 int ret;
1676 struct kretprobe_trace_entry_head field;
1677 struct trace_probe *tp;
1678
1679 tp = trace_probe_primary_from_call(event_call);
1680 if (WARN_ON_ONCE(!tp))
1681 return -ENOENT;
1682
1683 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1684 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1685
1686 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1687}
1688
1689#ifdef CONFIG_PERF_EVENTS
1690
1691/* Kprobe profile handler */
1692static int
1693kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1694{
1695 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1696 struct kprobe_trace_entry_head *entry;
1697 struct hlist_head *head;
1698 int size, __size, dsize;
1699 int rctx;
1700
1701 if (bpf_prog_array_valid(call)) {
1702 unsigned long orig_ip = instruction_pointer(regs);
1703 int ret;
1704
1705 ret = trace_call_bpf(call, regs);
1706
1707 /*
1708 * We need to check and see if we modified the pc of the
1709 * pt_regs, and if so return 1 so that we don't do the
1710 * single stepping.
1711 */
1712 if (orig_ip != instruction_pointer(regs))
1713 return 1;
1714 if (!ret)
1715 return 0;
1716 }
1717
1718 head = this_cpu_ptr(call->perf_events);
1719 if (hlist_empty(head))
1720 return 0;
1721
1722 dsize = __get_data_size(&tk->tp, regs, NULL);
1723 __size = sizeof(*entry) + tk->tp.size + dsize;
1724 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1725 size -= sizeof(u32);
1726
1727 entry = perf_trace_buf_alloc(size, NULL, &rctx);
1728 if (!entry)
1729 return 0;
1730
1731 entry->ip = (unsigned long)tk->rp.kp.addr;
1732 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize);
1733 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1734 head, NULL);
1735 return 0;
1736}
1737NOKPROBE_SYMBOL(kprobe_perf_func);
1738
1739/* Kretprobe profile handler */
1740static void
1741kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1742 struct pt_regs *regs)
1743{
1744 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1745 struct kretprobe_trace_entry_head *entry;
1746 struct hlist_head *head;
1747 int size, __size, dsize;
1748 int rctx;
1749
1750 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
1751 return;
1752
1753 head = this_cpu_ptr(call->perf_events);
1754 if (hlist_empty(head))
1755 return;
1756
1757 dsize = __get_data_size(&tk->tp, regs, ri->data);
1758 __size = sizeof(*entry) + tk->tp.size + dsize;
1759 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1760 size -= sizeof(u32);
1761
1762 entry = perf_trace_buf_alloc(size, NULL, &rctx);
1763 if (!entry)
1764 return;
1765
1766 entry->func = (unsigned long)tk->rp.kp.addr;
1767 entry->ret_ip = get_kretprobe_retaddr(ri);
1768 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize);
1769 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1770 head, NULL);
1771}
1772NOKPROBE_SYMBOL(kretprobe_perf_func);
1773
1774int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type,
1775 const char **symbol, u64 *probe_offset,
1776 u64 *probe_addr, unsigned long *missed,
1777 bool perf_type_tracepoint)
1778{
1779 const char *pevent = trace_event_name(event->tp_event);
1780 const char *group = event->tp_event->class->system;
1781 struct trace_kprobe *tk;
1782
1783 if (perf_type_tracepoint)
1784 tk = find_trace_kprobe(pevent, group);
1785 else
1786 tk = trace_kprobe_primary_from_call(event->tp_event);
1787 if (!tk)
1788 return -EINVAL;
1789
1790 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE
1791 : BPF_FD_TYPE_KPROBE;
1792 *probe_offset = tk->rp.kp.offset;
1793 *probe_addr = kallsyms_show_value(current_cred()) ?
1794 (unsigned long)tk->rp.kp.addr : 0;
1795 *symbol = tk->symbol;
1796 if (missed)
1797 *missed = trace_kprobe_missed(tk);
1798 return 0;
1799}
1800#endif /* CONFIG_PERF_EVENTS */
1801
1802/*
1803 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1804 *
1805 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1806 * lockless, but we can't race with this __init function.
1807 */
1808static int kprobe_register(struct trace_event_call *event,
1809 enum trace_reg type, void *data)
1810{
1811 struct trace_event_file *file = data;
1812
1813 switch (type) {
1814 case TRACE_REG_REGISTER:
1815 return enable_trace_kprobe(event, file);
1816 case TRACE_REG_UNREGISTER:
1817 return disable_trace_kprobe(event, file);
1818
1819#ifdef CONFIG_PERF_EVENTS
1820 case TRACE_REG_PERF_REGISTER:
1821 return enable_trace_kprobe(event, NULL);
1822 case TRACE_REG_PERF_UNREGISTER:
1823 return disable_trace_kprobe(event, NULL);
1824 case TRACE_REG_PERF_OPEN:
1825 case TRACE_REG_PERF_CLOSE:
1826 case TRACE_REG_PERF_ADD:
1827 case TRACE_REG_PERF_DEL:
1828 return 0;
1829#endif
1830 }
1831 return 0;
1832}
1833
1834static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1835{
1836 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1837 unsigned int flags = trace_probe_load_flag(&tk->tp);
1838 int ret = 0;
1839
1840 raw_cpu_inc(*tk->nhit);
1841
1842 if (flags & TP_FLAG_TRACE)
1843 kprobe_trace_func(tk, regs);
1844#ifdef CONFIG_PERF_EVENTS
1845 if (flags & TP_FLAG_PROFILE)
1846 ret = kprobe_perf_func(tk, regs);
1847#endif
1848 return ret;
1849}
1850NOKPROBE_SYMBOL(kprobe_dispatcher);
1851
1852static int
1853kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1854{
1855 struct kretprobe *rp = get_kretprobe(ri);
1856 struct trace_kprobe *tk;
1857 unsigned int flags;
1858
1859 /*
1860 * There is a small chance that get_kretprobe(ri) returns NULL when
1861 * the kretprobe is unregister on another CPU between kretprobe's
1862 * trampoline_handler and this function.
1863 */
1864 if (unlikely(!rp))
1865 return 0;
1866
1867 tk = container_of(rp, struct trace_kprobe, rp);
1868 raw_cpu_inc(*tk->nhit);
1869
1870 flags = trace_probe_load_flag(&tk->tp);
1871 if (flags & TP_FLAG_TRACE)
1872 kretprobe_trace_func(tk, ri, regs);
1873#ifdef CONFIG_PERF_EVENTS
1874 if (flags & TP_FLAG_PROFILE)
1875 kretprobe_perf_func(tk, ri, regs);
1876#endif
1877 return 0; /* We don't tweak kernel, so just return 0 */
1878}
1879NOKPROBE_SYMBOL(kretprobe_dispatcher);
1880
1881static struct trace_event_functions kretprobe_funcs = {
1882 .trace = print_kretprobe_event
1883};
1884
1885static struct trace_event_functions kprobe_funcs = {
1886 .trace = print_kprobe_event
1887};
1888
1889static struct trace_event_fields kretprobe_fields_array[] = {
1890 { .type = TRACE_FUNCTION_TYPE,
1891 .define_fields = kretprobe_event_define_fields },
1892 {}
1893};
1894
1895static struct trace_event_fields kprobe_fields_array[] = {
1896 { .type = TRACE_FUNCTION_TYPE,
1897 .define_fields = kprobe_event_define_fields },
1898 {}
1899};
1900
1901static inline void init_trace_event_call(struct trace_kprobe *tk)
1902{
1903 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1904
1905 if (trace_kprobe_is_return(tk)) {
1906 call->event.funcs = &kretprobe_funcs;
1907 call->class->fields_array = kretprobe_fields_array;
1908 } else {
1909 call->event.funcs = &kprobe_funcs;
1910 call->class->fields_array = kprobe_fields_array;
1911 }
1912
1913 call->flags = TRACE_EVENT_FL_KPROBE;
1914 call->class->reg = kprobe_register;
1915}
1916
1917static int register_kprobe_event(struct trace_kprobe *tk)
1918{
1919 init_trace_event_call(tk);
1920
1921 return trace_probe_register_event_call(&tk->tp);
1922}
1923
1924static int unregister_kprobe_event(struct trace_kprobe *tk)
1925{
1926 return trace_probe_unregister_event_call(&tk->tp);
1927}
1928
1929#ifdef CONFIG_PERF_EVENTS
1930
1931/* create a trace_kprobe, but don't add it to global lists */
1932struct trace_event_call *
1933create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1934 bool is_return)
1935{
1936 enum probe_print_type ptype;
1937 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL;
1938 int ret;
1939 char *event;
1940
1941 if (func) {
1942 ret = validate_probe_symbol(func);
1943 if (ret)
1944 return ERR_PTR(ret);
1945 }
1946
1947 /*
1948 * local trace_kprobes are not added to dyn_event, so they are never
1949 * searched in find_trace_kprobe(). Therefore, there is no concern of
1950 * duplicated name here.
1951 */
1952 event = func ? func : "DUMMY_EVENT";
1953
1954 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func,
1955 offs, 0 /* maxactive */, 0 /* nargs */,
1956 is_return);
1957
1958 if (IS_ERR(tk)) {
1959 pr_info("Failed to allocate trace_probe.(%d)\n",
1960 (int)PTR_ERR(tk));
1961 return ERR_CAST(tk);
1962 }
1963
1964 init_trace_event_call(tk);
1965
1966 ptype = trace_kprobe_is_return(tk) ?
1967 PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1968 if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0)
1969 return ERR_PTR(-ENOMEM);
1970
1971 ret = __register_trace_kprobe(tk);
1972 if (ret < 0)
1973 return ERR_PTR(ret);
1974
1975 return trace_probe_event_call(&(no_free_ptr(tk)->tp));
1976}
1977
1978void destroy_local_trace_kprobe(struct trace_event_call *event_call)
1979{
1980 struct trace_kprobe *tk;
1981
1982 tk = trace_kprobe_primary_from_call(event_call);
1983 if (unlikely(!tk))
1984 return;
1985
1986 if (trace_probe_is_enabled(&tk->tp)) {
1987 WARN_ON(1);
1988 return;
1989 }
1990
1991 __unregister_trace_kprobe(tk);
1992
1993 free_trace_kprobe(tk);
1994}
1995#endif /* CONFIG_PERF_EVENTS */
1996
1997static __init void enable_boot_kprobe_events(void)
1998{
1999 struct trace_array *tr = top_trace_array();
2000 struct trace_event_file *file;
2001 struct trace_kprobe *tk;
2002 struct dyn_event *pos;
2003
2004 if (trace_kprobe_list_empty())
2005 return;
2006
2007 guard(mutex)(&event_mutex);
2008 for_each_trace_kprobe(tk, pos) {
2009 list_for_each_entry(file, &tr->events, list)
2010 if (file->event_call == trace_probe_event_call(&tk->tp))
2011 trace_event_enable_disable(file, 1, 0);
2012 }
2013}
2014
2015static __init void setup_boot_kprobe_events(void)
2016{
2017 char *p, *cmd = kprobe_boot_events_buf;
2018 int ret;
2019
2020 strreplace(kprobe_boot_events_buf, ',', ' ');
2021
2022 while (cmd && *cmd != '\0') {
2023 p = strchr(cmd, ';');
2024 if (p)
2025 *p++ = '\0';
2026
2027 ret = create_or_delete_trace_kprobe(cmd);
2028 if (ret)
2029 pr_warn("Failed to add event(%d): %s\n", ret, cmd);
2030
2031 cmd = p;
2032 }
2033
2034 enable_boot_kprobe_events();
2035}
2036
2037/*
2038 * Register dynevent at core_initcall. This allows kernel to setup kprobe
2039 * events in postcore_initcall without tracefs.
2040 */
2041static __init int init_kprobe_trace_early(void)
2042{
2043 int ret;
2044
2045 ret = dyn_event_register(&trace_kprobe_ops);
2046 if (ret)
2047 return ret;
2048
2049 if (trace_kprobe_register_module_notifier())
2050 return -EINVAL;
2051
2052 return 0;
2053}
2054core_initcall(init_kprobe_trace_early);
2055
2056/* Make a tracefs interface for controlling probe points */
2057static __init int init_kprobe_trace(void)
2058{
2059 int ret;
2060
2061 ret = tracing_init_dentry();
2062 if (ret)
2063 return 0;
2064
2065 /* Event list interface */
2066 trace_create_file("kprobe_events", TRACE_MODE_WRITE,
2067 NULL, NULL, &kprobe_events_ops);
2068
2069 /* Profile interface */
2070 trace_create_file("kprobe_profile", TRACE_MODE_READ,
2071 NULL, NULL, &kprobe_profile_ops);
2072
2073 /* If no 'kprobe_event=' cmd is provided, return directly. */
2074 if (kprobe_boot_events_buf[0] == '\0')
2075 return 0;
2076
2077 setup_boot_kprobe_events();
2078
2079 return 0;
2080}
2081fs_initcall(init_kprobe_trace);
2082
2083
2084#ifdef CONFIG_FTRACE_STARTUP_TEST
2085static __init struct trace_event_file *
2086find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
2087{
2088 struct trace_event_file *file;
2089
2090 list_for_each_entry(file, &tr->events, list)
2091 if (file->event_call == trace_probe_event_call(&tk->tp))
2092 return file;
2093
2094 return NULL;
2095}
2096
2097/*
2098 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
2099 * stage, we can do this lockless.
2100 */
2101static __init int kprobe_trace_self_tests_init(void)
2102{
2103 int ret, warn = 0;
2104 int (*target)(int, int, int, int, int, int);
2105 struct trace_kprobe *tk;
2106 struct trace_event_file *file;
2107
2108 if (unlikely(tracing_disabled))
2109 return -ENODEV;
2110
2111 if (tracing_selftest_disabled)
2112 return 0;
2113
2114 target = kprobe_trace_selftest_target;
2115
2116 pr_info("Testing kprobe tracing: ");
2117
2118 ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)");
2119 if (WARN_ONCE(ret, "error on probing function entry.")) {
2120 warn++;
2121 } else {
2122 /* Enable trace point */
2123 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2124 if (WARN_ONCE(tk == NULL, "error on probing function entry.")) {
2125 warn++;
2126 } else {
2127 file = find_trace_probe_file(tk, top_trace_array());
2128 if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2129 warn++;
2130 } else
2131 enable_trace_kprobe(
2132 trace_probe_event_call(&tk->tp), file);
2133 }
2134 }
2135
2136 ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval");
2137 if (WARN_ONCE(ret, "error on probing function return.")) {
2138 warn++;
2139 } else {
2140 /* Enable trace point */
2141 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2142 if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) {
2143 warn++;
2144 } else {
2145 file = find_trace_probe_file(tk, top_trace_array());
2146 if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2147 warn++;
2148 } else
2149 enable_trace_kprobe(
2150 trace_probe_event_call(&tk->tp), file);
2151 }
2152 }
2153
2154 if (warn)
2155 goto end;
2156
2157 ret = target(1, 2, 3, 4, 5, 6);
2158
2159 /*
2160 * Not expecting an error here, the check is only to prevent the
2161 * optimizer from removing the call to target() as otherwise there
2162 * are no side-effects and the call is never performed.
2163 */
2164 if (ret != 21)
2165 warn++;
2166
2167 /* Disable trace points before removing it */
2168 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2169 if (WARN_ONCE(tk == NULL, "error on getting test probe.")) {
2170 warn++;
2171 } else {
2172 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
2173 "incorrect number of testprobe hits."))
2174 warn++;
2175
2176 file = find_trace_probe_file(tk, top_trace_array());
2177 if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2178 warn++;
2179 } else
2180 disable_trace_kprobe(
2181 trace_probe_event_call(&tk->tp), file);
2182 }
2183
2184 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2185 if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) {
2186 warn++;
2187 } else {
2188 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
2189 "incorrect number of testprobe2 hits."))
2190 warn++;
2191
2192 file = find_trace_probe_file(tk, top_trace_array());
2193 if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2194 warn++;
2195 } else
2196 disable_trace_kprobe(
2197 trace_probe_event_call(&tk->tp), file);
2198 }
2199
2200 ret = create_or_delete_trace_kprobe("-:testprobe");
2201 if (WARN_ONCE(ret, "error on deleting a probe."))
2202 warn++;
2203
2204 ret = create_or_delete_trace_kprobe("-:testprobe2");
2205 if (WARN_ONCE(ret, "error on deleting a probe."))
2206 warn++;
2207
2208
2209end:
2210 /*
2211 * Wait for the optimizer work to finish. Otherwise it might fiddle
2212 * with probes in already freed __init text.
2213 */
2214 wait_for_kprobe_optimizer();
2215 if (warn)
2216 pr_cont("NG: Some tests are failed. Please check them.\n");
2217 else
2218 pr_cont("OK\n");
2219 return 0;
2220}
2221
2222late_initcall(kprobe_trace_self_tests_init);
2223
2224#endif