projects/linux·blob uaccess-cleanups · refs

1030 lines · 23.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * ring buffer based function tracer
4 *
5 * Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
6 * Copyright (C) 2008 Ingo Molnar <[email protected]>
7 *
8 * Based on code from the latency_tracer, that is:
9 *
10 * Copyright (C) 2004-2006 Ingo Molnar
11 * Copyright (C) 2004 Nadia Yvette Chambers
12 */
13#include <linux/ring_buffer.h>
14#include <linux/debugfs.h>
15#include <linux/uaccess.h>
16#include <linux/ftrace.h>
17#include <linux/slab.h>
18#include <linux/fs.h>
19
20#include "trace.h"
21
22static void tracing_start_function_trace(struct trace_array *tr);
23static void tracing_stop_function_trace(struct trace_array *tr);
24static void
25function_trace_call(unsigned long ip, unsigned long parent_ip,
26 struct ftrace_ops *op, struct ftrace_regs *fregs);
27static void
28function_args_trace_call(unsigned long ip, unsigned long parent_ip,
29 struct ftrace_ops *op, struct ftrace_regs *fregs);
30static void
31function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
32 struct ftrace_ops *op, struct ftrace_regs *fregs);
33static void
34function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
35 struct ftrace_ops *op, struct ftrace_regs *fregs);
36static void
37function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
38 struct ftrace_ops *op,
39 struct ftrace_regs *fregs);
40static struct tracer_flags func_flags;
41
42/* Our option */
43enum {
44
45 TRACE_FUNC_NO_OPTS = 0x0, /* No flags set. */
46 TRACE_FUNC_OPT_STACK = 0x1,
47 TRACE_FUNC_OPT_NO_REPEATS = 0x2,
48 TRACE_FUNC_OPT_ARGS = 0x4,
49
50 /* Update this to next highest bit. */
51 TRACE_FUNC_OPT_HIGHEST_BIT = 0x8
52};
53
54#define TRACE_FUNC_OPT_MASK (TRACE_FUNC_OPT_HIGHEST_BIT - 1)
55
56int ftrace_allocate_ftrace_ops(struct trace_array *tr)
57{
58 struct ftrace_ops *ops;
59
60 /* The top level array uses the "global_ops" */
61 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
62 return 0;
63
64 ops = kzalloc_obj(*ops);
65 if (!ops)
66 return -ENOMEM;
67
68 /* Currently only the non stack version is supported */
69 ops->func = function_trace_call;
70 ops->flags = FTRACE_OPS_FL_PID;
71
72 tr->ops = ops;
73 ops->private = tr;
74
75 return 0;
76}
77
78void ftrace_free_ftrace_ops(struct trace_array *tr)
79{
80 kfree(tr->ops);
81 tr->ops = NULL;
82}
83
84int ftrace_create_function_files(struct trace_array *tr,
85 struct dentry *parent)
86{
87 int ret;
88 /*
89 * The top level array uses the "global_ops", and the files are
90 * created on boot up.
91 */
92 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
93 return 0;
94
95 if (!tr->ops)
96 return -EINVAL;
97
98 ret = allocate_fgraph_ops(tr, tr->ops);
99 if (ret) {
100 kfree(tr->ops);
101 return ret;
102 }
103
104 ftrace_create_filter_files(tr, parent);
105
106 return 0;
107}
108
109void ftrace_destroy_function_files(struct trace_array *tr)
110{
111 ftrace_destroy_filter_files(tr->ops);
112 ftrace_free_ftrace_ops(tr);
113 free_fgraph_ops(tr);
114}
115
116static ftrace_func_t select_trace_function(u32 flags_val)
117{
118 switch (flags_val & TRACE_FUNC_OPT_MASK) {
119 case TRACE_FUNC_NO_OPTS:
120 return function_trace_call;
121 case TRACE_FUNC_OPT_ARGS:
122 return function_args_trace_call;
123 case TRACE_FUNC_OPT_STACK:
124 return function_stack_trace_call;
125 case TRACE_FUNC_OPT_NO_REPEATS:
126 return function_no_repeats_trace_call;
127 case TRACE_FUNC_OPT_STACK | TRACE_FUNC_OPT_NO_REPEATS:
128 return function_stack_no_repeats_trace_call;
129 default:
130 return NULL;
131 }
132}
133
134static bool handle_func_repeats(struct trace_array *tr, u32 flags_val)
135{
136 if (!tr->last_func_repeats &&
137 (flags_val & TRACE_FUNC_OPT_NO_REPEATS)) {
138 tr->last_func_repeats = alloc_percpu(struct trace_func_repeats);
139 if (!tr->last_func_repeats)
140 return false;
141 }
142
143 return true;
144}
145
146static int function_trace_init(struct trace_array *tr)
147{
148 ftrace_func_t func;
149 /*
150 * Instance trace_arrays get their ops allocated
151 * at instance creation. Unless it failed
152 * the allocation.
153 */
154 if (!tr->ops)
155 return -ENOMEM;
156
157 func = select_trace_function(tr->current_trace_flags->val);
158 if (!func)
159 return -EINVAL;
160
161 if (!handle_func_repeats(tr, tr->current_trace_flags->val))
162 return -ENOMEM;
163
164 ftrace_init_array_ops(tr, func);
165
166 tr->array_buffer.cpu = raw_smp_processor_id();
167
168 tracing_start_cmdline_record();
169 tracing_start_function_trace(tr);
170 return 0;
171}
172
173static void function_trace_reset(struct trace_array *tr)
174{
175 tracing_stop_function_trace(tr);
176 tracing_stop_cmdline_record();
177 ftrace_reset_array_ops(tr);
178}
179
180static void function_trace_start(struct trace_array *tr)
181{
182 tracing_reset_online_cpus(&tr->array_buffer);
183}
184
185/* fregs are guaranteed not to be NULL if HAVE_DYNAMIC_FTRACE_WITH_ARGS is set */
186#if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS)
187static __always_inline unsigned long
188function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs)
189{
190 unsigned long true_parent_ip;
191 int idx = 0;
192
193 true_parent_ip = parent_ip;
194 if (unlikely(parent_ip == (unsigned long)&return_to_handler) && fregs)
195 true_parent_ip = ftrace_graph_ret_addr(current, &idx, parent_ip,
196 (unsigned long *)ftrace_regs_get_stack_pointer(fregs));
197 return true_parent_ip;
198}
199#else
200static __always_inline unsigned long
201function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs)
202{
203 return parent_ip;
204}
205#endif
206
207static void
208function_trace_call(unsigned long ip, unsigned long parent_ip,
209 struct ftrace_ops *op, struct ftrace_regs *fregs)
210{
211 struct trace_array *tr = op->private;
212 unsigned int trace_ctx;
213 int bit;
214
215 if (unlikely(!tr->function_enabled))
216 return;
217
218 bit = ftrace_test_recursion_trylock(ip, parent_ip);
219 if (bit < 0)
220 return;
221
222 parent_ip = function_get_true_parent_ip(parent_ip, fregs);
223
224 trace_ctx = tracing_gen_ctx_dec();
225
226 trace_function(tr, ip, parent_ip, trace_ctx, NULL);
227
228 ftrace_test_recursion_unlock(bit);
229}
230
231static void
232function_args_trace_call(unsigned long ip, unsigned long parent_ip,
233 struct ftrace_ops *op, struct ftrace_regs *fregs)
234{
235 struct trace_array *tr = op->private;
236 unsigned int trace_ctx;
237 int bit;
238
239 if (unlikely(!tr->function_enabled))
240 return;
241
242 bit = ftrace_test_recursion_trylock(ip, parent_ip);
243 if (bit < 0)
244 return;
245
246 trace_ctx = tracing_gen_ctx();
247
248 trace_function(tr, ip, parent_ip, trace_ctx, fregs);
249
250 ftrace_test_recursion_unlock(bit);
251}
252
253#ifdef CONFIG_UNWINDER_ORC
254/*
255 * Skip 2:
256 *
257 * function_stack_trace_call()
258 * ftrace_call()
259 */
260#define STACK_SKIP 2
261#else
262/*
263 * Skip 3:
264 * __trace_stack()
265 * function_stack_trace_call()
266 * ftrace_call()
267 */
268#define STACK_SKIP 3
269#endif
270
271static void
272function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
273 struct ftrace_ops *op, struct ftrace_regs *fregs)
274{
275 struct trace_array *tr = op->private;
276 struct trace_array_cpu *data;
277 unsigned long flags;
278 long disabled;
279 int cpu;
280 unsigned int trace_ctx;
281 int skip = STACK_SKIP;
282
283 if (unlikely(!tr->function_enabled))
284 return;
285
286 /*
287 * Need to use raw, since this must be called before the
288 * recursive protection is performed.
289 */
290 local_irq_save(flags);
291 parent_ip = function_get_true_parent_ip(parent_ip, fregs);
292 cpu = raw_smp_processor_id();
293 data = per_cpu_ptr(tr->array_buffer.data, cpu);
294 disabled = local_inc_return(&data->disabled);
295
296 if (likely(disabled == 1)) {
297 trace_ctx = tracing_gen_ctx_flags(flags);
298 trace_function(tr, ip, parent_ip, trace_ctx, NULL);
299#ifdef CONFIG_UNWINDER_FRAME_POINTER
300 if (ftrace_pids_enabled(op))
301 skip++;
302#endif
303 __trace_stack(tr, trace_ctx, skip);
304 }
305
306 local_dec(&data->disabled);
307 local_irq_restore(flags);
308}
309
310static inline bool is_repeat_check(struct trace_array *tr,
311 struct trace_func_repeats *last_info,
312 unsigned long ip, unsigned long parent_ip)
313{
314 if (last_info->ip == ip &&
315 last_info->parent_ip == parent_ip &&
316 last_info->count < U16_MAX) {
317 last_info->ts_last_call =
318 ring_buffer_time_stamp(tr->array_buffer.buffer);
319 last_info->count++;
320 return true;
321 }
322
323 return false;
324}
325
326static inline void process_repeats(struct trace_array *tr,
327 unsigned long ip, unsigned long parent_ip,
328 struct trace_func_repeats *last_info,
329 unsigned int trace_ctx)
330{
331 if (last_info->count) {
332 trace_last_func_repeats(tr, last_info, trace_ctx);
333 last_info->count = 0;
334 }
335
336 last_info->ip = ip;
337 last_info->parent_ip = parent_ip;
338}
339
340static void
341function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
342 struct ftrace_ops *op,
343 struct ftrace_regs *fregs)
344{
345 struct trace_func_repeats *last_info;
346 struct trace_array *tr = op->private;
347 unsigned int trace_ctx;
348 int bit;
349
350 if (unlikely(!tr->function_enabled))
351 return;
352
353 bit = ftrace_test_recursion_trylock(ip, parent_ip);
354 if (bit < 0)
355 return;
356
357 parent_ip = function_get_true_parent_ip(parent_ip, fregs);
358 if (!tracer_tracing_is_on(tr))
359 goto out;
360
361 /*
362 * An interrupt may happen at any place here. But as far as I can see,
363 * the only damage that this can cause is to mess up the repetition
364 * counter without valuable data being lost.
365 * TODO: think about a solution that is better than just hoping to be
366 * lucky.
367 */
368 last_info = this_cpu_ptr(tr->last_func_repeats);
369 if (is_repeat_check(tr, last_info, ip, parent_ip))
370 goto out;
371
372 trace_ctx = tracing_gen_ctx_dec();
373 process_repeats(tr, ip, parent_ip, last_info, trace_ctx);
374
375 trace_function(tr, ip, parent_ip, trace_ctx, NULL);
376
377out:
378 ftrace_test_recursion_unlock(bit);
379}
380
381static void
382function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
383 struct ftrace_ops *op,
384 struct ftrace_regs *fregs)
385{
386 struct trace_func_repeats *last_info;
387 struct trace_array *tr = op->private;
388 struct trace_array_cpu *data;
389 unsigned long flags;
390 long disabled;
391 int cpu;
392 unsigned int trace_ctx;
393
394 if (unlikely(!tr->function_enabled))
395 return;
396
397 /*
398 * Need to use raw, since this must be called before the
399 * recursive protection is performed.
400 */
401 local_irq_save(flags);
402 parent_ip = function_get_true_parent_ip(parent_ip, fregs);
403 cpu = raw_smp_processor_id();
404 data = per_cpu_ptr(tr->array_buffer.data, cpu);
405 disabled = local_inc_return(&data->disabled);
406
407 if (likely(disabled == 1)) {
408 last_info = per_cpu_ptr(tr->last_func_repeats, cpu);
409 if (is_repeat_check(tr, last_info, ip, parent_ip))
410 goto out;
411
412 trace_ctx = tracing_gen_ctx_flags(flags);
413 process_repeats(tr, ip, parent_ip, last_info, trace_ctx);
414
415 trace_function(tr, ip, parent_ip, trace_ctx, NULL);
416 __trace_stack(tr, trace_ctx, STACK_SKIP);
417 }
418
419 out:
420 local_dec(&data->disabled);
421 local_irq_restore(flags);
422}
423
424static struct tracer_opt func_opts[] = {
425#ifdef CONFIG_STACKTRACE
426 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
427#endif
428 { TRACER_OPT(func-no-repeats, TRACE_FUNC_OPT_NO_REPEATS) },
429#ifdef CONFIG_FUNCTION_TRACE_ARGS
430 { TRACER_OPT(func-args, TRACE_FUNC_OPT_ARGS) },
431#endif
432 { } /* Always set a last empty entry */
433};
434
435static struct tracer_flags func_flags = {
436 .val = TRACE_FUNC_NO_OPTS, /* By default: all flags disabled */
437 .opts = func_opts
438};
439
440static void tracing_start_function_trace(struct trace_array *tr)
441{
442 tr->function_enabled = 0;
443 register_ftrace_function(tr->ops);
444 tr->function_enabled = 1;
445}
446
447static void tracing_stop_function_trace(struct trace_array *tr)
448{
449 tr->function_enabled = 0;
450 unregister_ftrace_function(tr->ops);
451}
452
453static struct tracer function_trace;
454
455static int
456func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
457{
458 ftrace_func_t func;
459 u32 new_flags;
460
461 /* We can change this flag only when current tracer is function. */
462 if (tr->current_trace != &function_trace)
463 return 0;
464
465 /* Do nothing if already set. */
466 if (!!set == !!(tr->current_trace_flags->val & bit))
467 return 0;
468
469 new_flags = (tr->current_trace_flags->val & ~bit) | (set ? bit : 0);
470 func = select_trace_function(new_flags);
471 if (!func)
472 return -EINVAL;
473
474 /* Check if there's anything to change. */
475 if (tr->ops->func == func)
476 return 0;
477
478 if (!handle_func_repeats(tr, new_flags))
479 return -ENOMEM;
480
481 unregister_ftrace_function(tr->ops);
482 tr->ops->func = func;
483 register_ftrace_function(tr->ops);
484
485 return 0;
486}
487
488static struct tracer function_trace __tracer_data =
489{
490 .name = "function",
491 .init = function_trace_init,
492 .reset = function_trace_reset,
493 .start = function_trace_start,
494 .default_flags = &func_flags,
495 .set_flag = func_set_flag,
496 .allow_instances = true,
497#ifdef CONFIG_FTRACE_SELFTEST
498 .selftest = trace_selftest_startup_function,
499#endif
500};
501
502#ifdef CONFIG_DYNAMIC_FTRACE
503static void update_traceon_count(struct ftrace_probe_ops *ops,
504 unsigned long ip,
505 struct trace_array *tr, bool on,
506 void *data)
507{
508 struct ftrace_func_mapper *mapper = data;
509 long *count;
510 long old_count;
511
512 /*
513 * Tracing gets disabled (or enabled) once per count.
514 * This function can be called at the same time on multiple CPUs.
515 * It is fine if both disable (or enable) tracing, as disabling
516 * (or enabling) the second time doesn't do anything as the
517 * state of the tracer is already disabled (or enabled).
518 * What needs to be synchronized in this case is that the count
519 * only gets decremented once, even if the tracer is disabled
520 * (or enabled) twice, as the second one is really a nop.
521 *
522 * The memory barriers guarantee that we only decrement the
523 * counter once. First the count is read to a local variable
524 * and a read barrier is used to make sure that it is loaded
525 * before checking if the tracer is in the state we want.
526 * If the tracer is not in the state we want, then the count
527 * is guaranteed to be the old count.
528 *
529 * Next the tracer is set to the state we want (disabled or enabled)
530 * then a write memory barrier is used to make sure that
531 * the new state is visible before changing the counter by
532 * one minus the old counter. This guarantees that another CPU
533 * executing this code will see the new state before seeing
534 * the new counter value, and would not do anything if the new
535 * counter is seen.
536 *
537 * Note, there is no synchronization between this and a user
538 * setting the tracing_on file. But we currently don't care
539 * about that.
540 */
541 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
542 old_count = *count;
543
544 if (old_count <= 0)
545 return;
546
547 /* Make sure we see count before checking tracing state */
548 smp_rmb();
549
550 if (on == !!tracer_tracing_is_on(tr))
551 return;
552
553 if (on)
554 tracer_tracing_on(tr);
555 else
556 tracer_tracing_off(tr);
557
558 /* Make sure tracing state is visible before updating count */
559 smp_wmb();
560
561 *count = old_count - 1;
562}
563
564static void
565ftrace_traceon_count(unsigned long ip, unsigned long parent_ip,
566 struct trace_array *tr, struct ftrace_probe_ops *ops,
567 void *data)
568{
569 update_traceon_count(ops, ip, tr, 1, data);
570}
571
572static void
573ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip,
574 struct trace_array *tr, struct ftrace_probe_ops *ops,
575 void *data)
576{
577 update_traceon_count(ops, ip, tr, 0, data);
578}
579
580static void
581ftrace_traceon(unsigned long ip, unsigned long parent_ip,
582 struct trace_array *tr, struct ftrace_probe_ops *ops,
583 void *data)
584{
585 if (tracer_tracing_is_on(tr))
586 return;
587
588 tracer_tracing_on(tr);
589}
590
591static void
592ftrace_traceoff(unsigned long ip, unsigned long parent_ip,
593 struct trace_array *tr, struct ftrace_probe_ops *ops,
594 void *data)
595{
596 if (!tracer_tracing_is_on(tr))
597 return;
598
599 tracer_tracing_off(tr);
600}
601
602#ifdef CONFIG_UNWINDER_ORC
603/*
604 * Skip 3:
605 *
606 * function_trace_probe_call()
607 * ftrace_ops_assist_func()
608 * ftrace_call()
609 */
610#define FTRACE_STACK_SKIP 3
611#else
612/*
613 * Skip 5:
614 *
615 * __trace_stack()
616 * ftrace_stacktrace()
617 * function_trace_probe_call()
618 * ftrace_ops_assist_func()
619 * ftrace_call()
620 */
621#define FTRACE_STACK_SKIP 5
622#endif
623
624static __always_inline void trace_stack(struct trace_array *tr)
625{
626 __trace_stack(tr, tracing_gen_ctx_dec(), FTRACE_STACK_SKIP);
627}
628
629static void
630ftrace_stacktrace(unsigned long ip, unsigned long parent_ip,
631 struct trace_array *tr, struct ftrace_probe_ops *ops,
632 void *data)
633{
634 trace_stack(tr);
635}
636
637static void
638ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip,
639 struct trace_array *tr, struct ftrace_probe_ops *ops,
640 void *data)
641{
642 struct ftrace_func_mapper *mapper = data;
643 long *count;
644 long old_count;
645 long new_count;
646
647 if (!tracing_is_on())
648 return;
649
650 /* unlimited? */
651 if (!mapper) {
652 trace_stack(tr);
653 return;
654 }
655
656 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
657
658 /*
659 * Stack traces should only execute the number of times the
660 * user specified in the counter.
661 */
662 do {
663 old_count = *count;
664
665 if (!old_count)
666 return;
667
668 new_count = old_count - 1;
669 new_count = cmpxchg(count, old_count, new_count);
670 if (new_count == old_count)
671 trace_stack(tr);
672
673 if (!tracing_is_on())
674 return;
675
676 } while (new_count != old_count);
677}
678
679static int update_count(struct ftrace_probe_ops *ops, unsigned long ip,
680 void *data)
681{
682 struct ftrace_func_mapper *mapper = data;
683 long *count = NULL;
684
685 if (mapper)
686 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
687
688 if (count) {
689 if (*count <= 0)
690 return 0;
691 (*count)--;
692 }
693
694 return 1;
695}
696
697static void
698ftrace_dump_probe(unsigned long ip, unsigned long parent_ip,
699 struct trace_array *tr, struct ftrace_probe_ops *ops,
700 void *data)
701{
702 if (update_count(ops, ip, data))
703 ftrace_dump(DUMP_ALL);
704}
705
706/* Only dump the current CPU buffer. */
707static void
708ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip,
709 struct trace_array *tr, struct ftrace_probe_ops *ops,
710 void *data)
711{
712 if (update_count(ops, ip, data))
713 ftrace_dump(DUMP_ORIG);
714}
715
716static int
717ftrace_probe_print(const char *name, struct seq_file *m,
718 unsigned long ip, struct ftrace_probe_ops *ops,
719 void *data)
720{
721 struct ftrace_func_mapper *mapper = data;
722 long *count = NULL;
723
724 seq_printf(m, "%ps:%s", (void *)ip, name);
725
726 if (mapper)
727 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
728
729 if (count)
730 seq_printf(m, ":count=%ld\n", *count);
731 else
732 seq_puts(m, ":unlimited\n");
733
734 return 0;
735}
736
737static int
738ftrace_traceon_print(struct seq_file *m, unsigned long ip,
739 struct ftrace_probe_ops *ops,
740 void *data)
741{
742 return ftrace_probe_print("traceon", m, ip, ops, data);
743}
744
745static int
746ftrace_traceoff_print(struct seq_file *m, unsigned long ip,
747 struct ftrace_probe_ops *ops, void *data)
748{
749 return ftrace_probe_print("traceoff", m, ip, ops, data);
750}
751
752static int
753ftrace_stacktrace_print(struct seq_file *m, unsigned long ip,
754 struct ftrace_probe_ops *ops, void *data)
755{
756 return ftrace_probe_print("stacktrace", m, ip, ops, data);
757}
758
759static int
760ftrace_dump_print(struct seq_file *m, unsigned long ip,
761 struct ftrace_probe_ops *ops, void *data)
762{
763 return ftrace_probe_print("dump", m, ip, ops, data);
764}
765
766static int
767ftrace_cpudump_print(struct seq_file *m, unsigned long ip,
768 struct ftrace_probe_ops *ops, void *data)
769{
770 return ftrace_probe_print("cpudump", m, ip, ops, data);
771}
772
773
774static int
775ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
776 unsigned long ip, void *init_data, void **data)
777{
778 struct ftrace_func_mapper *mapper = *data;
779
780 if (!mapper) {
781 mapper = allocate_ftrace_func_mapper();
782 if (!mapper)
783 return -ENOMEM;
784 *data = mapper;
785 }
786
787 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
788}
789
790static void
791ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
792 unsigned long ip, void *data)
793{
794 struct ftrace_func_mapper *mapper = data;
795
796 if (!ip) {
797 free_ftrace_func_mapper(mapper, NULL);
798 return;
799 }
800
801 ftrace_func_mapper_remove_ip(mapper, ip);
802}
803
804static struct ftrace_probe_ops traceon_count_probe_ops = {
805 .func = ftrace_traceon_count,
806 .print = ftrace_traceon_print,
807 .init = ftrace_count_init,
808 .free = ftrace_count_free,
809};
810
811static struct ftrace_probe_ops traceoff_count_probe_ops = {
812 .func = ftrace_traceoff_count,
813 .print = ftrace_traceoff_print,
814 .init = ftrace_count_init,
815 .free = ftrace_count_free,
816};
817
818static struct ftrace_probe_ops stacktrace_count_probe_ops = {
819 .func = ftrace_stacktrace_count,
820 .print = ftrace_stacktrace_print,
821 .init = ftrace_count_init,
822 .free = ftrace_count_free,
823};
824
825static struct ftrace_probe_ops dump_probe_ops = {
826 .func = ftrace_dump_probe,
827 .print = ftrace_dump_print,
828 .init = ftrace_count_init,
829 .free = ftrace_count_free,
830};
831
832static struct ftrace_probe_ops cpudump_probe_ops = {
833 .func = ftrace_cpudump_probe,
834 .print = ftrace_cpudump_print,
835};
836
837static struct ftrace_probe_ops traceon_probe_ops = {
838 .func = ftrace_traceon,
839 .print = ftrace_traceon_print,
840};
841
842static struct ftrace_probe_ops traceoff_probe_ops = {
843 .func = ftrace_traceoff,
844 .print = ftrace_traceoff_print,
845};
846
847static struct ftrace_probe_ops stacktrace_probe_ops = {
848 .func = ftrace_stacktrace,
849 .print = ftrace_stacktrace_print,
850};
851
852static int
853ftrace_trace_probe_callback(struct trace_array *tr,
854 struct ftrace_probe_ops *ops,
855 struct ftrace_hash *hash, char *glob,
856 char *cmd, char *param, int enable)
857{
858 void *count = (void *)-1;
859 char *number;
860 int ret;
861
862 /* hash funcs only work with set_ftrace_filter */
863 if (!enable)
864 return -EINVAL;
865
866 if (glob[0] == '!')
867 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
868
869 if (!param)
870 goto out_reg;
871
872 number = strsep(&param, ":");
873
874 if (!strlen(number))
875 goto out_reg;
876
877 /*
878 * We use the callback data field (which is a pointer)
879 * as our counter.
880 */
881 ret = kstrtoul(number, 0, (unsigned long *)&count);
882 if (ret)
883 return ret;
884
885 out_reg:
886 ret = register_ftrace_function_probe(glob, tr, ops, count);
887
888 return ret < 0 ? ret : 0;
889}
890
891static int
892ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash,
893 char *glob, char *cmd, char *param, int enable)
894{
895 struct ftrace_probe_ops *ops;
896
897 if (!tr)
898 return -ENODEV;
899
900 /* we register both traceon and traceoff to this callback */
901 if (strcmp(cmd, "traceon") == 0)
902 ops = param ? &traceon_count_probe_ops : &traceon_probe_ops;
903 else
904 ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops;
905
906 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
907 param, enable);
908}
909
910static int
911ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash,
912 char *glob, char *cmd, char *param, int enable)
913{
914 struct ftrace_probe_ops *ops;
915
916 if (!tr)
917 return -ENODEV;
918
919 ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops;
920
921 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
922 param, enable);
923}
924
925static int
926ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash,
927 char *glob, char *cmd, char *param, int enable)
928{
929 struct ftrace_probe_ops *ops;
930
931 if (!tr)
932 return -ENODEV;
933
934 ops = &dump_probe_ops;
935
936 /* Only dump once. */
937 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
938 "1", enable);
939}
940
941static int
942ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash,
943 char *glob, char *cmd, char *param, int enable)
944{
945 struct ftrace_probe_ops *ops;
946
947 if (!tr)
948 return -ENODEV;
949
950 ops = &cpudump_probe_ops;
951
952 /* Only dump once. */
953 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
954 "1", enable);
955}
956
957static struct ftrace_func_command ftrace_traceon_cmd = {
958 .name = "traceon",
959 .func = ftrace_trace_onoff_callback,
960};
961
962static struct ftrace_func_command ftrace_traceoff_cmd = {
963 .name = "traceoff",
964 .func = ftrace_trace_onoff_callback,
965};
966
967static struct ftrace_func_command ftrace_stacktrace_cmd = {
968 .name = "stacktrace",
969 .func = ftrace_stacktrace_callback,
970};
971
972static struct ftrace_func_command ftrace_dump_cmd = {
973 .name = "dump",
974 .func = ftrace_dump_callback,
975};
976
977static struct ftrace_func_command ftrace_cpudump_cmd = {
978 .name = "cpudump",
979 .func = ftrace_cpudump_callback,
980};
981
982static int __init init_func_cmd_traceon(void)
983{
984 int ret;
985
986 ret = register_ftrace_command(&ftrace_traceoff_cmd);
987 if (ret)
988 return ret;
989
990 ret = register_ftrace_command(&ftrace_traceon_cmd);
991 if (ret)
992 goto out_free_traceoff;
993
994 ret = register_ftrace_command(&ftrace_stacktrace_cmd);
995 if (ret)
996 goto out_free_traceon;
997
998 ret = register_ftrace_command(&ftrace_dump_cmd);
999 if (ret)
1000 goto out_free_stacktrace;
1001
1002 ret = register_ftrace_command(&ftrace_cpudump_cmd);
1003 if (ret)
1004 goto out_free_dump;
1005
1006 return 0;
1007
1008 out_free_dump:
1009 unregister_ftrace_command(&ftrace_dump_cmd);
1010 out_free_stacktrace:
1011 unregister_ftrace_command(&ftrace_stacktrace_cmd);
1012 out_free_traceon:
1013 unregister_ftrace_command(&ftrace_traceon_cmd);
1014 out_free_traceoff:
1015 unregister_ftrace_command(&ftrace_traceoff_cmd);
1016
1017 return ret;
1018}
1019#else
1020static inline int init_func_cmd_traceon(void)
1021{
1022 return 0;
1023}
1024#endif /* CONFIG_DYNAMIC_FTRACE */
1025
1026__init int init_function_trace(void)
1027{
1028 init_func_cmd_traceon();
1029 return register_tracer(&function_trace);
1030}