projects/linux·blob uaccess-cleanups · refs

9470 lines · 230.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Infrastructure for profiling code inserted by 'gcc -pg'.
4 *
5 * Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
6 * Copyright (C) 2004-2008 Ingo Molnar <[email protected]>
7 *
8 * Originally ported from the -rt patch by:
9 * Copyright (C) 2007 Arnaldo Carvalho de Melo <[email protected]>
10 *
11 * Based on code in the latency_tracer, that is:
12 *
13 * Copyright (C) 2004-2006 Ingo Molnar
14 * Copyright (C) 2004 Nadia Yvette Chambers
15 */
16
17#include <linux/stop_machine.h>
18#include <linux/clocksource.h>
19#include <linux/sched/task.h>
20#include <linux/kallsyms.h>
21#include <linux/security.h>
22#include <linux/seq_file.h>
23#include <linux/tracefs.h>
24#include <linux/hardirq.h>
25#include <linux/kthread.h>
26#include <linux/uaccess.h>
27#include <linux/bsearch.h>
28#include <linux/module.h>
29#include <linux/ftrace.h>
30#include <linux/sysctl.h>
31#include <linux/slab.h>
32#include <linux/ctype.h>
33#include <linux/sort.h>
34#include <linux/list.h>
35#include <linux/hash.h>
36#include <linux/rcupdate.h>
37#include <linux/kprobes.h>
38
39#include <trace/events/sched.h>
40
41#include <asm/sections.h>
42#include <asm/setup.h>
43
44#include "ftrace_internal.h"
45#include "trace_output.h"
46#include "trace_stat.h"
47
48/* Flags that do not get reset */
49#define FTRACE_NOCLEAR_FLAGS (FTRACE_FL_DISABLED | FTRACE_FL_TOUCHED | \
50 FTRACE_FL_MODIFIED)
51
52#define FTRACE_INVALID_FUNCTION "__ftrace_invalid_address__"
53
54#define FTRACE_WARN_ON(cond) \
55 ({ \
56 int ___r = cond; \
57 if (WARN_ON(___r)) \
58 ftrace_kill(); \
59 ___r; \
60 })
61
62#define FTRACE_WARN_ON_ONCE(cond) \
63 ({ \
64 int ___r = cond; \
65 if (WARN_ON_ONCE(___r)) \
66 ftrace_kill(); \
67 ___r; \
68 })
69
70/* hash bits for specific function selection */
71#define FTRACE_HASH_MAX_BITS 12
72
73#ifdef CONFIG_DYNAMIC_FTRACE
74#define INIT_OPS_HASH(opsname) \
75 .func_hash = &opsname.local_hash, \
76 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), \
77 .subop_list = LIST_HEAD_INIT(opsname.subop_list),
78/* Used only to synchronize the initialization of ftrace_ops */
79static DEFINE_MUTEX(ops_mutex);
80#else
81#define INIT_OPS_HASH(opsname)
82#endif
83
84enum {
85 FTRACE_MODIFY_ENABLE_FL = (1 << 0),
86 FTRACE_MODIFY_MAY_SLEEP_FL = (1 << 1),
87};
88
89struct ftrace_ops ftrace_list_end __read_mostly = {
90 .func = ftrace_stub,
91 .flags = FTRACE_OPS_FL_STUB,
92 INIT_OPS_HASH(ftrace_list_end)
93};
94
95/* ftrace_enabled is a method to turn ftrace on or off */
96int ftrace_enabled __read_mostly;
97static int __maybe_unused last_ftrace_enabled;
98
99/* Current function tracing op */
100struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
101/* What to set function_trace_op to */
102static struct ftrace_ops *set_function_trace_op;
103
104bool ftrace_pids_enabled(struct ftrace_ops *ops)
105{
106 struct trace_array *tr;
107
108 if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
109 return false;
110
111 tr = ops->private;
112
113 return tr->function_pids != NULL || tr->function_no_pids != NULL;
114}
115
116static void ftrace_update_trampoline(struct ftrace_ops *ops);
117
118/*
119 * ftrace_disabled is set when an anomaly is discovered.
120 * ftrace_disabled is much stronger than ftrace_enabled.
121 */
122static int ftrace_disabled __read_mostly;
123
124DEFINE_MUTEX(ftrace_lock);
125
126struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = (struct ftrace_ops __rcu *)&ftrace_list_end;
127ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
128struct ftrace_ops global_ops;
129
130/* Defined by vmlinux.lds.h see the comment above arch_ftrace_ops_list_func for details */
131void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
132 struct ftrace_ops *op, struct ftrace_regs *fregs);
133
134#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
135/*
136 * Stub used to invoke the list ops without requiring a separate trampoline.
137 */
138const struct ftrace_ops ftrace_list_ops = {
139 .func = ftrace_ops_list_func,
140 .flags = FTRACE_OPS_FL_STUB,
141};
142
143static void ftrace_ops_nop_func(unsigned long ip, unsigned long parent_ip,
144 struct ftrace_ops *op,
145 struct ftrace_regs *fregs)
146{
147 /* do nothing */
148}
149
150/*
151 * Stub used when a call site is disabled. May be called transiently by threads
152 * which have made it into ftrace_caller but haven't yet recovered the ops at
153 * the point the call site is disabled.
154 */
155const struct ftrace_ops ftrace_nop_ops = {
156 .func = ftrace_ops_nop_func,
157 .flags = FTRACE_OPS_FL_STUB,
158};
159#endif
160
161static inline void ftrace_ops_init(struct ftrace_ops *ops)
162{
163#ifdef CONFIG_DYNAMIC_FTRACE
164 unsigned long flags = smp_load_acquire(&ops->flags);
165
166 if (!(flags & FTRACE_OPS_FL_INITIALIZED)) {
167 guard(mutex)(&ops_mutex);
168 /* Could have been initialized before lock taken */
169 if (unlikely(ops->flags & FTRACE_OPS_FL_INITIALIZED))
170 return;
171 mutex_init(&ops->local_hash.regex_lock);
172 INIT_LIST_HEAD(&ops->subop_list);
173 ops->func_hash = &ops->local_hash;
174 flags = ops->flags | FTRACE_OPS_FL_INITIALIZED;
175 smp_store_release(&ops->flags, flags);
176 }
177#endif
178}
179
180/* Call this function for when a callback filters on set_ftrace_pid */
181static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
182 struct ftrace_ops *op, struct ftrace_regs *fregs)
183{
184 struct trace_array *tr = op->private;
185 int pid;
186
187 if (tr) {
188 pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);
189 if (pid == FTRACE_PID_IGNORE)
190 return;
191 if (pid != FTRACE_PID_TRACE &&
192 pid != current->pid)
193 return;
194 }
195
196 op->saved_func(ip, parent_ip, op, fregs);
197}
198
199void ftrace_sync_ipi(void *data)
200{
201 /* Probably not needed, but do it anyway */
202 smp_rmb();
203}
204
205static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
206{
207 /*
208 * If this is a dynamic or RCU ops, or we force list func,
209 * then it needs to call the list anyway.
210 */
211 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||
212 FTRACE_FORCE_LIST_FUNC)
213 return ftrace_ops_list_func;
214
215 return ftrace_ops_get_func(ops);
216}
217
218static void update_ftrace_function(void)
219{
220 ftrace_func_t func;
221
222 /*
223 * Prepare the ftrace_ops that the arch callback will use.
224 * If there's only one ftrace_ops registered, the ftrace_ops_list
225 * will point to the ops we want.
226 */
227 set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
228 lockdep_is_held(&ftrace_lock));
229
230 /* If there's no ftrace_ops registered, just call the stub function */
231 if (set_function_trace_op == &ftrace_list_end) {
232 func = ftrace_stub;
233
234 /*
235 * If we are at the end of the list and this ops is
236 * recursion safe and not dynamic and the arch supports passing ops,
237 * then have the mcount trampoline call the function directly.
238 */
239 } else if (rcu_dereference_protected(ftrace_ops_list->next,
240 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
241 func = ftrace_ops_get_list_func(ftrace_ops_list);
242
243 } else {
244 /* Just use the default ftrace_ops */
245 set_function_trace_op = &ftrace_list_end;
246 func = ftrace_ops_list_func;
247 }
248
249 /* If there's no change, then do nothing more here */
250 if (ftrace_trace_function == func)
251 return;
252
253 /*
254 * If we are using the list function, it doesn't care
255 * about the function_trace_ops.
256 */
257 if (func == ftrace_ops_list_func) {
258 ftrace_trace_function = func;
259 /*
260 * Don't even bother setting function_trace_ops,
261 * it would be racy to do so anyway.
262 */
263 return;
264 }
265
266#ifndef CONFIG_DYNAMIC_FTRACE
267 /*
268 * For static tracing, we need to be a bit more careful.
269 * The function change takes affect immediately. Thus,
270 * we need to coordinate the setting of the function_trace_ops
271 * with the setting of the ftrace_trace_function.
272 *
273 * Set the function to the list ops, which will call the
274 * function we want, albeit indirectly, but it handles the
275 * ftrace_ops and doesn't depend on function_trace_op.
276 */
277 ftrace_trace_function = ftrace_ops_list_func;
278 /*
279 * Make sure all CPUs see this. Yes this is slow, but static
280 * tracing is slow and nasty to have enabled.
281 */
282 synchronize_rcu_tasks_rude();
283 /* Now all cpus are using the list ops. */
284 function_trace_op = set_function_trace_op;
285 /* Make sure the function_trace_op is visible on all CPUs */
286 smp_wmb();
287 /* Nasty way to force a rmb on all cpus */
288 smp_call_function(ftrace_sync_ipi, NULL, 1);
289 /* OK, we are all set to update the ftrace_trace_function now! */
290#endif /* !CONFIG_DYNAMIC_FTRACE */
291
292 ftrace_trace_function = func;
293}
294
295static void add_ftrace_ops(struct ftrace_ops __rcu **list,
296 struct ftrace_ops *ops)
297{
298 rcu_assign_pointer(ops->next, *list);
299
300 /*
301 * We are entering ops into the list but another
302 * CPU might be walking that list. We need to make sure
303 * the ops->next pointer is valid before another CPU sees
304 * the ops pointer included into the list.
305 */
306 rcu_assign_pointer(*list, ops);
307}
308
309static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
310 struct ftrace_ops *ops)
311{
312 struct ftrace_ops **p;
313
314 /*
315 * If we are removing the last function, then simply point
316 * to the ftrace_stub.
317 */
318 if (rcu_dereference_protected(*list,
319 lockdep_is_held(&ftrace_lock)) == ops &&
320 rcu_dereference_protected(ops->next,
321 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
322 rcu_assign_pointer(*list, &ftrace_list_end);
323 return 0;
324 }
325
326 for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
327 if (*p == ops)
328 break;
329
330 if (*p != ops)
331 return -1;
332
333 *p = (*p)->next;
334 return 0;
335}
336
337static void ftrace_update_trampoline(struct ftrace_ops *ops);
338
339int __register_ftrace_function(struct ftrace_ops *ops)
340{
341 if (ops->flags & FTRACE_OPS_FL_DELETED)
342 return -EINVAL;
343
344 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
345 return -EBUSY;
346
347#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
348 /*
349 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
350 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
351 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
352 */
353 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
354 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
355 return -EINVAL;
356
357 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
358 ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
359#endif
360 if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT))
361 return -EBUSY;
362
363 if (!is_kernel_core_data((unsigned long)ops))
364 ops->flags |= FTRACE_OPS_FL_DYNAMIC;
365
366 add_ftrace_ops(&ftrace_ops_list, ops);
367
368 /* Always save the function, and reset at unregistering */
369 ops->saved_func = ops->func;
370
371 if (ftrace_pids_enabled(ops))
372 ops->func = ftrace_pid_func;
373
374 ftrace_update_trampoline(ops);
375
376 if (ftrace_enabled)
377 update_ftrace_function();
378
379 return 0;
380}
381
382int __unregister_ftrace_function(struct ftrace_ops *ops)
383{
384 int ret;
385
386 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
387 return -EBUSY;
388
389 ret = remove_ftrace_ops(&ftrace_ops_list, ops);
390
391 if (ret < 0)
392 return ret;
393
394 if (ftrace_enabled)
395 update_ftrace_function();
396
397 ops->func = ops->saved_func;
398
399 return 0;
400}
401
402static void ftrace_update_pid_func(void)
403{
404 struct ftrace_ops *op;
405
406 /* Only do something if we are tracing something */
407 if (ftrace_trace_function == ftrace_stub)
408 return;
409
410 do_for_each_ftrace_op(op, ftrace_ops_list) {
411 if (op->flags & FTRACE_OPS_FL_PID) {
412 op->func = ftrace_pids_enabled(op) ?
413 ftrace_pid_func : op->saved_func;
414 ftrace_update_trampoline(op);
415 }
416 } while_for_each_ftrace_op(op);
417
418 fgraph_update_pid_func();
419
420 update_ftrace_function();
421}
422
423#ifdef CONFIG_FUNCTION_PROFILER
424struct ftrace_profile {
425 struct hlist_node node;
426 unsigned long ip;
427 unsigned long counter;
428#ifdef CONFIG_FUNCTION_GRAPH_TRACER
429 unsigned long long time;
430 unsigned long long time_squared;
431#endif
432};
433
434struct ftrace_profile_page {
435 struct ftrace_profile_page *next;
436 unsigned long index;
437 struct ftrace_profile records[];
438};
439
440struct ftrace_profile_stat {
441 atomic_t disabled;
442 struct hlist_head *hash;
443 struct ftrace_profile_page *pages;
444 struct ftrace_profile_page *start;
445 struct tracer_stat stat;
446};
447
448#define PROFILE_RECORDS_SIZE \
449 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
450
451#define PROFILES_PER_PAGE \
452 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
453
454static int ftrace_profile_enabled __read_mostly;
455
456/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
457static DEFINE_MUTEX(ftrace_profile_lock);
458
459static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
460
461#define FTRACE_PROFILE_HASH_BITS 10
462#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
463
464static void *
465function_stat_next(void *v, int idx)
466{
467 struct ftrace_profile *rec = v;
468 struct ftrace_profile_page *pg;
469
470 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
471
472 again:
473 if (idx != 0)
474 rec++;
475
476 if ((void *)rec >= (void *)&pg->records[pg->index]) {
477 pg = pg->next;
478 if (!pg)
479 return NULL;
480 rec = &pg->records[0];
481 if (!rec->counter)
482 goto again;
483 }
484
485 return rec;
486}
487
488static void *function_stat_start(struct tracer_stat *trace)
489{
490 struct ftrace_profile_stat *stat =
491 container_of(trace, struct ftrace_profile_stat, stat);
492
493 if (!stat || !stat->start)
494 return NULL;
495
496 return function_stat_next(&stat->start->records[0], 0);
497}
498
499#ifdef CONFIG_FUNCTION_GRAPH_TRACER
500/* function graph compares on total time */
501static int function_stat_cmp(const void *p1, const void *p2)
502{
503 const struct ftrace_profile *a = p1;
504 const struct ftrace_profile *b = p2;
505
506 if (a->time < b->time)
507 return -1;
508 if (a->time > b->time)
509 return 1;
510 else
511 return 0;
512}
513#else
514/* not function graph compares against hits */
515static int function_stat_cmp(const void *p1, const void *p2)
516{
517 const struct ftrace_profile *a = p1;
518 const struct ftrace_profile *b = p2;
519
520 if (a->counter < b->counter)
521 return -1;
522 if (a->counter > b->counter)
523 return 1;
524 else
525 return 0;
526}
527#endif
528
529static int function_stat_headers(struct seq_file *m)
530{
531#ifdef CONFIG_FUNCTION_GRAPH_TRACER
532 seq_puts(m, " Function "
533 "Hit Time Avg s^2\n"
534 " -------- "
535 "--- ---- --- ---\n");
536#else
537 seq_puts(m, " Function Hit\n"
538 " -------- ---\n");
539#endif
540 return 0;
541}
542
543static int function_stat_show(struct seq_file *m, void *v)
544{
545 struct trace_array *tr = trace_get_global_array();
546 struct ftrace_profile *rec = v;
547 const char *refsymbol = NULL;
548 char str[KSYM_SYMBOL_LEN];
549#ifdef CONFIG_FUNCTION_GRAPH_TRACER
550 static struct trace_seq s;
551 unsigned long long avg;
552 unsigned long long stddev;
553 unsigned long long stddev_denom;
554#endif
555 guard(mutex)(&ftrace_profile_lock);
556
557 /* we raced with function_profile_reset() */
558 if (unlikely(rec->counter == 0))
559 return -EBUSY;
560
561#ifdef CONFIG_FUNCTION_GRAPH_TRACER
562 avg = div64_ul(rec->time, rec->counter);
563 if (tracing_thresh && (avg < tracing_thresh))
564 return 0;
565#endif
566
567 if (tr->trace_flags & TRACE_ITER(PROF_TEXT_OFFSET)) {
568 unsigned long offset;
569
570 if (core_kernel_text(rec->ip)) {
571 refsymbol = "_text";
572 offset = rec->ip - (unsigned long)_text;
573 } else {
574 struct module *mod;
575
576 guard(rcu)();
577 mod = __module_text_address(rec->ip);
578 if (mod) {
579 refsymbol = mod->name;
580 /* Calculate offset from module's text entry address. */
581 offset = rec->ip - (unsigned long)mod->mem[MOD_TEXT].base;
582 }
583 }
584 if (refsymbol)
585 snprintf(str, sizeof(str), " %s+%#lx", refsymbol, offset);
586 }
587 if (!refsymbol)
588 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
589
590 seq_printf(m, " %-30.30s %10lu", str, rec->counter);
591
592#ifdef CONFIG_FUNCTION_GRAPH_TRACER
593 seq_puts(m, " ");
594
595 /*
596 * Variance formula:
597 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
598 * Maybe Welford's method is better here?
599 * Divide only by 1000 for ns^2 -> us^2 conversion.
600 * trace_print_graph_duration will divide by 1000 again.
601 */
602 stddev = 0;
603 stddev_denom = rec->counter * (rec->counter - 1) * 1000;
604 if (stddev_denom) {
605 stddev = rec->counter * rec->time_squared -
606 rec->time * rec->time;
607 stddev = div64_ul(stddev, stddev_denom);
608 }
609
610 trace_seq_init(&s);
611 trace_print_graph_duration(rec->time, &s);
612 trace_seq_puts(&s, " ");
613 trace_print_graph_duration(avg, &s);
614 trace_seq_puts(&s, " ");
615 trace_print_graph_duration(stddev, &s);
616 trace_print_seq(m, &s);
617#endif
618 seq_putc(m, '\n');
619
620 return 0;
621}
622
623static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
624{
625 struct ftrace_profile_page *pg;
626
627 pg = stat->pages = stat->start;
628
629 while (pg) {
630 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
631 pg->index = 0;
632 pg = pg->next;
633 }
634
635 memset(stat->hash, 0,
636 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
637}
638
639static int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
640{
641 struct ftrace_profile_page *pg;
642 int functions;
643 int pages;
644 int i;
645
646 /* If we already allocated, do nothing */
647 if (stat->pages)
648 return 0;
649
650 stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
651 if (!stat->pages)
652 return -ENOMEM;
653
654#ifdef CONFIG_DYNAMIC_FTRACE
655 functions = ftrace_update_tot_cnt;
656#else
657 /*
658 * We do not know the number of functions that exist because
659 * dynamic tracing is what counts them. With past experience
660 * we have around 20K functions. That should be more than enough.
661 * It is highly unlikely we will execute every function in
662 * the kernel.
663 */
664 functions = 20000;
665#endif
666
667 pg = stat->start = stat->pages;
668
669 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
670
671 for (i = 1; i < pages; i++) {
672 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
673 if (!pg->next)
674 goto out_free;
675 pg = pg->next;
676 }
677
678 return 0;
679
680 out_free:
681 pg = stat->start;
682 while (pg) {
683 unsigned long tmp = (unsigned long)pg;
684
685 pg = pg->next;
686 free_page(tmp);
687 }
688
689 stat->pages = NULL;
690 stat->start = NULL;
691
692 return -ENOMEM;
693}
694
695static int ftrace_profile_init_cpu(int cpu)
696{
697 struct ftrace_profile_stat *stat;
698 int size;
699
700 stat = &per_cpu(ftrace_profile_stats, cpu);
701
702 if (stat->hash) {
703 /* If the profile is already created, simply reset it */
704 ftrace_profile_reset(stat);
705 return 0;
706 }
707
708 /*
709 * We are profiling all functions, but usually only a few thousand
710 * functions are hit. We'll make a hash of 1024 items.
711 */
712 size = FTRACE_PROFILE_HASH_SIZE;
713
714 stat->hash = kzalloc_objs(struct hlist_head, size);
715
716 if (!stat->hash)
717 return -ENOMEM;
718
719 /* Preallocate the function profiling pages */
720 if (ftrace_profile_pages_init(stat) < 0) {
721 kfree(stat->hash);
722 stat->hash = NULL;
723 return -ENOMEM;
724 }
725
726 return 0;
727}
728
729static int ftrace_profile_init(void)
730{
731 int cpu;
732 int ret = 0;
733
734 for_each_possible_cpu(cpu) {
735 ret = ftrace_profile_init_cpu(cpu);
736 if (ret)
737 break;
738 }
739
740 return ret;
741}
742
743/* interrupts must be disabled */
744static struct ftrace_profile *
745ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
746{
747 struct ftrace_profile *rec;
748 struct hlist_head *hhd;
749 unsigned long key;
750
751 key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
752 hhd = &stat->hash[key];
753
754 if (hlist_empty(hhd))
755 return NULL;
756
757 hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
758 if (rec->ip == ip)
759 return rec;
760 }
761
762 return NULL;
763}
764
765static void ftrace_add_profile(struct ftrace_profile_stat *stat,
766 struct ftrace_profile *rec)
767{
768 unsigned long key;
769
770 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
771 hlist_add_head_rcu(&rec->node, &stat->hash[key]);
772}
773
774/*
775 * The memory is already allocated, this simply finds a new record to use.
776 */
777static struct ftrace_profile *
778ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
779{
780 struct ftrace_profile *rec = NULL;
781
782 /* prevent recursion (from NMIs) */
783 if (atomic_inc_return(&stat->disabled) != 1)
784 goto out;
785
786 /*
787 * Try to find the function again since an NMI
788 * could have added it
789 */
790 rec = ftrace_find_profiled_func(stat, ip);
791 if (rec)
792 goto out;
793
794 if (stat->pages->index == PROFILES_PER_PAGE) {
795 if (!stat->pages->next)
796 goto out;
797 stat->pages = stat->pages->next;
798 }
799
800 rec = &stat->pages->records[stat->pages->index++];
801 rec->ip = ip;
802 ftrace_add_profile(stat, rec);
803
804 out:
805 atomic_dec(&stat->disabled);
806
807 return rec;
808}
809
810static void
811function_profile_call(unsigned long ip, unsigned long parent_ip,
812 struct ftrace_ops *ops, struct ftrace_regs *fregs)
813{
814 struct ftrace_profile_stat *stat;
815 struct ftrace_profile *rec;
816
817 if (!ftrace_profile_enabled)
818 return;
819
820 guard(preempt_notrace)();
821
822 stat = this_cpu_ptr(&ftrace_profile_stats);
823 if (!stat->hash || !ftrace_profile_enabled)
824 return;
825
826 rec = ftrace_find_profiled_func(stat, ip);
827 if (!rec) {
828 rec = ftrace_profile_alloc(stat, ip);
829 if (!rec)
830 return;
831 }
832
833 rec->counter++;
834}
835
836#ifdef CONFIG_FUNCTION_GRAPH_TRACER
837static bool fgraph_graph_time = true;
838
839void ftrace_graph_graph_time_control(bool enable)
840{
841 fgraph_graph_time = enable;
842}
843
844struct profile_fgraph_data {
845 unsigned long long calltime;
846 unsigned long long subtime;
847 unsigned long long sleeptime;
848};
849
850static int profile_graph_entry(struct ftrace_graph_ent *trace,
851 struct fgraph_ops *gops,
852 struct ftrace_regs *fregs)
853{
854 struct profile_fgraph_data *profile_data;
855
856 function_profile_call(trace->func, 0, NULL, NULL);
857
858 /* If function graph is shutting down, ret_stack can be NULL */
859 if (!current->ret_stack)
860 return 0;
861
862 profile_data = fgraph_reserve_data(gops->idx, sizeof(*profile_data));
863 if (!profile_data)
864 return 0;
865
866 profile_data->subtime = 0;
867 profile_data->sleeptime = current->ftrace_sleeptime;
868 profile_data->calltime = trace_clock_local();
869
870 return 1;
871}
872
873bool fprofile_no_sleep_time;
874
875static void profile_graph_return(struct ftrace_graph_ret *trace,
876 struct fgraph_ops *gops,
877 struct ftrace_regs *fregs)
878{
879 struct profile_fgraph_data *profile_data;
880 struct ftrace_profile_stat *stat;
881 unsigned long long calltime;
882 unsigned long long rettime = trace_clock_local();
883 struct ftrace_profile *rec;
884 int size;
885
886 guard(preempt_notrace)();
887
888 stat = this_cpu_ptr(&ftrace_profile_stats);
889 if (!stat->hash || !ftrace_profile_enabled)
890 return;
891
892 profile_data = fgraph_retrieve_data(gops->idx, &size);
893
894 /* If the calltime was zero'd ignore it */
895 if (!profile_data || !profile_data->calltime)
896 return;
897
898 calltime = rettime - profile_data->calltime;
899
900 if (fprofile_no_sleep_time) {
901 if (current->ftrace_sleeptime)
902 calltime -= current->ftrace_sleeptime - profile_data->sleeptime;
903 }
904
905 if (!fgraph_graph_time) {
906 struct profile_fgraph_data *parent_data;
907
908 /* Append this call time to the parent time to subtract */
909 parent_data = fgraph_retrieve_parent_data(gops->idx, &size, 1);
910 if (parent_data)
911 parent_data->subtime += calltime;
912
913 if (profile_data->subtime && profile_data->subtime < calltime)
914 calltime -= profile_data->subtime;
915 else
916 calltime = 0;
917 }
918
919 rec = ftrace_find_profiled_func(stat, trace->func);
920 if (rec) {
921 rec->time += calltime;
922 rec->time_squared += calltime * calltime;
923 }
924}
925
926static struct fgraph_ops fprofiler_ops = {
927 .entryfunc = &profile_graph_entry,
928 .retfunc = &profile_graph_return,
929};
930
931static int register_ftrace_profiler(void)
932{
933 ftrace_ops_set_global_filter(&fprofiler_ops.ops);
934 return register_ftrace_graph(&fprofiler_ops);
935}
936
937static void unregister_ftrace_profiler(void)
938{
939 unregister_ftrace_graph(&fprofiler_ops);
940}
941#else
942static struct ftrace_ops ftrace_profile_ops __read_mostly = {
943 .func = function_profile_call,
944};
945
946static int register_ftrace_profiler(void)
947{
948 ftrace_ops_set_global_filter(&ftrace_profile_ops);
949 return register_ftrace_function(&ftrace_profile_ops);
950}
951
952static void unregister_ftrace_profiler(void)
953{
954 unregister_ftrace_function(&ftrace_profile_ops);
955}
956#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
957
958static ssize_t
959ftrace_profile_write(struct file *filp, const char __user *ubuf,
960 size_t cnt, loff_t *ppos)
961{
962 unsigned long val;
963 int ret;
964
965 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
966 if (ret)
967 return ret;
968
969 val = !!val;
970
971 guard(mutex)(&ftrace_profile_lock);
972 if (ftrace_profile_enabled ^ val) {
973 if (val) {
974 ret = ftrace_profile_init();
975 if (ret < 0)
976 return ret;
977
978 ret = register_ftrace_profiler();
979 if (ret < 0)
980 return ret;
981 ftrace_profile_enabled = 1;
982 } else {
983 ftrace_profile_enabled = 0;
984 /*
985 * unregister_ftrace_profiler calls stop_machine
986 * so this acts like an synchronize_rcu.
987 */
988 unregister_ftrace_profiler();
989 }
990 }
991
992 *ppos += cnt;
993
994 return cnt;
995}
996
997static ssize_t
998ftrace_profile_read(struct file *filp, char __user *ubuf,
999 size_t cnt, loff_t *ppos)
1000{
1001 char buf[64]; /* big enough to hold a number */
1002 int r;
1003
1004 r = sprintf(buf, "%u\n", ftrace_profile_enabled);
1005 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1006}
1007
1008static const struct file_operations ftrace_profile_fops = {
1009 .open = tracing_open_generic,
1010 .read = ftrace_profile_read,
1011 .write = ftrace_profile_write,
1012 .llseek = default_llseek,
1013};
1014
1015/* used to initialize the real stat files */
1016static struct tracer_stat function_stats __initdata = {
1017 .name = "functions",
1018 .stat_start = function_stat_start,
1019 .stat_next = function_stat_next,
1020 .stat_cmp = function_stat_cmp,
1021 .stat_headers = function_stat_headers,
1022 .stat_show = function_stat_show
1023};
1024
1025static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1026{
1027 struct ftrace_profile_stat *stat;
1028 char *name;
1029 int ret;
1030 int cpu;
1031
1032 for_each_possible_cpu(cpu) {
1033 stat = &per_cpu(ftrace_profile_stats, cpu);
1034
1035 name = kasprintf(GFP_KERNEL, "function%d", cpu);
1036 if (!name) {
1037 /*
1038 * The files created are permanent, if something happens
1039 * we still do not free memory.
1040 */
1041 WARN(1,
1042 "Could not allocate stat file for cpu %d\n",
1043 cpu);
1044 return;
1045 }
1046 stat->stat = function_stats;
1047 stat->stat.name = name;
1048 ret = register_stat_tracer(&stat->stat);
1049 if (ret) {
1050 WARN(1,
1051 "Could not register function stat for cpu %d\n",
1052 cpu);
1053 kfree(name);
1054 return;
1055 }
1056 }
1057
1058 trace_create_file("function_profile_enabled",
1059 TRACE_MODE_WRITE, d_tracer, NULL,
1060 &ftrace_profile_fops);
1061}
1062
1063#else /* CONFIG_FUNCTION_PROFILER */
1064static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1065{
1066}
1067#endif /* CONFIG_FUNCTION_PROFILER */
1068
1069#ifdef CONFIG_DYNAMIC_FTRACE
1070
1071static struct ftrace_ops *removed_ops;
1072
1073/*
1074 * Set when doing a global update, like enabling all recs or disabling them.
1075 * It is not set when just updating a single ftrace_ops.
1076 */
1077static bool update_all_ops;
1078
1079struct ftrace_func_probe {
1080 struct ftrace_probe_ops *probe_ops;
1081 struct ftrace_ops ops;
1082 struct trace_array *tr;
1083 struct list_head list;
1084 void *data;
1085 int ref;
1086};
1087
1088/*
1089 * We make these constant because no one should touch them,
1090 * but they are used as the default "empty hash", to avoid allocating
1091 * it all the time. These are in a read only section such that if
1092 * anyone does try to modify it, it will cause an exception.
1093 */
1094static const struct hlist_head empty_buckets[1];
1095static const struct ftrace_hash empty_hash = {
1096 .buckets = (struct hlist_head *)empty_buckets,
1097};
1098#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
1099
1100struct ftrace_ops global_ops = {
1101 .func = ftrace_stub,
1102 .local_hash.notrace_hash = EMPTY_HASH,
1103 .local_hash.filter_hash = EMPTY_HASH,
1104 INIT_OPS_HASH(global_ops)
1105 .flags = FTRACE_OPS_FL_INITIALIZED |
1106 FTRACE_OPS_FL_PID,
1107};
1108
1109/*
1110 * parser_lock - Protects trace_parser state against concurrent operations.
1111 * Held across trace_get_user() and subsequent buffer parsing to prevent races.
1112 */
1113static DEFINE_MUTEX(parser_lock);
1114
1115/*
1116 * Used by the stack unwinder to know about dynamic ftrace trampolines.
1117 */
1118struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr)
1119{
1120 struct ftrace_ops *op = NULL;
1121
1122 /*
1123 * Some of the ops may be dynamically allocated,
1124 * they are freed after a synchronize_rcu().
1125 */
1126 preempt_disable_notrace();
1127
1128 do_for_each_ftrace_op(op, ftrace_ops_list) {
1129 /*
1130 * This is to check for dynamically allocated trampolines.
1131 * Trampolines that are in kernel text will have
1132 * core_kernel_text() return true.
1133 */
1134 if (op->trampoline && op->trampoline_size)
1135 if (addr >= op->trampoline &&
1136 addr < op->trampoline + op->trampoline_size) {
1137 preempt_enable_notrace();
1138 return op;
1139 }
1140 } while_for_each_ftrace_op(op);
1141 preempt_enable_notrace();
1142
1143 return NULL;
1144}
1145
1146/*
1147 * This is used by __kernel_text_address() to return true if the
1148 * address is on a dynamically allocated trampoline that would
1149 * not return true for either core_kernel_text() or
1150 * is_module_text_address().
1151 */
1152bool is_ftrace_trampoline(unsigned long addr)
1153{
1154 return ftrace_ops_trampoline(addr) != NULL;
1155}
1156
1157struct ftrace_page {
1158 struct ftrace_page *next;
1159 struct dyn_ftrace *records;
1160 int index;
1161 int order;
1162};
1163
1164#define ENTRY_SIZE sizeof(struct dyn_ftrace)
1165#define ENTRIES_PER_PAGE_GROUP(order) ((PAGE_SIZE << (order)) / ENTRY_SIZE)
1166
1167static struct ftrace_page *ftrace_pages_start;
1168static struct ftrace_page *ftrace_pages;
1169
1170static __always_inline unsigned long
1171ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
1172{
1173 if (hash->size_bits > 0)
1174 return hash_long(ip, hash->size_bits);
1175
1176 return 0;
1177}
1178
1179/* Only use this function if ftrace_hash_empty() has already been tested */
1180static __always_inline struct ftrace_func_entry *
1181__ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1182{
1183 unsigned long key;
1184 struct ftrace_func_entry *entry;
1185 struct hlist_head *hhd;
1186
1187 key = ftrace_hash_key(hash, ip);
1188 hhd = &hash->buckets[key];
1189
1190 hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1191 if (entry->ip == ip)
1192 return entry;
1193 }
1194 return NULL;
1195}
1196
1197/**
1198 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
1199 * @hash: The hash to look at
1200 * @ip: The instruction pointer to test
1201 *
1202 * Search a given @hash to see if a given instruction pointer (@ip)
1203 * exists in it.
1204 *
1205 * Returns: the entry that holds the @ip if found. NULL otherwise.
1206 */
1207struct ftrace_func_entry *
1208ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1209{
1210 if (ftrace_hash_empty(hash))
1211 return NULL;
1212
1213 return __ftrace_lookup_ip(hash, ip);
1214}
1215
1216void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry)
1217{
1218 struct hlist_head *hhd;
1219 unsigned long key;
1220
1221 key = ftrace_hash_key(hash, entry->ip);
1222 hhd = &hash->buckets[key];
1223 hlist_add_head(&entry->hlist, hhd);
1224 hash->count++;
1225}
1226
1227struct ftrace_func_entry *
1228add_ftrace_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigned long direct)
1229{
1230 struct ftrace_func_entry *entry;
1231
1232 entry = kmalloc_obj(*entry);
1233 if (!entry)
1234 return NULL;
1235
1236 entry->ip = ip;
1237 entry->direct = direct;
1238 add_ftrace_hash_entry(hash, entry);
1239
1240 return entry;
1241}
1242
1243static struct ftrace_func_entry *
1244add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1245{
1246 return add_ftrace_hash_entry_direct(hash, ip, 0);
1247}
1248
1249static void
1250free_hash_entry(struct ftrace_hash *hash,
1251 struct ftrace_func_entry *entry)
1252{
1253 hlist_del(&entry->hlist);
1254 kfree(entry);
1255 hash->count--;
1256}
1257
1258static void
1259remove_hash_entry(struct ftrace_hash *hash,
1260 struct ftrace_func_entry *entry)
1261{
1262 hlist_del_rcu(&entry->hlist);
1263 hash->count--;
1264}
1265
1266void ftrace_hash_remove(struct ftrace_hash *hash)
1267{
1268 struct ftrace_func_entry *entry;
1269 struct hlist_head *hhd;
1270 struct hlist_node *tn;
1271 int size;
1272 int i;
1273
1274 if (!hash || !hash->count)
1275 return;
1276 size = 1 << hash->size_bits;
1277 for (i = 0; i < size; i++) {
1278 hhd = &hash->buckets[i];
1279 hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1280 remove_hash_entry(hash, entry);
1281 }
1282 FTRACE_WARN_ON(hash->count);
1283}
1284
1285static void ftrace_hash_clear(struct ftrace_hash *hash)
1286{
1287 struct hlist_head *hhd;
1288 struct hlist_node *tn;
1289 struct ftrace_func_entry *entry;
1290 int size = 1 << hash->size_bits;
1291 int i;
1292
1293 if (!hash->count)
1294 return;
1295
1296 for (i = 0; i < size; i++) {
1297 hhd = &hash->buckets[i];
1298 hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1299 free_hash_entry(hash, entry);
1300 }
1301 FTRACE_WARN_ON(hash->count);
1302}
1303
1304static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
1305{
1306 list_del(&ftrace_mod->list);
1307 kfree(ftrace_mod->module);
1308 kfree(ftrace_mod->func);
1309 kfree(ftrace_mod);
1310}
1311
1312static void clear_ftrace_mod_list(struct list_head *head)
1313{
1314 struct ftrace_mod_load *p, *n;
1315
1316 /* stack tracer isn't supported yet */
1317 if (!head)
1318 return;
1319
1320 mutex_lock(&ftrace_lock);
1321 list_for_each_entry_safe(p, n, head, list)
1322 free_ftrace_mod(p);
1323 mutex_unlock(&ftrace_lock);
1324}
1325
1326void free_ftrace_hash(struct ftrace_hash *hash)
1327{
1328 if (!hash || hash == EMPTY_HASH)
1329 return;
1330 ftrace_hash_clear(hash);
1331 kfree(hash->buckets);
1332 kfree(hash);
1333}
1334
1335static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1336{
1337 struct ftrace_hash *hash;
1338
1339 hash = container_of(rcu, struct ftrace_hash, rcu);
1340 free_ftrace_hash(hash);
1341}
1342
1343static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1344{
1345 if (!hash || hash == EMPTY_HASH)
1346 return;
1347 call_rcu(&hash->rcu, __free_ftrace_hash_rcu);
1348}
1349
1350/**
1351 * ftrace_free_filter - remove all filters for an ftrace_ops
1352 * @ops: the ops to remove the filters from
1353 */
1354void ftrace_free_filter(struct ftrace_ops *ops)
1355{
1356 ftrace_ops_init(ops);
1357 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
1358 return;
1359 free_ftrace_hash(ops->func_hash->filter_hash);
1360 free_ftrace_hash(ops->func_hash->notrace_hash);
1361 ops->func_hash->filter_hash = EMPTY_HASH;
1362 ops->func_hash->notrace_hash = EMPTY_HASH;
1363}
1364EXPORT_SYMBOL_GPL(ftrace_free_filter);
1365
1366struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1367{
1368 struct ftrace_hash *hash;
1369 int size;
1370
1371 hash = kzalloc_obj(*hash);
1372 if (!hash)
1373 return NULL;
1374
1375 size = 1 << size_bits;
1376 hash->buckets = kzalloc_objs(*hash->buckets, size);
1377
1378 if (!hash->buckets) {
1379 kfree(hash);
1380 return NULL;
1381 }
1382
1383 hash->size_bits = size_bits;
1384
1385 return hash;
1386}
1387
1388/* Used to save filters on functions for modules not loaded yet */
1389static int ftrace_add_mod(struct trace_array *tr,
1390 const char *func, const char *module,
1391 int enable)
1392{
1393 struct ftrace_mod_load *ftrace_mod;
1394 struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;
1395
1396 ftrace_mod = kzalloc_obj(*ftrace_mod);
1397 if (!ftrace_mod)
1398 return -ENOMEM;
1399
1400 INIT_LIST_HEAD(&ftrace_mod->list);
1401 ftrace_mod->func = kstrdup(func, GFP_KERNEL);
1402 ftrace_mod->module = kstrdup(module, GFP_KERNEL);
1403 ftrace_mod->enable = enable;
1404
1405 if (!ftrace_mod->func || !ftrace_mod->module)
1406 goto out_free;
1407
1408 list_add(&ftrace_mod->list, mod_head);
1409
1410 return 0;
1411
1412 out_free:
1413 free_ftrace_mod(ftrace_mod);
1414
1415 return -ENOMEM;
1416}
1417
1418static struct ftrace_hash *
1419alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1420{
1421 struct ftrace_func_entry *entry;
1422 struct ftrace_hash *new_hash;
1423 int size;
1424 int i;
1425
1426 new_hash = alloc_ftrace_hash(size_bits);
1427 if (!new_hash)
1428 return NULL;
1429
1430 if (hash)
1431 new_hash->flags = hash->flags;
1432
1433 /* Empty hash? */
1434 if (ftrace_hash_empty(hash))
1435 return new_hash;
1436
1437 size = 1 << hash->size_bits;
1438 for (i = 0; i < size; i++) {
1439 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1440 if (add_ftrace_hash_entry_direct(new_hash, entry->ip, entry->direct) == NULL)
1441 goto free_hash;
1442 }
1443 }
1444
1445 FTRACE_WARN_ON(new_hash->count != hash->count);
1446
1447 return new_hash;
1448
1449 free_hash:
1450 free_ftrace_hash(new_hash);
1451 return NULL;
1452}
1453
1454static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops);
1455static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops);
1456
1457static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1458 struct ftrace_hash *new_hash);
1459
1460/*
1461 * Allocate a new hash and remove entries from @src and move them to the new hash.
1462 * On success, the @src hash will be empty and should be freed.
1463 */
1464static struct ftrace_hash *__move_hash(struct ftrace_hash *src, int size)
1465{
1466 struct ftrace_func_entry *entry;
1467 struct ftrace_hash *new_hash;
1468 struct hlist_head *hhd;
1469 struct hlist_node *tn;
1470 int bits = 0;
1471 int i;
1472
1473 /*
1474 * Use around half the size (max bit of it), but
1475 * a minimum of 2 is fine (as size of 0 or 1 both give 1 for bits).
1476 */
1477 bits = fls(size / 2);
1478
1479 /* Don't allocate too much */
1480 if (bits > FTRACE_HASH_MAX_BITS)
1481 bits = FTRACE_HASH_MAX_BITS;
1482
1483 new_hash = alloc_ftrace_hash(bits);
1484 if (!new_hash)
1485 return NULL;
1486
1487 new_hash->flags = src->flags;
1488
1489 size = 1 << src->size_bits;
1490 for (i = 0; i < size; i++) {
1491 hhd = &src->buckets[i];
1492 hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1493 remove_hash_entry(src, entry);
1494 add_ftrace_hash_entry(new_hash, entry);
1495 }
1496 }
1497 return new_hash;
1498}
1499
1500/* Move the @src entries to a newly allocated hash */
1501static struct ftrace_hash *
1502__ftrace_hash_move(struct ftrace_hash *src)
1503{
1504 int size = src->count;
1505
1506 /*
1507 * If the new source is empty, just return the empty_hash.
1508 */
1509 if (ftrace_hash_empty(src))
1510 return EMPTY_HASH;
1511
1512 return __move_hash(src, size);
1513}
1514
1515/**
1516 * ftrace_hash_move - move a new hash to a filter and do updates
1517 * @ops: The ops with the hash that @dst points to
1518 * @enable: True if for the filter hash, false for the notrace hash
1519 * @dst: Points to the @ops hash that should be updated
1520 * @src: The hash to update @dst with
1521 *
1522 * This is called when an ftrace_ops hash is being updated and the
1523 * the kernel needs to reflect this. Note, this only updates the kernel
1524 * function callbacks if the @ops is enabled (not to be confused with
1525 * @enable above). If the @ops is enabled, its hash determines what
1526 * callbacks get called. This function gets called when the @ops hash
1527 * is updated and it requires new callbacks.
1528 *
1529 * On success the elements of @src is moved to @dst, and @dst is updated
1530 * properly, as well as the functions determined by the @ops hashes
1531 * are now calling the @ops callback function.
1532 *
1533 * Regardless of return type, @src should be freed with free_ftrace_hash().
1534 */
1535static int
1536ftrace_hash_move(struct ftrace_ops *ops, int enable,
1537 struct ftrace_hash **dst, struct ftrace_hash *src)
1538{
1539 struct ftrace_hash *new_hash;
1540 int ret;
1541
1542 /* Reject setting notrace hash on IPMODIFY ftrace_ops */
1543 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
1544 return -EINVAL;
1545
1546 new_hash = __ftrace_hash_move(src);
1547 if (!new_hash)
1548 return -ENOMEM;
1549
1550 /* Make sure this can be applied if it is IPMODIFY ftrace_ops */
1551 if (enable) {
1552 /* IPMODIFY should be updated only when filter_hash updating */
1553 ret = ftrace_hash_ipmodify_update(ops, new_hash);
1554 if (ret < 0) {
1555 free_ftrace_hash(new_hash);
1556 return ret;
1557 }
1558 }
1559
1560 /*
1561 * Remove the current set, update the hash and add
1562 * them back.
1563 */
1564 ftrace_hash_rec_disable_modify(ops);
1565
1566 rcu_assign_pointer(*dst, new_hash);
1567
1568 ftrace_hash_rec_enable_modify(ops);
1569
1570 return 0;
1571}
1572
1573static bool hash_contains_ip(unsigned long ip,
1574 struct ftrace_ops_hash *hash)
1575{
1576 /*
1577 * The function record is a match if it exists in the filter
1578 * hash and not in the notrace hash. Note, an empty hash is
1579 * considered a match for the filter hash, but an empty
1580 * notrace hash is considered not in the notrace hash.
1581 */
1582 return (ftrace_hash_empty(hash->filter_hash) ||
1583 __ftrace_lookup_ip(hash->filter_hash, ip)) &&
1584 (ftrace_hash_empty(hash->notrace_hash) ||
1585 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1586}
1587
1588/*
1589 * Test the hashes for this ops to see if we want to call
1590 * the ops->func or not.
1591 *
1592 * It's a match if the ip is in the ops->filter_hash or
1593 * the filter_hash does not exist or is empty,
1594 * AND
1595 * the ip is not in the ops->notrace_hash.
1596 *
1597 * This needs to be called with preemption disabled as
1598 * the hashes are freed with call_rcu().
1599 */
1600int
1601ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1602{
1603 struct ftrace_ops_hash hash;
1604 int ret;
1605
1606#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1607 /*
1608 * There's a small race when adding ops that the ftrace handler
1609 * that wants regs, may be called without them. We can not
1610 * allow that handler to be called if regs is NULL.
1611 */
1612 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
1613 return 0;
1614#endif
1615
1616 rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
1617 rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
1618
1619 if (hash_contains_ip(ip, &hash))
1620 ret = 1;
1621 else
1622 ret = 0;
1623
1624 return ret;
1625}
1626
1627/*
1628 * This is a double for. Do not use 'break' to break out of the loop,
1629 * you must use a goto.
1630 */
1631#define do_for_each_ftrace_rec(pg, rec) \
1632 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1633 int _____i; \
1634 for (_____i = 0; _____i < pg->index; _____i++) { \
1635 rec = &pg->records[_____i];
1636
1637#define while_for_each_ftrace_rec() \
1638 } \
1639 }
1640
1641
1642static int ftrace_cmp_recs(const void *a, const void *b)
1643{
1644 const struct dyn_ftrace *key = a;
1645 const struct dyn_ftrace *rec = b;
1646
1647 if (key->flags < rec->ip)
1648 return -1;
1649 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1650 return 1;
1651 return 0;
1652}
1653
1654static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end)
1655{
1656 struct ftrace_page *pg;
1657 struct dyn_ftrace *rec = NULL;
1658 struct dyn_ftrace key;
1659
1660 key.ip = start;
1661 key.flags = end; /* overload flags, as it is unsigned long */
1662
1663 for (pg = ftrace_pages_start; pg; pg = pg->next) {
1664 if (pg->index == 0 ||
1665 end < pg->records[0].ip ||
1666 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1667 continue;
1668 rec = bsearch(&key, pg->records, pg->index,
1669 sizeof(struct dyn_ftrace),
1670 ftrace_cmp_recs);
1671 if (rec)
1672 break;
1673 }
1674 return rec;
1675}
1676
1677/**
1678 * ftrace_location_range - return the first address of a traced location
1679 * if it touches the given ip range
1680 * @start: start of range to search.
1681 * @end: end of range to search (inclusive). @end points to the last byte
1682 * to check.
1683 *
1684 * Returns: rec->ip if the related ftrace location is a least partly within
1685 * the given address range. That is, the first address of the instruction
1686 * that is either a NOP or call to the function tracer. It checks the ftrace
1687 * internal tables to determine if the address belongs or not.
1688 */
1689unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1690{
1691 struct dyn_ftrace *rec;
1692 unsigned long ip = 0;
1693
1694 rcu_read_lock();
1695 rec = lookup_rec(start, end);
1696 if (rec)
1697 ip = rec->ip;
1698 rcu_read_unlock();
1699
1700 return ip;
1701}
1702
1703/**
1704 * ftrace_location - return the ftrace location
1705 * @ip: the instruction pointer to check
1706 *
1707 * Returns:
1708 * * If @ip matches the ftrace location, return @ip.
1709 * * If @ip matches sym+0, return sym's ftrace location.
1710 * * Otherwise, return 0.
1711 */
1712unsigned long ftrace_location(unsigned long ip)
1713{
1714 unsigned long loc;
1715 unsigned long offset;
1716 unsigned long size;
1717
1718 loc = ftrace_location_range(ip, ip);
1719 if (!loc) {
1720 if (!kallsyms_lookup_size_offset(ip, &size, &offset))
1721 return 0;
1722
1723 /* map sym+0 to __fentry__ */
1724 if (!offset)
1725 loc = ftrace_location_range(ip, ip + size - 1);
1726 }
1727 return loc;
1728}
1729
1730/**
1731 * ftrace_text_reserved - return true if range contains an ftrace location
1732 * @start: start of range to search
1733 * @end: end of range to search (inclusive). @end points to the last byte to check.
1734 *
1735 * Returns: 1 if @start and @end contains a ftrace location.
1736 * That is, the instruction that is either a NOP or call to
1737 * the function tracer. It checks the ftrace internal tables to
1738 * determine if the address belongs or not.
1739 */
1740int ftrace_text_reserved(const void *start, const void *end)
1741{
1742 unsigned long ret;
1743
1744 ret = ftrace_location_range((unsigned long)start,
1745 (unsigned long)end);
1746
1747 return (int)!!ret;
1748}
1749
1750/* Test if ops registered to this rec needs regs */
1751static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
1752{
1753 struct ftrace_ops *ops;
1754 bool keep_regs = false;
1755
1756 for (ops = ftrace_ops_list;
1757 ops != &ftrace_list_end; ops = ops->next) {
1758 /* pass rec in as regs to have non-NULL val */
1759 if (ftrace_ops_test(ops, rec->ip, rec)) {
1760 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1761 keep_regs = true;
1762 break;
1763 }
1764 }
1765 }
1766
1767 return keep_regs;
1768}
1769
1770static struct ftrace_ops *
1771ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1772static struct ftrace_ops *
1773ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude);
1774static struct ftrace_ops *
1775ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1776
1777static bool skip_record(struct dyn_ftrace *rec)
1778{
1779 /*
1780 * At boot up, weak functions are set to disable. Function tracing
1781 * can be enabled before they are, and they still need to be disabled now.
1782 * If the record is disabled, still continue if it is marked as already
1783 * enabled (this is needed to keep the accounting working).
1784 */
1785 return rec->flags & FTRACE_FL_DISABLED &&
1786 !(rec->flags & FTRACE_FL_ENABLED);
1787}
1788
1789/*
1790 * This is the main engine to the ftrace updates to the dyn_ftrace records.
1791 *
1792 * It will iterate through all the available ftrace functions
1793 * (the ones that ftrace can have callbacks to) and set the flags
1794 * in the associated dyn_ftrace records.
1795 *
1796 * @inc: If true, the functions associated to @ops are added to
1797 * the dyn_ftrace records, otherwise they are removed.
1798 */
1799static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1800 bool inc)
1801{
1802 struct ftrace_hash *hash;
1803 struct ftrace_hash *notrace_hash;
1804 struct ftrace_page *pg;
1805 struct dyn_ftrace *rec;
1806 bool update = false;
1807 int count = 0;
1808 int all = false;
1809
1810 /* Only update if the ops has been registered */
1811 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1812 return false;
1813
1814 /*
1815 * If the count is zero, we update all records.
1816 * Otherwise we just update the items in the hash.
1817 */
1818 hash = ops->func_hash->filter_hash;
1819 notrace_hash = ops->func_hash->notrace_hash;
1820 if (ftrace_hash_empty(hash))
1821 all = true;
1822
1823 do_for_each_ftrace_rec(pg, rec) {
1824 int in_notrace_hash = 0;
1825 int in_hash = 0;
1826 int match = 0;
1827
1828 if (skip_record(rec))
1829 continue;
1830
1831 if (all) {
1832 /*
1833 * Only the filter_hash affects all records.
1834 * Update if the record is not in the notrace hash.
1835 */
1836 if (!notrace_hash || !ftrace_lookup_ip(notrace_hash, rec->ip))
1837 match = 1;
1838 } else {
1839 in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1840 in_notrace_hash = !!ftrace_lookup_ip(notrace_hash, rec->ip);
1841
1842 /*
1843 * We want to match all functions that are in the hash but
1844 * not in the other hash.
1845 */
1846 if (in_hash && !in_notrace_hash)
1847 match = 1;
1848 }
1849 if (!match)
1850 continue;
1851
1852 if (inc) {
1853 rec->flags++;
1854 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1855 return false;
1856
1857 if (ops->flags & FTRACE_OPS_FL_DIRECT)
1858 rec->flags |= FTRACE_FL_DIRECT;
1859
1860 /*
1861 * If there's only a single callback registered to a
1862 * function, and the ops has a trampoline registered
1863 * for it, then we can call it directly.
1864 */
1865 if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1866 rec->flags |= FTRACE_FL_TRAMP;
1867 else
1868 /*
1869 * If we are adding another function callback
1870 * to this function, and the previous had a
1871 * custom trampoline in use, then we need to go
1872 * back to the default trampoline.
1873 */
1874 rec->flags &= ~FTRACE_FL_TRAMP;
1875
1876 /*
1877 * If any ops wants regs saved for this function
1878 * then all ops will get saved regs.
1879 */
1880 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1881 rec->flags |= FTRACE_FL_REGS;
1882 } else {
1883 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1884 return false;
1885 rec->flags--;
1886
1887 /*
1888 * Only the internal direct_ops should have the
1889 * DIRECT flag set. Thus, if it is removing a
1890 * function, then that function should no longer
1891 * be direct.
1892 */
1893 if (ops->flags & FTRACE_OPS_FL_DIRECT)
1894 rec->flags &= ~FTRACE_FL_DIRECT;
1895
1896 /*
1897 * If the rec had REGS enabled and the ops that is
1898 * being removed had REGS set, then see if there is
1899 * still any ops for this record that wants regs.
1900 * If not, we can stop recording them.
1901 */
1902 if (ftrace_rec_count(rec) > 0 &&
1903 rec->flags & FTRACE_FL_REGS &&
1904 ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1905 if (!test_rec_ops_needs_regs(rec))
1906 rec->flags &= ~FTRACE_FL_REGS;
1907 }
1908
1909 /*
1910 * The TRAMP needs to be set only if rec count
1911 * is decremented to one, and the ops that is
1912 * left has a trampoline. As TRAMP can only be
1913 * enabled if there is only a single ops attached
1914 * to it.
1915 */
1916 if (ftrace_rec_count(rec) == 1 &&
1917 ftrace_find_tramp_ops_any_other(rec, ops))
1918 rec->flags |= FTRACE_FL_TRAMP;
1919 else
1920 rec->flags &= ~FTRACE_FL_TRAMP;
1921
1922 /*
1923 * flags will be cleared in ftrace_check_record()
1924 * if rec count is zero.
1925 */
1926 }
1927
1928 /*
1929 * If the rec has a single associated ops, and ops->func can be
1930 * called directly, allow the call site to call via the ops.
1931 */
1932 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) &&
1933 ftrace_rec_count(rec) == 1 &&
1934 ftrace_ops_get_func(ops) == ops->func)
1935 rec->flags |= FTRACE_FL_CALL_OPS;
1936 else
1937 rec->flags &= ~FTRACE_FL_CALL_OPS;
1938
1939 count++;
1940
1941 /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
1942 update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE;
1943
1944 /* Shortcut, if we handled all records, we are done. */
1945 if (!all && count == hash->count)
1946 return update;
1947 } while_for_each_ftrace_rec();
1948
1949 return update;
1950}
1951
1952/*
1953 * This is called when an ops is removed from tracing. It will decrement
1954 * the counters of the dyn_ftrace records for all the functions that
1955 * the @ops attached to.
1956 */
1957static bool ftrace_hash_rec_disable(struct ftrace_ops *ops)
1958{
1959 return __ftrace_hash_rec_update(ops, false);
1960}
1961
1962/*
1963 * This is called when an ops is added to tracing. It will increment
1964 * the counters of the dyn_ftrace records for all the functions that
1965 * the @ops attached to.
1966 */
1967static bool ftrace_hash_rec_enable(struct ftrace_ops *ops)
1968{
1969 return __ftrace_hash_rec_update(ops, true);
1970}
1971
1972/*
1973 * This function will update what functions @ops traces when its filter
1974 * changes.
1975 *
1976 * The @inc states if the @ops callbacks are going to be added or removed.
1977 * When one of the @ops hashes are updated to a "new_hash" the dyn_ftrace
1978 * records are update via:
1979 *
1980 * ftrace_hash_rec_disable_modify(ops);
1981 * ops->hash = new_hash
1982 * ftrace_hash_rec_enable_modify(ops);
1983 *
1984 * Where the @ops is removed from all the records it is tracing using
1985 * its old hash. The @ops hash is updated to the new hash, and then
1986 * the @ops is added back to the records so that it is tracing all
1987 * the new functions.
1988 */
1989static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, bool inc)
1990{
1991 struct ftrace_ops *op;
1992
1993 __ftrace_hash_rec_update(ops, inc);
1994
1995 if (ops->func_hash != &global_ops.local_hash)
1996 return;
1997
1998 /*
1999 * If the ops shares the global_ops hash, then we need to update
2000 * all ops that are enabled and use this hash.
2001 */
2002 do_for_each_ftrace_op(op, ftrace_ops_list) {
2003 /* Already done */
2004 if (op == ops)
2005 continue;
2006 if (op->func_hash == &global_ops.local_hash)
2007 __ftrace_hash_rec_update(op, inc);
2008 } while_for_each_ftrace_op(op);
2009}
2010
2011static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops)
2012{
2013 ftrace_hash_rec_update_modify(ops, false);
2014}
2015
2016static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops)
2017{
2018 ftrace_hash_rec_update_modify(ops, true);
2019}
2020
2021/*
2022 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
2023 * or no-needed to update, -EBUSY if it detects a conflict of the flag
2024 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
2025 * Note that old_hash and new_hash has below meanings
2026 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
2027 * - If the hash is EMPTY_HASH, it hits nothing
2028 * - Anything else hits the recs which match the hash entries.
2029 *
2030 * DIRECT ops does not have IPMODIFY flag, but we still need to check it
2031 * against functions with FTRACE_FL_IPMODIFY. If there is any overlap, call
2032 * ops_func(SHARE_IPMODIFY_SELF) to make sure current ops can share with
2033 * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate
2034 * the return value to the caller and eventually to the owner of the DIRECT
2035 * ops.
2036 */
2037static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
2038 struct ftrace_hash *old_hash,
2039 struct ftrace_hash *new_hash,
2040 bool update_target)
2041{
2042 struct ftrace_page *pg;
2043 struct dyn_ftrace *rec, *end = NULL;
2044 int in_old, in_new;
2045 bool is_ipmodify, is_direct;
2046
2047 /* Only update if the ops has been registered */
2048 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
2049 return 0;
2050
2051 is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY;
2052 is_direct = ops->flags & FTRACE_OPS_FL_DIRECT;
2053
2054 /* neither IPMODIFY nor DIRECT, skip */
2055 if (!is_ipmodify && !is_direct)
2056 return 0;
2057
2058 if (WARN_ON_ONCE(is_ipmodify && is_direct))
2059 return 0;
2060
2061 /*
2062 * Since the IPMODIFY and DIRECT are very address sensitive
2063 * actions, we do not allow ftrace_ops to set all functions to new
2064 * hash.
2065 */
2066 if (!new_hash || !old_hash)
2067 return -EINVAL;
2068
2069 /* Update rec->flags */
2070 do_for_each_ftrace_rec(pg, rec) {
2071
2072 if (rec->flags & FTRACE_FL_DISABLED)
2073 continue;
2074
2075 /*
2076 * Unless we are updating the target of a direct function,
2077 * we only need to update differences of filter_hash
2078 */
2079 in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
2080 in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
2081 if (!update_target && (in_old == in_new))
2082 continue;
2083
2084 if (in_new) {
2085 if (rec->flags & FTRACE_FL_IPMODIFY) {
2086 int ret;
2087
2088 /* Cannot have two ipmodify on same rec */
2089 if (is_ipmodify)
2090 goto rollback;
2091
2092 /*
2093 * If this is called by __modify_ftrace_direct()
2094 * then it is only changing where the direct
2095 * pointer is jumping to, and the record already
2096 * points to a direct trampoline. If it isn't,
2097 * then it is a bug to update ipmodify on a direct
2098 * caller.
2099 */
2100 FTRACE_WARN_ON(!update_target &&
2101 (rec->flags & FTRACE_FL_DIRECT));
2102
2103 /*
2104 * Another ops with IPMODIFY is already
2105 * attached. We are now attaching a direct
2106 * ops. Run SHARE_IPMODIFY_SELF, to check
2107 * whether sharing is supported.
2108 */
2109 if (!ops->ops_func)
2110 return -EBUSY;
2111 ret = ops->ops_func(ops, rec->ip, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF);
2112 if (ret)
2113 return ret;
2114 } else if (is_ipmodify) {
2115 rec->flags |= FTRACE_FL_IPMODIFY;
2116 }
2117 } else if (is_ipmodify) {
2118 rec->flags &= ~FTRACE_FL_IPMODIFY;
2119 }
2120 } while_for_each_ftrace_rec();
2121
2122 return 0;
2123
2124rollback:
2125 end = rec;
2126
2127 /* Roll back what we did above */
2128 do_for_each_ftrace_rec(pg, rec) {
2129
2130 if (rec->flags & FTRACE_FL_DISABLED)
2131 continue;
2132
2133 if (rec == end)
2134 return -EBUSY;
2135
2136 in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
2137 in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
2138 if (in_old == in_new)
2139 continue;
2140
2141 if (in_new)
2142 rec->flags &= ~FTRACE_FL_IPMODIFY;
2143 else
2144 rec->flags |= FTRACE_FL_IPMODIFY;
2145 } while_for_each_ftrace_rec();
2146
2147 return -EBUSY;
2148}
2149
2150static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
2151{
2152 struct ftrace_hash *hash = ops->func_hash->filter_hash;
2153
2154 if (ftrace_hash_empty(hash))
2155 hash = NULL;
2156
2157 return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash, false);
2158}
2159
2160/* Disabling always succeeds */
2161static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
2162{
2163 struct ftrace_hash *hash = ops->func_hash->filter_hash;
2164
2165 if (ftrace_hash_empty(hash))
2166 hash = NULL;
2167
2168 __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH, false);
2169}
2170
2171static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
2172 struct ftrace_hash *new_hash)
2173{
2174 struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
2175
2176 if (ftrace_hash_empty(old_hash))
2177 old_hash = NULL;
2178
2179 if (ftrace_hash_empty(new_hash))
2180 new_hash = NULL;
2181
2182 return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash, false);
2183}
2184
2185static void print_ip_ins(const char *fmt, const unsigned char *p)
2186{
2187 char ins[MCOUNT_INSN_SIZE];
2188
2189 if (copy_from_kernel_nofault(ins, p, MCOUNT_INSN_SIZE)) {
2190 printk(KERN_CONT "%s[FAULT] %px\n", fmt, p);
2191 return;
2192 }
2193
2194 printk(KERN_CONT "%s", fmt);
2195 pr_cont("%*phC", MCOUNT_INSN_SIZE, ins);
2196}
2197
2198enum ftrace_bug_type ftrace_bug_type;
2199const void *ftrace_expected;
2200
2201static void print_bug_type(void)
2202{
2203 switch (ftrace_bug_type) {
2204 case FTRACE_BUG_UNKNOWN:
2205 break;
2206 case FTRACE_BUG_INIT:
2207 pr_info("Initializing ftrace call sites\n");
2208 break;
2209 case FTRACE_BUG_NOP:
2210 pr_info("Setting ftrace call site to NOP\n");
2211 break;
2212 case FTRACE_BUG_CALL:
2213 pr_info("Setting ftrace call site to call ftrace function\n");
2214 break;
2215 case FTRACE_BUG_UPDATE:
2216 pr_info("Updating ftrace call site to call a different ftrace function\n");
2217 break;
2218 }
2219}
2220
2221/**
2222 * ftrace_bug - report and shutdown function tracer
2223 * @failed: The failed type (EFAULT, EINVAL, EPERM)
2224 * @rec: The record that failed
2225 *
2226 * The arch code that enables or disables the function tracing
2227 * can call ftrace_bug() when it has detected a problem in
2228 * modifying the code. @failed should be one of either:
2229 * EFAULT - if the problem happens on reading the @ip address
2230 * EINVAL - if what is read at @ip is not what was expected
2231 * EPERM - if the problem happens on writing to the @ip address
2232 */
2233void ftrace_bug(int failed, struct dyn_ftrace *rec)
2234{
2235 unsigned long ip = rec ? rec->ip : 0;
2236
2237 pr_info("------------[ ftrace bug ]------------\n");
2238
2239 switch (failed) {
2240 case -EFAULT:
2241 pr_info("ftrace faulted on modifying ");
2242 print_ip_sym(KERN_INFO, ip);
2243 break;
2244 case -EINVAL:
2245 pr_info("ftrace failed to modify ");
2246 print_ip_sym(KERN_INFO, ip);
2247 print_ip_ins(" actual: ", (unsigned char *)ip);
2248 pr_cont("\n");
2249 if (ftrace_expected) {
2250 print_ip_ins(" expected: ", ftrace_expected);
2251 pr_cont("\n");
2252 }
2253 break;
2254 case -EPERM:
2255 pr_info("ftrace faulted on writing ");
2256 print_ip_sym(KERN_INFO, ip);
2257 break;
2258 default:
2259 pr_info("ftrace faulted on unknown error ");
2260 print_ip_sym(KERN_INFO, ip);
2261 }
2262 print_bug_type();
2263 if (rec) {
2264 struct ftrace_ops *ops = NULL;
2265
2266 pr_info("ftrace record flags: %lx\n", rec->flags);
2267 pr_cont(" (%ld)%s%s", ftrace_rec_count(rec),
2268 rec->flags & FTRACE_FL_REGS ? " R" : " ",
2269 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ");
2270 if (rec->flags & FTRACE_FL_TRAMP_EN) {
2271 ops = ftrace_find_tramp_ops_any(rec);
2272 if (ops) {
2273 do {
2274 pr_cont("\ttramp: %pS (%pS)",
2275 (void *)ops->trampoline,
2276 (void *)ops->func);
2277 ops = ftrace_find_tramp_ops_next(rec, ops);
2278 } while (ops);
2279 } else
2280 pr_cont("\ttramp: ERROR!");
2281
2282 }
2283 ip = ftrace_get_addr_curr(rec);
2284 pr_cont("\n expected tramp: %lx\n", ip);
2285 }
2286
2287 FTRACE_WARN_ON_ONCE(1);
2288}
2289
2290static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
2291{
2292 unsigned long flag = 0UL;
2293
2294 ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2295
2296 if (skip_record(rec))
2297 return FTRACE_UPDATE_IGNORE;
2298
2299 /*
2300 * If we are updating calls:
2301 *
2302 * If the record has a ref count, then we need to enable it
2303 * because someone is using it.
2304 *
2305 * Otherwise we make sure its disabled.
2306 *
2307 * If we are disabling calls, then disable all records that
2308 * are enabled.
2309 */
2310 if (enable && ftrace_rec_count(rec))
2311 flag = FTRACE_FL_ENABLED;
2312
2313 /*
2314 * If enabling and the REGS flag does not match the REGS_EN, or
2315 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
2316 * this record. Set flags to fail the compare against ENABLED.
2317 * Same for direct calls.
2318 */
2319 if (flag) {
2320 if (!(rec->flags & FTRACE_FL_REGS) !=
2321 !(rec->flags & FTRACE_FL_REGS_EN))
2322 flag |= FTRACE_FL_REGS;
2323
2324 if (!(rec->flags & FTRACE_FL_TRAMP) !=
2325 !(rec->flags & FTRACE_FL_TRAMP_EN))
2326 flag |= FTRACE_FL_TRAMP;
2327
2328 /*
2329 * Direct calls are special, as count matters.
2330 * We must test the record for direct, if the
2331 * DIRECT and DIRECT_EN do not match, but only
2332 * if the count is 1. That's because, if the
2333 * count is something other than one, we do not
2334 * want the direct enabled (it will be done via the
2335 * direct helper). But if DIRECT_EN is set, and
2336 * the count is not one, we need to clear it.
2337 *
2338 */
2339 if (ftrace_rec_count(rec) == 1) {
2340 if (!(rec->flags & FTRACE_FL_DIRECT) !=
2341 !(rec->flags & FTRACE_FL_DIRECT_EN))
2342 flag |= FTRACE_FL_DIRECT;
2343 } else if (rec->flags & FTRACE_FL_DIRECT_EN) {
2344 flag |= FTRACE_FL_DIRECT;
2345 }
2346
2347 /*
2348 * Ops calls are special, as count matters.
2349 * As with direct calls, they must only be enabled when count
2350 * is one, otherwise they'll be handled via the list ops.
2351 */
2352 if (ftrace_rec_count(rec) == 1) {
2353 if (!(rec->flags & FTRACE_FL_CALL_OPS) !=
2354 !(rec->flags & FTRACE_FL_CALL_OPS_EN))
2355 flag |= FTRACE_FL_CALL_OPS;
2356 } else if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
2357 flag |= FTRACE_FL_CALL_OPS;
2358 }
2359 }
2360
2361 /* If the state of this record hasn't changed, then do nothing */
2362 if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2363 return FTRACE_UPDATE_IGNORE;
2364
2365 if (flag) {
2366 /* Save off if rec is being enabled (for return value) */
2367 flag ^= rec->flags & FTRACE_FL_ENABLED;
2368
2369 if (update) {
2370 rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED;
2371 if (flag & FTRACE_FL_REGS) {
2372 if (rec->flags & FTRACE_FL_REGS)
2373 rec->flags |= FTRACE_FL_REGS_EN;
2374 else
2375 rec->flags &= ~FTRACE_FL_REGS_EN;
2376 }
2377 if (flag & FTRACE_FL_TRAMP) {
2378 if (rec->flags & FTRACE_FL_TRAMP)
2379 rec->flags |= FTRACE_FL_TRAMP_EN;
2380 else
2381 rec->flags &= ~FTRACE_FL_TRAMP_EN;
2382 }
2383
2384 /* Keep track of anything that modifies the function */
2385 if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY))
2386 rec->flags |= FTRACE_FL_MODIFIED;
2387
2388 if (flag & FTRACE_FL_DIRECT) {
2389 /*
2390 * If there's only one user (direct_ops helper)
2391 * then we can call the direct function
2392 * directly (no ftrace trampoline).
2393 */
2394 if (ftrace_rec_count(rec) == 1) {
2395 if (rec->flags & FTRACE_FL_DIRECT)
2396 rec->flags |= FTRACE_FL_DIRECT_EN;
2397 else
2398 rec->flags &= ~FTRACE_FL_DIRECT_EN;
2399 } else {
2400 /*
2401 * Can only call directly if there's
2402 * only one callback to the function.
2403 */
2404 rec->flags &= ~FTRACE_FL_DIRECT_EN;
2405 }
2406 }
2407
2408 if (flag & FTRACE_FL_CALL_OPS) {
2409 if (ftrace_rec_count(rec) == 1) {
2410 if (rec->flags & FTRACE_FL_CALL_OPS)
2411 rec->flags |= FTRACE_FL_CALL_OPS_EN;
2412 else
2413 rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
2414 } else {
2415 /*
2416 * Can only call directly if there's
2417 * only one set of associated ops.
2418 */
2419 rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
2420 }
2421 }
2422 }
2423
2424 /*
2425 * If this record is being updated from a nop, then
2426 * return UPDATE_MAKE_CALL.
2427 * Otherwise,
2428 * return UPDATE_MODIFY_CALL to tell the caller to convert
2429 * from the save regs, to a non-save regs function or
2430 * vice versa, or from a trampoline call.
2431 */
2432 if (flag & FTRACE_FL_ENABLED) {
2433 ftrace_bug_type = FTRACE_BUG_CALL;
2434 return FTRACE_UPDATE_MAKE_CALL;
2435 }
2436
2437 ftrace_bug_type = FTRACE_BUG_UPDATE;
2438 return FTRACE_UPDATE_MODIFY_CALL;
2439 }
2440
2441 if (update) {
2442 /* If there's no more users, clear all flags */
2443 if (!ftrace_rec_count(rec))
2444 rec->flags &= FTRACE_NOCLEAR_FLAGS;
2445 else
2446 /*
2447 * Just disable the record, but keep the ops TRAMP
2448 * and REGS states. The _EN flags must be disabled though.
2449 */
2450 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
2451 FTRACE_FL_REGS_EN | FTRACE_FL_DIRECT_EN |
2452 FTRACE_FL_CALL_OPS_EN);
2453 }
2454
2455 ftrace_bug_type = FTRACE_BUG_NOP;
2456 return FTRACE_UPDATE_MAKE_NOP;
2457}
2458
2459/**
2460 * ftrace_update_record - set a record that now is tracing or not
2461 * @rec: the record to update
2462 * @enable: set to true if the record is tracing, false to force disable
2463 *
2464 * The records that represent all functions that can be traced need
2465 * to be updated when tracing has been enabled.
2466 */
2467int ftrace_update_record(struct dyn_ftrace *rec, bool enable)
2468{
2469 return ftrace_check_record(rec, enable, true);
2470}
2471
2472/**
2473 * ftrace_test_record - check if the record has been enabled or not
2474 * @rec: the record to test
2475 * @enable: set to true to check if enabled, false if it is disabled
2476 *
2477 * The arch code may need to test if a record is already set to
2478 * tracing to determine how to modify the function code that it
2479 * represents.
2480 */
2481int ftrace_test_record(struct dyn_ftrace *rec, bool enable)
2482{
2483 return ftrace_check_record(rec, enable, false);
2484}
2485
2486static struct ftrace_ops *
2487ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
2488{
2489 struct ftrace_ops *op;
2490 unsigned long ip = rec->ip;
2491
2492 do_for_each_ftrace_op(op, ftrace_ops_list) {
2493
2494 if (!op->trampoline)
2495 continue;
2496
2497 if (hash_contains_ip(ip, op->func_hash))
2498 return op;
2499 } while_for_each_ftrace_op(op);
2500
2501 return NULL;
2502}
2503
2504static struct ftrace_ops *
2505ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude)
2506{
2507 struct ftrace_ops *op;
2508 unsigned long ip = rec->ip;
2509
2510 do_for_each_ftrace_op(op, ftrace_ops_list) {
2511
2512 if (op == op_exclude || !op->trampoline)
2513 continue;
2514
2515 if (hash_contains_ip(ip, op->func_hash))
2516 return op;
2517 } while_for_each_ftrace_op(op);
2518
2519 return NULL;
2520}
2521
2522static struct ftrace_ops *
2523ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
2524 struct ftrace_ops *op)
2525{
2526 unsigned long ip = rec->ip;
2527
2528 while_for_each_ftrace_op(op) {
2529
2530 if (!op->trampoline)
2531 continue;
2532
2533 if (hash_contains_ip(ip, op->func_hash))
2534 return op;
2535 }
2536
2537 return NULL;
2538}
2539
2540static struct ftrace_ops *
2541ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
2542{
2543 struct ftrace_ops *op;
2544 unsigned long ip = rec->ip;
2545
2546 /*
2547 * Need to check removed ops first.
2548 * If they are being removed, and this rec has a tramp,
2549 * and this rec is in the ops list, then it would be the
2550 * one with the tramp.
2551 */
2552 if (removed_ops) {
2553 if (hash_contains_ip(ip, &removed_ops->old_hash))
2554 return removed_ops;
2555 }
2556
2557 /*
2558 * Need to find the current trampoline for a rec.
2559 * Now, a trampoline is only attached to a rec if there
2560 * was a single 'ops' attached to it. But this can be called
2561 * when we are adding another op to the rec or removing the
2562 * current one. Thus, if the op is being added, we can
2563 * ignore it because it hasn't attached itself to the rec
2564 * yet.
2565 *
2566 * If an ops is being modified (hooking to different functions)
2567 * then we don't care about the new functions that are being
2568 * added, just the old ones (that are probably being removed).
2569 *
2570 * If we are adding an ops to a function that already is using
2571 * a trampoline, it needs to be removed (trampolines are only
2572 * for single ops connected), then an ops that is not being
2573 * modified also needs to be checked.
2574 */
2575 do_for_each_ftrace_op(op, ftrace_ops_list) {
2576
2577 if (!op->trampoline)
2578 continue;
2579
2580 /*
2581 * If the ops is being added, it hasn't gotten to
2582 * the point to be removed from this tree yet.
2583 */
2584 if (op->flags & FTRACE_OPS_FL_ADDING)
2585 continue;
2586
2587
2588 /*
2589 * If the ops is being modified and is in the old
2590 * hash, then it is probably being removed from this
2591 * function.
2592 */
2593 if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
2594 hash_contains_ip(ip, &op->old_hash))
2595 return op;
2596 /*
2597 * If the ops is not being added or modified, and it's
2598 * in its normal filter hash, then this must be the one
2599 * we want!
2600 */
2601 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
2602 hash_contains_ip(ip, op->func_hash))
2603 return op;
2604
2605 } while_for_each_ftrace_op(op);
2606
2607 return NULL;
2608}
2609
2610static struct ftrace_ops *
2611ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
2612{
2613 struct ftrace_ops *op;
2614 unsigned long ip = rec->ip;
2615
2616 do_for_each_ftrace_op(op, ftrace_ops_list) {
2617 /* pass rec in as regs to have non-NULL val */
2618 if (hash_contains_ip(ip, op->func_hash))
2619 return op;
2620 } while_for_each_ftrace_op(op);
2621
2622 return NULL;
2623}
2624
2625struct ftrace_ops *
2626ftrace_find_unique_ops(struct dyn_ftrace *rec)
2627{
2628 struct ftrace_ops *op, *found = NULL;
2629 unsigned long ip = rec->ip;
2630
2631 do_for_each_ftrace_op(op, ftrace_ops_list) {
2632
2633 if (hash_contains_ip(ip, op->func_hash)) {
2634 if (found)
2635 return NULL;
2636 found = op;
2637 }
2638
2639 } while_for_each_ftrace_op(op);
2640
2641 return found;
2642}
2643
2644#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
2645/* Protected by rcu_tasks for reading, and direct_mutex for writing */
2646static struct ftrace_hash __rcu *direct_functions = EMPTY_HASH;
2647static DEFINE_MUTEX(direct_mutex);
2648
2649/*
2650 * Search the direct_functions hash to see if the given instruction pointer
2651 * has a direct caller attached to it.
2652 */
2653unsigned long ftrace_find_rec_direct(unsigned long ip)
2654{
2655 struct ftrace_func_entry *entry;
2656
2657 guard(preempt_notrace)();
2658 entry = __ftrace_lookup_ip(rcu_dereference_sched(direct_functions), ip);
2659 if (!entry)
2660 return 0;
2661
2662 return entry->direct;
2663}
2664
2665static void call_direct_funcs(unsigned long ip, unsigned long pip,
2666 struct ftrace_ops *ops, struct ftrace_regs *fregs)
2667{
2668 unsigned long addr;
2669
2670#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS
2671 addr = ftrace_find_rec_direct(ip);
2672#else
2673 addr = READ_ONCE(ops->direct_call);
2674#endif
2675 if (!addr)
2676 return;
2677
2678 arch_ftrace_set_direct_caller(fregs, addr);
2679}
2680#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
2681
2682/**
2683 * ftrace_get_addr_new - Get the call address to set to
2684 * @rec: The ftrace record descriptor
2685 *
2686 * If the record has the FTRACE_FL_REGS set, that means that it
2687 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
2688 * is not set, then it wants to convert to the normal callback.
2689 *
2690 * Returns: the address of the trampoline to set to
2691 */
2692unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
2693{
2694 struct ftrace_ops *ops;
2695 unsigned long addr;
2696
2697 if ((rec->flags & FTRACE_FL_DIRECT) &&
2698 (ftrace_rec_count(rec) == 1)) {
2699 addr = ftrace_find_rec_direct(rec->ip);
2700 if (addr)
2701 return addr;
2702 WARN_ON_ONCE(1);
2703 }
2704
2705 /* Trampolines take precedence over regs */
2706 if (rec->flags & FTRACE_FL_TRAMP) {
2707 ops = ftrace_find_tramp_ops_new(rec);
2708 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2709 pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
2710 (void *)rec->ip, (void *)rec->ip, rec->flags);
2711 /* Ftrace is shutting down, return anything */
2712 return (unsigned long)FTRACE_ADDR;
2713 }
2714 return ops->trampoline;
2715 }
2716
2717 if (rec->flags & FTRACE_FL_REGS)
2718 return (unsigned long)FTRACE_REGS_ADDR;
2719 else
2720 return (unsigned long)FTRACE_ADDR;
2721}
2722
2723/**
2724 * ftrace_get_addr_curr - Get the call address that is already there
2725 * @rec: The ftrace record descriptor
2726 *
2727 * The FTRACE_FL_REGS_EN is set when the record already points to
2728 * a function that saves all the regs. Basically the '_EN' version
2729 * represents the current state of the function.
2730 *
2731 * Returns: the address of the trampoline that is currently being called
2732 */
2733unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
2734{
2735 struct ftrace_ops *ops;
2736 unsigned long addr;
2737
2738 /* Direct calls take precedence over trampolines */
2739 if (rec->flags & FTRACE_FL_DIRECT_EN) {
2740 addr = ftrace_find_rec_direct(rec->ip);
2741 if (addr)
2742 return addr;
2743 WARN_ON_ONCE(1);
2744 }
2745
2746 /* Trampolines take precedence over regs */
2747 if (rec->flags & FTRACE_FL_TRAMP_EN) {
2748 ops = ftrace_find_tramp_ops_curr(rec);
2749 if (FTRACE_WARN_ON(!ops)) {
2750 pr_warn("Bad trampoline accounting at: %p (%pS)\n",
2751 (void *)rec->ip, (void *)rec->ip);
2752 /* Ftrace is shutting down, return anything */
2753 return (unsigned long)FTRACE_ADDR;
2754 }
2755 return ops->trampoline;
2756 }
2757
2758 if (rec->flags & FTRACE_FL_REGS_EN)
2759 return (unsigned long)FTRACE_REGS_ADDR;
2760 else
2761 return (unsigned long)FTRACE_ADDR;
2762}
2763
2764static int
2765__ftrace_replace_code(struct dyn_ftrace *rec, bool enable)
2766{
2767 unsigned long ftrace_old_addr;
2768 unsigned long ftrace_addr;
2769 int ret;
2770
2771 ftrace_addr = ftrace_get_addr_new(rec);
2772
2773 /* This needs to be done before we call ftrace_update_record */
2774 ftrace_old_addr = ftrace_get_addr_curr(rec);
2775
2776 ret = ftrace_update_record(rec, enable);
2777
2778 ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2779
2780 switch (ret) {
2781 case FTRACE_UPDATE_IGNORE:
2782 return 0;
2783
2784 case FTRACE_UPDATE_MAKE_CALL:
2785 ftrace_bug_type = FTRACE_BUG_CALL;
2786 return ftrace_make_call(rec, ftrace_addr);
2787
2788 case FTRACE_UPDATE_MAKE_NOP:
2789 ftrace_bug_type = FTRACE_BUG_NOP;
2790 return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2791
2792 case FTRACE_UPDATE_MODIFY_CALL:
2793 ftrace_bug_type = FTRACE_BUG_UPDATE;
2794 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2795 }
2796
2797 return -1; /* unknown ftrace bug */
2798}
2799
2800void __weak ftrace_replace_code(int mod_flags)
2801{
2802 struct dyn_ftrace *rec;
2803 struct ftrace_page *pg;
2804 bool enable = mod_flags & FTRACE_MODIFY_ENABLE_FL;
2805 int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL;
2806 int failed;
2807
2808 if (unlikely(ftrace_disabled))
2809 return;
2810
2811 do_for_each_ftrace_rec(pg, rec) {
2812
2813 if (skip_record(rec))
2814 continue;
2815
2816 failed = __ftrace_replace_code(rec, enable);
2817 if (failed) {
2818 ftrace_bug(failed, rec);
2819 /* Stop processing */
2820 return;
2821 }
2822 if (schedulable)
2823 cond_resched();
2824 } while_for_each_ftrace_rec();
2825}
2826
2827struct ftrace_rec_iter {
2828 struct ftrace_page *pg;
2829 int index;
2830};
2831
2832/**
2833 * ftrace_rec_iter_start - start up iterating over traced functions
2834 *
2835 * Returns: an iterator handle that is used to iterate over all
2836 * the records that represent address locations where functions
2837 * are traced.
2838 *
2839 * May return NULL if no records are available.
2840 */
2841struct ftrace_rec_iter *ftrace_rec_iter_start(void)
2842{
2843 /*
2844 * We only use a single iterator.
2845 * Protected by the ftrace_lock mutex.
2846 */
2847 static struct ftrace_rec_iter ftrace_rec_iter;
2848 struct ftrace_rec_iter *iter = &ftrace_rec_iter;
2849
2850 iter->pg = ftrace_pages_start;
2851 iter->index = 0;
2852
2853 /* Could have empty pages */
2854 while (iter->pg && !iter->pg->index)
2855 iter->pg = iter->pg->next;
2856
2857 if (!iter->pg)
2858 return NULL;
2859
2860 return iter;
2861}
2862
2863/**
2864 * ftrace_rec_iter_next - get the next record to process.
2865 * @iter: The handle to the iterator.
2866 *
2867 * Returns: the next iterator after the given iterator @iter.
2868 */
2869struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
2870{
2871 iter->index++;
2872
2873 if (iter->index >= iter->pg->index) {
2874 iter->pg = iter->pg->next;
2875 iter->index = 0;
2876
2877 /* Could have empty pages */
2878 while (iter->pg && !iter->pg->index)
2879 iter->pg = iter->pg->next;
2880 }
2881
2882 if (!iter->pg)
2883 return NULL;
2884
2885 return iter;
2886}
2887
2888/**
2889 * ftrace_rec_iter_record - get the record at the iterator location
2890 * @iter: The current iterator location
2891 *
2892 * Returns: the record that the current @iter is at.
2893 */
2894struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
2895{
2896 return &iter->pg->records[iter->index];
2897}
2898
2899static int
2900ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec)
2901{
2902 int ret;
2903
2904 if (unlikely(ftrace_disabled))
2905 return 0;
2906
2907 ret = ftrace_init_nop(mod, rec);
2908 if (ret) {
2909 ftrace_bug_type = FTRACE_BUG_INIT;
2910 ftrace_bug(ret, rec);
2911 return 0;
2912 }
2913 return 1;
2914}
2915
2916/*
2917 * archs can override this function if they must do something
2918 * before the modifying code is performed.
2919 */
2920void __weak ftrace_arch_code_modify_prepare(void)
2921{
2922}
2923
2924/*
2925 * archs can override this function if they must do something
2926 * after the modifying code is performed.
2927 */
2928void __weak ftrace_arch_code_modify_post_process(void)
2929{
2930}
2931
2932static int update_ftrace_func(ftrace_func_t func)
2933{
2934 static ftrace_func_t save_func;
2935
2936 /* Avoid updating if it hasn't changed */
2937 if (func == save_func)
2938 return 0;
2939
2940 save_func = func;
2941
2942 return ftrace_update_ftrace_func(func);
2943}
2944
2945void ftrace_modify_all_code(int command)
2946{
2947 int update = command & FTRACE_UPDATE_TRACE_FUNC;
2948 int mod_flags = 0;
2949 int err = 0;
2950
2951 if (command & FTRACE_MAY_SLEEP)
2952 mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL;
2953
2954 /*
2955 * If the ftrace_caller calls a ftrace_ops func directly,
2956 * we need to make sure that it only traces functions it
2957 * expects to trace. When doing the switch of functions,
2958 * we need to update to the ftrace_ops_list_func first
2959 * before the transition between old and new calls are set,
2960 * as the ftrace_ops_list_func will check the ops hashes
2961 * to make sure the ops are having the right functions
2962 * traced.
2963 */
2964 if (update) {
2965 err = update_ftrace_func(ftrace_ops_list_func);
2966 if (FTRACE_WARN_ON(err))
2967 return;
2968 }
2969
2970 if (command & FTRACE_UPDATE_CALLS)
2971 ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL);
2972 else if (command & FTRACE_DISABLE_CALLS)
2973 ftrace_replace_code(mod_flags);
2974
2975 if (update && ftrace_trace_function != ftrace_ops_list_func) {
2976 function_trace_op = set_function_trace_op;
2977 smp_wmb();
2978 /* If irqs are disabled, we are in stop machine */
2979 if (!irqs_disabled())
2980 smp_call_function(ftrace_sync_ipi, NULL, 1);
2981 err = update_ftrace_func(ftrace_trace_function);
2982 if (FTRACE_WARN_ON(err))
2983 return;
2984 }
2985
2986 if (command & FTRACE_START_FUNC_RET)
2987 err = ftrace_enable_ftrace_graph_caller();
2988 else if (command & FTRACE_STOP_FUNC_RET)
2989 err = ftrace_disable_ftrace_graph_caller();
2990 FTRACE_WARN_ON(err);
2991}
2992
2993static int __ftrace_modify_code(void *data)
2994{
2995 int *command = data;
2996
2997 ftrace_modify_all_code(*command);
2998
2999 return 0;
3000}
3001
3002/**
3003 * ftrace_run_stop_machine - go back to the stop machine method
3004 * @command: The command to tell ftrace what to do
3005 *
3006 * If an arch needs to fall back to the stop machine method, the
3007 * it can call this function.
3008 */
3009void ftrace_run_stop_machine(int command)
3010{
3011 stop_machine(__ftrace_modify_code, &command, NULL);
3012}
3013
3014/**
3015 * arch_ftrace_update_code - modify the code to trace or not trace
3016 * @command: The command that needs to be done
3017 *
3018 * Archs can override this function if it does not need to
3019 * run stop_machine() to modify code.
3020 */
3021void __weak arch_ftrace_update_code(int command)
3022{
3023 ftrace_run_stop_machine(command);
3024}
3025
3026static void ftrace_run_update_code(int command)
3027{
3028 ftrace_arch_code_modify_prepare();
3029
3030 /*
3031 * By default we use stop_machine() to modify the code.
3032 * But archs can do what ever they want as long as it
3033 * is safe. The stop_machine() is the safest, but also
3034 * produces the most overhead.
3035 */
3036 arch_ftrace_update_code(command);
3037
3038 ftrace_arch_code_modify_post_process();
3039}
3040
3041static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
3042 struct ftrace_ops_hash *old_hash)
3043{
3044 ops->flags |= FTRACE_OPS_FL_MODIFYING;
3045 ops->old_hash.filter_hash = old_hash->filter_hash;
3046 ops->old_hash.notrace_hash = old_hash->notrace_hash;
3047 ftrace_run_update_code(command);
3048 ops->old_hash.filter_hash = NULL;
3049 ops->old_hash.notrace_hash = NULL;
3050 ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
3051}
3052
3053static ftrace_func_t saved_ftrace_func;
3054static int ftrace_start_up;
3055
3056void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
3057{
3058}
3059
3060/* List of trace_ops that have allocated trampolines */
3061static LIST_HEAD(ftrace_ops_trampoline_list);
3062
3063static void ftrace_add_trampoline_to_kallsyms(struct ftrace_ops *ops)
3064{
3065 lockdep_assert_held(&ftrace_lock);
3066 list_add_rcu(&ops->list, &ftrace_ops_trampoline_list);
3067}
3068
3069static void ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops *ops)
3070{
3071 lockdep_assert_held(&ftrace_lock);
3072 list_del_rcu(&ops->list);
3073 synchronize_rcu();
3074}
3075
3076/*
3077 * "__builtin__ftrace" is used as a module name in /proc/kallsyms for symbols
3078 * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is
3079 * not a module.
3080 */
3081#define FTRACE_TRAMPOLINE_MOD "__builtin__ftrace"
3082#define FTRACE_TRAMPOLINE_SYM "ftrace_trampoline"
3083
3084static void ftrace_trampoline_free(struct ftrace_ops *ops)
3085{
3086 if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) &&
3087 ops->trampoline) {
3088 /*
3089 * Record the text poke event before the ksymbol unregister
3090 * event.
3091 */
3092 perf_event_text_poke((void *)ops->trampoline,
3093 (void *)ops->trampoline,
3094 ops->trampoline_size, NULL, 0);
3095 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
3096 ops->trampoline, ops->trampoline_size,
3097 true, FTRACE_TRAMPOLINE_SYM);
3098 /* Remove from kallsyms after the perf events */
3099 ftrace_remove_trampoline_from_kallsyms(ops);
3100 }
3101
3102 arch_ftrace_trampoline_free(ops);
3103}
3104
3105static void ftrace_startup_enable(int command)
3106{
3107 if (saved_ftrace_func != ftrace_trace_function) {
3108 saved_ftrace_func = ftrace_trace_function;
3109 command |= FTRACE_UPDATE_TRACE_FUNC;
3110 }
3111
3112 if (!command || !ftrace_enabled)
3113 return;
3114
3115 ftrace_run_update_code(command);
3116}
3117
3118static void ftrace_startup_all(int command)
3119{
3120 update_all_ops = true;
3121 ftrace_startup_enable(command);
3122 update_all_ops = false;
3123}
3124
3125int ftrace_startup(struct ftrace_ops *ops, int command)
3126{
3127 int ret;
3128
3129 if (unlikely(ftrace_disabled))
3130 return -ENODEV;
3131
3132 ret = __register_ftrace_function(ops);
3133 if (ret)
3134 return ret;
3135
3136 ftrace_start_up++;
3137
3138 /*
3139 * Note that ftrace probes uses this to start up
3140 * and modify functions it will probe. But we still
3141 * set the ADDING flag for modification, as probes
3142 * do not have trampolines. If they add them in the
3143 * future, then the probes will need to distinguish
3144 * between adding and updating probes.
3145 */
3146 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
3147
3148 ret = ftrace_hash_ipmodify_enable(ops);
3149 if (ret < 0) {
3150 /* Rollback registration process */
3151 __unregister_ftrace_function(ops);
3152 ftrace_start_up--;
3153 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
3154 if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
3155 ftrace_trampoline_free(ops);
3156 return ret;
3157 }
3158
3159 if (ftrace_hash_rec_enable(ops))
3160 command |= FTRACE_UPDATE_CALLS;
3161
3162 ftrace_startup_enable(command);
3163
3164 /*
3165 * If ftrace is in an undefined state, we just remove ops from list
3166 * to prevent the NULL pointer, instead of totally rolling it back and
3167 * free trampoline, because those actions could cause further damage.
3168 */
3169 if (unlikely(ftrace_disabled)) {
3170 __unregister_ftrace_function(ops);
3171 return -ENODEV;
3172 }
3173
3174 ops->flags &= ~FTRACE_OPS_FL_ADDING;
3175
3176 return 0;
3177}
3178
3179int ftrace_shutdown(struct ftrace_ops *ops, int command)
3180{
3181 int ret;
3182
3183 if (unlikely(ftrace_disabled))
3184 return -ENODEV;
3185
3186 ret = __unregister_ftrace_function(ops);
3187 if (ret)
3188 return ret;
3189
3190 ftrace_start_up--;
3191 /*
3192 * Just warn in case of unbalance, no need to kill ftrace, it's not
3193 * critical but the ftrace_call callers may be never nopped again after
3194 * further ftrace uses.
3195 */
3196 WARN_ON_ONCE(ftrace_start_up < 0);
3197
3198 /* Disabling ipmodify never fails */
3199 ftrace_hash_ipmodify_disable(ops);
3200
3201 if (ftrace_hash_rec_disable(ops))
3202 command |= FTRACE_UPDATE_CALLS;
3203
3204 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
3205
3206 if (saved_ftrace_func != ftrace_trace_function) {
3207 saved_ftrace_func = ftrace_trace_function;
3208 command |= FTRACE_UPDATE_TRACE_FUNC;
3209 }
3210
3211 if (!command || !ftrace_enabled)
3212 goto out;
3213
3214 /*
3215 * If the ops uses a trampoline, then it needs to be
3216 * tested first on update.
3217 */
3218 ops->flags |= FTRACE_OPS_FL_REMOVING;
3219 removed_ops = ops;
3220
3221 /* The trampoline logic checks the old hashes */
3222 ops->old_hash.filter_hash = ops->func_hash->filter_hash;
3223 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
3224
3225 ftrace_run_update_code(command);
3226
3227 /*
3228 * If there's no more ops registered with ftrace, run a
3229 * sanity check to make sure all rec flags are cleared.
3230 */
3231 if (rcu_dereference_protected(ftrace_ops_list,
3232 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
3233 struct ftrace_page *pg;
3234 struct dyn_ftrace *rec;
3235
3236 do_for_each_ftrace_rec(pg, rec) {
3237 if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS))
3238 pr_warn(" %pS flags:%lx\n",
3239 (void *)rec->ip, rec->flags);
3240 } while_for_each_ftrace_rec();
3241 }
3242
3243 ops->old_hash.filter_hash = NULL;
3244 ops->old_hash.notrace_hash = NULL;
3245
3246 removed_ops = NULL;
3247 ops->flags &= ~FTRACE_OPS_FL_REMOVING;
3248
3249out:
3250 /*
3251 * Dynamic ops may be freed, we must make sure that all
3252 * callers are done before leaving this function.
3253 */
3254 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) {
3255 /*
3256 * We need to do a hard force of sched synchronization.
3257 * This is because we use preempt_disable() to do RCU, but
3258 * the function tracers can be called where RCU is not watching
3259 * (like before user_exit()). We can not rely on the RCU
3260 * infrastructure to do the synchronization, thus we must do it
3261 * ourselves.
3262 */
3263 synchronize_rcu_tasks_rude();
3264
3265 /*
3266 * When the kernel is preemptive, tasks can be preempted
3267 * while on a ftrace trampoline. Just scheduling a task on
3268 * a CPU is not good enough to flush them. Calling
3269 * synchronize_rcu_tasks() will wait for those tasks to
3270 * execute and either schedule voluntarily or enter user space.
3271 */
3272 synchronize_rcu_tasks();
3273
3274 ftrace_trampoline_free(ops);
3275 }
3276
3277 return 0;
3278}
3279
3280/* Simply make a copy of @src and return it */
3281static struct ftrace_hash *copy_hash(struct ftrace_hash *src)
3282{
3283 if (ftrace_hash_empty(src))
3284 return EMPTY_HASH;
3285
3286 return alloc_and_copy_ftrace_hash(src->size_bits, src);
3287}
3288
3289/*
3290 * Append @new_hash entries to @hash:
3291 *
3292 * If @hash is the EMPTY_HASH then it traces all functions and nothing
3293 * needs to be done.
3294 *
3295 * If @new_hash is the EMPTY_HASH, then make *hash the EMPTY_HASH so
3296 * that it traces everything.
3297 *
3298 * Otherwise, go through all of @new_hash and add anything that @hash
3299 * doesn't already have, to @hash.
3300 *
3301 * The filter_hash updates uses just the append_hash() function
3302 * and the notrace_hash does not.
3303 */
3304static int append_hash(struct ftrace_hash **hash, struct ftrace_hash *new_hash,
3305 int size_bits)
3306{
3307 struct ftrace_func_entry *entry;
3308 int size;
3309 int i;
3310
3311 if (*hash) {
3312 /* An empty hash does everything */
3313 if (ftrace_hash_empty(*hash))
3314 return 0;
3315 } else {
3316 *hash = alloc_ftrace_hash(size_bits);
3317 if (!*hash)
3318 return -ENOMEM;
3319 }
3320
3321 /* If new_hash has everything make hash have everything */
3322 if (ftrace_hash_empty(new_hash)) {
3323 free_ftrace_hash(*hash);
3324 *hash = EMPTY_HASH;
3325 return 0;
3326 }
3327
3328 size = 1 << new_hash->size_bits;
3329 for (i = 0; i < size; i++) {
3330 hlist_for_each_entry(entry, &new_hash->buckets[i], hlist) {
3331 /* Only add if not already in hash */
3332 if (!__ftrace_lookup_ip(*hash, entry->ip) &&
3333 add_hash_entry(*hash, entry->ip) == NULL)
3334 return -ENOMEM;
3335 }
3336 }
3337 return 0;
3338}
3339
3340/*
3341 * Remove functions from @hash that are in @notrace_hash
3342 */
3343static void remove_hash(struct ftrace_hash *hash, struct ftrace_hash *notrace_hash)
3344{
3345 struct ftrace_func_entry *entry;
3346 struct hlist_node *tmp;
3347 int size;
3348 int i;
3349
3350 /* If the notrace hash is empty, there's nothing to do */
3351 if (ftrace_hash_empty(notrace_hash))
3352 return;
3353
3354 size = 1 << hash->size_bits;
3355 for (i = 0; i < size; i++) {
3356 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
3357 if (!__ftrace_lookup_ip(notrace_hash, entry->ip))
3358 continue;
3359 remove_hash_entry(hash, entry);
3360 kfree(entry);
3361 }
3362 }
3363}
3364
3365/*
3366 * Add to @hash only those that are in both @new_hash1 and @new_hash2
3367 *
3368 * The notrace_hash updates uses just the intersect_hash() function
3369 * and the filter_hash does not.
3370 */
3371static int intersect_hash(struct ftrace_hash **hash, struct ftrace_hash *new_hash1,
3372 struct ftrace_hash *new_hash2)
3373{
3374 struct ftrace_func_entry *entry;
3375 int size;
3376 int i;
3377
3378 /*
3379 * If new_hash1 or new_hash2 is the EMPTY_HASH then make the hash
3380 * empty as well as empty for notrace means none are notraced.
3381 */
3382 if (ftrace_hash_empty(new_hash1) || ftrace_hash_empty(new_hash2)) {
3383 free_ftrace_hash(*hash);
3384 *hash = EMPTY_HASH;
3385 return 0;
3386 }
3387
3388 size = 1 << new_hash1->size_bits;
3389 for (i = 0; i < size; i++) {
3390 hlist_for_each_entry(entry, &new_hash1->buckets[i], hlist) {
3391 /* Only add if in both @new_hash1 and @new_hash2 */
3392 if (__ftrace_lookup_ip(new_hash2, entry->ip) &&
3393 add_hash_entry(*hash, entry->ip) == NULL)
3394 return -ENOMEM;
3395 }
3396 }
3397 /* If nothing intersects, make it the empty set */
3398 if (ftrace_hash_empty(*hash)) {
3399 free_ftrace_hash(*hash);
3400 *hash = EMPTY_HASH;
3401 }
3402 return 0;
3403}
3404
3405static bool ops_equal(struct ftrace_hash *A, struct ftrace_hash *B)
3406{
3407 struct ftrace_func_entry *entry;
3408 int size;
3409 int i;
3410
3411 if (ftrace_hash_empty(A))
3412 return ftrace_hash_empty(B);
3413
3414 if (ftrace_hash_empty(B))
3415 return ftrace_hash_empty(A);
3416
3417 if (A->count != B->count)
3418 return false;
3419
3420 size = 1 << A->size_bits;
3421 for (i = 0; i < size; i++) {
3422 hlist_for_each_entry(entry, &A->buckets[i], hlist) {
3423 if (!__ftrace_lookup_ip(B, entry->ip))
3424 return false;
3425 }
3426 }
3427
3428 return true;
3429}
3430
3431static void ftrace_ops_update_code(struct ftrace_ops *ops,
3432 struct ftrace_ops_hash *old_hash);
3433
3434static int __ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
3435 struct ftrace_hash **orig_hash,
3436 struct ftrace_hash *hash,
3437 int enable)
3438{
3439 struct ftrace_ops_hash old_hash_ops;
3440 struct ftrace_hash *old_hash;
3441 int ret;
3442
3443 old_hash = *orig_hash;
3444 old_hash_ops.filter_hash = ops->func_hash->filter_hash;
3445 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
3446 ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3447 if (!ret) {
3448 ftrace_ops_update_code(ops, &old_hash_ops);
3449 free_ftrace_hash_rcu(old_hash);
3450 }
3451 return ret;
3452}
3453
3454static int ftrace_update_ops(struct ftrace_ops *ops, struct ftrace_hash *filter_hash,
3455 struct ftrace_hash *notrace_hash)
3456{
3457 int ret;
3458
3459 if (!ops_equal(filter_hash, ops->func_hash->filter_hash)) {
3460 ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->filter_hash,
3461 filter_hash, 1);
3462 if (ret < 0)
3463 return ret;
3464 }
3465
3466 if (!ops_equal(notrace_hash, ops->func_hash->notrace_hash)) {
3467 ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->notrace_hash,
3468 notrace_hash, 0);
3469 if (ret < 0)
3470 return ret;
3471 }
3472
3473 return 0;
3474}
3475
3476static int add_first_hash(struct ftrace_hash **filter_hash, struct ftrace_hash **notrace_hash,
3477 struct ftrace_ops_hash *func_hash)
3478{
3479 /* If the filter hash is not empty, simply remove the nohash from it */
3480 if (!ftrace_hash_empty(func_hash->filter_hash)) {
3481 *filter_hash = copy_hash(func_hash->filter_hash);
3482 if (!*filter_hash)
3483 return -ENOMEM;
3484 remove_hash(*filter_hash, func_hash->notrace_hash);
3485 *notrace_hash = EMPTY_HASH;
3486
3487 } else {
3488 *notrace_hash = copy_hash(func_hash->notrace_hash);
3489 if (!*notrace_hash)
3490 return -ENOMEM;
3491 *filter_hash = EMPTY_HASH;
3492 }
3493 return 0;
3494}
3495
3496static int add_next_hash(struct ftrace_hash **filter_hash, struct ftrace_hash **notrace_hash,
3497 struct ftrace_ops_hash *ops_hash, struct ftrace_ops_hash *subops_hash)
3498{
3499 int size_bits;
3500 int ret;
3501
3502 /* If the subops trace all functions so must the main ops */
3503 if (ftrace_hash_empty(ops_hash->filter_hash) ||
3504 ftrace_hash_empty(subops_hash->filter_hash)) {
3505 *filter_hash = EMPTY_HASH;
3506 } else {
3507 /*
3508 * The main ops filter hash is not empty, so its
3509 * notrace_hash had better be, as the notrace hash
3510 * is only used for empty main filter hashes.
3511 */
3512 WARN_ON_ONCE(!ftrace_hash_empty(ops_hash->notrace_hash));
3513
3514 size_bits = max(ops_hash->filter_hash->size_bits,
3515 subops_hash->filter_hash->size_bits);
3516
3517 /* Copy the subops hash */
3518 *filter_hash = alloc_and_copy_ftrace_hash(size_bits, subops_hash->filter_hash);
3519 if (!*filter_hash)
3520 return -ENOMEM;
3521 /* Remove any notrace functions from the copy */
3522 remove_hash(*filter_hash, subops_hash->notrace_hash);
3523
3524 ret = append_hash(filter_hash, ops_hash->filter_hash,
3525 size_bits);
3526 if (ret < 0) {
3527 free_ftrace_hash(*filter_hash);
3528 *filter_hash = EMPTY_HASH;
3529 return ret;
3530 }
3531 }
3532
3533 /*
3534 * Only process notrace hashes if the main filter hash is empty
3535 * (tracing all functions), otherwise the filter hash will just
3536 * remove the notrace hash functions, and the notrace hash is
3537 * not needed.
3538 */
3539 if (ftrace_hash_empty(*filter_hash)) {
3540 /*
3541 * Intersect the notrace functions. That is, if two
3542 * subops are not tracing a set of functions, the
3543 * main ops will only not trace the functions that are
3544 * in both subops, but has to trace the functions that
3545 * are only notrace in one of the subops, for the other
3546 * subops to be able to trace them.
3547 */
3548 size_bits = max(ops_hash->notrace_hash->size_bits,
3549 subops_hash->notrace_hash->size_bits);
3550 *notrace_hash = alloc_ftrace_hash(size_bits);
3551 if (!*notrace_hash)
3552 return -ENOMEM;
3553
3554 ret = intersect_hash(notrace_hash, ops_hash->notrace_hash,
3555 subops_hash->notrace_hash);
3556 if (ret < 0) {
3557 free_ftrace_hash(*notrace_hash);
3558 *notrace_hash = EMPTY_HASH;
3559 return ret;
3560 }
3561 }
3562 return 0;
3563}
3564
3565/**
3566 * ftrace_startup_subops - enable tracing for subops of an ops
3567 * @ops: Manager ops (used to pick all the functions of its subops)
3568 * @subops: A new ops to add to @ops
3569 * @command: Extra commands to use to enable tracing
3570 *
3571 * The @ops is a manager @ops that has the filter that includes all the functions
3572 * that its list of subops are tracing. Adding a new @subops will add the
3573 * functions of @subops to @ops.
3574 */
3575int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command)
3576{
3577 struct ftrace_hash *filter_hash = EMPTY_HASH;
3578 struct ftrace_hash *notrace_hash = EMPTY_HASH;
3579 struct ftrace_hash *save_filter_hash;
3580 struct ftrace_hash *save_notrace_hash;
3581 int ret;
3582
3583 if (unlikely(ftrace_disabled))
3584 return -ENODEV;
3585
3586 ftrace_ops_init(ops);
3587 ftrace_ops_init(subops);
3588
3589 if (WARN_ON_ONCE(subops->flags & FTRACE_OPS_FL_ENABLED))
3590 return -EBUSY;
3591
3592 /* Make everything canonical (Just in case!) */
3593 if (!ops->func_hash->filter_hash)
3594 ops->func_hash->filter_hash = EMPTY_HASH;
3595 if (!ops->func_hash->notrace_hash)
3596 ops->func_hash->notrace_hash = EMPTY_HASH;
3597 if (!subops->func_hash->filter_hash)
3598 subops->func_hash->filter_hash = EMPTY_HASH;
3599 if (!subops->func_hash->notrace_hash)
3600 subops->func_hash->notrace_hash = EMPTY_HASH;
3601
3602 /* For the first subops to ops just enable it normally */
3603 if (list_empty(&ops->subop_list)) {
3604
3605 /* The ops was empty, should have empty hashes */
3606 WARN_ON_ONCE(!ftrace_hash_empty(ops->func_hash->filter_hash));
3607 WARN_ON_ONCE(!ftrace_hash_empty(ops->func_hash->notrace_hash));
3608
3609 ret = add_first_hash(&filter_hash, &notrace_hash, subops->func_hash);
3610 if (ret < 0)
3611 return ret;
3612
3613 save_filter_hash = ops->func_hash->filter_hash;
3614 save_notrace_hash = ops->func_hash->notrace_hash;
3615
3616 ops->func_hash->filter_hash = filter_hash;
3617 ops->func_hash->notrace_hash = notrace_hash;
3618 list_add(&subops->list, &ops->subop_list);
3619 ret = ftrace_startup(ops, command);
3620 if (ret < 0) {
3621 list_del(&subops->list);
3622 ops->func_hash->filter_hash = save_filter_hash;
3623 ops->func_hash->notrace_hash = save_notrace_hash;
3624 free_ftrace_hash(filter_hash);
3625 free_ftrace_hash(notrace_hash);
3626 } else {
3627 free_ftrace_hash(save_filter_hash);
3628 free_ftrace_hash(save_notrace_hash);
3629 subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP;
3630 subops->managed = ops;
3631 }
3632 return ret;
3633 }
3634
3635 /*
3636 * Here there's already something attached. Here are the rules:
3637 * If the new subops and main ops filter hashes are not empty:
3638 * o Make a copy of the subops filter hash
3639 * o Remove all functions in the nohash from it.
3640 * o Add in the main hash filter functions
3641 * o Remove any of these functions from the main notrace hash
3642 */
3643
3644 ret = add_next_hash(&filter_hash, &notrace_hash, ops->func_hash, subops->func_hash);
3645 if (ret < 0)
3646 return ret;
3647
3648 list_add(&subops->list, &ops->subop_list);
3649
3650 ret = ftrace_update_ops(ops, filter_hash, notrace_hash);
3651 free_ftrace_hash(filter_hash);
3652 free_ftrace_hash(notrace_hash);
3653 if (ret < 0) {
3654 list_del(&subops->list);
3655 } else {
3656 subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP;
3657 subops->managed = ops;
3658 }
3659 return ret;
3660}
3661
3662static int rebuild_hashes(struct ftrace_hash **filter_hash, struct ftrace_hash **notrace_hash,
3663 struct ftrace_ops *ops)
3664{
3665 struct ftrace_ops_hash temp_hash;
3666 struct ftrace_ops *subops;
3667 bool first = true;
3668 int ret;
3669
3670 temp_hash.filter_hash = EMPTY_HASH;
3671 temp_hash.notrace_hash = EMPTY_HASH;
3672
3673 list_for_each_entry(subops, &ops->subop_list, list) {
3674 *filter_hash = EMPTY_HASH;
3675 *notrace_hash = EMPTY_HASH;
3676
3677 if (first) {
3678 ret = add_first_hash(filter_hash, notrace_hash, subops->func_hash);
3679 if (ret < 0)
3680 return ret;
3681 first = false;
3682 } else {
3683 ret = add_next_hash(filter_hash, notrace_hash,
3684 &temp_hash, subops->func_hash);
3685 if (ret < 0) {
3686 free_ftrace_hash(temp_hash.filter_hash);
3687 free_ftrace_hash(temp_hash.notrace_hash);
3688 return ret;
3689 }
3690 }
3691
3692 free_ftrace_hash(temp_hash.filter_hash);
3693 free_ftrace_hash(temp_hash.notrace_hash);
3694
3695 temp_hash.filter_hash = *filter_hash;
3696 temp_hash.notrace_hash = *notrace_hash;
3697 }
3698 return 0;
3699}
3700
3701/**
3702 * ftrace_shutdown_subops - Remove a subops from a manager ops
3703 * @ops: A manager ops to remove @subops from
3704 * @subops: The subops to remove from @ops
3705 * @command: Any extra command flags to add to modifying the text
3706 *
3707 * Removes the functions being traced by the @subops from @ops. Note, it
3708 * will not affect functions that are being traced by other subops that
3709 * still exist in @ops.
3710 *
3711 * If the last subops is removed from @ops, then @ops is shutdown normally.
3712 */
3713int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command)
3714{
3715 struct ftrace_hash *filter_hash = EMPTY_HASH;
3716 struct ftrace_hash *notrace_hash = EMPTY_HASH;
3717 int ret;
3718
3719 if (unlikely(ftrace_disabled))
3720 return -ENODEV;
3721
3722 if (WARN_ON_ONCE(!(subops->flags & FTRACE_OPS_FL_ENABLED)))
3723 return -EINVAL;
3724
3725 list_del(&subops->list);
3726
3727 if (list_empty(&ops->subop_list)) {
3728 /* Last one, just disable the current ops */
3729
3730 ret = ftrace_shutdown(ops, command);
3731 if (ret < 0) {
3732 list_add(&subops->list, &ops->subop_list);
3733 return ret;
3734 }
3735
3736 subops->flags &= ~FTRACE_OPS_FL_ENABLED;
3737
3738 free_ftrace_hash(ops->func_hash->filter_hash);
3739 free_ftrace_hash(ops->func_hash->notrace_hash);
3740 ops->func_hash->filter_hash = EMPTY_HASH;
3741 ops->func_hash->notrace_hash = EMPTY_HASH;
3742 subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP);
3743 subops->managed = NULL;
3744
3745 return 0;
3746 }
3747
3748 /* Rebuild the hashes without subops */
3749 ret = rebuild_hashes(&filter_hash, &notrace_hash, ops);
3750 if (ret < 0)
3751 return ret;
3752
3753 ret = ftrace_update_ops(ops, filter_hash, notrace_hash);
3754 if (ret < 0) {
3755 list_add(&subops->list, &ops->subop_list);
3756 } else {
3757 subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP);
3758 subops->managed = NULL;
3759 }
3760 free_ftrace_hash(filter_hash);
3761 free_ftrace_hash(notrace_hash);
3762 return ret;
3763}
3764
3765static int ftrace_hash_move_and_update_subops(struct ftrace_ops *subops,
3766 struct ftrace_hash **orig_subhash,
3767 struct ftrace_hash *hash)
3768{
3769 struct ftrace_ops *ops = subops->managed;
3770 struct ftrace_hash *notrace_hash;
3771 struct ftrace_hash *filter_hash;
3772 struct ftrace_hash *save_hash;
3773 struct ftrace_hash *new_hash;
3774 int ret;
3775
3776 /* Manager ops can not be subops (yet) */
3777 if (WARN_ON_ONCE(!ops || ops->flags & FTRACE_OPS_FL_SUBOP))
3778 return -EINVAL;
3779
3780 /* Move the new hash over to the subops hash */
3781 save_hash = *orig_subhash;
3782 *orig_subhash = __ftrace_hash_move(hash);
3783 if (!*orig_subhash) {
3784 *orig_subhash = save_hash;
3785 return -ENOMEM;
3786 }
3787
3788 ret = rebuild_hashes(&filter_hash, &notrace_hash, ops);
3789 if (!ret) {
3790 ret = ftrace_update_ops(ops, filter_hash, notrace_hash);
3791 free_ftrace_hash(filter_hash);
3792 free_ftrace_hash(notrace_hash);
3793 }
3794
3795 if (ret) {
3796 /* Put back the original hash */
3797 new_hash = *orig_subhash;
3798 *orig_subhash = save_hash;
3799 free_ftrace_hash_rcu(new_hash);
3800 } else {
3801 free_ftrace_hash_rcu(save_hash);
3802 }
3803 return ret;
3804}
3805
3806
3807u64 ftrace_update_time;
3808u64 ftrace_total_mod_time;
3809unsigned long ftrace_update_tot_cnt;
3810unsigned long ftrace_number_of_pages;
3811unsigned long ftrace_number_of_groups;
3812
3813static inline int ops_traces_mod(struct ftrace_ops *ops)
3814{
3815 /*
3816 * Filter_hash being empty will default to trace module.
3817 * But notrace hash requires a test of individual module functions.
3818 */
3819 return ftrace_hash_empty(ops->func_hash->filter_hash) &&
3820 ftrace_hash_empty(ops->func_hash->notrace_hash);
3821}
3822
3823static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
3824{
3825 bool init_nop = ftrace_need_init_nop();
3826 struct ftrace_page *pg;
3827 struct dyn_ftrace *p;
3828 u64 start, stop, update_time;
3829 unsigned long update_cnt = 0;
3830 unsigned long rec_flags = 0;
3831 int i;
3832
3833 start = ftrace_now(raw_smp_processor_id());
3834
3835 /*
3836 * When a module is loaded, this function is called to convert
3837 * the calls to mcount in its text to nops, and also to create
3838 * an entry in the ftrace data. Now, if ftrace is activated
3839 * after this call, but before the module sets its text to
3840 * read-only, the modification of enabling ftrace can fail if
3841 * the read-only is done while ftrace is converting the calls.
3842 * To prevent this, the module's records are set as disabled
3843 * and will be enabled after the call to set the module's text
3844 * to read-only.
3845 */
3846 if (mod)
3847 rec_flags |= FTRACE_FL_DISABLED;
3848
3849 for (pg = new_pgs; pg; pg = pg->next) {
3850
3851 for (i = 0; i < pg->index; i++) {
3852
3853 /* If something went wrong, bail without enabling anything */
3854 if (unlikely(ftrace_disabled))
3855 return -1;
3856
3857 p = &pg->records[i];
3858 p->flags = rec_flags;
3859
3860 /*
3861 * Do the initial record conversion from mcount jump
3862 * to the NOP instructions.
3863 */
3864 if (init_nop && !ftrace_nop_initialize(mod, p))
3865 break;
3866
3867 update_cnt++;
3868 }
3869 }
3870
3871 stop = ftrace_now(raw_smp_processor_id());
3872 update_time = stop - start;
3873 if (mod)
3874 ftrace_total_mod_time += update_time;
3875 else
3876 ftrace_update_time = update_time;
3877 ftrace_update_tot_cnt += update_cnt;
3878
3879 return 0;
3880}
3881
3882static int ftrace_allocate_records(struct ftrace_page *pg, int count,
3883 unsigned long *num_pages)
3884{
3885 int order;
3886 int pages;
3887 int cnt;
3888
3889 if (WARN_ON(!count))
3890 return -EINVAL;
3891
3892 /* We want to fill as much as possible, with no empty pages */
3893 pages = DIV_ROUND_UP(count * ENTRY_SIZE, PAGE_SIZE);
3894 order = fls(pages) - 1;
3895
3896 again:
3897 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
3898
3899 if (!pg->records) {
3900 /* if we can't allocate this size, try something smaller */
3901 if (!order)
3902 return -ENOMEM;
3903 order--;
3904 goto again;
3905 }
3906
3907 ftrace_number_of_pages += 1 << order;
3908 *num_pages += 1 << order;
3909 ftrace_number_of_groups++;
3910
3911 cnt = ENTRIES_PER_PAGE_GROUP(order);
3912 pg->order = order;
3913
3914 if (cnt > count)
3915 cnt = count;
3916
3917 return cnt;
3918}
3919
3920static void ftrace_free_pages(struct ftrace_page *pages)
3921{
3922 struct ftrace_page *pg = pages;
3923
3924 while (pg) {
3925 if (pg->records) {
3926 free_pages((unsigned long)pg->records, pg->order);
3927 ftrace_number_of_pages -= 1 << pg->order;
3928 }
3929 pages = pg->next;
3930 kfree(pg);
3931 pg = pages;
3932 ftrace_number_of_groups--;
3933 }
3934}
3935
3936static struct ftrace_page *
3937ftrace_allocate_pages(unsigned long num_to_init, unsigned long *num_pages)
3938{
3939 struct ftrace_page *start_pg;
3940 struct ftrace_page *pg;
3941 int cnt;
3942
3943 *num_pages = 0;
3944
3945 if (!num_to_init)
3946 return NULL;
3947
3948 start_pg = pg = kzalloc_obj(*pg);
3949 if (!pg)
3950 return NULL;
3951
3952 /*
3953 * Try to allocate as much as possible in one continues
3954 * location that fills in all of the space. We want to
3955 * waste as little space as possible.
3956 */
3957 for (;;) {
3958 cnt = ftrace_allocate_records(pg, num_to_init, num_pages);
3959 if (cnt < 0)
3960 goto free_pages;
3961
3962 num_to_init -= cnt;
3963 if (!num_to_init)
3964 break;
3965
3966 pg->next = kzalloc_obj(*pg);
3967 if (!pg->next)
3968 goto free_pages;
3969
3970 pg = pg->next;
3971 }
3972
3973 return start_pg;
3974
3975 free_pages:
3976 ftrace_free_pages(start_pg);
3977 pr_info("ftrace: FAILED to allocate memory for functions\n");
3978 return NULL;
3979}
3980
3981#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
3982
3983struct ftrace_iterator {
3984 loff_t pos;
3985 loff_t func_pos;
3986 loff_t mod_pos;
3987 struct ftrace_page *pg;
3988 struct dyn_ftrace *func;
3989 struct ftrace_func_probe *probe;
3990 struct ftrace_func_entry *probe_entry;
3991 struct trace_parser parser;
3992 struct ftrace_hash *hash;
3993 struct ftrace_ops *ops;
3994 struct trace_array *tr;
3995 struct list_head *mod_list;
3996 int pidx;
3997 int idx;
3998 unsigned flags;
3999};
4000
4001static void *
4002t_probe_next(struct seq_file *m, loff_t *pos)
4003{
4004 struct ftrace_iterator *iter = m->private;
4005 struct trace_array *tr = iter->ops->private;
4006 struct list_head *func_probes;
4007 struct ftrace_hash *hash;
4008 struct list_head *next;
4009 struct hlist_node *hnd = NULL;
4010 struct hlist_head *hhd;
4011 int size;
4012
4013 (*pos)++;
4014 iter->pos = *pos;
4015
4016 if (!tr)
4017 return NULL;
4018
4019 func_probes = &tr->func_probes;
4020 if (list_empty(func_probes))
4021 return NULL;
4022
4023 if (!iter->probe) {
4024 next = func_probes->next;
4025 iter->probe = list_entry(next, struct ftrace_func_probe, list);
4026 }
4027
4028 if (iter->probe_entry)
4029 hnd = &iter->probe_entry->hlist;
4030
4031 hash = iter->probe->ops.func_hash->filter_hash;
4032
4033 /*
4034 * A probe being registered may temporarily have an empty hash
4035 * and it's at the end of the func_probes list.
4036 */
4037 if (!hash || hash == EMPTY_HASH)
4038 return NULL;
4039
4040 size = 1 << hash->size_bits;
4041
4042 retry:
4043 if (iter->pidx >= size) {
4044 if (iter->probe->list.next == func_probes)
4045 return NULL;
4046 next = iter->probe->list.next;
4047 iter->probe = list_entry(next, struct ftrace_func_probe, list);
4048 hash = iter->probe->ops.func_hash->filter_hash;
4049 size = 1 << hash->size_bits;
4050 iter->pidx = 0;
4051 }
4052
4053 hhd = &hash->buckets[iter->pidx];
4054
4055 if (hlist_empty(hhd)) {
4056 iter->pidx++;
4057 hnd = NULL;
4058 goto retry;
4059 }
4060
4061 if (!hnd)
4062 hnd = hhd->first;
4063 else {
4064 hnd = hnd->next;
4065 if (!hnd) {
4066 iter->pidx++;
4067 goto retry;
4068 }
4069 }
4070
4071 if (WARN_ON_ONCE(!hnd))
4072 return NULL;
4073
4074 iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
4075
4076 return iter;
4077}
4078
4079static void *t_probe_start(struct seq_file *m, loff_t *pos)
4080{
4081 struct ftrace_iterator *iter = m->private;
4082 void *p = NULL;
4083 loff_t l;
4084
4085 if (!(iter->flags & FTRACE_ITER_DO_PROBES))
4086 return NULL;
4087
4088 if (iter->mod_pos > *pos)
4089 return NULL;
4090
4091 iter->probe = NULL;
4092 iter->probe_entry = NULL;
4093 iter->pidx = 0;
4094 for (l = 0; l <= (*pos - iter->mod_pos); ) {
4095 p = t_probe_next(m, &l);
4096 if (!p)
4097 break;
4098 }
4099 if (!p)
4100 return NULL;
4101
4102 /* Only set this if we have an item */
4103 iter->flags |= FTRACE_ITER_PROBE;
4104
4105 return iter;
4106}
4107
4108static int
4109t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
4110{
4111 struct ftrace_func_entry *probe_entry;
4112 struct ftrace_probe_ops *probe_ops;
4113 struct ftrace_func_probe *probe;
4114
4115 probe = iter->probe;
4116 probe_entry = iter->probe_entry;
4117
4118 if (WARN_ON_ONCE(!probe || !probe_entry))
4119 return -EIO;
4120
4121 probe_ops = probe->probe_ops;
4122
4123 if (probe_ops->print)
4124 return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
4125
4126 seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
4127 (void *)probe_ops->func);
4128
4129 return 0;
4130}
4131
4132static void *
4133t_mod_next(struct seq_file *m, loff_t *pos)
4134{
4135 struct ftrace_iterator *iter = m->private;
4136 struct trace_array *tr = iter->tr;
4137
4138 (*pos)++;
4139 iter->pos = *pos;
4140
4141 iter->mod_list = iter->mod_list->next;
4142
4143 if (iter->mod_list == &tr->mod_trace ||
4144 iter->mod_list == &tr->mod_notrace) {
4145 iter->flags &= ~FTRACE_ITER_MOD;
4146 return NULL;
4147 }
4148
4149 iter->mod_pos = *pos;
4150
4151 return iter;
4152}
4153
4154static void *t_mod_start(struct seq_file *m, loff_t *pos)
4155{
4156 struct ftrace_iterator *iter = m->private;
4157 void *p = NULL;
4158 loff_t l;
4159
4160 if (iter->func_pos > *pos)
4161 return NULL;
4162
4163 iter->mod_pos = iter->func_pos;
4164
4165 /* probes are only available if tr is set */
4166 if (!iter->tr)
4167 return NULL;
4168
4169 for (l = 0; l <= (*pos - iter->func_pos); ) {
4170 p = t_mod_next(m, &l);
4171 if (!p)
4172 break;
4173 }
4174 if (!p) {
4175 iter->flags &= ~FTRACE_ITER_MOD;
4176 return t_probe_start(m, pos);
4177 }
4178
4179 /* Only set this if we have an item */
4180 iter->flags |= FTRACE_ITER_MOD;
4181
4182 return iter;
4183}
4184
4185static int
4186t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
4187{
4188 struct ftrace_mod_load *ftrace_mod;
4189 struct trace_array *tr = iter->tr;
4190
4191 if (WARN_ON_ONCE(!iter->mod_list) ||
4192 iter->mod_list == &tr->mod_trace ||
4193 iter->mod_list == &tr->mod_notrace)
4194 return -EIO;
4195
4196 ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);
4197
4198 if (ftrace_mod->func)
4199 seq_printf(m, "%s", ftrace_mod->func);
4200 else
4201 seq_putc(m, '*');
4202
4203 seq_printf(m, ":mod:%s\n", ftrace_mod->module);
4204
4205 return 0;
4206}
4207
4208static void *
4209t_func_next(struct seq_file *m, loff_t *pos)
4210{
4211 struct ftrace_iterator *iter = m->private;
4212 struct dyn_ftrace *rec = NULL;
4213
4214 (*pos)++;
4215
4216 retry:
4217 if (iter->idx >= iter->pg->index) {
4218 if (iter->pg->next) {
4219 iter->pg = iter->pg->next;
4220 iter->idx = 0;
4221 goto retry;
4222 }
4223 } else {
4224 rec = &iter->pg->records[iter->idx++];
4225 if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
4226 !ftrace_lookup_ip(iter->hash, rec->ip)) ||
4227
4228 ((iter->flags & FTRACE_ITER_ENABLED) &&
4229 !(rec->flags & FTRACE_FL_ENABLED)) ||
4230
4231 ((iter->flags & FTRACE_ITER_TOUCHED) &&
4232 !(rec->flags & FTRACE_FL_TOUCHED))) {
4233
4234 rec = NULL;
4235 goto retry;
4236 }
4237 }
4238
4239 if (!rec)
4240 return NULL;
4241
4242 iter->pos = iter->func_pos = *pos;
4243 iter->func = rec;
4244
4245 return iter;
4246}
4247
4248static void *
4249t_next(struct seq_file *m, void *v, loff_t *pos)
4250{
4251 struct ftrace_iterator *iter = m->private;
4252 loff_t l = *pos; /* t_probe_start() must use original pos */
4253 void *ret;
4254
4255 if (unlikely(ftrace_disabled))
4256 return NULL;
4257
4258 if (iter->flags & FTRACE_ITER_PROBE)
4259 return t_probe_next(m, pos);
4260
4261 if (iter->flags & FTRACE_ITER_MOD)
4262 return t_mod_next(m, pos);
4263
4264 if (iter->flags & FTRACE_ITER_PRINTALL) {
4265 /* next must increment pos, and t_probe_start does not */
4266 (*pos)++;
4267 return t_mod_start(m, &l);
4268 }
4269
4270 ret = t_func_next(m, pos);
4271
4272 if (!ret)
4273 return t_mod_start(m, &l);
4274
4275 return ret;
4276}
4277
4278static void reset_iter_read(struct ftrace_iterator *iter)
4279{
4280 iter->pos = 0;
4281 iter->func_pos = 0;
4282 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
4283}
4284
4285static void *t_start(struct seq_file *m, loff_t *pos)
4286{
4287 struct ftrace_iterator *iter = m->private;
4288 void *p = NULL;
4289 loff_t l;
4290
4291 mutex_lock(&ftrace_lock);
4292
4293 if (unlikely(ftrace_disabled))
4294 return NULL;
4295
4296 /*
4297 * If an lseek was done, then reset and start from beginning.
4298 */
4299 if (*pos < iter->pos)
4300 reset_iter_read(iter);
4301
4302 /*
4303 * For set_ftrace_filter reading, if we have the filter
4304 * off, we can short cut and just print out that all
4305 * functions are enabled.
4306 */
4307 if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
4308 ftrace_hash_empty(iter->hash)) {
4309 iter->func_pos = 1; /* Account for the message */
4310 if (*pos > 0)
4311 return t_mod_start(m, pos);
4312 iter->flags |= FTRACE_ITER_PRINTALL;
4313 /* reset in case of seek/pread */
4314 iter->flags &= ~FTRACE_ITER_PROBE;
4315 return iter;
4316 }
4317
4318 if (iter->flags & FTRACE_ITER_MOD)
4319 return t_mod_start(m, pos);
4320
4321 /*
4322 * Unfortunately, we need to restart at ftrace_pages_start
4323 * every time we let go of the ftrace_mutex. This is because
4324 * those pointers can change without the lock.
4325 */
4326 iter->pg = ftrace_pages_start;
4327 iter->idx = 0;
4328 for (l = 0; l <= *pos; ) {
4329 p = t_func_next(m, &l);
4330 if (!p)
4331 break;
4332 }
4333
4334 if (!p)
4335 return t_mod_start(m, pos);
4336
4337 return iter;
4338}
4339
4340static void t_stop(struct seq_file *m, void *p)
4341{
4342 mutex_unlock(&ftrace_lock);
4343}
4344
4345void * __weak
4346arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
4347{
4348 return NULL;
4349}
4350
4351static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
4352 struct dyn_ftrace *rec)
4353{
4354 void *ptr;
4355
4356 ptr = arch_ftrace_trampoline_func(ops, rec);
4357 if (ptr)
4358 seq_printf(m, " ->%pS", ptr);
4359}
4360
4361#ifdef FTRACE_MCOUNT_MAX_OFFSET
4362/*
4363 * Weak functions can still have an mcount/fentry that is saved in
4364 * the __mcount_loc section. These can be detected by having a
4365 * symbol offset of greater than FTRACE_MCOUNT_MAX_OFFSET, as the
4366 * symbol found by kallsyms is not the function that the mcount/fentry
4367 * is part of. The offset is much greater in these cases.
4368 *
4369 * Test the record to make sure that the ip points to a valid kallsyms
4370 * and if not, mark it disabled.
4371 */
4372static int test_for_valid_rec(struct dyn_ftrace *rec)
4373{
4374 char str[KSYM_SYMBOL_LEN];
4375 unsigned long offset;
4376 const char *ret;
4377
4378 ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str);
4379
4380 /* Weak functions can cause invalid addresses */
4381 if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) {
4382 rec->flags |= FTRACE_FL_DISABLED;
4383 return 0;
4384 }
4385 return 1;
4386}
4387
4388static struct workqueue_struct *ftrace_check_wq __initdata;
4389static struct work_struct ftrace_check_work __initdata;
4390
4391/*
4392 * Scan all the mcount/fentry entries to make sure they are valid.
4393 */
4394static __init void ftrace_check_work_func(struct work_struct *work)
4395{
4396 struct ftrace_page *pg;
4397 struct dyn_ftrace *rec;
4398
4399 mutex_lock(&ftrace_lock);
4400 do_for_each_ftrace_rec(pg, rec) {
4401 test_for_valid_rec(rec);
4402 } while_for_each_ftrace_rec();
4403 mutex_unlock(&ftrace_lock);
4404}
4405
4406static int __init ftrace_check_for_weak_functions(void)
4407{
4408 INIT_WORK(&ftrace_check_work, ftrace_check_work_func);
4409
4410 ftrace_check_wq = alloc_workqueue("ftrace_check_wq", WQ_UNBOUND, 0);
4411
4412 queue_work(ftrace_check_wq, &ftrace_check_work);
4413 return 0;
4414}
4415
4416static int __init ftrace_check_sync(void)
4417{
4418 /* Make sure the ftrace_check updates are finished */
4419 if (ftrace_check_wq)
4420 destroy_workqueue(ftrace_check_wq);
4421 return 0;
4422}
4423
4424late_initcall_sync(ftrace_check_sync);
4425subsys_initcall(ftrace_check_for_weak_functions);
4426
4427static int print_rec(struct seq_file *m, unsigned long ip)
4428{
4429 unsigned long offset;
4430 char str[KSYM_SYMBOL_LEN];
4431 char *modname;
4432 const char *ret;
4433
4434 ret = kallsyms_lookup(ip, NULL, &offset, &modname, str);
4435 /* Weak functions can cause invalid addresses */
4436 if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) {
4437 snprintf(str, KSYM_SYMBOL_LEN, "%s_%ld",
4438 FTRACE_INVALID_FUNCTION, offset);
4439 ret = NULL;
4440 }
4441
4442 seq_puts(m, str);
4443 if (modname)
4444 seq_printf(m, " [%s]", modname);
4445 return ret == NULL ? -1 : 0;
4446}
4447#else
4448static inline int test_for_valid_rec(struct dyn_ftrace *rec)
4449{
4450 return 1;
4451}
4452
4453static inline int print_rec(struct seq_file *m, unsigned long ip)
4454{
4455 seq_printf(m, "%ps", (void *)ip);
4456 return 0;
4457}
4458#endif
4459
4460static void print_subops(struct seq_file *m, struct ftrace_ops *ops, struct dyn_ftrace *rec)
4461{
4462 struct ftrace_ops *subops;
4463 bool first = true;
4464
4465 list_for_each_entry(subops, &ops->subop_list, list) {
4466 if (!((subops->flags & FTRACE_OPS_FL_ENABLED) &&
4467 hash_contains_ip(rec->ip, subops->func_hash)))
4468 continue;
4469 if (first) {
4470 seq_printf(m, "\tsubops:");
4471 first = false;
4472 }
4473#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4474 if (subops->flags & FTRACE_OPS_FL_GRAPH) {
4475 struct fgraph_ops *gops;
4476
4477 gops = container_of(subops, struct fgraph_ops, ops);
4478 seq_printf(m, " {ent:%pS ret:%pS}",
4479 (void *)gops->entryfunc,
4480 (void *)gops->retfunc);
4481 continue;
4482 }
4483#endif
4484 if (subops->trampoline) {
4485 seq_printf(m, " {%pS (%pS)}",
4486 (void *)subops->trampoline,
4487 (void *)subops->func);
4488 add_trampoline_func(m, subops, rec);
4489 } else {
4490 seq_printf(m, " {%pS}",
4491 (void *)subops->func);
4492 }
4493 }
4494}
4495
4496static int t_show(struct seq_file *m, void *v)
4497{
4498 struct ftrace_iterator *iter = m->private;
4499 struct dyn_ftrace *rec;
4500
4501 if (iter->flags & FTRACE_ITER_PROBE)
4502 return t_probe_show(m, iter);
4503
4504 if (iter->flags & FTRACE_ITER_MOD)
4505 return t_mod_show(m, iter);
4506
4507 if (iter->flags & FTRACE_ITER_PRINTALL) {
4508 if (iter->flags & FTRACE_ITER_NOTRACE)
4509 seq_puts(m, "#### no functions disabled ####\n");
4510 else
4511 seq_puts(m, "#### all functions enabled ####\n");
4512 return 0;
4513 }
4514
4515 rec = iter->func;
4516
4517 if (!rec)
4518 return 0;
4519
4520 if (iter->flags & FTRACE_ITER_ADDRS)
4521 seq_printf(m, "%lx ", rec->ip);
4522
4523 if (print_rec(m, rec->ip)) {
4524 /* This should only happen when a rec is disabled */
4525 WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED));
4526 seq_putc(m, '\n');
4527 return 0;
4528 }
4529
4530 if (iter->flags & (FTRACE_ITER_ENABLED | FTRACE_ITER_TOUCHED)) {
4531 struct ftrace_ops *ops;
4532
4533 seq_printf(m, " (%ld)%s%s%s%s%s",
4534 ftrace_rec_count(rec),
4535 rec->flags & FTRACE_FL_REGS ? " R" : " ",
4536 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ",
4537 rec->flags & FTRACE_FL_DIRECT ? " D" : " ",
4538 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ",
4539 rec->flags & FTRACE_FL_MODIFIED ? " M " : " ");
4540 if (rec->flags & FTRACE_FL_TRAMP_EN) {
4541 ops = ftrace_find_tramp_ops_any(rec);
4542 if (ops) {
4543 do {
4544 seq_printf(m, "\ttramp: %pS (%pS)",
4545 (void *)ops->trampoline,
4546 (void *)ops->func);
4547 add_trampoline_func(m, ops, rec);
4548 print_subops(m, ops, rec);
4549 ops = ftrace_find_tramp_ops_next(rec, ops);
4550 } while (ops);
4551 } else
4552 seq_puts(m, "\ttramp: ERROR!");
4553 } else {
4554 add_trampoline_func(m, NULL, rec);
4555 }
4556 if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
4557 ops = ftrace_find_unique_ops(rec);
4558 if (ops) {
4559 seq_printf(m, "\tops: %pS (%pS)",
4560 ops, ops->func);
4561 print_subops(m, ops, rec);
4562 } else {
4563 seq_puts(m, "\tops: ERROR!");
4564 }
4565 }
4566 if (rec->flags & FTRACE_FL_DIRECT) {
4567 unsigned long direct;
4568
4569 direct = ftrace_find_rec_direct(rec->ip);
4570 if (direct) {
4571 seq_printf(m, "\n\tdirect%s-->%pS",
4572 ftrace_is_jmp(direct) ? "(jmp)" : "",
4573 (void *)ftrace_jmp_get(direct));
4574 }
4575 }
4576 }
4577
4578 seq_putc(m, '\n');
4579
4580 return 0;
4581}
4582
4583static const struct seq_operations show_ftrace_seq_ops = {
4584 .start = t_start,
4585 .next = t_next,
4586 .stop = t_stop,
4587 .show = t_show,
4588};
4589
4590static int
4591ftrace_avail_open(struct inode *inode, struct file *file)
4592{
4593 struct ftrace_iterator *iter;
4594 int ret;
4595
4596 ret = security_locked_down(LOCKDOWN_TRACEFS);
4597 if (ret)
4598 return ret;
4599
4600 if (unlikely(ftrace_disabled))
4601 return -ENODEV;
4602
4603 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
4604 if (!iter)
4605 return -ENOMEM;
4606
4607 iter->pg = ftrace_pages_start;
4608 iter->ops = &global_ops;
4609
4610 return 0;
4611}
4612
4613static int
4614ftrace_enabled_open(struct inode *inode, struct file *file)
4615{
4616 struct ftrace_iterator *iter;
4617
4618 /*
4619 * This shows us what functions are currently being
4620 * traced and by what. Not sure if we want lockdown
4621 * to hide such critical information for an admin.
4622 * Although, perhaps it can show information we don't
4623 * want people to see, but if something is tracing
4624 * something, we probably want to know about it.
4625 */
4626
4627 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
4628 if (!iter)
4629 return -ENOMEM;
4630
4631 iter->pg = ftrace_pages_start;
4632 iter->flags = FTRACE_ITER_ENABLED;
4633 iter->ops = &global_ops;
4634
4635 return 0;
4636}
4637
4638static int
4639ftrace_touched_open(struct inode *inode, struct file *file)
4640{
4641 struct ftrace_iterator *iter;
4642
4643 /*
4644 * This shows us what functions have ever been enabled
4645 * (traced, direct, patched, etc). Not sure if we want lockdown
4646 * to hide such critical information for an admin.
4647 * Although, perhaps it can show information we don't
4648 * want people to see, but if something had traced
4649 * something, we probably want to know about it.
4650 */
4651
4652 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
4653 if (!iter)
4654 return -ENOMEM;
4655
4656 iter->pg = ftrace_pages_start;
4657 iter->flags = FTRACE_ITER_TOUCHED;
4658 iter->ops = &global_ops;
4659
4660 return 0;
4661}
4662
4663static int
4664ftrace_avail_addrs_open(struct inode *inode, struct file *file)
4665{
4666 struct ftrace_iterator *iter;
4667 int ret;
4668
4669 ret = security_locked_down(LOCKDOWN_TRACEFS);
4670 if (ret)
4671 return ret;
4672
4673 if (unlikely(ftrace_disabled))
4674 return -ENODEV;
4675
4676 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
4677 if (!iter)
4678 return -ENOMEM;
4679
4680 iter->pg = ftrace_pages_start;
4681 iter->flags = FTRACE_ITER_ADDRS;
4682 iter->ops = &global_ops;
4683
4684 return 0;
4685}
4686
4687/**
4688 * ftrace_regex_open - initialize function tracer filter files
4689 * @tr: The trace_array that holds the ftrace_ops [optional]
4690 * @ops: The ftrace_ops that hold the hash filters [optional]
4691 * @flag: The type of filter to process
4692 * @inode: The inode, usually passed in to your open routine
4693 * @file: The file, usually passed in to your open routine
4694 *
4695 * ftrace_regex_open() initializes the filter files for the
4696 * @ops. Depending on @flag it may process the filter hash or
4697 * the notrace hash of @ops. With this called from the open
4698 * routine, you can use ftrace_filter_write() for the write
4699 * routine if @flag has FTRACE_ITER_FILTER set, or
4700 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
4701 * tracing_lseek() should be used as the lseek routine, and
4702 * release must call ftrace_regex_release().
4703 *
4704 * Note, If @tr is not NULL, its reference has to be taken before
4705 * @ops may be referenced.
4706 * If @ops is NULL and @tr is not, then @tr->ops is used.
4707 * If @tr is NULL and @ops is not then @ops->private is uesd for @tr.
4708 * If both @tr and @ops are NULL, then the &global_ops is
4709 * to be used, and @tr will be the global_ops.private pointer.
4710 *
4711 * Returns: 0 on success or a negative errno value on failure
4712 */
4713int
4714ftrace_regex_open(struct trace_array *tr, struct ftrace_ops *ops, int flag,
4715 struct inode *inode, struct file *file)
4716{
4717 struct ftrace_iterator *iter = NULL;
4718 struct ftrace_hash *hash;
4719 struct list_head *mod_head;
4720 int ret = -ENODEV;
4721
4722 if (unlikely(ftrace_disabled))
4723 return -ENODEV;
4724
4725 if (!tr) {
4726 if (!ops)
4727 ops = &global_ops;
4728 tr = ops->private;
4729 }
4730
4731 if (tracing_check_open_get_tr(tr))
4732 return -ENODEV;
4733
4734 if (!ops)
4735 ops = tr->ops;
4736
4737 if (WARN_ON_ONCE(!ops))
4738 goto out;
4739
4740 ftrace_ops_init(ops);
4741
4742 ret = -ENOMEM;
4743 iter = kzalloc_obj(*iter);
4744 if (!iter)
4745 goto out;
4746
4747 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX))
4748 goto out;
4749
4750 iter->ops = ops;
4751 iter->flags = flag;
4752 iter->tr = tr;
4753
4754 mutex_lock(&ops->func_hash->regex_lock);
4755
4756 if (flag & FTRACE_ITER_NOTRACE) {
4757 hash = ops->func_hash->notrace_hash;
4758 mod_head = tr ? &tr->mod_notrace : NULL;
4759 } else {
4760 hash = ops->func_hash->filter_hash;
4761 mod_head = tr ? &tr->mod_trace : NULL;
4762 }
4763
4764 iter->mod_list = mod_head;
4765
4766 if (file->f_mode & FMODE_WRITE) {
4767 const int size_bits = FTRACE_HASH_DEFAULT_BITS;
4768
4769 if (file->f_flags & O_TRUNC) {
4770 iter->hash = alloc_ftrace_hash(size_bits);
4771 clear_ftrace_mod_list(mod_head);
4772 } else {
4773 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
4774 }
4775 } else {
4776 if (hash)
4777 iter->hash = alloc_and_copy_ftrace_hash(hash->size_bits, hash);
4778 else
4779 iter->hash = EMPTY_HASH;
4780 }
4781
4782 if (!iter->hash) {
4783 trace_parser_put(&iter->parser);
4784 goto out_unlock;
4785 }
4786
4787 ret = 0;
4788
4789 if (file->f_mode & FMODE_READ) {
4790 iter->pg = ftrace_pages_start;
4791
4792 ret = seq_open(file, &show_ftrace_seq_ops);
4793 if (!ret) {
4794 struct seq_file *m = file->private_data;
4795 m->private = iter;
4796 } else {
4797 /* Failed */
4798 free_ftrace_hash(iter->hash);
4799 trace_parser_put(&iter->parser);
4800 }
4801 } else
4802 file->private_data = iter;
4803
4804 out_unlock:
4805 mutex_unlock(&ops->func_hash->regex_lock);
4806
4807 out:
4808 if (ret) {
4809 kfree(iter);
4810 if (tr)
4811 trace_array_put(tr);
4812 }
4813
4814 return ret;
4815}
4816
4817static int
4818ftrace_filter_open(struct inode *inode, struct file *file)
4819{
4820 struct trace_array *tr = inode->i_private;
4821
4822 return ftrace_regex_open(tr, NULL,
4823 FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
4824 inode, file);
4825}
4826
4827static int
4828ftrace_notrace_open(struct inode *inode, struct file *file)
4829{
4830 struct trace_array *tr = inode->i_private;
4831
4832 return ftrace_regex_open(tr, NULL, FTRACE_ITER_NOTRACE,
4833 inode, file);
4834}
4835
4836/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
4837struct ftrace_glob {
4838 char *search;
4839 unsigned len;
4840 int type;
4841};
4842
4843/*
4844 * If symbols in an architecture don't correspond exactly to the user-visible
4845 * name of what they represent, it is possible to define this function to
4846 * perform the necessary adjustments.
4847*/
4848char * __weak arch_ftrace_match_adjust(char *str, const char *search)
4849{
4850 return str;
4851}
4852
4853static int ftrace_match(char *str, struct ftrace_glob *g)
4854{
4855 int matched = 0;
4856 int slen;
4857
4858 str = arch_ftrace_match_adjust(str, g->search);
4859
4860 switch (g->type) {
4861 case MATCH_FULL:
4862 if (strcmp(str, g->search) == 0)
4863 matched = 1;
4864 break;
4865 case MATCH_FRONT_ONLY:
4866 if (strncmp(str, g->search, g->len) == 0)
4867 matched = 1;
4868 break;
4869 case MATCH_MIDDLE_ONLY:
4870 if (strstr(str, g->search))
4871 matched = 1;
4872 break;
4873 case MATCH_END_ONLY:
4874 slen = strlen(str);
4875 if (slen >= g->len &&
4876 memcmp(str + slen - g->len, g->search, g->len) == 0)
4877 matched = 1;
4878 break;
4879 case MATCH_GLOB:
4880 if (glob_match(g->search, str))
4881 matched = 1;
4882 break;
4883 }
4884
4885 return matched;
4886}
4887
4888static int
4889enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
4890{
4891 struct ftrace_func_entry *entry;
4892 int ret = 0;
4893
4894 entry = ftrace_lookup_ip(hash, rec->ip);
4895 if (clear_filter) {
4896 /* Do nothing if it doesn't exist */
4897 if (!entry)
4898 return 0;
4899
4900 free_hash_entry(hash, entry);
4901 } else {
4902 /* Do nothing if it exists */
4903 if (entry)
4904 return 0;
4905 if (add_hash_entry(hash, rec->ip) == NULL)
4906 ret = -ENOMEM;
4907 }
4908 return ret;
4909}
4910
4911static int
4912add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g,
4913 int clear_filter)
4914{
4915 long index;
4916 struct ftrace_page *pg;
4917 struct dyn_ftrace *rec;
4918
4919 /* The index starts at 1 */
4920 if (kstrtoul(func_g->search, 0, &index) || --index < 0)
4921 return 0;
4922
4923 do_for_each_ftrace_rec(pg, rec) {
4924 if (pg->index <= index) {
4925 index -= pg->index;
4926 /* this is a double loop, break goes to the next page */
4927 break;
4928 }
4929 rec = &pg->records[index];
4930 enter_record(hash, rec, clear_filter);
4931 return 1;
4932 } while_for_each_ftrace_rec();
4933 return 0;
4934}
4935
4936#ifdef FTRACE_MCOUNT_MAX_OFFSET
4937static int lookup_ip(unsigned long ip, char **modname, char *str)
4938{
4939 unsigned long offset;
4940
4941 kallsyms_lookup(ip, NULL, &offset, modname, str);
4942 if (offset > FTRACE_MCOUNT_MAX_OFFSET)
4943 return -1;
4944 return 0;
4945}
4946#else
4947static int lookup_ip(unsigned long ip, char **modname, char *str)
4948{
4949 kallsyms_lookup(ip, NULL, NULL, modname, str);
4950 return 0;
4951}
4952#endif
4953
4954static int
4955ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
4956 struct ftrace_glob *mod_g, int exclude_mod)
4957{
4958 char str[KSYM_SYMBOL_LEN];
4959 char *modname;
4960
4961 if (lookup_ip(rec->ip, &modname, str)) {
4962 /* This should only happen when a rec is disabled */
4963 WARN_ON_ONCE(system_state == SYSTEM_RUNNING &&
4964 !(rec->flags & FTRACE_FL_DISABLED));
4965 return 0;
4966 }
4967
4968 if (mod_g) {
4969 int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;
4970
4971 /* blank module name to match all modules */
4972 if (!mod_g->len) {
4973 /* blank module globbing: modname xor exclude_mod */
4974 if (!exclude_mod != !modname)
4975 goto func_match;
4976 return 0;
4977 }
4978
4979 /*
4980 * exclude_mod is set to trace everything but the given
4981 * module. If it is set and the module matches, then
4982 * return 0. If it is not set, and the module doesn't match
4983 * also return 0. Otherwise, check the function to see if
4984 * that matches.
4985 */
4986 if (!mod_matches == !exclude_mod)
4987 return 0;
4988func_match:
4989 /* blank search means to match all funcs in the mod */
4990 if (!func_g->len)
4991 return 1;
4992 }
4993
4994 return ftrace_match(str, func_g);
4995}
4996
4997static int
4998match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
4999{
5000 struct ftrace_page *pg;
5001 struct dyn_ftrace *rec;
5002 struct ftrace_glob func_g = { .type = MATCH_FULL };
5003 struct ftrace_glob mod_g = { .type = MATCH_FULL };
5004 struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
5005 int exclude_mod = 0;
5006 int found = 0;
5007 int ret;
5008 int clear_filter = 0;
5009
5010 if (func) {
5011 func_g.type = filter_parse_regex(func, len, &func_g.search,
5012 &clear_filter);
5013 func_g.len = strlen(func_g.search);
5014 }
5015
5016 if (mod) {
5017 mod_g.type = filter_parse_regex(mod, strlen(mod),
5018 &mod_g.search, &exclude_mod);
5019 mod_g.len = strlen(mod_g.search);
5020 }
5021
5022 guard(mutex)(&ftrace_lock);
5023
5024 if (unlikely(ftrace_disabled))
5025 return 0;
5026
5027 if (func_g.type == MATCH_INDEX)
5028 return add_rec_by_index(hash, &func_g, clear_filter);
5029
5030 do_for_each_ftrace_rec(pg, rec) {
5031
5032 if (rec->flags & FTRACE_FL_DISABLED)
5033 continue;
5034
5035 if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
5036 ret = enter_record(hash, rec, clear_filter);
5037 if (ret < 0)
5038 return ret;
5039 found = 1;
5040 }
5041 cond_resched();
5042 } while_for_each_ftrace_rec();
5043
5044 return found;
5045}
5046
5047static int
5048ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
5049{
5050 return match_records(hash, buff, len, NULL);
5051}
5052
5053static void ftrace_ops_update_code(struct ftrace_ops *ops,
5054 struct ftrace_ops_hash *old_hash)
5055{
5056 struct ftrace_ops *op;
5057
5058 if (!ftrace_enabled)
5059 return;
5060
5061 if (ops->flags & FTRACE_OPS_FL_ENABLED) {
5062 ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
5063 return;
5064 }
5065
5066 /*
5067 * If this is the shared global_ops filter, then we need to
5068 * check if there is another ops that shares it, is enabled.
5069 * If so, we still need to run the modify code.
5070 */
5071 if (ops->func_hash != &global_ops.local_hash)
5072 return;
5073
5074 do_for_each_ftrace_op(op, ftrace_ops_list) {
5075 if (op->func_hash == &global_ops.local_hash &&
5076 op->flags & FTRACE_OPS_FL_ENABLED) {
5077 ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
5078 /* Only need to do this once */
5079 return;
5080 }
5081 } while_for_each_ftrace_op(op);
5082}
5083
5084static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
5085 struct ftrace_hash **orig_hash,
5086 struct ftrace_hash *hash,
5087 int enable)
5088{
5089 if (ops->flags & FTRACE_OPS_FL_SUBOP)
5090 return ftrace_hash_move_and_update_subops(ops, orig_hash, hash);
5091
5092 /*
5093 * If this ops is not enabled, it could be sharing its filters
5094 * with a subop. If that's the case, update the subop instead of
5095 * this ops. Shared filters are only allowed to have one ops set
5096 * at a time, and if we update the ops that is not enabled,
5097 * it will not affect subops that share it.
5098 */
5099 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) {
5100 struct ftrace_ops *op;
5101
5102 /* Check if any other manager subops maps to this hash */
5103 do_for_each_ftrace_op(op, ftrace_ops_list) {
5104 struct ftrace_ops *subops;
5105
5106 list_for_each_entry(subops, &op->subop_list, list) {
5107 if ((subops->flags & FTRACE_OPS_FL_ENABLED) &&
5108 subops->func_hash == ops->func_hash) {
5109 return ftrace_hash_move_and_update_subops(subops, orig_hash, hash);
5110 }
5111 }
5112 } while_for_each_ftrace_op(op);
5113 }
5114
5115 return __ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
5116}
5117
5118static int cache_mod(struct trace_array *tr,
5119 const char *func, char *module, int enable)
5120{
5121 struct ftrace_mod_load *ftrace_mod, *n;
5122 struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
5123
5124 guard(mutex)(&ftrace_lock);
5125
5126 /* We do not cache inverse filters */
5127 if (func[0] == '!') {
5128 int ret = -EINVAL;
5129
5130 func++;
5131
5132 /* Look to remove this hash */
5133 list_for_each_entry_safe(ftrace_mod, n, head, list) {
5134 if (strcmp(ftrace_mod->module, module) != 0)
5135 continue;
5136
5137 /* no func matches all */
5138 if (strcmp(func, "*") == 0 ||
5139 (ftrace_mod->func &&
5140 strcmp(ftrace_mod->func, func) == 0)) {
5141 ret = 0;
5142 free_ftrace_mod(ftrace_mod);
5143 continue;
5144 }
5145 }
5146 return ret;
5147 }
5148
5149 /* We only care about modules that have not been loaded yet */
5150 if (module_exists(module))
5151 return -EINVAL;
5152
5153 /* Save this string off, and execute it when the module is loaded */
5154 return ftrace_add_mod(tr, func, module, enable);
5155}
5156
5157#ifdef CONFIG_MODULES
5158static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
5159 char *mod, bool enable)
5160{
5161 struct ftrace_mod_load *ftrace_mod, *n;
5162 struct ftrace_hash **orig_hash, *new_hash;
5163 LIST_HEAD(process_mods);
5164 char *func;
5165
5166 mutex_lock(&ops->func_hash->regex_lock);
5167
5168 if (enable)
5169 orig_hash = &ops->func_hash->filter_hash;
5170 else
5171 orig_hash = &ops->func_hash->notrace_hash;
5172
5173 new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
5174 *orig_hash);
5175 if (!new_hash)
5176 goto out; /* warn? */
5177
5178 mutex_lock(&ftrace_lock);
5179
5180 list_for_each_entry_safe(ftrace_mod, n, head, list) {
5181
5182 if (strcmp(ftrace_mod->module, mod) != 0)
5183 continue;
5184
5185 if (ftrace_mod->func)
5186 func = kstrdup(ftrace_mod->func, GFP_KERNEL);
5187 else
5188 func = kstrdup("*", GFP_KERNEL);
5189
5190 if (!func) /* warn? */
5191 continue;
5192
5193 list_move(&ftrace_mod->list, &process_mods);
5194
5195 /* Use the newly allocated func, as it may be "*" */
5196 kfree(ftrace_mod->func);
5197 ftrace_mod->func = func;
5198 }
5199
5200 mutex_unlock(&ftrace_lock);
5201
5202 list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {
5203
5204 func = ftrace_mod->func;
5205
5206 /* Grabs ftrace_lock, which is why we have this extra step */
5207 match_records(new_hash, func, strlen(func), mod);
5208 free_ftrace_mod(ftrace_mod);
5209 }
5210
5211 if (enable && list_empty(head))
5212 new_hash->flags &= ~FTRACE_HASH_FL_MOD;
5213
5214 mutex_lock(&ftrace_lock);
5215
5216 ftrace_hash_move_and_update_ops(ops, orig_hash,
5217 new_hash, enable);
5218 mutex_unlock(&ftrace_lock);
5219
5220 out:
5221 mutex_unlock(&ops->func_hash->regex_lock);
5222
5223 free_ftrace_hash(new_hash);
5224}
5225
5226static void process_cached_mods(const char *mod_name)
5227{
5228 struct trace_array *tr;
5229 char *mod;
5230
5231 mod = kstrdup(mod_name, GFP_KERNEL);
5232 if (!mod)
5233 return;
5234
5235 mutex_lock(&trace_types_lock);
5236 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5237 if (!list_empty(&tr->mod_trace))
5238 process_mod_list(&tr->mod_trace, tr->ops, mod, true);
5239 if (!list_empty(&tr->mod_notrace))
5240 process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
5241 }
5242 mutex_unlock(&trace_types_lock);
5243
5244 kfree(mod);
5245}
5246#endif
5247
5248/*
5249 * We register the module command as a template to show others how
5250 * to register the a command as well.
5251 */
5252
5253static int
5254ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
5255 char *func_orig, char *cmd, char *module, int enable)
5256{
5257 char *func;
5258 int ret;
5259
5260 if (!tr)
5261 return -ENODEV;
5262
5263 /* match_records() modifies func, and we need the original */
5264 func = kstrdup(func_orig, GFP_KERNEL);
5265 if (!func)
5266 return -ENOMEM;
5267
5268 /*
5269 * cmd == 'mod' because we only registered this func
5270 * for the 'mod' ftrace_func_command.
5271 * But if you register one func with multiple commands,
5272 * you can tell which command was used by the cmd
5273 * parameter.
5274 */
5275 ret = match_records(hash, func, strlen(func), module);
5276 kfree(func);
5277
5278 if (!ret)
5279 return cache_mod(tr, func_orig, module, enable);
5280 if (ret < 0)
5281 return ret;
5282 return 0;
5283}
5284
5285static struct ftrace_func_command ftrace_mod_cmd = {
5286 .name = "mod",
5287 .func = ftrace_mod_callback,
5288};
5289
5290static int __init ftrace_mod_cmd_init(void)
5291{
5292 return register_ftrace_command(&ftrace_mod_cmd);
5293}
5294core_initcall(ftrace_mod_cmd_init);
5295
5296static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
5297 struct ftrace_ops *op, struct ftrace_regs *fregs)
5298{
5299 struct ftrace_probe_ops *probe_ops;
5300 struct ftrace_func_probe *probe;
5301
5302 probe = container_of(op, struct ftrace_func_probe, ops);
5303 probe_ops = probe->probe_ops;
5304
5305 /*
5306 * Disable preemption for these calls to prevent a RCU grace
5307 * period. This syncs the hash iteration and freeing of items
5308 * on the hash. rcu_read_lock is too dangerous here.
5309 */
5310 preempt_disable_notrace();
5311 probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
5312 preempt_enable_notrace();
5313}
5314
5315struct ftrace_func_map {
5316 struct ftrace_func_entry entry;
5317 void *data;
5318};
5319
5320/*
5321 * Note, ftrace_func_mapper is freed by free_ftrace_hash(&mapper->hash).
5322 * The hash field must be the first field.
5323 */
5324struct ftrace_func_mapper {
5325 struct ftrace_hash hash; /* Must be first! */
5326};
5327
5328/**
5329 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
5330 *
5331 * Returns: a ftrace_func_mapper descriptor that can be used to map ips to data.
5332 */
5333struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
5334{
5335 struct ftrace_hash *hash;
5336
5337 /*
5338 * The mapper is simply a ftrace_hash, but since the entries
5339 * in the hash are not ftrace_func_entry type, we define it
5340 * as a separate structure.
5341 */
5342 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
5343 return (struct ftrace_func_mapper *)hash;
5344}
5345
5346/**
5347 * ftrace_func_mapper_find_ip - Find some data mapped to an ip
5348 * @mapper: The mapper that has the ip maps
5349 * @ip: the instruction pointer to find the data for
5350 *
5351 * Returns: the data mapped to @ip if found otherwise NULL. The return
5352 * is actually the address of the mapper data pointer. The address is
5353 * returned for use cases where the data is no bigger than a long, and
5354 * the user can use the data pointer as its data instead of having to
5355 * allocate more memory for the reference.
5356 */
5357void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
5358 unsigned long ip)
5359{
5360 struct ftrace_func_entry *entry;
5361 struct ftrace_func_map *map;
5362
5363 entry = ftrace_lookup_ip(&mapper->hash, ip);
5364 if (!entry)
5365 return NULL;
5366
5367 map = (struct ftrace_func_map *)entry;
5368 return &map->data;
5369}
5370
5371/**
5372 * ftrace_func_mapper_add_ip - Map some data to an ip
5373 * @mapper: The mapper that has the ip maps
5374 * @ip: The instruction pointer address to map @data to
5375 * @data: The data to map to @ip
5376 *
5377 * Returns: 0 on success otherwise an error.
5378 */
5379int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
5380 unsigned long ip, void *data)
5381{
5382 struct ftrace_func_entry *entry;
5383 struct ftrace_func_map *map;
5384
5385 entry = ftrace_lookup_ip(&mapper->hash, ip);
5386 if (entry)
5387 return -EBUSY;
5388
5389 map = kmalloc_obj(*map);
5390 if (!map)
5391 return -ENOMEM;
5392
5393 map->entry.ip = ip;
5394 map->data = data;
5395
5396 add_ftrace_hash_entry(&mapper->hash, &map->entry);
5397
5398 return 0;
5399}
5400
5401/**
5402 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
5403 * @mapper: The mapper that has the ip maps
5404 * @ip: The instruction pointer address to remove the data from
5405 *
5406 * Returns: the data if it is found, otherwise NULL.
5407 * Note, if the data pointer is used as the data itself, (see
5408 * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
5409 * if the data pointer was set to zero.
5410 */
5411void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
5412 unsigned long ip)
5413{
5414 struct ftrace_func_entry *entry;
5415 struct ftrace_func_map *map;
5416 void *data;
5417
5418 entry = ftrace_lookup_ip(&mapper->hash, ip);
5419 if (!entry)
5420 return NULL;
5421
5422 map = (struct ftrace_func_map *)entry;
5423 data = map->data;
5424
5425 remove_hash_entry(&mapper->hash, entry);
5426 kfree(entry);
5427
5428 return data;
5429}
5430
5431/**
5432 * free_ftrace_func_mapper - free a mapping of ips and data
5433 * @mapper: The mapper that has the ip maps
5434 * @free_func: A function to be called on each data item.
5435 *
5436 * This is used to free the function mapper. The @free_func is optional
5437 * and can be used if the data needs to be freed as well.
5438 */
5439void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
5440 ftrace_mapper_func free_func)
5441{
5442 struct ftrace_func_entry *entry;
5443 struct ftrace_func_map *map;
5444 struct hlist_head *hhd;
5445 int size, i;
5446
5447 if (!mapper)
5448 return;
5449
5450 if (free_func && mapper->hash.count) {
5451 size = 1 << mapper->hash.size_bits;
5452 for (i = 0; i < size; i++) {
5453 hhd = &mapper->hash.buckets[i];
5454 hlist_for_each_entry(entry, hhd, hlist) {
5455 map = (struct ftrace_func_map *)entry;
5456 free_func(map);
5457 }
5458 }
5459 }
5460 /* This also frees the mapper itself */
5461 free_ftrace_hash(&mapper->hash);
5462}
5463
5464static void release_probe(struct ftrace_func_probe *probe)
5465{
5466 struct ftrace_probe_ops *probe_ops;
5467
5468 guard(mutex)(&ftrace_lock);
5469
5470 WARN_ON(probe->ref <= 0);
5471
5472 /* Subtract the ref that was used to protect this instance */
5473 probe->ref--;
5474
5475 if (!probe->ref) {
5476 probe_ops = probe->probe_ops;
5477 /*
5478 * Sending zero as ip tells probe_ops to free
5479 * the probe->data itself
5480 */
5481 if (probe_ops->free)
5482 probe_ops->free(probe_ops, probe->tr, 0, probe->data);
5483 list_del(&probe->list);
5484 kfree(probe);
5485 }
5486}
5487
5488static void acquire_probe_locked(struct ftrace_func_probe *probe)
5489{
5490 /*
5491 * Add one ref to keep it from being freed when releasing the
5492 * ftrace_lock mutex.
5493 */
5494 probe->ref++;
5495}
5496
5497int
5498register_ftrace_function_probe(char *glob, struct trace_array *tr,
5499 struct ftrace_probe_ops *probe_ops,
5500 void *data)
5501{
5502 struct ftrace_func_probe *probe = NULL, *iter;
5503 struct ftrace_func_entry *entry;
5504 struct ftrace_hash **orig_hash;
5505 struct ftrace_hash *old_hash;
5506 struct ftrace_hash *hash;
5507 int count = 0;
5508 int size;
5509 int ret;
5510 int i;
5511
5512 if (WARN_ON(!tr))
5513 return -EINVAL;
5514
5515 /* We do not support '!' for function probes */
5516 if (WARN_ON(glob[0] == '!'))
5517 return -EINVAL;
5518
5519
5520 mutex_lock(&ftrace_lock);
5521 /* Check if the probe_ops is already registered */
5522 list_for_each_entry(iter, &tr->func_probes, list) {
5523 if (iter->probe_ops == probe_ops) {
5524 probe = iter;
5525 break;
5526 }
5527 }
5528 if (!probe) {
5529 probe = kzalloc_obj(*probe);
5530 if (!probe) {
5531 mutex_unlock(&ftrace_lock);
5532 return -ENOMEM;
5533 }
5534 probe->probe_ops = probe_ops;
5535 probe->ops.func = function_trace_probe_call;
5536 probe->tr = tr;
5537 ftrace_ops_init(&probe->ops);
5538 list_add(&probe->list, &tr->func_probes);
5539 }
5540
5541 acquire_probe_locked(probe);
5542
5543 mutex_unlock(&ftrace_lock);
5544
5545 /*
5546 * Note, there's a small window here that the func_hash->filter_hash
5547 * may be NULL or empty. Need to be careful when reading the loop.
5548 */
5549 mutex_lock(&probe->ops.func_hash->regex_lock);
5550
5551 orig_hash = &probe->ops.func_hash->filter_hash;
5552 old_hash = *orig_hash;
5553 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
5554
5555 if (!hash) {
5556 ret = -ENOMEM;
5557 goto out;
5558 }
5559
5560 ret = ftrace_match_records(hash, glob, strlen(glob));
5561
5562 /* Nothing found? */
5563 if (!ret)
5564 ret = -EINVAL;
5565
5566 if (ret < 0)
5567 goto out;
5568
5569 size = 1 << hash->size_bits;
5570 for (i = 0; i < size; i++) {
5571 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5572 if (ftrace_lookup_ip(old_hash, entry->ip))
5573 continue;
5574 /*
5575 * The caller might want to do something special
5576 * for each function we find. We call the callback
5577 * to give the caller an opportunity to do so.
5578 */
5579 if (probe_ops->init) {
5580 ret = probe_ops->init(probe_ops, tr,
5581 entry->ip, data,
5582 &probe->data);
5583 if (ret < 0) {
5584 if (probe_ops->free && count)
5585 probe_ops->free(probe_ops, tr,
5586 0, probe->data);
5587 probe->data = NULL;
5588 goto out;
5589 }
5590 }
5591 count++;
5592 }
5593 }
5594
5595 mutex_lock(&ftrace_lock);
5596
5597 if (!count) {
5598 /* Nothing was added? */
5599 ret = -EINVAL;
5600 goto out_unlock;
5601 }
5602
5603 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
5604 hash, 1);
5605 if (ret < 0)
5606 goto err_unlock;
5607
5608 /* One ref for each new function traced */
5609 probe->ref += count;
5610
5611 if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
5612 ret = ftrace_startup(&probe->ops, 0);
5613
5614 out_unlock:
5615 mutex_unlock(&ftrace_lock);
5616
5617 if (!ret)
5618 ret = count;
5619 out:
5620 mutex_unlock(&probe->ops.func_hash->regex_lock);
5621 free_ftrace_hash(hash);
5622
5623 release_probe(probe);
5624
5625 return ret;
5626
5627 err_unlock:
5628 if (!probe_ops->free || !count)
5629 goto out_unlock;
5630
5631 /* Failed to do the move, need to call the free functions */
5632 for (i = 0; i < size; i++) {
5633 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5634 if (ftrace_lookup_ip(old_hash, entry->ip))
5635 continue;
5636 probe_ops->free(probe_ops, tr, entry->ip, probe->data);
5637 }
5638 }
5639 goto out_unlock;
5640}
5641
5642int
5643unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
5644 struct ftrace_probe_ops *probe_ops)
5645{
5646 struct ftrace_func_probe *probe = NULL, *iter;
5647 struct ftrace_ops_hash old_hash_ops;
5648 struct ftrace_func_entry *entry;
5649 struct ftrace_glob func_g;
5650 struct ftrace_hash **orig_hash;
5651 struct ftrace_hash *old_hash;
5652 struct ftrace_hash *hash = NULL;
5653 struct hlist_node *tmp;
5654 struct hlist_head hhd;
5655 char str[KSYM_SYMBOL_LEN];
5656 int count = 0;
5657 int i, ret = -ENODEV;
5658 int size;
5659
5660 if (!glob || !strlen(glob) || !strcmp(glob, "*"))
5661 func_g.search = NULL;
5662 else {
5663 int not;
5664
5665 func_g.type = filter_parse_regex(glob, strlen(glob),
5666 &func_g.search, &not);
5667 func_g.len = strlen(func_g.search);
5668
5669 /* we do not support '!' for function probes */
5670 if (WARN_ON(not))
5671 return -EINVAL;
5672 }
5673
5674 mutex_lock(&ftrace_lock);
5675 /* Check if the probe_ops is already registered */
5676 list_for_each_entry(iter, &tr->func_probes, list) {
5677 if (iter->probe_ops == probe_ops) {
5678 probe = iter;
5679 break;
5680 }
5681 }
5682 if (!probe)
5683 goto err_unlock_ftrace;
5684
5685 ret = -EINVAL;
5686 if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
5687 goto err_unlock_ftrace;
5688
5689 acquire_probe_locked(probe);
5690
5691 mutex_unlock(&ftrace_lock);
5692
5693 mutex_lock(&probe->ops.func_hash->regex_lock);
5694
5695 orig_hash = &probe->ops.func_hash->filter_hash;
5696 old_hash = *orig_hash;
5697
5698 if (ftrace_hash_empty(old_hash))
5699 goto out_unlock;
5700
5701 old_hash_ops.filter_hash = old_hash;
5702 /* Probes only have filters */
5703 old_hash_ops.notrace_hash = NULL;
5704
5705 ret = -ENOMEM;
5706 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
5707 if (!hash)
5708 goto out_unlock;
5709
5710 INIT_HLIST_HEAD(&hhd);
5711
5712 size = 1 << hash->size_bits;
5713 for (i = 0; i < size; i++) {
5714 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
5715
5716 if (func_g.search) {
5717 kallsyms_lookup(entry->ip, NULL, NULL,
5718 NULL, str);
5719 if (!ftrace_match(str, &func_g))
5720 continue;
5721 }
5722 count++;
5723 remove_hash_entry(hash, entry);
5724 hlist_add_head(&entry->hlist, &hhd);
5725 }
5726 }
5727
5728 /* Nothing found? */
5729 if (!count) {
5730 ret = -EINVAL;
5731 goto out_unlock;
5732 }
5733
5734 mutex_lock(&ftrace_lock);
5735
5736 WARN_ON(probe->ref < count);
5737
5738 probe->ref -= count;
5739
5740 if (ftrace_hash_empty(hash))
5741 ftrace_shutdown(&probe->ops, 0);
5742
5743 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
5744 hash, 1);
5745
5746 /* still need to update the function call sites */
5747 if (ftrace_enabled && !ftrace_hash_empty(hash))
5748 ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
5749 &old_hash_ops);
5750 synchronize_rcu();
5751
5752 hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
5753 hlist_del(&entry->hlist);
5754 if (probe_ops->free)
5755 probe_ops->free(probe_ops, tr, entry->ip, probe->data);
5756 kfree(entry);
5757 }
5758 mutex_unlock(&ftrace_lock);
5759
5760 out_unlock:
5761 mutex_unlock(&probe->ops.func_hash->regex_lock);
5762 free_ftrace_hash(hash);
5763
5764 release_probe(probe);
5765
5766 return ret;
5767
5768 err_unlock_ftrace:
5769 mutex_unlock(&ftrace_lock);
5770 return ret;
5771}
5772
5773void clear_ftrace_function_probes(struct trace_array *tr)
5774{
5775 struct ftrace_func_probe *probe, *n;
5776
5777 list_for_each_entry_safe(probe, n, &tr->func_probes, list)
5778 unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
5779}
5780
5781static LIST_HEAD(ftrace_commands);
5782static DEFINE_MUTEX(ftrace_cmd_mutex);
5783
5784/*
5785 * Currently we only register ftrace commands from __init, so mark this
5786 * __init too.
5787 */
5788__init int register_ftrace_command(struct ftrace_func_command *cmd)
5789{
5790 struct ftrace_func_command *p;
5791
5792 guard(mutex)(&ftrace_cmd_mutex);
5793 list_for_each_entry(p, &ftrace_commands, list) {
5794 if (strcmp(cmd->name, p->name) == 0)
5795 return -EBUSY;
5796 }
5797 list_add(&cmd->list, &ftrace_commands);
5798
5799 return 0;
5800}
5801
5802/*
5803 * Currently we only unregister ftrace commands from __init, so mark
5804 * this __init too.
5805 */
5806__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
5807{
5808 struct ftrace_func_command *p, *n;
5809
5810 guard(mutex)(&ftrace_cmd_mutex);
5811
5812 list_for_each_entry_safe(p, n, &ftrace_commands, list) {
5813 if (strcmp(cmd->name, p->name) == 0) {
5814 list_del_init(&p->list);
5815 return 0;
5816 }
5817 }
5818
5819 return -ENODEV;
5820}
5821
5822static int ftrace_process_regex(struct ftrace_iterator *iter,
5823 char *buff, int len, int enable)
5824{
5825 struct ftrace_hash *hash = iter->hash;
5826 struct trace_array *tr = iter->ops->private;
5827 char *func, *command, *next = buff;
5828 struct ftrace_func_command *p;
5829 int ret;
5830
5831 func = strsep(&next, ":");
5832
5833 if (!next) {
5834 ret = ftrace_match_records(hash, func, len);
5835 if (!ret)
5836 ret = -EINVAL;
5837 if (ret < 0)
5838 return ret;
5839 return 0;
5840 }
5841
5842 /* command found */
5843
5844 command = strsep(&next, ":");
5845
5846 guard(mutex)(&ftrace_cmd_mutex);
5847
5848 list_for_each_entry(p, &ftrace_commands, list) {
5849 if (strcmp(p->name, command) == 0)
5850 return p->func(tr, hash, func, command, next, enable);
5851 }
5852
5853 return -EINVAL;
5854}
5855
5856static ssize_t
5857ftrace_regex_write(struct file *file, const char __user *ubuf,
5858 size_t cnt, loff_t *ppos, int enable)
5859{
5860 struct ftrace_iterator *iter;
5861 struct trace_parser *parser;
5862 ssize_t ret, read;
5863
5864 if (!cnt)
5865 return 0;
5866
5867 if (file->f_mode & FMODE_READ) {
5868 struct seq_file *m = file->private_data;
5869 iter = m->private;
5870 } else
5871 iter = file->private_data;
5872
5873 if (unlikely(ftrace_disabled))
5874 return -ENODEV;
5875
5876 /* iter->hash is a local copy, so we don't need regex_lock */
5877
5878 parser = &iter->parser;
5879
5880 guard(mutex)(&parser_lock);
5881 read = trace_get_user(parser, ubuf, cnt, ppos);
5882
5883 if (read >= 0 && trace_parser_loaded(parser) &&
5884 !trace_parser_cont(parser)) {
5885 ret = ftrace_process_regex(iter, parser->buffer,
5886 parser->idx, enable);
5887 trace_parser_clear(parser);
5888 if (ret < 0)
5889 return ret;
5890 }
5891
5892 return read;
5893}
5894
5895ssize_t
5896ftrace_filter_write(struct file *file, const char __user *ubuf,
5897 size_t cnt, loff_t *ppos)
5898{
5899 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
5900}
5901
5902ssize_t
5903ftrace_notrace_write(struct file *file, const char __user *ubuf,
5904 size_t cnt, loff_t *ppos)
5905{
5906 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
5907}
5908
5909static int
5910__ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
5911{
5912 struct ftrace_func_entry *entry;
5913
5914 ip = ftrace_location(ip);
5915 if (!ip)
5916 return -EINVAL;
5917
5918 if (remove) {
5919 entry = ftrace_lookup_ip(hash, ip);
5920 if (!entry)
5921 return -ENOENT;
5922 free_hash_entry(hash, entry);
5923 return 0;
5924 } else if (__ftrace_lookup_ip(hash, ip) != NULL) {
5925 /* Already exists */
5926 return 0;
5927 }
5928
5929 entry = add_hash_entry(hash, ip);
5930 return entry ? 0 : -ENOMEM;
5931}
5932
5933static int
5934ftrace_match_addr(struct ftrace_hash *hash, unsigned long *ips,
5935 unsigned int cnt, int remove)
5936{
5937 unsigned int i;
5938 int err;
5939
5940 for (i = 0; i < cnt; i++) {
5941 err = __ftrace_match_addr(hash, ips[i], remove);
5942 if (err) {
5943 /*
5944 * This expects the @hash is a temporary hash and if this
5945 * fails the caller must free the @hash.
5946 */
5947 return err;
5948 }
5949 }
5950 return 0;
5951}
5952
5953static int
5954ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
5955 unsigned long *ips, unsigned int cnt,
5956 int remove, int reset, int enable, char *mod)
5957{
5958 struct ftrace_hash **orig_hash;
5959 struct ftrace_hash *hash;
5960 int ret;
5961
5962 if (unlikely(ftrace_disabled))
5963 return -ENODEV;
5964
5965 mutex_lock(&ops->func_hash->regex_lock);
5966
5967 if (enable)
5968 orig_hash = &ops->func_hash->filter_hash;
5969 else
5970 orig_hash = &ops->func_hash->notrace_hash;
5971
5972 if (reset)
5973 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
5974 else
5975 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
5976
5977 if (!hash) {
5978 ret = -ENOMEM;
5979 goto out_regex_unlock;
5980 }
5981
5982 if (buf && !match_records(hash, buf, len, mod)) {
5983 /* If this was for a module and nothing was enabled, flag it */
5984 if (mod)
5985 (*orig_hash)->flags |= FTRACE_HASH_FL_MOD;
5986
5987 /*
5988 * Even if it is a mod, return error to let caller know
5989 * nothing was added
5990 */
5991 ret = -EINVAL;
5992 goto out_regex_unlock;
5993 }
5994 if (ips) {
5995 ret = ftrace_match_addr(hash, ips, cnt, remove);
5996 if (ret < 0)
5997 goto out_regex_unlock;
5998 }
5999
6000 mutex_lock(&ftrace_lock);
6001 ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
6002 mutex_unlock(&ftrace_lock);
6003
6004 out_regex_unlock:
6005 mutex_unlock(&ops->func_hash->regex_lock);
6006
6007 free_ftrace_hash(hash);
6008 return ret;
6009}
6010
6011static int
6012ftrace_set_addr(struct ftrace_ops *ops, unsigned long *ips, unsigned int cnt,
6013 int remove, int reset, int enable)
6014{
6015 return ftrace_set_hash(ops, NULL, 0, ips, cnt, remove, reset, enable, NULL);
6016}
6017
6018#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
6019
6020static int register_ftrace_function_nolock(struct ftrace_ops *ops);
6021
6022/*
6023 * If there are multiple ftrace_ops, use SAVE_REGS by default, so that direct
6024 * call will be jumped from ftrace_regs_caller. Only if the architecture does
6025 * not support ftrace_regs_caller but direct_call, use SAVE_ARGS so that it
6026 * jumps from ftrace_caller for multiple ftrace_ops.
6027 */
6028#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS
6029#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_ARGS)
6030#else
6031#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS)
6032#endif
6033
6034static int check_direct_multi(struct ftrace_ops *ops)
6035{
6036 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
6037 return -EINVAL;
6038 if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS)
6039 return -EINVAL;
6040 return 0;
6041}
6042
6043static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long addr)
6044{
6045 struct ftrace_func_entry *entry, *del;
6046 int size, i;
6047
6048 size = 1 << hash->size_bits;
6049 for (i = 0; i < size; i++) {
6050 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
6051 del = __ftrace_lookup_ip(direct_functions, entry->ip);
6052 if (del && ftrace_jmp_get(del->direct) ==
6053 ftrace_jmp_get(addr)) {
6054 remove_hash_entry(direct_functions, del);
6055 kfree(del);
6056 }
6057 }
6058 }
6059}
6060
6061static void register_ftrace_direct_cb(struct rcu_head *rhp)
6062{
6063 struct ftrace_hash *fhp = container_of(rhp, struct ftrace_hash, rcu);
6064
6065 free_ftrace_hash(fhp);
6066}
6067
6068static void reset_direct(struct ftrace_ops *ops, unsigned long addr)
6069{
6070 struct ftrace_hash *hash = ops->func_hash->filter_hash;
6071
6072 remove_direct_functions_hash(hash, addr);
6073
6074 /* cleanup for possible another register call */
6075 ops->func = NULL;
6076 ops->trampoline = 0;
6077}
6078
6079/**
6080 * register_ftrace_direct - Call a custom trampoline directly
6081 * for multiple functions registered in @ops
6082 * @ops: The address of the struct ftrace_ops object
6083 * @addr: The address of the trampoline to call at @ops functions
6084 *
6085 * This is used to connect a direct calls to @addr from the nop locations
6086 * of the functions registered in @ops (with by ftrace_set_filter_ip
6087 * function).
6088 *
6089 * The location that it calls (@addr) must be able to handle a direct call,
6090 * and save the parameters of the function being traced, and restore them
6091 * (or inject new ones if needed), before returning.
6092 *
6093 * Returns:
6094 * 0 on success
6095 * -EINVAL - The @ops object was already registered with this call or
6096 * when there are no functions in @ops object.
6097 * -EBUSY - Another direct function is already attached (there can be only one)
6098 * -ENODEV - @ip does not point to a ftrace nop location (or not supported)
6099 * -ENOMEM - There was an allocation failure.
6100 */
6101int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
6102{
6103 struct ftrace_hash *hash, *new_hash = NULL, *free_hash = NULL;
6104 struct ftrace_func_entry *entry, *new;
6105 int err = -EBUSY, size, i;
6106
6107 if (ops->func || ops->trampoline)
6108 return -EINVAL;
6109 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
6110 return -EINVAL;
6111 if (ops->flags & FTRACE_OPS_FL_ENABLED)
6112 return -EINVAL;
6113
6114 hash = ops->func_hash->filter_hash;
6115 if (ftrace_hash_empty(hash))
6116 return -EINVAL;
6117
6118 mutex_lock(&direct_mutex);
6119
6120 /* Make sure requested entries are not already registered.. */
6121 size = 1 << hash->size_bits;
6122 for (i = 0; i < size; i++) {
6123 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
6124 if (ftrace_find_rec_direct(entry->ip))
6125 goto out_unlock;
6126 }
6127 }
6128
6129 err = -ENOMEM;
6130
6131 /* Make a copy hash to place the new and the old entries in */
6132 size = hash->count + direct_functions->count;
6133 size = fls(size);
6134 if (size > FTRACE_HASH_MAX_BITS)
6135 size = FTRACE_HASH_MAX_BITS;
6136 new_hash = alloc_ftrace_hash(size);
6137 if (!new_hash)
6138 goto out_unlock;
6139
6140 /* Now copy over the existing direct entries */
6141 size = 1 << direct_functions->size_bits;
6142 for (i = 0; i < size; i++) {
6143 hlist_for_each_entry(entry, &direct_functions->buckets[i], hlist) {
6144 new = add_hash_entry(new_hash, entry->ip);
6145 if (!new)
6146 goto out_unlock;
6147 new->direct = entry->direct;
6148 }
6149 }
6150
6151 /* ... and add the new entries */
6152 size = 1 << hash->size_bits;
6153 for (i = 0; i < size; i++) {
6154 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
6155 new = add_hash_entry(new_hash, entry->ip);
6156 if (!new)
6157 goto out_unlock;
6158 /* Update both the copy and the hash entry */
6159 new->direct = addr;
6160 entry->direct = addr;
6161 }
6162 }
6163
6164 free_hash = direct_functions;
6165 rcu_assign_pointer(direct_functions, new_hash);
6166 new_hash = NULL;
6167
6168 ops->func = call_direct_funcs;
6169 ops->flags |= MULTI_FLAGS;
6170 ops->trampoline = FTRACE_REGS_ADDR;
6171 ops->direct_call = addr;
6172
6173 err = register_ftrace_function_nolock(ops);
6174 if (err)
6175 reset_direct(ops, addr);
6176
6177 out_unlock:
6178 mutex_unlock(&direct_mutex);
6179
6180 if (free_hash && free_hash != EMPTY_HASH)
6181 call_rcu_tasks(&free_hash->rcu, register_ftrace_direct_cb);
6182
6183 if (new_hash)
6184 free_ftrace_hash(new_hash);
6185
6186 return err;
6187}
6188EXPORT_SYMBOL_GPL(register_ftrace_direct);
6189
6190/**
6191 * unregister_ftrace_direct - Remove calls to custom trampoline
6192 * previously registered by register_ftrace_direct for @ops object.
6193 * @ops: The address of the struct ftrace_ops object
6194 * @addr: The address of the direct function that is called by the @ops functions
6195 * @free_filters: Set to true to remove all filters for the ftrace_ops, false otherwise
6196 *
6197 * This is used to remove a direct calls to @addr from the nop locations
6198 * of the functions registered in @ops (with by ftrace_set_filter_ip
6199 * function).
6200 *
6201 * Returns:
6202 * 0 on success
6203 * -EINVAL - The @ops object was not properly registered.
6204 */
6205int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
6206 bool free_filters)
6207{
6208 int err;
6209
6210 if (check_direct_multi(ops))
6211 return -EINVAL;
6212 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
6213 return -EINVAL;
6214
6215 mutex_lock(&direct_mutex);
6216 err = unregister_ftrace_function(ops);
6217 reset_direct(ops, addr);
6218 mutex_unlock(&direct_mutex);
6219
6220 if (free_filters)
6221 ftrace_free_filter(ops);
6222 return err;
6223}
6224EXPORT_SYMBOL_GPL(unregister_ftrace_direct);
6225
6226static int
6227__modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
6228{
6229 struct ftrace_hash *hash = ops->func_hash->filter_hash;
6230 struct ftrace_func_entry *entry, *iter;
6231 static struct ftrace_ops tmp_ops = {
6232 .func = ftrace_stub,
6233 .flags = FTRACE_OPS_FL_STUB,
6234 };
6235 int i, size;
6236 int err;
6237
6238 lockdep_assert_held_once(&direct_mutex);
6239
6240 /* Enable the tmp_ops to have the same functions as the direct ops */
6241 ftrace_ops_init(&tmp_ops);
6242 tmp_ops.func_hash = ops->func_hash;
6243 tmp_ops.direct_call = addr;
6244
6245 err = register_ftrace_function_nolock(&tmp_ops);
6246 if (err)
6247 return err;
6248
6249 /*
6250 * Call __ftrace_hash_update_ipmodify() here, so that we can call
6251 * ops->ops_func for the ops. This is needed because the above
6252 * register_ftrace_function_nolock() worked on tmp_ops.
6253 */
6254 err = __ftrace_hash_update_ipmodify(ops, hash, hash, true);
6255 if (err)
6256 goto out;
6257
6258 /*
6259 * Now the ftrace_ops_list_func() is called to do the direct callers.
6260 * We can safely change the direct functions attached to each entry.
6261 */
6262 mutex_lock(&ftrace_lock);
6263
6264 size = 1 << hash->size_bits;
6265 for (i = 0; i < size; i++) {
6266 hlist_for_each_entry(iter, &hash->buckets[i], hlist) {
6267 entry = __ftrace_lookup_ip(direct_functions, iter->ip);
6268 if (!entry)
6269 continue;
6270 entry->direct = addr;
6271 }
6272 }
6273 /* Prevent store tearing if a trampoline concurrently accesses the value */
6274 WRITE_ONCE(ops->direct_call, addr);
6275
6276 mutex_unlock(&ftrace_lock);
6277
6278out:
6279 /* Removing the tmp_ops will add the updated direct callers to the functions */
6280 unregister_ftrace_function(&tmp_ops);
6281
6282 return err;
6283}
6284
6285/**
6286 * modify_ftrace_direct_nolock - Modify an existing direct 'multi' call
6287 * to call something else
6288 * @ops: The address of the struct ftrace_ops object
6289 * @addr: The address of the new trampoline to call at @ops functions
6290 *
6291 * This is used to unregister currently registered direct caller and
6292 * register new one @addr on functions registered in @ops object.
6293 *
6294 * Note there's window between ftrace_shutdown and ftrace_startup calls
6295 * where there will be no callbacks called.
6296 *
6297 * Caller should already have direct_mutex locked, so we don't lock
6298 * direct_mutex here.
6299 *
6300 * Returns: zero on success. Non zero on error, which includes:
6301 * -EINVAL - The @ops object was not properly registered.
6302 */
6303int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
6304{
6305 if (check_direct_multi(ops))
6306 return -EINVAL;
6307 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
6308 return -EINVAL;
6309
6310 return __modify_ftrace_direct(ops, addr);
6311}
6312EXPORT_SYMBOL_GPL(modify_ftrace_direct_nolock);
6313
6314/**
6315 * modify_ftrace_direct - Modify an existing direct 'multi' call
6316 * to call something else
6317 * @ops: The address of the struct ftrace_ops object
6318 * @addr: The address of the new trampoline to call at @ops functions
6319 *
6320 * This is used to unregister currently registered direct caller and
6321 * register new one @addr on functions registered in @ops object.
6322 *
6323 * Note there's window between ftrace_shutdown and ftrace_startup calls
6324 * where there will be no callbacks called.
6325 *
6326 * Returns: zero on success. Non zero on error, which includes:
6327 * -EINVAL - The @ops object was not properly registered.
6328 */
6329int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
6330{
6331 int err;
6332
6333 if (check_direct_multi(ops))
6334 return -EINVAL;
6335 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
6336 return -EINVAL;
6337
6338 mutex_lock(&direct_mutex);
6339 err = __modify_ftrace_direct(ops, addr);
6340 mutex_unlock(&direct_mutex);
6341 return err;
6342}
6343EXPORT_SYMBOL_GPL(modify_ftrace_direct);
6344
6345static inline unsigned long hash_count(struct ftrace_hash *hash)
6346{
6347 return hash ? hash->count : 0;
6348}
6349
6350unsigned long ftrace_hash_count(struct ftrace_hash *hash)
6351{
6352 return hash_count(hash);
6353}
6354
6355/**
6356 * hash_add - adds two struct ftrace_hash and returns the result
6357 * @a: struct ftrace_hash object
6358 * @b: struct ftrace_hash object
6359 *
6360 * Returns struct ftrace_hash object on success, NULL on error.
6361 */
6362static struct ftrace_hash *hash_add(struct ftrace_hash *a, struct ftrace_hash *b)
6363{
6364 struct ftrace_func_entry *entry;
6365 struct ftrace_hash *add;
6366 int size;
6367
6368 size = hash_count(a) + hash_count(b);
6369 if (size > 32)
6370 size = 32;
6371
6372 add = alloc_and_copy_ftrace_hash(fls(size), a);
6373 if (!add)
6374 return NULL;
6375
6376 size = 1 << b->size_bits;
6377 for (int i = 0; i < size; i++) {
6378 hlist_for_each_entry(entry, &b->buckets[i], hlist) {
6379 if (add_ftrace_hash_entry_direct(add, entry->ip, entry->direct) == NULL) {
6380 free_ftrace_hash(add);
6381 return NULL;
6382 }
6383 }
6384 }
6385 return add;
6386}
6387
6388/**
6389 * update_ftrace_direct_add - Updates @ops by adding direct
6390 * callers provided in @hash
6391 * @ops: The address of the struct ftrace_ops object
6392 * @hash: The address of the struct ftrace_hash object
6393 *
6394 * This is used to add custom direct callers (ip -> addr) to @ops,
6395 * specified in @hash. The @ops will be either registered or updated.
6396 *
6397 * Returns: zero on success. Non zero on error, which includes:
6398 * -EINVAL - The @hash is empty
6399 */
6400int update_ftrace_direct_add(struct ftrace_ops *ops, struct ftrace_hash *hash)
6401{
6402 struct ftrace_hash *old_direct_functions = NULL;
6403 struct ftrace_hash *new_direct_functions;
6404 struct ftrace_hash *old_filter_hash;
6405 struct ftrace_hash *new_filter_hash = NULL;
6406 struct ftrace_func_entry *entry;
6407 int err = -EINVAL;
6408 int size;
6409 bool reg;
6410
6411 if (!hash_count(hash))
6412 return -EINVAL;
6413
6414 mutex_lock(&direct_mutex);
6415
6416 /* Make sure requested entries are not already registered. */
6417 size = 1 << hash->size_bits;
6418 for (int i = 0; i < size; i++) {
6419 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
6420 if (__ftrace_lookup_ip(direct_functions, entry->ip))
6421 goto out_unlock;
6422 }
6423 }
6424
6425 old_filter_hash = ops->func_hash ? ops->func_hash->filter_hash : NULL;
6426
6427 /* If there's nothing in filter_hash we need to register the ops. */
6428 reg = hash_count(old_filter_hash) == 0;
6429 if (reg) {
6430 if (ops->func || ops->trampoline)
6431 goto out_unlock;
6432 if (ops->flags & FTRACE_OPS_FL_ENABLED)
6433 goto out_unlock;
6434 }
6435
6436 err = -ENOMEM;
6437 new_filter_hash = hash_add(old_filter_hash, hash);
6438 if (!new_filter_hash)
6439 goto out_unlock;
6440
6441 new_direct_functions = hash_add(direct_functions, hash);
6442 if (!new_direct_functions)
6443 goto out_unlock;
6444
6445 old_direct_functions = direct_functions;
6446 rcu_assign_pointer(direct_functions, new_direct_functions);
6447
6448 if (reg) {
6449 ops->func = call_direct_funcs;
6450 ops->flags |= MULTI_FLAGS;
6451 ops->trampoline = FTRACE_REGS_ADDR;
6452 ops->local_hash.filter_hash = new_filter_hash;
6453
6454 err = register_ftrace_function_nolock(ops);
6455 if (err) {
6456 /* restore old filter on error */
6457 ops->local_hash.filter_hash = old_filter_hash;
6458
6459 /* cleanup for possible another register call */
6460 ops->func = NULL;
6461 ops->trampoline = 0;
6462 } else {
6463 new_filter_hash = old_filter_hash;
6464 }
6465 } else {
6466 guard(mutex)(&ftrace_lock);
6467 err = ftrace_update_ops(ops, new_filter_hash, EMPTY_HASH);
6468 /*
6469 * new_filter_hash is dup-ed, so we need to release it anyway,
6470 * old_filter_hash either stays on error or is already released
6471 */
6472 }
6473
6474 if (err) {
6475 /* reset direct_functions and free the new one */
6476 rcu_assign_pointer(direct_functions, old_direct_functions);
6477 old_direct_functions = new_direct_functions;
6478 }
6479
6480 out_unlock:
6481 mutex_unlock(&direct_mutex);
6482
6483 if (old_direct_functions && old_direct_functions != EMPTY_HASH)
6484 call_rcu_tasks(&old_direct_functions->rcu, register_ftrace_direct_cb);
6485 free_ftrace_hash(new_filter_hash);
6486
6487 return err;
6488}
6489
6490/**
6491 * hash_sub - substracts @b from @a and returns the result
6492 * @a: struct ftrace_hash object
6493 * @b: struct ftrace_hash object
6494 *
6495 * Returns struct ftrace_hash object on success, NULL on error.
6496 */
6497static struct ftrace_hash *hash_sub(struct ftrace_hash *a, struct ftrace_hash *b)
6498{
6499 struct ftrace_func_entry *entry, *del;
6500 struct ftrace_hash *sub;
6501 int size;
6502
6503 sub = alloc_and_copy_ftrace_hash(a->size_bits, a);
6504 if (!sub)
6505 return NULL;
6506
6507 size = 1 << b->size_bits;
6508 for (int i = 0; i < size; i++) {
6509 hlist_for_each_entry(entry, &b->buckets[i], hlist) {
6510 del = __ftrace_lookup_ip(sub, entry->ip);
6511 if (WARN_ON_ONCE(!del)) {
6512 free_ftrace_hash(sub);
6513 return NULL;
6514 }
6515 remove_hash_entry(sub, del);
6516 kfree(del);
6517 }
6518 }
6519 return sub;
6520}
6521
6522/**
6523 * update_ftrace_direct_del - Updates @ops by removing its direct
6524 * callers provided in @hash
6525 * @ops: The address of the struct ftrace_ops object
6526 * @hash: The address of the struct ftrace_hash object
6527 *
6528 * This is used to delete custom direct callers (ip -> addr) in
6529 * @ops specified via @hash. The @ops will be either unregistered
6530 * updated.
6531 *
6532 * Returns: zero on success. Non zero on error, which includes:
6533 * -EINVAL - The @hash is empty
6534 * -EINVAL - The @ops is not registered
6535 */
6536int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash)
6537{
6538 struct ftrace_hash *old_direct_functions = NULL;
6539 struct ftrace_hash *new_direct_functions;
6540 struct ftrace_hash *new_filter_hash = NULL;
6541 struct ftrace_hash *old_filter_hash;
6542 struct ftrace_hash *direct_hash;
6543 struct ftrace_func_entry *entry;
6544 struct ftrace_func_entry *del;
6545 unsigned long size;
6546 int err = -EINVAL;
6547
6548 if (!hash_count(hash))
6549 return -EINVAL;
6550 if (check_direct_multi(ops))
6551 return -EINVAL;
6552 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
6553 return -EINVAL;
6554
6555 mutex_lock(&direct_mutex);
6556
6557 direct_hash = rcu_dereference_protected(direct_functions, lockdep_is_held(&direct_mutex));
6558 if (direct_hash == EMPTY_HASH)
6559 goto out_unlock;
6560
6561 old_filter_hash = ops->func_hash ? ops->func_hash->filter_hash : NULL;
6562
6563 if (!hash_count(old_filter_hash))
6564 goto out_unlock;
6565
6566 /* Make sure requested entries are already registered. */
6567 size = 1 << hash->size_bits;
6568 for (int i = 0; i < size; i++) {
6569 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
6570 del = __ftrace_lookup_ip(direct_hash, entry->ip);
6571 if (!del || del->direct != entry->direct)
6572 goto out_unlock;
6573 }
6574 }
6575
6576 err = -ENOMEM;
6577 new_filter_hash = hash_sub(old_filter_hash, hash);
6578 if (!new_filter_hash)
6579 goto out_unlock;
6580
6581 new_direct_functions = hash_sub(direct_hash, hash);
6582 if (!new_direct_functions)
6583 goto out_unlock;
6584
6585 /* If there's nothing left, we need to unregister the ops. */
6586 if (ftrace_hash_empty(new_filter_hash)) {
6587 err = unregister_ftrace_function(ops);
6588 if (!err) {
6589 /* cleanup for possible another register call */
6590 ops->func = NULL;
6591 ops->trampoline = 0;
6592 ftrace_free_filter(ops);
6593 ops->func_hash->filter_hash = NULL;
6594 }
6595 } else {
6596 guard(mutex)(&ftrace_lock);
6597 err = ftrace_update_ops(ops, new_filter_hash, EMPTY_HASH);
6598 /*
6599 * new_filter_hash is dup-ed, so we need to release it anyway,
6600 * old_filter_hash either stays on error or is already released
6601 */
6602 }
6603
6604 if (err) {
6605 /* free the new_direct_functions */
6606 old_direct_functions = new_direct_functions;
6607 } else {
6608 old_direct_functions = direct_hash;
6609 rcu_assign_pointer(direct_functions, new_direct_functions);
6610 }
6611
6612 out_unlock:
6613 mutex_unlock(&direct_mutex);
6614
6615 if (old_direct_functions && old_direct_functions != EMPTY_HASH)
6616 call_rcu_tasks(&old_direct_functions->rcu, register_ftrace_direct_cb);
6617 free_ftrace_hash(new_filter_hash);
6618
6619 return err;
6620}
6621
6622/**
6623 * update_ftrace_direct_mod - Updates @ops by modifing its direct
6624 * callers provided in @hash
6625 * @ops: The address of the struct ftrace_ops object
6626 * @hash: The address of the struct ftrace_hash object
6627 * @do_direct_lock: If true lock the direct_mutex
6628 *
6629 * This is used to modify custom direct callers (ip -> addr) in
6630 * @ops specified via @hash.
6631 *
6632 * This can be called from within ftrace ops_func callback with
6633 * direct_mutex already locked, in which case @do_direct_lock
6634 * needs to be false.
6635 *
6636 * Returns: zero on success. Non zero on error, which includes:
6637 * -EINVAL - The @hash is empty
6638 * -EINVAL - The @ops is not registered
6639 */
6640int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, bool do_direct_lock)
6641{
6642 struct ftrace_func_entry *entry, *tmp;
6643 static struct ftrace_ops tmp_ops = {
6644 .func = ftrace_stub,
6645 .flags = FTRACE_OPS_FL_STUB,
6646 };
6647 struct ftrace_hash *direct_hash;
6648 struct ftrace_hash *orig_hash;
6649 unsigned long size, i;
6650 int err = -EINVAL;
6651
6652 if (!hash_count(hash))
6653 return -EINVAL;
6654 if (check_direct_multi(ops))
6655 return -EINVAL;
6656 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
6657 return -EINVAL;
6658
6659 /*
6660 * We can be called from within ops_func callback with direct_mutex
6661 * already taken.
6662 */
6663 if (do_direct_lock)
6664 mutex_lock(&direct_mutex);
6665 else
6666 lockdep_assert_held_once(&direct_mutex);
6667
6668 direct_hash = rcu_dereference_protected(direct_functions, lockdep_is_held(&direct_mutex));
6669 if (direct_hash == EMPTY_HASH)
6670 goto unlock;
6671
6672 orig_hash = ops->func_hash ? ops->func_hash->filter_hash : NULL;
6673 if (!orig_hash)
6674 goto unlock;
6675
6676 /* Enable the tmp_ops to have the same functions as the hash object. */
6677 ftrace_ops_init(&tmp_ops);
6678 rcu_assign_pointer(tmp_ops.func_hash->filter_hash, hash);
6679
6680 err = register_ftrace_function_nolock(&tmp_ops);
6681 if (err)
6682 goto unlock;
6683
6684 /*
6685 * Call __ftrace_hash_update_ipmodify() here, so that we can call
6686 * ops->ops_func for the ops. This is needed because the above
6687 * register_ftrace_function_nolock() worked on tmp_ops.
6688 */
6689 err = __ftrace_hash_update_ipmodify(ops, orig_hash, orig_hash, true);
6690 if (err)
6691 goto out;
6692
6693 /*
6694 * Now the ftrace_ops_list_func() is called to do the direct callers.
6695 * We can safely change the direct functions attached to each entry.
6696 */
6697 mutex_lock(&ftrace_lock);
6698
6699 size = 1 << hash->size_bits;
6700 for (i = 0; i < size; i++) {
6701 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
6702 tmp = __ftrace_lookup_ip(direct_hash, entry->ip);
6703 if (!tmp)
6704 continue;
6705 tmp->direct = entry->direct;
6706 }
6707 }
6708
6709 mutex_unlock(&ftrace_lock);
6710
6711out:
6712 /* Removing the tmp_ops will add the updated direct callers to the functions */
6713 unregister_ftrace_function(&tmp_ops);
6714
6715unlock:
6716 if (do_direct_lock)
6717 mutex_unlock(&direct_mutex);
6718 return err;
6719}
6720
6721#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
6722
6723/**
6724 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
6725 * @ops: the ops to set the filter with
6726 * @ip: the address to add to or remove from the filter.
6727 * @remove: non zero to remove the ip from the filter
6728 * @reset: non zero to reset all filters before applying this filter.
6729 *
6730 * Filters denote which functions should be enabled when tracing is enabled
6731 * If @ip is NULL, it fails to update filter.
6732 *
6733 * This can allocate memory which must be freed before @ops can be freed,
6734 * either by removing each filtered addr or by using
6735 * ftrace_free_filter(@ops).
6736 */
6737int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
6738 int remove, int reset)
6739{
6740 ftrace_ops_init(ops);
6741 return ftrace_set_addr(ops, &ip, 1, remove, reset, 1);
6742}
6743EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
6744
6745/**
6746 * ftrace_set_filter_ips - set functions to filter on in ftrace by addresses
6747 * @ops: the ops to set the filter with
6748 * @ips: the array of addresses to add to or remove from the filter.
6749 * @cnt: the number of addresses in @ips
6750 * @remove: non zero to remove ips from the filter
6751 * @reset: non zero to reset all filters before applying this filter.
6752 *
6753 * Filters denote which functions should be enabled when tracing is enabled
6754 * If @ips array or any ip specified within is NULL , it fails to update filter.
6755 *
6756 * This can allocate memory which must be freed before @ops can be freed,
6757 * either by removing each filtered addr or by using
6758 * ftrace_free_filter(@ops).
6759*/
6760int ftrace_set_filter_ips(struct ftrace_ops *ops, unsigned long *ips,
6761 unsigned int cnt, int remove, int reset)
6762{
6763 ftrace_ops_init(ops);
6764 return ftrace_set_addr(ops, ips, cnt, remove, reset, 1);
6765}
6766EXPORT_SYMBOL_GPL(ftrace_set_filter_ips);
6767
6768/**
6769 * ftrace_ops_set_global_filter - setup ops to use global filters
6770 * @ops: the ops which will use the global filters
6771 *
6772 * ftrace users who need global function trace filtering should call this.
6773 * It can set the global filter only if ops were not initialized before.
6774 */
6775void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
6776{
6777 if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
6778 return;
6779
6780 ftrace_ops_init(ops);
6781 ops->func_hash = &global_ops.local_hash;
6782}
6783EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
6784
6785static int
6786ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
6787 int reset, int enable)
6788{
6789 char *mod = NULL, *func, *command, *next = buf;
6790 char *tmp __free(kfree) = NULL;
6791 struct trace_array *tr = ops->private;
6792 int ret;
6793
6794 func = strsep(&next, ":");
6795
6796 /* This can also handle :mod: parsing */
6797 if (next) {
6798 if (!tr)
6799 return -EINVAL;
6800
6801 command = strsep(&next, ":");
6802 if (strcmp(command, "mod") != 0)
6803 return -EINVAL;
6804
6805 mod = next;
6806 len = command - func;
6807 /* Save the original func as ftrace_set_hash() can modify it */
6808 tmp = kstrdup(func, GFP_KERNEL);
6809 }
6810
6811 ret = ftrace_set_hash(ops, func, len, NULL, 0, 0, reset, enable, mod);
6812
6813 if (tr && mod && ret < 0) {
6814 /* Did tmp fail to allocate? */
6815 if (!tmp)
6816 return -ENOMEM;
6817 ret = cache_mod(tr, tmp, mod, enable);
6818 }
6819
6820 return ret;
6821}
6822
6823/**
6824 * ftrace_set_filter - set a function to filter on in ftrace
6825 * @ops: the ops to set the filter with
6826 * @buf: the string that holds the function filter text.
6827 * @len: the length of the string.
6828 * @reset: non-zero to reset all filters before applying this filter.
6829 *
6830 * Filters denote which functions should be enabled when tracing is enabled.
6831 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
6832 *
6833 * This can allocate memory which must be freed before @ops can be freed,
6834 * either by removing each filtered addr or by using
6835 * ftrace_free_filter(@ops).
6836 */
6837int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
6838 int len, int reset)
6839{
6840 ftrace_ops_init(ops);
6841 return ftrace_set_regex(ops, buf, len, reset, 1);
6842}
6843EXPORT_SYMBOL_GPL(ftrace_set_filter);
6844
6845/**
6846 * ftrace_set_notrace - set a function to not trace in ftrace
6847 * @ops: the ops to set the notrace filter with
6848 * @buf: the string that holds the function notrace text.
6849 * @len: the length of the string.
6850 * @reset: non-zero to reset all filters before applying this filter.
6851 *
6852 * Notrace Filters denote which functions should not be enabled when tracing
6853 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
6854 * for tracing.
6855 *
6856 * This can allocate memory which must be freed before @ops can be freed,
6857 * either by removing each filtered addr or by using
6858 * ftrace_free_filter(@ops).
6859 */
6860int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
6861 int len, int reset)
6862{
6863 ftrace_ops_init(ops);
6864 return ftrace_set_regex(ops, buf, len, reset, 0);
6865}
6866EXPORT_SYMBOL_GPL(ftrace_set_notrace);
6867/**
6868 * ftrace_set_global_filter - set a function to filter on with global tracers
6869 * @buf: the string that holds the function filter text.
6870 * @len: the length of the string.
6871 * @reset: non-zero to reset all filters before applying this filter.
6872 *
6873 * Filters denote which functions should be enabled when tracing is enabled.
6874 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
6875 */
6876void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
6877{
6878 ftrace_set_regex(&global_ops, buf, len, reset, 1);
6879}
6880EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
6881
6882/**
6883 * ftrace_set_global_notrace - set a function to not trace with global tracers
6884 * @buf: the string that holds the function notrace text.
6885 * @len: the length of the string.
6886 * @reset: non-zero to reset all filters before applying this filter.
6887 *
6888 * Notrace Filters denote which functions should not be enabled when tracing
6889 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
6890 * for tracing.
6891 */
6892void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
6893{
6894 ftrace_set_regex(&global_ops, buf, len, reset, 0);
6895}
6896EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
6897
6898/*
6899 * command line interface to allow users to set filters on boot up.
6900 */
6901#define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
6902static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
6903static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
6904
6905/* Used by function selftest to not test if filter is set */
6906bool ftrace_filter_param __initdata;
6907
6908static int __init set_ftrace_notrace(char *str)
6909{
6910 ftrace_filter_param = true;
6911 trace_append_boot_param(ftrace_notrace_buf, str, ',',
6912 FTRACE_FILTER_SIZE);
6913 return 1;
6914}
6915__setup("ftrace_notrace=", set_ftrace_notrace);
6916
6917static int __init set_ftrace_filter(char *str)
6918{
6919 ftrace_filter_param = true;
6920 trace_append_boot_param(ftrace_filter_buf, str, ',',
6921 FTRACE_FILTER_SIZE);
6922 return 1;
6923}
6924__setup("ftrace_filter=", set_ftrace_filter);
6925
6926#ifdef CONFIG_FUNCTION_GRAPH_TRACER
6927static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
6928static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
6929static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
6930
6931static int __init set_graph_function(char *str)
6932{
6933 trace_append_boot_param(ftrace_graph_buf, str, ',',
6934 FTRACE_FILTER_SIZE);
6935 return 1;
6936}
6937__setup("ftrace_graph_filter=", set_graph_function);
6938
6939static int __init set_graph_notrace_function(char *str)
6940{
6941 trace_append_boot_param(ftrace_graph_notrace_buf, str, ',',
6942 FTRACE_FILTER_SIZE);
6943 return 1;
6944}
6945__setup("ftrace_graph_notrace=", set_graph_notrace_function);
6946
6947static int __init set_graph_max_depth_function(char *str)
6948{
6949 if (!str || kstrtouint(str, 0, &fgraph_max_depth))
6950 return 0;
6951 return 1;
6952}
6953__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
6954
6955static void __init set_ftrace_early_graph(char *buf, int enable)
6956{
6957 int ret;
6958 char *func;
6959 struct ftrace_hash *hash;
6960
6961 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
6962 if (MEM_FAIL(!hash, "Failed to allocate hash\n"))
6963 return;
6964
6965 while (buf) {
6966 func = strsep(&buf, ",");
6967 /* we allow only one expression at a time */
6968 ret = ftrace_graph_set_hash(hash, func);
6969 if (ret)
6970 printk(KERN_DEBUG "ftrace: function %s not "
6971 "traceable\n", func);
6972 }
6973
6974 if (enable)
6975 ftrace_graph_hash = hash;
6976 else
6977 ftrace_graph_notrace_hash = hash;
6978}
6979#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
6980
6981void __init
6982ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
6983{
6984 char *func;
6985
6986 ftrace_ops_init(ops);
6987
6988 /* The trace_array is needed for caching module function filters */
6989 if (!ops->private) {
6990 struct trace_array *tr = trace_get_global_array();
6991
6992 ops->private = tr;
6993 ftrace_init_trace_array(tr);
6994 }
6995
6996 while (buf) {
6997 func = strsep(&buf, ",");
6998 ftrace_set_regex(ops, func, strlen(func), 0, enable);
6999 }
7000}
7001
7002static void __init set_ftrace_early_filters(void)
7003{
7004 if (ftrace_filter_buf[0])
7005 ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
7006 if (ftrace_notrace_buf[0])
7007 ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
7008#ifdef CONFIG_FUNCTION_GRAPH_TRACER
7009 if (ftrace_graph_buf[0])
7010 set_ftrace_early_graph(ftrace_graph_buf, 1);
7011 if (ftrace_graph_notrace_buf[0])
7012 set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
7013#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
7014}
7015
7016int ftrace_regex_release(struct inode *inode, struct file *file)
7017{
7018 struct seq_file *m = (struct seq_file *)file->private_data;
7019 struct ftrace_iterator *iter;
7020 struct ftrace_hash **orig_hash;
7021 struct trace_parser *parser;
7022 int filter_hash;
7023
7024 if (file->f_mode & FMODE_READ) {
7025 iter = m->private;
7026 seq_release(inode, file);
7027 } else
7028 iter = file->private_data;
7029
7030 parser = &iter->parser;
7031 mutex_lock(&parser_lock);
7032 if (trace_parser_loaded(parser)) {
7033 int enable = !(iter->flags & FTRACE_ITER_NOTRACE);
7034
7035 ftrace_process_regex(iter, parser->buffer,
7036 parser->idx, enable);
7037 }
7038 mutex_unlock(&parser_lock);
7039
7040 trace_parser_put(parser);
7041
7042 mutex_lock(&iter->ops->func_hash->regex_lock);
7043
7044 if (file->f_mode & FMODE_WRITE) {
7045 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
7046
7047 if (filter_hash) {
7048 orig_hash = &iter->ops->func_hash->filter_hash;
7049 if (iter->tr) {
7050 if (list_empty(&iter->tr->mod_trace))
7051 iter->hash->flags &= ~FTRACE_HASH_FL_MOD;
7052 else
7053 iter->hash->flags |= FTRACE_HASH_FL_MOD;
7054 }
7055 } else
7056 orig_hash = &iter->ops->func_hash->notrace_hash;
7057
7058 mutex_lock(&ftrace_lock);
7059 ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
7060 iter->hash, filter_hash);
7061 mutex_unlock(&ftrace_lock);
7062 }
7063
7064 mutex_unlock(&iter->ops->func_hash->regex_lock);
7065 free_ftrace_hash(iter->hash);
7066 if (iter->tr)
7067 trace_array_put(iter->tr);
7068 kfree(iter);
7069
7070 return 0;
7071}
7072
7073static const struct file_operations ftrace_avail_fops = {
7074 .open = ftrace_avail_open,
7075 .read = seq_read,
7076 .llseek = seq_lseek,
7077 .release = seq_release_private,
7078};
7079
7080static const struct file_operations ftrace_enabled_fops = {
7081 .open = ftrace_enabled_open,
7082 .read = seq_read,
7083 .llseek = seq_lseek,
7084 .release = seq_release_private,
7085};
7086
7087static const struct file_operations ftrace_touched_fops = {
7088 .open = ftrace_touched_open,
7089 .read = seq_read,
7090 .llseek = seq_lseek,
7091 .release = seq_release_private,
7092};
7093
7094static const struct file_operations ftrace_avail_addrs_fops = {
7095 .open = ftrace_avail_addrs_open,
7096 .read = seq_read,
7097 .llseek = seq_lseek,
7098 .release = seq_release_private,
7099};
7100
7101static const struct file_operations ftrace_filter_fops = {
7102 .open = ftrace_filter_open,
7103 .read = seq_read,
7104 .write = ftrace_filter_write,
7105 .llseek = tracing_lseek,
7106 .release = ftrace_regex_release,
7107};
7108
7109static const struct file_operations ftrace_notrace_fops = {
7110 .open = ftrace_notrace_open,
7111 .read = seq_read,
7112 .write = ftrace_notrace_write,
7113 .llseek = tracing_lseek,
7114 .release = ftrace_regex_release,
7115};
7116
7117#ifdef CONFIG_FUNCTION_GRAPH_TRACER
7118
7119static DEFINE_MUTEX(graph_lock);
7120
7121struct ftrace_hash __rcu *ftrace_graph_hash = EMPTY_HASH;
7122struct ftrace_hash __rcu *ftrace_graph_notrace_hash = EMPTY_HASH;
7123
7124enum graph_filter_type {
7125 GRAPH_FILTER_NOTRACE = 0,
7126 GRAPH_FILTER_FUNCTION,
7127};
7128
7129#define FTRACE_GRAPH_EMPTY ((void *)1)
7130
7131struct ftrace_graph_data {
7132 struct ftrace_hash *hash;
7133 struct ftrace_func_entry *entry;
7134 int idx; /* for hash table iteration */
7135 enum graph_filter_type type;
7136 struct ftrace_hash *new_hash;
7137 const struct seq_operations *seq_ops;
7138 struct trace_parser parser;
7139};
7140
7141static void *
7142__g_next(struct seq_file *m, loff_t *pos)
7143{
7144 struct ftrace_graph_data *fgd = m->private;
7145 struct ftrace_func_entry *entry = fgd->entry;
7146 struct hlist_head *head;
7147 int i, idx = fgd->idx;
7148
7149 if (*pos >= fgd->hash->count)
7150 return NULL;
7151
7152 if (entry) {
7153 hlist_for_each_entry_continue(entry, hlist) {
7154 fgd->entry = entry;
7155 return entry;
7156 }
7157
7158 idx++;
7159 }
7160
7161 for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
7162 head = &fgd->hash->buckets[i];
7163 hlist_for_each_entry(entry, head, hlist) {
7164 fgd->entry = entry;
7165 fgd->idx = i;
7166 return entry;
7167 }
7168 }
7169 return NULL;
7170}
7171
7172static void *
7173g_next(struct seq_file *m, void *v, loff_t *pos)
7174{
7175 (*pos)++;
7176 return __g_next(m, pos);
7177}
7178
7179static void *g_start(struct seq_file *m, loff_t *pos)
7180{
7181 struct ftrace_graph_data *fgd = m->private;
7182
7183 mutex_lock(&graph_lock);
7184
7185 if (fgd->type == GRAPH_FILTER_FUNCTION)
7186 fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
7187 lockdep_is_held(&graph_lock));
7188 else
7189 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
7190 lockdep_is_held(&graph_lock));
7191
7192 /* Nothing, tell g_show to print all functions are enabled */
7193 if (ftrace_hash_empty(fgd->hash) && !*pos)
7194 return FTRACE_GRAPH_EMPTY;
7195
7196 fgd->idx = 0;
7197 fgd->entry = NULL;
7198 return __g_next(m, pos);
7199}
7200
7201static void g_stop(struct seq_file *m, void *p)
7202{
7203 mutex_unlock(&graph_lock);
7204}
7205
7206static int g_show(struct seq_file *m, void *v)
7207{
7208 struct ftrace_func_entry *entry = v;
7209
7210 if (!entry)
7211 return 0;
7212
7213 if (entry == FTRACE_GRAPH_EMPTY) {
7214 struct ftrace_graph_data *fgd = m->private;
7215
7216 if (fgd->type == GRAPH_FILTER_FUNCTION)
7217 seq_puts(m, "#### all functions enabled ####\n");
7218 else
7219 seq_puts(m, "#### no functions disabled ####\n");
7220 return 0;
7221 }
7222
7223 seq_printf(m, "%ps\n", (void *)entry->ip);
7224
7225 return 0;
7226}
7227
7228static const struct seq_operations ftrace_graph_seq_ops = {
7229 .start = g_start,
7230 .next = g_next,
7231 .stop = g_stop,
7232 .show = g_show,
7233};
7234
7235static int
7236__ftrace_graph_open(struct inode *inode, struct file *file,
7237 struct ftrace_graph_data *fgd)
7238{
7239 int ret;
7240 struct ftrace_hash *new_hash = NULL;
7241
7242 ret = security_locked_down(LOCKDOWN_TRACEFS);
7243 if (ret)
7244 return ret;
7245
7246 if (file->f_mode & FMODE_WRITE) {
7247 const int size_bits = FTRACE_HASH_DEFAULT_BITS;
7248
7249 if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
7250 return -ENOMEM;
7251
7252 if (file->f_flags & O_TRUNC)
7253 new_hash = alloc_ftrace_hash(size_bits);
7254 else
7255 new_hash = alloc_and_copy_ftrace_hash(size_bits,
7256 fgd->hash);
7257 if (!new_hash) {
7258 ret = -ENOMEM;
7259 goto out;
7260 }
7261 }
7262
7263 if (file->f_mode & FMODE_READ) {
7264 ret = seq_open(file, &ftrace_graph_seq_ops);
7265 if (!ret) {
7266 struct seq_file *m = file->private_data;
7267 m->private = fgd;
7268 } else {
7269 /* Failed */
7270 free_ftrace_hash(new_hash);
7271 new_hash = NULL;
7272 }
7273 } else
7274 file->private_data = fgd;
7275
7276out:
7277 if (ret < 0 && file->f_mode & FMODE_WRITE)
7278 trace_parser_put(&fgd->parser);
7279
7280 fgd->new_hash = new_hash;
7281
7282 /*
7283 * All uses of fgd->hash must be taken with the graph_lock
7284 * held. The graph_lock is going to be released, so force
7285 * fgd->hash to be reinitialized when it is taken again.
7286 */
7287 fgd->hash = NULL;
7288
7289 return ret;
7290}
7291
7292static int
7293ftrace_graph_open(struct inode *inode, struct file *file)
7294{
7295 struct ftrace_graph_data *fgd;
7296 int ret;
7297
7298 if (unlikely(ftrace_disabled))
7299 return -ENODEV;
7300
7301 fgd = kmalloc_obj(*fgd);
7302 if (fgd == NULL)
7303 return -ENOMEM;
7304
7305 mutex_lock(&graph_lock);
7306
7307 fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
7308 lockdep_is_held(&graph_lock));
7309 fgd->type = GRAPH_FILTER_FUNCTION;
7310 fgd->seq_ops = &ftrace_graph_seq_ops;
7311
7312 ret = __ftrace_graph_open(inode, file, fgd);
7313 if (ret < 0)
7314 kfree(fgd);
7315
7316 mutex_unlock(&graph_lock);
7317 return ret;
7318}
7319
7320static int
7321ftrace_graph_notrace_open(struct inode *inode, struct file *file)
7322{
7323 struct ftrace_graph_data *fgd;
7324 int ret;
7325
7326 if (unlikely(ftrace_disabled))
7327 return -ENODEV;
7328
7329 fgd = kmalloc_obj(*fgd);
7330 if (fgd == NULL)
7331 return -ENOMEM;
7332
7333 mutex_lock(&graph_lock);
7334
7335 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
7336 lockdep_is_held(&graph_lock));
7337 fgd->type = GRAPH_FILTER_NOTRACE;
7338 fgd->seq_ops = &ftrace_graph_seq_ops;
7339
7340 ret = __ftrace_graph_open(inode, file, fgd);
7341 if (ret < 0)
7342 kfree(fgd);
7343
7344 mutex_unlock(&graph_lock);
7345 return ret;
7346}
7347
7348static int
7349ftrace_graph_release(struct inode *inode, struct file *file)
7350{
7351 struct ftrace_graph_data *fgd;
7352 struct ftrace_hash *old_hash, *new_hash;
7353 struct trace_parser *parser;
7354 int ret = 0;
7355
7356 if (file->f_mode & FMODE_READ) {
7357 struct seq_file *m = file->private_data;
7358
7359 fgd = m->private;
7360 seq_release(inode, file);
7361 } else {
7362 fgd = file->private_data;
7363 }
7364
7365
7366 if (file->f_mode & FMODE_WRITE) {
7367
7368 parser = &fgd->parser;
7369
7370 mutex_lock(&parser_lock);
7371 if (trace_parser_loaded((parser))) {
7372 ret = ftrace_graph_set_hash(fgd->new_hash,
7373 parser->buffer);
7374 }
7375 mutex_unlock(&parser_lock);
7376
7377 trace_parser_put(parser);
7378
7379 new_hash = __ftrace_hash_move(fgd->new_hash);
7380 if (!new_hash) {
7381 ret = -ENOMEM;
7382 goto out;
7383 }
7384
7385 mutex_lock(&graph_lock);
7386
7387 if (fgd->type == GRAPH_FILTER_FUNCTION) {
7388 old_hash = rcu_dereference_protected(ftrace_graph_hash,
7389 lockdep_is_held(&graph_lock));
7390 rcu_assign_pointer(ftrace_graph_hash, new_hash);
7391 } else {
7392 old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
7393 lockdep_is_held(&graph_lock));
7394 rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
7395 }
7396
7397 mutex_unlock(&graph_lock);
7398
7399 /*
7400 * We need to do a hard force of sched synchronization.
7401 * This is because we use preempt_disable() to do RCU, but
7402 * the function tracers can be called where RCU is not watching
7403 * (like before user_exit()). We can not rely on the RCU
7404 * infrastructure to do the synchronization, thus we must do it
7405 * ourselves.
7406 */
7407 if (old_hash != EMPTY_HASH)
7408 synchronize_rcu_tasks_rude();
7409
7410 free_ftrace_hash(old_hash);
7411 }
7412
7413 out:
7414 free_ftrace_hash(fgd->new_hash);
7415 kfree(fgd);
7416
7417 return ret;
7418}
7419
7420static int
7421ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
7422{
7423 struct ftrace_glob func_g;
7424 struct dyn_ftrace *rec;
7425 struct ftrace_page *pg;
7426 struct ftrace_func_entry *entry;
7427 int fail = 1;
7428 int not;
7429
7430 /* decode regex */
7431 func_g.type = filter_parse_regex(buffer, strlen(buffer),
7432 &func_g.search, &not);
7433
7434 func_g.len = strlen(func_g.search);
7435
7436 guard(mutex)(&ftrace_lock);
7437
7438 if (unlikely(ftrace_disabled))
7439 return -ENODEV;
7440
7441 do_for_each_ftrace_rec(pg, rec) {
7442
7443 if (rec->flags & FTRACE_FL_DISABLED)
7444 continue;
7445
7446 if (ftrace_match_record(rec, &func_g, NULL, 0)) {
7447 entry = ftrace_lookup_ip(hash, rec->ip);
7448
7449 if (!not) {
7450 fail = 0;
7451
7452 if (entry)
7453 continue;
7454 if (add_hash_entry(hash, rec->ip) == NULL)
7455 return 0;
7456 } else {
7457 if (entry) {
7458 free_hash_entry(hash, entry);
7459 fail = 0;
7460 }
7461 }
7462 }
7463 cond_resched();
7464 } while_for_each_ftrace_rec();
7465
7466 return fail ? -EINVAL : 0;
7467}
7468
7469static ssize_t
7470ftrace_graph_write(struct file *file, const char __user *ubuf,
7471 size_t cnt, loff_t *ppos)
7472{
7473 ssize_t read, ret = 0;
7474 struct ftrace_graph_data *fgd = file->private_data;
7475 struct trace_parser *parser;
7476
7477 if (!cnt)
7478 return 0;
7479
7480 /* Read mode uses seq functions */
7481 if (file->f_mode & FMODE_READ) {
7482 struct seq_file *m = file->private_data;
7483 fgd = m->private;
7484 }
7485
7486 parser = &fgd->parser;
7487
7488 guard(mutex)(&parser_lock);
7489 read = trace_get_user(parser, ubuf, cnt, ppos);
7490
7491 if (read >= 0 && trace_parser_loaded(parser) &&
7492 !trace_parser_cont(parser)) {
7493
7494 ret = ftrace_graph_set_hash(fgd->new_hash,
7495 parser->buffer);
7496 trace_parser_clear(parser);
7497 }
7498
7499 if (!ret)
7500 ret = read;
7501
7502 return ret;
7503}
7504
7505static const struct file_operations ftrace_graph_fops = {
7506 .open = ftrace_graph_open,
7507 .read = seq_read,
7508 .write = ftrace_graph_write,
7509 .llseek = tracing_lseek,
7510 .release = ftrace_graph_release,
7511};
7512
7513static const struct file_operations ftrace_graph_notrace_fops = {
7514 .open = ftrace_graph_notrace_open,
7515 .read = seq_read,
7516 .write = ftrace_graph_write,
7517 .llseek = tracing_lseek,
7518 .release = ftrace_graph_release,
7519};
7520#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
7521
7522void ftrace_create_filter_files(struct trace_array *tr,
7523 struct dentry *parent)
7524{
7525
7526 trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent,
7527 tr, &ftrace_filter_fops);
7528
7529 trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent,
7530 tr, &ftrace_notrace_fops);
7531}
7532
7533/*
7534 * The name "destroy_filter_files" is really a misnomer. Although
7535 * in the future, it may actually delete the files, but this is
7536 * really intended to make sure the ops passed in are disabled
7537 * and that when this function returns, the caller is free to
7538 * free the ops.
7539 *
7540 * The "destroy" name is only to match the "create" name that this
7541 * should be paired with.
7542 */
7543void ftrace_destroy_filter_files(struct ftrace_ops *ops)
7544{
7545 mutex_lock(&ftrace_lock);
7546 if (ops->flags & FTRACE_OPS_FL_ENABLED)
7547 ftrace_shutdown(ops, 0);
7548 ops->flags |= FTRACE_OPS_FL_DELETED;
7549 ftrace_free_filter(ops);
7550 mutex_unlock(&ftrace_lock);
7551}
7552
7553static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
7554{
7555 trace_create_file("available_filter_functions", TRACE_MODE_READ,
7556 d_tracer, NULL, &ftrace_avail_fops);
7557
7558 trace_create_file("available_filter_functions_addrs", TRACE_MODE_READ,
7559 d_tracer, NULL, &ftrace_avail_addrs_fops);
7560
7561 trace_create_file("enabled_functions", TRACE_MODE_READ,
7562 d_tracer, NULL, &ftrace_enabled_fops);
7563
7564 trace_create_file("touched_functions", TRACE_MODE_READ,
7565 d_tracer, NULL, &ftrace_touched_fops);
7566
7567 ftrace_create_filter_files(NULL, d_tracer);
7568
7569#ifdef CONFIG_FUNCTION_GRAPH_TRACER
7570 trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer,
7571 NULL,
7572 &ftrace_graph_fops);
7573 trace_create_file("set_graph_notrace", TRACE_MODE_WRITE, d_tracer,
7574 NULL,
7575 &ftrace_graph_notrace_fops);
7576#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
7577
7578 return 0;
7579}
7580
7581static int ftrace_cmp_ips(const void *a, const void *b)
7582{
7583 const unsigned long *ipa = a;
7584 const unsigned long *ipb = b;
7585
7586 if (*ipa > *ipb)
7587 return 1;
7588 if (*ipa < *ipb)
7589 return -1;
7590 return 0;
7591}
7592
7593#ifdef CONFIG_FTRACE_SORT_STARTUP_TEST
7594static void test_is_sorted(unsigned long *start, unsigned long count)
7595{
7596 int i;
7597
7598 for (i = 1; i < count; i++) {
7599 if (WARN(start[i - 1] > start[i],
7600 "[%d] %pS at %lx is not sorted with %pS at %lx\n", i,
7601 (void *)start[i - 1], start[i - 1],
7602 (void *)start[i], start[i]))
7603 break;
7604 }
7605 if (i == count)
7606 pr_info("ftrace section at %px sorted properly\n", start);
7607}
7608#else
7609static void test_is_sorted(unsigned long *start, unsigned long count)
7610{
7611}
7612#endif
7613
7614static int ftrace_process_locs(struct module *mod,
7615 unsigned long *start,
7616 unsigned long *end)
7617{
7618 struct ftrace_page *pg_unuse = NULL;
7619 struct ftrace_page *start_pg;
7620 struct ftrace_page *pg;
7621 struct dyn_ftrace *rec;
7622 unsigned long skipped = 0;
7623 unsigned long count;
7624 unsigned long *p;
7625 unsigned long addr;
7626 unsigned long flags = 0; /* Shut up gcc */
7627 unsigned long pages;
7628 int ret = -ENOMEM;
7629
7630 count = end - start;
7631
7632 if (!count)
7633 return 0;
7634
7635 /*
7636 * Sorting mcount in vmlinux at build time depend on
7637 * CONFIG_BUILDTIME_MCOUNT_SORT, while mcount loc in
7638 * modules can not be sorted at build time.
7639 */
7640 if (!IS_ENABLED(CONFIG_BUILDTIME_MCOUNT_SORT) || mod) {
7641 sort(start, count, sizeof(*start),
7642 ftrace_cmp_ips, NULL);
7643 } else {
7644 test_is_sorted(start, count);
7645 }
7646
7647 start_pg = ftrace_allocate_pages(count, &pages);
7648 if (!start_pg)
7649 return -ENOMEM;
7650
7651 mutex_lock(&ftrace_lock);
7652
7653 /*
7654 * Core and each module needs their own pages, as
7655 * modules will free them when they are removed.
7656 * Force a new page to be allocated for modules.
7657 */
7658 if (!mod) {
7659 WARN_ON(ftrace_pages || ftrace_pages_start);
7660 /* First initialization */
7661 ftrace_pages = ftrace_pages_start = start_pg;
7662 } else {
7663 if (!ftrace_pages)
7664 goto out;
7665
7666 if (WARN_ON(ftrace_pages->next)) {
7667 /* Hmm, we have free pages? */
7668 while (ftrace_pages->next)
7669 ftrace_pages = ftrace_pages->next;
7670 }
7671
7672 ftrace_pages->next = start_pg;
7673 }
7674
7675 p = start;
7676 pg = start_pg;
7677 while (p < end) {
7678 unsigned long end_offset;
7679
7680 addr = *p++;
7681
7682 /*
7683 * Some architecture linkers will pad between
7684 * the different mcount_loc sections of different
7685 * object files to satisfy alignments.
7686 * Skip any NULL pointers.
7687 */
7688 if (!addr) {
7689 skipped++;
7690 continue;
7691 }
7692
7693 /*
7694 * If this is core kernel, make sure the address is in core
7695 * or inittext, as weak functions get zeroed and KASLR can
7696 * move them to something other than zero. It just will not
7697 * move it to an area where kernel text is.
7698 */
7699 if (!mod && !(is_kernel_text(addr) || is_kernel_inittext(addr))) {
7700 skipped++;
7701 continue;
7702 }
7703
7704 addr = ftrace_call_adjust(addr);
7705
7706 end_offset = (pg->index+1) * sizeof(pg->records[0]);
7707 if (end_offset > PAGE_SIZE << pg->order) {
7708 /* We should have allocated enough */
7709 if (WARN_ON(!pg->next))
7710 break;
7711 pg = pg->next;
7712 }
7713
7714 rec = &pg->records[pg->index++];
7715 rec->ip = addr;
7716 }
7717
7718 if (pg->next) {
7719 pg_unuse = pg->next;
7720 pg->next = NULL;
7721 }
7722
7723 /* Assign the last page to ftrace_pages */
7724 ftrace_pages = pg;
7725
7726 /*
7727 * We only need to disable interrupts on start up
7728 * because we are modifying code that an interrupt
7729 * may execute, and the modification is not atomic.
7730 * But for modules, nothing runs the code we modify
7731 * until we are finished with it, and there's no
7732 * reason to cause large interrupt latencies while we do it.
7733 */
7734 if (!mod)
7735 local_irq_save(flags);
7736 ftrace_update_code(mod, start_pg);
7737 if (!mod)
7738 local_irq_restore(flags);
7739 ret = 0;
7740 out:
7741 mutex_unlock(&ftrace_lock);
7742
7743 /* We should have used all pages unless we skipped some */
7744 if (pg_unuse) {
7745 unsigned long pg_remaining, remaining = 0;
7746 long skip;
7747
7748 /* Count the number of entries unused and compare it to skipped. */
7749 pg_remaining = ENTRIES_PER_PAGE_GROUP(pg->order) - pg->index;
7750
7751 if (!WARN(skipped < pg_remaining, "Extra allocated pages for ftrace")) {
7752
7753 skip = skipped - pg_remaining;
7754
7755 for (pg = pg_unuse; pg && skip > 0; pg = pg->next) {
7756 remaining += 1 << pg->order;
7757 skip -= ENTRIES_PER_PAGE_GROUP(pg->order);
7758 }
7759
7760 pages -= remaining;
7761
7762 /*
7763 * Check to see if the number of pages remaining would
7764 * just fit the number of entries skipped.
7765 */
7766 WARN(pg || skip > 0, "Extra allocated pages for ftrace: %lu with %lu skipped",
7767 remaining, skipped);
7768 }
7769 /* Need to synchronize with ftrace_location_range() */
7770 synchronize_rcu();
7771 ftrace_free_pages(pg_unuse);
7772 }
7773
7774 if (!mod) {
7775 count -= skipped;
7776 pr_info("ftrace: allocating %ld entries in %ld pages\n",
7777 count, pages);
7778 }
7779
7780 return ret;
7781}
7782
7783struct ftrace_mod_func {
7784 struct list_head list;
7785 char *name;
7786 unsigned long ip;
7787 unsigned int size;
7788};
7789
7790struct ftrace_mod_map {
7791 struct rcu_head rcu;
7792 struct list_head list;
7793 struct module *mod;
7794 unsigned long start_addr;
7795 unsigned long end_addr;
7796 struct list_head funcs;
7797 unsigned int num_funcs;
7798};
7799
7800static int ftrace_get_trampoline_kallsym(unsigned int symnum,
7801 unsigned long *value, char *type,
7802 char *name, char *module_name,
7803 int *exported)
7804{
7805 struct ftrace_ops *op;
7806
7807 list_for_each_entry_rcu(op, &ftrace_ops_trampoline_list, list) {
7808 if (!op->trampoline || symnum--)
7809 continue;
7810 *value = op->trampoline;
7811 *type = 't';
7812 strscpy(name, FTRACE_TRAMPOLINE_SYM, KSYM_NAME_LEN);
7813 strscpy(module_name, FTRACE_TRAMPOLINE_MOD, MODULE_NAME_LEN);
7814 *exported = 0;
7815 return 0;
7816 }
7817
7818 return -ERANGE;
7819}
7820
7821#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) || defined(CONFIG_MODULES)
7822/*
7823 * Check if the current ops references the given ip.
7824 *
7825 * If the ops traces all functions, then it was already accounted for.
7826 * If the ops does not trace the current record function, skip it.
7827 * If the ops ignores the function via notrace filter, skip it.
7828 */
7829static bool
7830ops_references_ip(struct ftrace_ops *ops, unsigned long ip)
7831{
7832 /* If ops isn't enabled, ignore it */
7833 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
7834 return false;
7835
7836 /* If ops traces all then it includes this function */
7837 if (ops_traces_mod(ops))
7838 return true;
7839
7840 /* The function must be in the filter */
7841 if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
7842 !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip))
7843 return false;
7844
7845 /* If in notrace hash, we ignore it too */
7846 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip))
7847 return false;
7848
7849 return true;
7850}
7851#endif
7852
7853#ifdef CONFIG_MODULES
7854
7855#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
7856
7857static LIST_HEAD(ftrace_mod_maps);
7858
7859static int referenced_filters(struct dyn_ftrace *rec)
7860{
7861 struct ftrace_ops *ops;
7862 int cnt = 0;
7863
7864 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
7865 if (ops_references_ip(ops, rec->ip)) {
7866 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT))
7867 continue;
7868 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY))
7869 continue;
7870 cnt++;
7871 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
7872 rec->flags |= FTRACE_FL_REGS;
7873 if (cnt == 1 && ops->trampoline)
7874 rec->flags |= FTRACE_FL_TRAMP;
7875 else
7876 rec->flags &= ~FTRACE_FL_TRAMP;
7877 }
7878 }
7879
7880 return cnt;
7881}
7882
7883static void
7884clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)
7885{
7886 struct ftrace_func_entry *entry;
7887 struct dyn_ftrace *rec;
7888 int i;
7889
7890 if (ftrace_hash_empty(hash))
7891 return;
7892
7893 for (i = 0; i < pg->index; i++) {
7894 rec = &pg->records[i];
7895 entry = __ftrace_lookup_ip(hash, rec->ip);
7896 /*
7897 * Do not allow this rec to match again.
7898 * Yeah, it may waste some memory, but will be removed
7899 * if/when the hash is modified again.
7900 */
7901 if (entry)
7902 entry->ip = 0;
7903 }
7904}
7905
7906/* Clear any records from hashes */
7907static void clear_mod_from_hashes(struct ftrace_page *pg)
7908{
7909 struct trace_array *tr;
7910
7911 mutex_lock(&trace_types_lock);
7912 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7913 if (!tr->ops || !tr->ops->func_hash)
7914 continue;
7915 mutex_lock(&tr->ops->func_hash->regex_lock);
7916 clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);
7917 clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);
7918 mutex_unlock(&tr->ops->func_hash->regex_lock);
7919 }
7920 mutex_unlock(&trace_types_lock);
7921}
7922
7923static void ftrace_free_mod_map(struct rcu_head *rcu)
7924{
7925 struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu);
7926 struct ftrace_mod_func *mod_func;
7927 struct ftrace_mod_func *n;
7928
7929 /* All the contents of mod_map are now not visible to readers */
7930 list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) {
7931 kfree(mod_func->name);
7932 list_del(&mod_func->list);
7933 kfree(mod_func);
7934 }
7935
7936 kfree(mod_map);
7937}
7938
7939void ftrace_release_mod(struct module *mod)
7940{
7941 struct ftrace_mod_map *mod_map;
7942 struct ftrace_mod_map *n;
7943 struct dyn_ftrace *rec;
7944 struct ftrace_page **last_pg;
7945 struct ftrace_page *tmp_page = NULL;
7946 struct ftrace_page *pg;
7947
7948 mutex_lock(&ftrace_lock);
7949
7950 /*
7951 * To avoid the UAF problem after the module is unloaded, the
7952 * 'mod_map' resource needs to be released unconditionally.
7953 */
7954 list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) {
7955 if (mod_map->mod == mod) {
7956 list_del_rcu(&mod_map->list);
7957 call_rcu(&mod_map->rcu, ftrace_free_mod_map);
7958 break;
7959 }
7960 }
7961
7962 if (ftrace_disabled)
7963 goto out_unlock;
7964
7965 /*
7966 * Each module has its own ftrace_pages, remove
7967 * them from the list.
7968 */
7969 last_pg = &ftrace_pages_start;
7970 for (pg = ftrace_pages_start; pg; pg = *last_pg) {
7971 rec = &pg->records[0];
7972 if (within_module(rec->ip, mod)) {
7973 /*
7974 * As core pages are first, the first
7975 * page should never be a module page.
7976 */
7977 if (WARN_ON(pg == ftrace_pages_start))
7978 goto out_unlock;
7979
7980 /* Check if we are deleting the last page */
7981 if (pg == ftrace_pages)
7982 ftrace_pages = next_to_ftrace_page(last_pg);
7983
7984 ftrace_update_tot_cnt -= pg->index;
7985 *last_pg = pg->next;
7986
7987 pg->next = tmp_page;
7988 tmp_page = pg;
7989 } else
7990 last_pg = &pg->next;
7991 }
7992 out_unlock:
7993 mutex_unlock(&ftrace_lock);
7994
7995 /* Need to synchronize with ftrace_location_range() */
7996 if (tmp_page)
7997 synchronize_rcu();
7998 for (pg = tmp_page; pg; pg = tmp_page) {
7999
8000 /* Needs to be called outside of ftrace_lock */
8001 clear_mod_from_hashes(pg);
8002
8003 if (pg->records) {
8004 free_pages((unsigned long)pg->records, pg->order);
8005 ftrace_number_of_pages -= 1 << pg->order;
8006 }
8007 tmp_page = pg->next;
8008 kfree(pg);
8009 ftrace_number_of_groups--;
8010 }
8011}
8012
8013void ftrace_module_enable(struct module *mod)
8014{
8015 struct dyn_ftrace *rec;
8016 struct ftrace_page *pg;
8017
8018 mutex_lock(&ftrace_lock);
8019
8020 if (ftrace_disabled)
8021 goto out_unlock;
8022
8023 /*
8024 * If the tracing is enabled, go ahead and enable the record.
8025 *
8026 * The reason not to enable the record immediately is the
8027 * inherent check of ftrace_make_nop/ftrace_make_call for
8028 * correct previous instructions. Making first the NOP
8029 * conversion puts the module to the correct state, thus
8030 * passing the ftrace_make_call check.
8031 *
8032 * We also delay this to after the module code already set the
8033 * text to read-only, as we now need to set it back to read-write
8034 * so that we can modify the text.
8035 */
8036 if (ftrace_start_up)
8037 ftrace_arch_code_modify_prepare();
8038
8039 do_for_each_ftrace_rec(pg, rec) {
8040 int cnt;
8041 /*
8042 * do_for_each_ftrace_rec() is a double loop.
8043 * module text shares the pg. If a record is
8044 * not part of this module, then skip this pg,
8045 * which the "break" will do.
8046 */
8047 if (!within_module(rec->ip, mod))
8048 break;
8049
8050 cond_resched();
8051
8052 /* Weak functions should still be ignored */
8053 if (!test_for_valid_rec(rec)) {
8054 /* Clear all other flags. Should not be enabled anyway */
8055 rec->flags = FTRACE_FL_DISABLED;
8056 continue;
8057 }
8058
8059 cnt = 0;
8060
8061 /*
8062 * When adding a module, we need to check if tracers are
8063 * currently enabled and if they are, and can trace this record,
8064 * we need to enable the module functions as well as update the
8065 * reference counts for those function records.
8066 */
8067 if (ftrace_start_up)
8068 cnt += referenced_filters(rec);
8069
8070 rec->flags &= ~FTRACE_FL_DISABLED;
8071 rec->flags += cnt;
8072
8073 if (ftrace_start_up && cnt) {
8074 int failed = __ftrace_replace_code(rec, 1);
8075 if (failed) {
8076 ftrace_bug(failed, rec);
8077 goto out_loop;
8078 }
8079 }
8080
8081 } while_for_each_ftrace_rec();
8082
8083 out_loop:
8084 if (ftrace_start_up)
8085 ftrace_arch_code_modify_post_process();
8086
8087 out_unlock:
8088 mutex_unlock(&ftrace_lock);
8089
8090 process_cached_mods(mod->name);
8091}
8092
8093void ftrace_module_init(struct module *mod)
8094{
8095 int ret;
8096
8097 if (ftrace_disabled || !mod->num_ftrace_callsites)
8098 return;
8099
8100 ret = ftrace_process_locs(mod, mod->ftrace_callsites,
8101 mod->ftrace_callsites + mod->num_ftrace_callsites);
8102 if (ret)
8103 pr_warn("ftrace: failed to allocate entries for module '%s' functions\n",
8104 mod->name);
8105}
8106
8107static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
8108 struct dyn_ftrace *rec)
8109{
8110 struct ftrace_mod_func *mod_func;
8111 unsigned long symsize;
8112 unsigned long offset;
8113 char str[KSYM_SYMBOL_LEN];
8114 char *modname;
8115 const char *ret;
8116
8117 ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
8118 if (!ret)
8119 return;
8120
8121 mod_func = kmalloc_obj(*mod_func);
8122 if (!mod_func)
8123 return;
8124
8125 mod_func->name = kstrdup(str, GFP_KERNEL);
8126 if (!mod_func->name) {
8127 kfree(mod_func);
8128 return;
8129 }
8130
8131 mod_func->ip = rec->ip - offset;
8132 mod_func->size = symsize;
8133
8134 mod_map->num_funcs++;
8135
8136 list_add_rcu(&mod_func->list, &mod_map->funcs);
8137}
8138
8139static struct ftrace_mod_map *
8140allocate_ftrace_mod_map(struct module *mod,
8141 unsigned long start, unsigned long end)
8142{
8143 struct ftrace_mod_map *mod_map;
8144
8145 if (ftrace_disabled)
8146 return NULL;
8147
8148 mod_map = kmalloc_obj(*mod_map);
8149 if (!mod_map)
8150 return NULL;
8151
8152 mod_map->mod = mod;
8153 mod_map->start_addr = start;
8154 mod_map->end_addr = end;
8155 mod_map->num_funcs = 0;
8156
8157 INIT_LIST_HEAD_RCU(&mod_map->funcs);
8158
8159 list_add_rcu(&mod_map->list, &ftrace_mod_maps);
8160
8161 return mod_map;
8162}
8163
8164static int
8165ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
8166 unsigned long addr, unsigned long *size,
8167 unsigned long *off, char *sym)
8168{
8169 struct ftrace_mod_func *found_func = NULL;
8170 struct ftrace_mod_func *mod_func;
8171
8172 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
8173 if (addr >= mod_func->ip &&
8174 addr < mod_func->ip + mod_func->size) {
8175 found_func = mod_func;
8176 break;
8177 }
8178 }
8179
8180 if (found_func) {
8181 if (size)
8182 *size = found_func->size;
8183 if (off)
8184 *off = addr - found_func->ip;
8185 return strscpy(sym, found_func->name, KSYM_NAME_LEN);
8186 }
8187
8188 return 0;
8189}
8190
8191int
8192ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
8193 unsigned long *off, char **modname,
8194 const unsigned char **modbuildid, char *sym)
8195{
8196 struct ftrace_mod_map *mod_map;
8197 int ret = 0;
8198
8199 /* mod_map is freed via call_rcu() */
8200 preempt_disable();
8201 list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
8202 ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym);
8203 if (ret) {
8204 if (modname)
8205 *modname = mod_map->mod->name;
8206 if (modbuildid)
8207 *modbuildid = module_buildid(mod_map->mod);
8208 break;
8209 }
8210 }
8211 preempt_enable();
8212
8213 return ret;
8214}
8215
8216int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
8217 char *type, char *name,
8218 char *module_name, int *exported)
8219{
8220 struct ftrace_mod_map *mod_map;
8221 struct ftrace_mod_func *mod_func;
8222 int ret;
8223
8224 preempt_disable();
8225 list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
8226
8227 if (symnum >= mod_map->num_funcs) {
8228 symnum -= mod_map->num_funcs;
8229 continue;
8230 }
8231
8232 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
8233 if (symnum > 1) {
8234 symnum--;
8235 continue;
8236 }
8237
8238 *value = mod_func->ip;
8239 *type = 'T';
8240 strscpy(name, mod_func->name, KSYM_NAME_LEN);
8241 strscpy(module_name, mod_map->mod->name, MODULE_NAME_LEN);
8242 *exported = 1;
8243 preempt_enable();
8244 return 0;
8245 }
8246 WARN_ON(1);
8247 break;
8248 }
8249 ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
8250 module_name, exported);
8251 preempt_enable();
8252 return ret;
8253}
8254
8255#else
8256static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
8257 struct dyn_ftrace *rec) { }
8258static inline struct ftrace_mod_map *
8259allocate_ftrace_mod_map(struct module *mod,
8260 unsigned long start, unsigned long end)
8261{
8262 return NULL;
8263}
8264int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
8265 char *type, char *name, char *module_name,
8266 int *exported)
8267{
8268 int ret;
8269
8270 preempt_disable();
8271 ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
8272 module_name, exported);
8273 preempt_enable();
8274 return ret;
8275}
8276#endif /* CONFIG_MODULES */
8277
8278struct ftrace_init_func {
8279 struct list_head list;
8280 unsigned long ip;
8281};
8282
8283/* Clear any init ips from hashes */
8284static void
8285clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash)
8286{
8287 struct ftrace_func_entry *entry;
8288
8289 entry = ftrace_lookup_ip(hash, func->ip);
8290 /*
8291 * Do not allow this rec to match again.
8292 * Yeah, it may waste some memory, but will be removed
8293 * if/when the hash is modified again.
8294 */
8295 if (entry)
8296 entry->ip = 0;
8297}
8298
8299static void
8300clear_func_from_hashes(struct ftrace_init_func *func)
8301{
8302 struct trace_array *tr;
8303
8304 mutex_lock(&trace_types_lock);
8305 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8306 if (!tr->ops || !tr->ops->func_hash)
8307 continue;
8308 mutex_lock(&tr->ops->func_hash->regex_lock);
8309 clear_func_from_hash(func, tr->ops->func_hash->filter_hash);
8310 clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);
8311 mutex_unlock(&tr->ops->func_hash->regex_lock);
8312 }
8313 mutex_unlock(&trace_types_lock);
8314}
8315
8316static void add_to_clear_hash_list(struct list_head *clear_list,
8317 struct dyn_ftrace *rec)
8318{
8319 struct ftrace_init_func *func;
8320
8321 func = kmalloc_obj(*func);
8322 if (!func) {
8323 MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n");
8324 return;
8325 }
8326
8327 func->ip = rec->ip;
8328 list_add(&func->list, clear_list);
8329}
8330
8331void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
8332{
8333 unsigned long start = (unsigned long)(start_ptr);
8334 /* end is inclusive and end_ptr is exclusive */
8335 unsigned long end = (unsigned long)(end_ptr) - 1;
8336 struct ftrace_page **last_pg = &ftrace_pages_start;
8337 struct ftrace_page *tmp_page = NULL;
8338 struct ftrace_page *pg;
8339 struct dyn_ftrace *rec;
8340 struct dyn_ftrace key;
8341 struct ftrace_mod_map *mod_map = NULL;
8342 struct ftrace_init_func *func, *func_next;
8343 LIST_HEAD(clear_hash);
8344
8345 if (start_ptr >= end_ptr)
8346 return;
8347
8348 key.ip = start;
8349 key.flags = end; /* overload flags, as it is unsigned long */
8350
8351 mutex_lock(&ftrace_lock);
8352
8353 /*
8354 * If we are freeing module init memory, then check if
8355 * any tracer is active. If so, we need to save a mapping of
8356 * the module functions being freed with the address.
8357 */
8358 if (mod && ftrace_ops_list != &ftrace_list_end)
8359 mod_map = allocate_ftrace_mod_map(mod, start, end);
8360
8361 for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
8362 if (end < pg->records[0].ip ||
8363 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
8364 continue;
8365 again:
8366 rec = bsearch(&key, pg->records, pg->index,
8367 sizeof(struct dyn_ftrace),
8368 ftrace_cmp_recs);
8369 if (!rec)
8370 continue;
8371
8372 /* rec will be cleared from hashes after ftrace_lock unlock */
8373 add_to_clear_hash_list(&clear_hash, rec);
8374
8375 if (mod_map)
8376 save_ftrace_mod_rec(mod_map, rec);
8377
8378 pg->index--;
8379 ftrace_update_tot_cnt--;
8380 if (!pg->index) {
8381 *last_pg = pg->next;
8382 pg->next = tmp_page;
8383 tmp_page = pg;
8384 pg = container_of(last_pg, struct ftrace_page, next);
8385 if (!(*last_pg))
8386 ftrace_pages = pg;
8387 continue;
8388 }
8389 memmove(rec, rec + 1,
8390 (pg->index - (rec - pg->records)) * sizeof(*rec));
8391 /* More than one function may be in this block */
8392 goto again;
8393 }
8394 mutex_unlock(&ftrace_lock);
8395
8396 list_for_each_entry_safe(func, func_next, &clear_hash, list) {
8397 clear_func_from_hashes(func);
8398 kfree(func);
8399 }
8400 /* Need to synchronize with ftrace_location_range() */
8401 if (tmp_page) {
8402 synchronize_rcu();
8403 ftrace_free_pages(tmp_page);
8404 }
8405}
8406
8407void __init ftrace_free_init_mem(void)
8408{
8409 void *start = (void *)(&__init_begin);
8410 void *end = (void *)(&__init_end);
8411
8412 ftrace_boot_snapshot();
8413
8414 ftrace_free_mem(NULL, start, end);
8415}
8416
8417int __init __weak ftrace_dyn_arch_init(void)
8418{
8419 return 0;
8420}
8421
8422void __init ftrace_init(void)
8423{
8424 extern unsigned long __start_mcount_loc[];
8425 extern unsigned long __stop_mcount_loc[];
8426 unsigned long count, flags;
8427 int ret;
8428
8429 local_irq_save(flags);
8430 ret = ftrace_dyn_arch_init();
8431 local_irq_restore(flags);
8432 if (ret)
8433 goto failed;
8434
8435 count = __stop_mcount_loc - __start_mcount_loc;
8436 if (!count) {
8437 pr_info("ftrace: No functions to be traced?\n");
8438 goto failed;
8439 }
8440
8441 ret = ftrace_process_locs(NULL,
8442 __start_mcount_loc,
8443 __stop_mcount_loc);
8444 if (ret) {
8445 pr_warn("ftrace: failed to allocate entries for functions\n");
8446 goto failed;
8447 }
8448
8449 pr_info("ftrace: allocated %ld pages with %ld groups\n",
8450 ftrace_number_of_pages, ftrace_number_of_groups);
8451
8452 last_ftrace_enabled = ftrace_enabled = 1;
8453
8454 set_ftrace_early_filters();
8455
8456 return;
8457 failed:
8458 ftrace_disabled = 1;
8459}
8460
8461/* Do nothing if arch does not support this */
8462void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
8463{
8464}
8465
8466static void ftrace_update_trampoline(struct ftrace_ops *ops)
8467{
8468 unsigned long trampoline = ops->trampoline;
8469
8470 arch_ftrace_update_trampoline(ops);
8471 if (ops->trampoline && ops->trampoline != trampoline &&
8472 (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) {
8473 /* Add to kallsyms before the perf events */
8474 ftrace_add_trampoline_to_kallsyms(ops);
8475 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
8476 ops->trampoline, ops->trampoline_size, false,
8477 FTRACE_TRAMPOLINE_SYM);
8478 /*
8479 * Record the perf text poke event after the ksymbol register
8480 * event.
8481 */
8482 perf_event_text_poke((void *)ops->trampoline, NULL, 0,
8483 (void *)ops->trampoline,
8484 ops->trampoline_size);
8485 }
8486}
8487
8488void ftrace_init_trace_array(struct trace_array *tr)
8489{
8490 if (tr->flags & TRACE_ARRAY_FL_MOD_INIT)
8491 return;
8492
8493 INIT_LIST_HEAD(&tr->func_probes);
8494 INIT_LIST_HEAD(&tr->mod_trace);
8495 INIT_LIST_HEAD(&tr->mod_notrace);
8496
8497 tr->flags |= TRACE_ARRAY_FL_MOD_INIT;
8498}
8499#else
8500
8501struct ftrace_ops global_ops = {
8502 .func = ftrace_stub,
8503 .flags = FTRACE_OPS_FL_INITIALIZED |
8504 FTRACE_OPS_FL_PID,
8505};
8506
8507static int __init ftrace_nodyn_init(void)
8508{
8509 ftrace_enabled = 1;
8510 return 0;
8511}
8512core_initcall(ftrace_nodyn_init);
8513
8514static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
8515static inline void ftrace_startup_all(int command) { }
8516
8517static void ftrace_update_trampoline(struct ftrace_ops *ops)
8518{
8519}
8520
8521#endif /* CONFIG_DYNAMIC_FTRACE */
8522
8523__init void ftrace_init_global_array_ops(struct trace_array *tr)
8524{
8525 tr->ops = &global_ops;
8526 if (!global_ops.private)
8527 global_ops.private = tr;
8528 ftrace_init_trace_array(tr);
8529 init_array_fgraph_ops(tr, tr->ops);
8530}
8531
8532void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
8533{
8534 /* If we filter on pids, update to use the pid function */
8535 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
8536 if (WARN_ON(tr->ops->func != ftrace_stub))
8537 printk("ftrace ops had %pS for function\n",
8538 tr->ops->func);
8539 }
8540 tr->ops->func = func;
8541 tr->ops->private = tr;
8542}
8543
8544void ftrace_reset_array_ops(struct trace_array *tr)
8545{
8546 tr->ops->func = ftrace_stub;
8547}
8548
8549static nokprobe_inline void
8550__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
8551 struct ftrace_ops *ignored, struct ftrace_regs *fregs)
8552{
8553 struct pt_regs *regs = ftrace_get_regs(fregs);
8554 struct ftrace_ops *op;
8555 int bit;
8556
8557 /*
8558 * The ftrace_test_and_set_recursion() will disable preemption,
8559 * which is required since some of the ops may be dynamically
8560 * allocated, they must be freed after a synchronize_rcu().
8561 */
8562 bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START);
8563 if (bit < 0)
8564 return;
8565
8566 do_for_each_ftrace_op(op, ftrace_ops_list) {
8567 /* Stub functions don't need to be called nor tested */
8568 if (op->flags & FTRACE_OPS_FL_STUB)
8569 continue;
8570 /*
8571 * Check the following for each ops before calling their func:
8572 * if RCU flag is set, then rcu_is_watching() must be true
8573 * Otherwise test if the ip matches the ops filter
8574 *
8575 * If any of the above fails then the op->func() is not executed.
8576 */
8577 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
8578 ftrace_ops_test(op, ip, regs)) {
8579 if (FTRACE_WARN_ON(!op->func)) {
8580 pr_warn("op=%p %pS\n", op, op);
8581 goto out;
8582 }
8583 op->func(ip, parent_ip, op, fregs);
8584 }
8585 } while_for_each_ftrace_op(op);
8586out:
8587 trace_clear_recursion(bit);
8588}
8589
8590/*
8591 * Some archs only support passing ip and parent_ip. Even though
8592 * the list function ignores the op parameter, we do not want any
8593 * C side effects, where a function is called without the caller
8594 * sending a third parameter.
8595 * Archs are to support both the regs and ftrace_ops at the same time.
8596 * If they support ftrace_ops, it is assumed they support regs.
8597 * If call backs want to use regs, they must either check for regs
8598 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
8599 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
8600 * An architecture can pass partial regs with ftrace_ops and still
8601 * set the ARCH_SUPPORTS_FTRACE_OPS.
8602 *
8603 * In vmlinux.lds.h, ftrace_ops_list_func() is defined to be
8604 * arch_ftrace_ops_list_func.
8605 */
8606#if ARCH_SUPPORTS_FTRACE_OPS
8607void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
8608 struct ftrace_ops *op, struct ftrace_regs *fregs)
8609{
8610 kmsan_unpoison_memory(fregs, ftrace_regs_size());
8611 __ftrace_ops_list_func(ip, parent_ip, NULL, fregs);
8612}
8613#else
8614void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
8615{
8616 __ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
8617}
8618#endif
8619NOKPROBE_SYMBOL(arch_ftrace_ops_list_func);
8620
8621/*
8622 * If there's only one function registered but it does not support
8623 * recursion, needs RCU protection, then this function will be called
8624 * by the mcount trampoline.
8625 */
8626static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
8627 struct ftrace_ops *op, struct ftrace_regs *fregs)
8628{
8629 int bit;
8630
8631 bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START);
8632 if (bit < 0)
8633 return;
8634
8635 if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching())
8636 op->func(ip, parent_ip, op, fregs);
8637
8638 trace_clear_recursion(bit);
8639}
8640NOKPROBE_SYMBOL(ftrace_ops_assist_func);
8641
8642/**
8643 * ftrace_ops_get_func - get the function a trampoline should call
8644 * @ops: the ops to get the function for
8645 *
8646 * Normally the mcount trampoline will call the ops->func, but there
8647 * are times that it should not. For example, if the ops does not
8648 * have its own recursion protection, then it should call the
8649 * ftrace_ops_assist_func() instead.
8650 *
8651 * Returns: the function that the trampoline should call for @ops.
8652 */
8653ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
8654{
8655 /*
8656 * If the function does not handle recursion or needs to be RCU safe,
8657 * then we need to call the assist handler.
8658 */
8659 if (ops->flags & (FTRACE_OPS_FL_RECURSION |
8660 FTRACE_OPS_FL_RCU))
8661 return ftrace_ops_assist_func;
8662
8663 return ops->func;
8664}
8665
8666static void
8667ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
8668 struct task_struct *prev,
8669 struct task_struct *next,
8670 unsigned int prev_state)
8671{
8672 struct trace_array *tr = data;
8673 struct trace_pid_list *pid_list;
8674 struct trace_pid_list *no_pid_list;
8675
8676 pid_list = rcu_dereference_sched(tr->function_pids);
8677 no_pid_list = rcu_dereference_sched(tr->function_no_pids);
8678
8679 if (trace_ignore_this_task(pid_list, no_pid_list, next))
8680 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
8681 FTRACE_PID_IGNORE);
8682 else
8683 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
8684 next->pid);
8685}
8686
8687static void
8688ftrace_pid_follow_sched_process_fork(void *data,
8689 struct task_struct *self,
8690 struct task_struct *task)
8691{
8692 struct trace_pid_list *pid_list;
8693 struct trace_array *tr = data;
8694
8695 guard(preempt)();
8696 pid_list = rcu_dereference_sched(tr->function_pids);
8697 trace_filter_add_remove_task(pid_list, self, task);
8698
8699 pid_list = rcu_dereference_sched(tr->function_no_pids);
8700 trace_filter_add_remove_task(pid_list, self, task);
8701}
8702
8703static void
8704ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
8705{
8706 struct trace_pid_list *pid_list;
8707 struct trace_array *tr = data;
8708
8709 guard(preempt)();
8710 pid_list = rcu_dereference_sched(tr->function_pids);
8711 trace_filter_add_remove_task(pid_list, NULL, task);
8712
8713 pid_list = rcu_dereference_sched(tr->function_no_pids);
8714 trace_filter_add_remove_task(pid_list, NULL, task);
8715}
8716
8717void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
8718{
8719 if (enable) {
8720 register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
8721 tr);
8722 register_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
8723 tr);
8724 } else {
8725 unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
8726 tr);
8727 unregister_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
8728 tr);
8729 }
8730}
8731
8732static void clear_ftrace_pids(struct trace_array *tr, int type)
8733{
8734 struct trace_pid_list *pid_list;
8735 struct trace_pid_list *no_pid_list;
8736 int cpu;
8737
8738 pid_list = rcu_dereference_protected(tr->function_pids,
8739 lockdep_is_held(&ftrace_lock));
8740 no_pid_list = rcu_dereference_protected(tr->function_no_pids,
8741 lockdep_is_held(&ftrace_lock));
8742
8743 /* Make sure there's something to do */
8744 if (!pid_type_enabled(type, pid_list, no_pid_list))
8745 return;
8746
8747 /* See if the pids still need to be checked after this */
8748 if (!still_need_pid_events(type, pid_list, no_pid_list)) {
8749 unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
8750 for_each_possible_cpu(cpu)
8751 per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE;
8752 }
8753
8754 if (type & TRACE_PIDS)
8755 rcu_assign_pointer(tr->function_pids, NULL);
8756
8757 if (type & TRACE_NO_PIDS)
8758 rcu_assign_pointer(tr->function_no_pids, NULL);
8759
8760 /* Wait till all users are no longer using pid filtering */
8761 synchronize_rcu();
8762
8763 if ((type & TRACE_PIDS) && pid_list)
8764 trace_pid_list_free(pid_list);
8765
8766 if ((type & TRACE_NO_PIDS) && no_pid_list)
8767 trace_pid_list_free(no_pid_list);
8768}
8769
8770void ftrace_clear_pids(struct trace_array *tr)
8771{
8772 mutex_lock(&ftrace_lock);
8773
8774 clear_ftrace_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
8775
8776 mutex_unlock(&ftrace_lock);
8777}
8778
8779static void ftrace_pid_reset(struct trace_array *tr, int type)
8780{
8781 mutex_lock(&ftrace_lock);
8782 clear_ftrace_pids(tr, type);
8783
8784 ftrace_update_pid_func();
8785 ftrace_startup_all(0);
8786
8787 mutex_unlock(&ftrace_lock);
8788}
8789
8790/* Greater than any max PID */
8791#define FTRACE_NO_PIDS (void *)(PID_MAX_LIMIT + 1)
8792
8793static void *fpid_start(struct seq_file *m, loff_t *pos)
8794 __acquires(RCU)
8795{
8796 struct trace_pid_list *pid_list;
8797 struct trace_array *tr = m->private;
8798
8799 mutex_lock(&ftrace_lock);
8800 rcu_read_lock_sched();
8801
8802 pid_list = rcu_dereference_sched(tr->function_pids);
8803
8804 if (!pid_list)
8805 return !(*pos) ? FTRACE_NO_PIDS : NULL;
8806
8807 return trace_pid_start(pid_list, pos);
8808}
8809
8810static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
8811{
8812 struct trace_array *tr = m->private;
8813 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
8814
8815 if (v == FTRACE_NO_PIDS) {
8816 (*pos)++;
8817 return NULL;
8818 }
8819 return trace_pid_next(pid_list, v, pos);
8820}
8821
8822static void fpid_stop(struct seq_file *m, void *p)
8823 __releases(RCU)
8824{
8825 rcu_read_unlock_sched();
8826 mutex_unlock(&ftrace_lock);
8827}
8828
8829static int fpid_show(struct seq_file *m, void *v)
8830{
8831 if (v == FTRACE_NO_PIDS) {
8832 seq_puts(m, "no pid\n");
8833 return 0;
8834 }
8835
8836 return trace_pid_show(m, v);
8837}
8838
8839static const struct seq_operations ftrace_pid_sops = {
8840 .start = fpid_start,
8841 .next = fpid_next,
8842 .stop = fpid_stop,
8843 .show = fpid_show,
8844};
8845
8846static void *fnpid_start(struct seq_file *m, loff_t *pos)
8847 __acquires(RCU)
8848{
8849 struct trace_pid_list *pid_list;
8850 struct trace_array *tr = m->private;
8851
8852 mutex_lock(&ftrace_lock);
8853 rcu_read_lock_sched();
8854
8855 pid_list = rcu_dereference_sched(tr->function_no_pids);
8856
8857 if (!pid_list)
8858 return !(*pos) ? FTRACE_NO_PIDS : NULL;
8859
8860 return trace_pid_start(pid_list, pos);
8861}
8862
8863static void *fnpid_next(struct seq_file *m, void *v, loff_t *pos)
8864{
8865 struct trace_array *tr = m->private;
8866 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids);
8867
8868 if (v == FTRACE_NO_PIDS) {
8869 (*pos)++;
8870 return NULL;
8871 }
8872 return trace_pid_next(pid_list, v, pos);
8873}
8874
8875static const struct seq_operations ftrace_no_pid_sops = {
8876 .start = fnpid_start,
8877 .next = fnpid_next,
8878 .stop = fpid_stop,
8879 .show = fpid_show,
8880};
8881
8882static int pid_open(struct inode *inode, struct file *file, int type)
8883{
8884 const struct seq_operations *seq_ops;
8885 struct trace_array *tr = inode->i_private;
8886 struct seq_file *m;
8887 int ret = 0;
8888
8889 ret = tracing_check_open_get_tr(tr);
8890 if (ret)
8891 return ret;
8892
8893 if ((file->f_mode & FMODE_WRITE) &&
8894 (file->f_flags & O_TRUNC))
8895 ftrace_pid_reset(tr, type);
8896
8897 switch (type) {
8898 case TRACE_PIDS:
8899 seq_ops = &ftrace_pid_sops;
8900 break;
8901 case TRACE_NO_PIDS:
8902 seq_ops = &ftrace_no_pid_sops;
8903 break;
8904 default:
8905 trace_array_put(tr);
8906 WARN_ON_ONCE(1);
8907 return -EINVAL;
8908 }
8909
8910 ret = seq_open(file, seq_ops);
8911 if (ret < 0) {
8912 trace_array_put(tr);
8913 } else {
8914 m = file->private_data;
8915 /* copy tr over to seq ops */
8916 m->private = tr;
8917 }
8918
8919 return ret;
8920}
8921
8922static int
8923ftrace_pid_open(struct inode *inode, struct file *file)
8924{
8925 return pid_open(inode, file, TRACE_PIDS);
8926}
8927
8928static int
8929ftrace_no_pid_open(struct inode *inode, struct file *file)
8930{
8931 return pid_open(inode, file, TRACE_NO_PIDS);
8932}
8933
8934static void ignore_task_cpu(void *data)
8935{
8936 struct trace_array *tr = data;
8937 struct trace_pid_list *pid_list;
8938 struct trace_pid_list *no_pid_list;
8939
8940 /*
8941 * This function is called by on_each_cpu() while the
8942 * event_mutex is held.
8943 */
8944 pid_list = rcu_dereference_protected(tr->function_pids,
8945 mutex_is_locked(&ftrace_lock));
8946 no_pid_list = rcu_dereference_protected(tr->function_no_pids,
8947 mutex_is_locked(&ftrace_lock));
8948
8949 if (trace_ignore_this_task(pid_list, no_pid_list, current))
8950 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
8951 FTRACE_PID_IGNORE);
8952 else
8953 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
8954 current->pid);
8955}
8956
8957static ssize_t
8958pid_write(struct file *filp, const char __user *ubuf,
8959 size_t cnt, loff_t *ppos, int type)
8960{
8961 struct seq_file *m = filp->private_data;
8962 struct trace_array *tr = m->private;
8963 struct trace_pid_list *filtered_pids;
8964 struct trace_pid_list *other_pids;
8965 struct trace_pid_list *pid_list;
8966 ssize_t ret;
8967
8968 if (!cnt)
8969 return 0;
8970
8971 guard(mutex)(&ftrace_lock);
8972
8973 switch (type) {
8974 case TRACE_PIDS:
8975 filtered_pids = rcu_dereference_protected(tr->function_pids,
8976 lockdep_is_held(&ftrace_lock));
8977 other_pids = rcu_dereference_protected(tr->function_no_pids,
8978 lockdep_is_held(&ftrace_lock));
8979 break;
8980 case TRACE_NO_PIDS:
8981 filtered_pids = rcu_dereference_protected(tr->function_no_pids,
8982 lockdep_is_held(&ftrace_lock));
8983 other_pids = rcu_dereference_protected(tr->function_pids,
8984 lockdep_is_held(&ftrace_lock));
8985 break;
8986 default:
8987 WARN_ON_ONCE(1);
8988 return -EINVAL;
8989 }
8990
8991 ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
8992 if (ret < 0)
8993 return ret;
8994
8995 switch (type) {
8996 case TRACE_PIDS:
8997 rcu_assign_pointer(tr->function_pids, pid_list);
8998 break;
8999 case TRACE_NO_PIDS:
9000 rcu_assign_pointer(tr->function_no_pids, pid_list);
9001 break;
9002 }
9003
9004
9005 if (filtered_pids) {
9006 synchronize_rcu();
9007 trace_pid_list_free(filtered_pids);
9008 } else if (pid_list && !other_pids) {
9009 /* Register a probe to set whether to ignore the tracing of a task */
9010 register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
9011 }
9012
9013 /*
9014 * Ignoring of pids is done at task switch. But we have to
9015 * check for those tasks that are currently running.
9016 * Always do this in case a pid was appended or removed.
9017 */
9018 on_each_cpu(ignore_task_cpu, tr, 1);
9019
9020 ftrace_update_pid_func();
9021 ftrace_startup_all(0);
9022
9023 *ppos += ret;
9024
9025 return ret;
9026}
9027
9028static ssize_t
9029ftrace_pid_write(struct file *filp, const char __user *ubuf,
9030 size_t cnt, loff_t *ppos)
9031{
9032 return pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
9033}
9034
9035static ssize_t
9036ftrace_no_pid_write(struct file *filp, const char __user *ubuf,
9037 size_t cnt, loff_t *ppos)
9038{
9039 return pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
9040}
9041
9042static int
9043ftrace_pid_release(struct inode *inode, struct file *file)
9044{
9045 struct trace_array *tr = inode->i_private;
9046
9047 trace_array_put(tr);
9048
9049 return seq_release(inode, file);
9050}
9051
9052static const struct file_operations ftrace_pid_fops = {
9053 .open = ftrace_pid_open,
9054 .write = ftrace_pid_write,
9055 .read = seq_read,
9056 .llseek = tracing_lseek,
9057 .release = ftrace_pid_release,
9058};
9059
9060static const struct file_operations ftrace_no_pid_fops = {
9061 .open = ftrace_no_pid_open,
9062 .write = ftrace_no_pid_write,
9063 .read = seq_read,
9064 .llseek = tracing_lseek,
9065 .release = ftrace_pid_release,
9066};
9067
9068void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
9069{
9070 trace_create_file("set_ftrace_pid", TRACE_MODE_WRITE, d_tracer,
9071 tr, &ftrace_pid_fops);
9072 trace_create_file("set_ftrace_notrace_pid", TRACE_MODE_WRITE,
9073 d_tracer, tr, &ftrace_no_pid_fops);
9074}
9075
9076void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
9077 struct dentry *d_tracer)
9078{
9079 /* Only the top level directory has the dyn_tracefs and profile */
9080 WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
9081
9082 ftrace_init_dyn_tracefs(d_tracer);
9083 ftrace_profile_tracefs(d_tracer);
9084}
9085
9086/**
9087 * ftrace_kill - kill ftrace
9088 *
9089 * This function should be used by panic code. It stops ftrace
9090 * but in a not so nice way. If you need to simply kill ftrace
9091 * from a non-atomic section, use ftrace_kill.
9092 */
9093void ftrace_kill(void)
9094{
9095 ftrace_disabled = 1;
9096 ftrace_enabled = 0;
9097 ftrace_trace_function = ftrace_stub;
9098 kprobe_ftrace_kill();
9099}
9100
9101/**
9102 * ftrace_is_dead - Test if ftrace is dead or not.
9103 *
9104 * Returns: 1 if ftrace is "dead", zero otherwise.
9105 */
9106int ftrace_is_dead(void)
9107{
9108 return ftrace_disabled;
9109}
9110
9111#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
9112/*
9113 * When registering ftrace_ops with IPMODIFY, it is necessary to make sure
9114 * it doesn't conflict with any direct ftrace_ops. If there is existing
9115 * direct ftrace_ops on a kernel function being patched, call
9116 * FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER on it to enable sharing.
9117 *
9118 * @ops: ftrace_ops being registered.
9119 *
9120 * Returns:
9121 * 0 on success;
9122 * Negative on failure.
9123 */
9124static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops)
9125{
9126 struct ftrace_func_entry *entry;
9127 struct ftrace_hash *hash;
9128 struct ftrace_ops *op;
9129 int size, i, ret;
9130
9131 lockdep_assert_held_once(&direct_mutex);
9132
9133 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
9134 return 0;
9135
9136 hash = ops->func_hash->filter_hash;
9137 size = 1 << hash->size_bits;
9138 for (i = 0; i < size; i++) {
9139 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
9140 unsigned long ip = entry->ip;
9141 bool found_op = false;
9142
9143 mutex_lock(&ftrace_lock);
9144 do_for_each_ftrace_op(op, ftrace_ops_list) {
9145 if (!(op->flags & FTRACE_OPS_FL_DIRECT))
9146 continue;
9147 if (ops_references_ip(op, ip)) {
9148 found_op = true;
9149 break;
9150 }
9151 } while_for_each_ftrace_op(op);
9152 mutex_unlock(&ftrace_lock);
9153
9154 if (found_op) {
9155 if (!op->ops_func)
9156 return -EBUSY;
9157
9158 ret = op->ops_func(op, ip, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER);
9159 if (ret)
9160 return ret;
9161 }
9162 }
9163 }
9164
9165 return 0;
9166}
9167
9168/*
9169 * Similar to prepare_direct_functions_for_ipmodify, clean up after ops
9170 * with IPMODIFY is unregistered. The cleanup is optional for most DIRECT
9171 * ops.
9172 */
9173static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops)
9174{
9175 struct ftrace_func_entry *entry;
9176 struct ftrace_hash *hash;
9177 struct ftrace_ops *op;
9178 int size, i;
9179
9180 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
9181 return;
9182
9183 mutex_lock(&direct_mutex);
9184
9185 hash = ops->func_hash->filter_hash;
9186 size = 1 << hash->size_bits;
9187 for (i = 0; i < size; i++) {
9188 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
9189 unsigned long ip = entry->ip;
9190 bool found_op = false;
9191
9192 mutex_lock(&ftrace_lock);
9193 do_for_each_ftrace_op(op, ftrace_ops_list) {
9194 if (!(op->flags & FTRACE_OPS_FL_DIRECT))
9195 continue;
9196 if (ops_references_ip(op, ip)) {
9197 found_op = true;
9198 break;
9199 }
9200 } while_for_each_ftrace_op(op);
9201 mutex_unlock(&ftrace_lock);
9202
9203 /* The cleanup is optional, ignore any errors */
9204 if (found_op && op->ops_func)
9205 op->ops_func(op, ip, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER);
9206 }
9207 }
9208 mutex_unlock(&direct_mutex);
9209}
9210
9211#define lock_direct_mutex() mutex_lock(&direct_mutex)
9212#define unlock_direct_mutex() mutex_unlock(&direct_mutex)
9213
9214#else /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
9215
9216static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops)
9217{
9218 return 0;
9219}
9220
9221static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops)
9222{
9223}
9224
9225#define lock_direct_mutex() do { } while (0)
9226#define unlock_direct_mutex() do { } while (0)
9227
9228#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
9229
9230/*
9231 * Similar to register_ftrace_function, except we don't lock direct_mutex.
9232 */
9233static int register_ftrace_function_nolock(struct ftrace_ops *ops)
9234{
9235 int ret;
9236
9237 ftrace_ops_init(ops);
9238
9239 mutex_lock(&ftrace_lock);
9240
9241 ret = ftrace_startup(ops, 0);
9242
9243 mutex_unlock(&ftrace_lock);
9244
9245 return ret;
9246}
9247
9248/**
9249 * register_ftrace_function - register a function for profiling
9250 * @ops: ops structure that holds the function for profiling.
9251 *
9252 * Register a function to be called by all functions in the
9253 * kernel.
9254 *
9255 * Note: @ops->func and all the functions it calls must be labeled
9256 * with "notrace", otherwise it will go into a
9257 * recursive loop.
9258 */
9259int register_ftrace_function(struct ftrace_ops *ops)
9260{
9261 int ret;
9262
9263 lock_direct_mutex();
9264 ret = prepare_direct_functions_for_ipmodify(ops);
9265 if (ret < 0)
9266 goto out_unlock;
9267
9268 ret = register_ftrace_function_nolock(ops);
9269
9270out_unlock:
9271 unlock_direct_mutex();
9272 return ret;
9273}
9274EXPORT_SYMBOL_GPL(register_ftrace_function);
9275
9276/**
9277 * unregister_ftrace_function - unregister a function for profiling.
9278 * @ops: ops structure that holds the function to unregister
9279 *
9280 * Unregister a function that was added to be called by ftrace profiling.
9281 */
9282int unregister_ftrace_function(struct ftrace_ops *ops)
9283{
9284 int ret;
9285
9286 mutex_lock(&ftrace_lock);
9287 ret = ftrace_shutdown(ops, 0);
9288 mutex_unlock(&ftrace_lock);
9289
9290 cleanup_direct_functions_after_ipmodify(ops);
9291 return ret;
9292}
9293EXPORT_SYMBOL_GPL(unregister_ftrace_function);
9294
9295static int symbols_cmp(const void *a, const void *b)
9296{
9297 const char **str_a = (const char **) a;
9298 const char **str_b = (const char **) b;
9299
9300 return strcmp(*str_a, *str_b);
9301}
9302
9303struct kallsyms_data {
9304 unsigned long *addrs;
9305 const char **syms;
9306 size_t cnt;
9307 size_t found;
9308};
9309
9310/* This function gets called for all kernel and module symbols
9311 * and returns 1 in case we resolved all the requested symbols,
9312 * 0 otherwise.
9313 */
9314static int kallsyms_callback(void *data, const char *name, unsigned long addr)
9315{
9316 struct kallsyms_data *args = data;
9317 const char **sym;
9318 int idx;
9319
9320 sym = bsearch(&name, args->syms, args->cnt, sizeof(*args->syms), symbols_cmp);
9321 if (!sym)
9322 return 0;
9323
9324 idx = sym - args->syms;
9325 if (args->addrs[idx])
9326 return 0;
9327
9328 if (!ftrace_location(addr))
9329 return 0;
9330
9331 args->addrs[idx] = addr;
9332 args->found++;
9333 return args->found == args->cnt ? 1 : 0;
9334}
9335
9336/**
9337 * ftrace_lookup_symbols - Lookup addresses for array of symbols
9338 *
9339 * @sorted_syms: array of symbols pointers symbols to resolve,
9340 * must be alphabetically sorted
9341 * @cnt: number of symbols/addresses in @syms/@addrs arrays
9342 * @addrs: array for storing resulting addresses
9343 *
9344 * This function looks up addresses for array of symbols provided in
9345 * @syms array (must be alphabetically sorted) and stores them in
9346 * @addrs array, which needs to be big enough to store at least @cnt
9347 * addresses.
9348 *
9349 * For a single symbol (cnt == 1), uses kallsyms_lookup_name() which
9350 * performs an O(log N) binary search via the sorted kallsyms index.
9351 * This avoids the full O(N) linear scan over all kernel symbols that
9352 * the multi-symbol path requires.
9353 *
9354 * For multiple symbols, uses a single-pass linear scan via
9355 * kallsyms_on_each_symbol() with binary search into the sorted input
9356 * array.
9357 *
9358 * Returns: 0 if all provided symbols are found, -ESRCH otherwise.
9359 */
9360int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs)
9361{
9362 struct kallsyms_data args;
9363 int found_all;
9364
9365 /* Fast path: single symbol uses O(log N) binary search */
9366 if (cnt == 1) {
9367 addrs[0] = kallsyms_lookup_name(sorted_syms[0]);
9368 if (addrs[0] && ftrace_location(addrs[0]))
9369 return 0;
9370 /*
9371 * Binary lookup can fail for duplicate symbol names
9372 * where the first match is not ftrace-instrumented.
9373 * Retry with linear scan.
9374 */
9375 }
9376
9377 /* Batch path: single-pass O(N) linear scan */
9378 memset(addrs, 0, sizeof(*addrs) * cnt);
9379 args.addrs = addrs;
9380 args.syms = sorted_syms;
9381 args.cnt = cnt;
9382 args.found = 0;
9383
9384 found_all = kallsyms_on_each_symbol(kallsyms_callback, &args);
9385 if (found_all)
9386 return 0;
9387 found_all = module_kallsyms_on_each_symbol(NULL, kallsyms_callback, &args);
9388 return found_all ? 0 : -ESRCH;
9389}
9390
9391#ifdef CONFIG_SYSCTL
9392
9393#ifdef CONFIG_DYNAMIC_FTRACE
9394static void ftrace_startup_sysctl(void)
9395{
9396 int command;
9397
9398 if (unlikely(ftrace_disabled))
9399 return;
9400
9401 /* Force update next time */
9402 saved_ftrace_func = NULL;
9403 /* ftrace_start_up is true if we want ftrace running */
9404 if (ftrace_start_up) {
9405 command = FTRACE_UPDATE_CALLS;
9406 if (ftrace_graph_active)
9407 command |= FTRACE_START_FUNC_RET;
9408 ftrace_startup_enable(command);
9409 }
9410}
9411
9412#else
9413# define ftrace_startup_sysctl() do { } while (0)
9414#endif /* CONFIG_DYNAMIC_FTRACE */
9415
9416static int
9417ftrace_enable_sysctl(const struct ctl_table *table, int write,
9418 void *buffer, size_t *lenp, loff_t *ppos)
9419{
9420 int ret;
9421
9422 guard(mutex)(&ftrace_lock);
9423
9424 if (unlikely(ftrace_disabled))
9425 return -ENODEV;
9426
9427 ret = proc_dointvec(table, write, buffer, lenp, ppos);
9428
9429 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
9430 return ret;
9431
9432 if (ftrace_enabled) {
9433
9434 /* we are starting ftrace again */
9435 if (rcu_dereference_protected(ftrace_ops_list,
9436 lockdep_is_held(&ftrace_lock)) != &ftrace_list_end)
9437 update_ftrace_function();
9438
9439 ftrace_startup_sysctl();
9440
9441 } else {
9442 /*
9443 * Disabling ftrace at runtime via this knob is deprecated.
9444 */
9445 ftrace_enabled = true;
9446 pr_warn_once("The ftrace_enabled file is deprecated and no longer disables ftrace\n");
9447 return -EOPNOTSUPP;
9448 }
9449
9450 last_ftrace_enabled = !!ftrace_enabled;
9451 return 0;
9452}
9453
9454static const struct ctl_table ftrace_sysctls[] = {
9455 {
9456 .procname = "ftrace_enabled",
9457 .data = &ftrace_enabled,
9458 .maxlen = sizeof(int),
9459 .mode = 0644,
9460 .proc_handler = ftrace_enable_sysctl,
9461 },
9462};
9463
9464static int __init ftrace_sysctl_init(void)
9465{
9466 register_sysctl_init("kernel", ftrace_sysctls);
9467 return 0;
9468}
9469late_initcall(ftrace_sysctl_init);
9470#endif