projects/linux·blob v7.0 · refs

972 lines · 24.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fprobe - Simple ftrace probe wrapper for function entry.
4 */
5#define pr_fmt(fmt) "fprobe: " fmt
6
7#include <linux/err.h>
8#include <linux/fprobe.h>
9#include <linux/kallsyms.h>
10#include <linux/kprobes.h>
11#include <linux/list.h>
12#include <linux/mutex.h>
13#include <linux/rhashtable.h>
14#include <linux/slab.h>
15#include <linux/sort.h>
16
17#include <asm/fprobe.h>
18
19#include "trace.h"
20
21#define FPROBE_IP_HASH_BITS 8
22#define FPROBE_IP_TABLE_SIZE (1 << FPROBE_IP_HASH_BITS)
23
24#define FPROBE_HASH_BITS 6
25#define FPROBE_TABLE_SIZE (1 << FPROBE_HASH_BITS)
26
27#define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2))
28
29/*
30 * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still
31 * exists. The key is the address of fprobe instance.
32 * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe
33 * instance related to the function address. The key is the ftrace IP
34 * address.
35 *
36 * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp
37 * are set NULL and delete those from both hash tables (by hlist_del_rcu).
38 * After an RCU grace period, the fprobe_hlist itself will be released.
39 *
40 * fprobe_table and fprobe_ip_table can be accessed from either
41 * - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held.
42 * - RCU hlist traversal under disabling preempt
43 */
44static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE];
45static struct rhltable fprobe_ip_table;
46static DEFINE_MUTEX(fprobe_mutex);
47static struct fgraph_ops fprobe_graph_ops;
48
49static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed)
50{
51 return hash_ptr(*(unsigned long **)data, 32);
52}
53
54static int fprobe_node_cmp(struct rhashtable_compare_arg *arg,
55 const void *ptr)
56{
57 unsigned long key = *(unsigned long *)arg->key;
58 const struct fprobe_hlist_node *n = ptr;
59
60 return n->addr != key;
61}
62
63static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed)
64{
65 const struct fprobe_hlist_node *n = data;
66
67 return hash_ptr((void *)n->addr, 32);
68}
69
70static const struct rhashtable_params fprobe_rht_params = {
71 .head_offset = offsetof(struct fprobe_hlist_node, hlist),
72 .key_offset = offsetof(struct fprobe_hlist_node, addr),
73 .key_len = sizeof_field(struct fprobe_hlist_node, addr),
74 .hashfn = fprobe_node_hashfn,
75 .obj_hashfn = fprobe_node_obj_hashfn,
76 .obj_cmpfn = fprobe_node_cmp,
77 .automatic_shrinking = true,
78};
79
80/* Node insertion and deletion requires the fprobe_mutex */
81static int insert_fprobe_node(struct fprobe_hlist_node *node)
82{
83 lockdep_assert_held(&fprobe_mutex);
84
85 return rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params);
86}
87
88/* Return true if there are synonims */
89static bool delete_fprobe_node(struct fprobe_hlist_node *node)
90{
91 lockdep_assert_held(&fprobe_mutex);
92 bool ret;
93
94 /* Avoid double deleting */
95 if (READ_ONCE(node->fp) != NULL) {
96 WRITE_ONCE(node->fp, NULL);
97 rhltable_remove(&fprobe_ip_table, &node->hlist,
98 fprobe_rht_params);
99 }
100
101 rcu_read_lock();
102 ret = !!rhltable_lookup(&fprobe_ip_table, &node->addr,
103 fprobe_rht_params);
104 rcu_read_unlock();
105
106 return ret;
107}
108
109/* Check existence of the fprobe */
110static bool is_fprobe_still_exist(struct fprobe *fp)
111{
112 struct hlist_head *head;
113 struct fprobe_hlist *fph;
114
115 head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];
116 hlist_for_each_entry_rcu(fph, head, hlist,
117 lockdep_is_held(&fprobe_mutex)) {
118 if (fph->fp == fp)
119 return true;
120 }
121 return false;
122}
123NOKPROBE_SYMBOL(is_fprobe_still_exist);
124
125static int add_fprobe_hash(struct fprobe *fp)
126{
127 struct fprobe_hlist *fph = fp->hlist_array;
128 struct hlist_head *head;
129
130 lockdep_assert_held(&fprobe_mutex);
131
132 if (WARN_ON_ONCE(!fph))
133 return -EINVAL;
134
135 if (is_fprobe_still_exist(fp))
136 return -EEXIST;
137
138 head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];
139 hlist_add_head_rcu(&fp->hlist_array->hlist, head);
140 return 0;
141}
142
143static int del_fprobe_hash(struct fprobe *fp)
144{
145 struct fprobe_hlist *fph = fp->hlist_array;
146
147 lockdep_assert_held(&fprobe_mutex);
148
149 if (WARN_ON_ONCE(!fph))
150 return -EINVAL;
151
152 if (!is_fprobe_still_exist(fp))
153 return -ENOENT;
154
155 fph->fp = NULL;
156 hlist_del_rcu(&fph->hlist);
157 return 0;
158}
159
160#ifdef ARCH_DEFINE_ENCODE_FPROBE_HEADER
161
162/* The arch should encode fprobe_header info into one unsigned long */
163#define FPROBE_HEADER_SIZE_IN_LONG 1
164
165static inline bool write_fprobe_header(unsigned long *stack,
166 struct fprobe *fp, unsigned int size_words)
167{
168 if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD ||
169 !arch_fprobe_header_encodable(fp)))
170 return false;
171
172 *stack = arch_encode_fprobe_header(fp, size_words);
173 return true;
174}
175
176static inline void read_fprobe_header(unsigned long *stack,
177 struct fprobe **fp, unsigned int *size_words)
178{
179 *fp = arch_decode_fprobe_header_fp(*stack);
180 *size_words = arch_decode_fprobe_header_size(*stack);
181}
182
183#else
184
185/* Generic fprobe_header */
186struct __fprobe_header {
187 struct fprobe *fp;
188 unsigned long size_words;
189} __packed;
190
191#define FPROBE_HEADER_SIZE_IN_LONG SIZE_IN_LONG(sizeof(struct __fprobe_header))
192
193static inline bool write_fprobe_header(unsigned long *stack,
194 struct fprobe *fp, unsigned int size_words)
195{
196 struct __fprobe_header *fph = (struct __fprobe_header *)stack;
197
198 if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD))
199 return false;
200
201 fph->fp = fp;
202 fph->size_words = size_words;
203 return true;
204}
205
206static inline void read_fprobe_header(unsigned long *stack,
207 struct fprobe **fp, unsigned int *size_words)
208{
209 struct __fprobe_header *fph = (struct __fprobe_header *)stack;
210
211 *fp = fph->fp;
212 *size_words = fph->size_words;
213}
214
215#endif
216
217/*
218 * fprobe shadow stack management:
219 * Since fprobe shares a single fgraph_ops, it needs to share the stack entry
220 * among the probes on the same function exit. Note that a new probe can be
221 * registered before a target function is returning, we can not use the hash
222 * table to find the corresponding probes. Thus the probe address is stored on
223 * the shadow stack with its entry data size.
224 *
225 */
226static inline int __fprobe_handler(unsigned long ip, unsigned long parent_ip,
227 struct fprobe *fp, struct ftrace_regs *fregs,
228 void *data)
229{
230 if (!fp->entry_handler)
231 return 0;
232
233 return fp->entry_handler(fp, ip, parent_ip, fregs, data);
234}
235
236static inline int __fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
237 struct fprobe *fp, struct ftrace_regs *fregs,
238 void *data)
239{
240 int ret;
241 /*
242 * This user handler is shared with other kprobes and is not expected to be
243 * called recursively. So if any other kprobe handler is running, this will
244 * exit as kprobe does. See the section 'Share the callbacks with kprobes'
245 * in Documentation/trace/fprobe.rst for more information.
246 */
247 if (unlikely(kprobe_running())) {
248 fp->nmissed++;
249 return 0;
250 }
251
252 kprobe_busy_begin();
253 ret = __fprobe_handler(ip, parent_ip, fp, fregs, data);
254 kprobe_busy_end();
255 return ret;
256}
257
258#if defined(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) || defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS)
259/* ftrace_ops callback, this processes fprobes which have only entry_handler. */
260static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip,
261 struct ftrace_ops *ops, struct ftrace_regs *fregs)
262{
263 struct fprobe_hlist_node *node;
264 struct rhlist_head *head, *pos;
265 struct fprobe *fp;
266 int bit;
267
268 bit = ftrace_test_recursion_trylock(ip, parent_ip);
269 if (bit < 0)
270 return;
271
272 /*
273 * ftrace_test_recursion_trylock() disables preemption, but
274 * rhltable_lookup() checks whether rcu_read_lcok is held.
275 * So we take rcu_read_lock() here.
276 */
277 rcu_read_lock();
278 head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params);
279
280 rhl_for_each_entry_rcu(node, pos, head, hlist) {
281 if (node->addr != ip)
282 break;
283 fp = READ_ONCE(node->fp);
284 if (unlikely(!fp || fprobe_disabled(fp) || fp->exit_handler))
285 continue;
286
287 if (fprobe_shared_with_kprobes(fp))
288 __fprobe_kprobe_handler(ip, parent_ip, fp, fregs, NULL);
289 else
290 __fprobe_handler(ip, parent_ip, fp, fregs, NULL);
291 }
292 rcu_read_unlock();
293 ftrace_test_recursion_unlock(bit);
294}
295NOKPROBE_SYMBOL(fprobe_ftrace_entry);
296
297static struct ftrace_ops fprobe_ftrace_ops = {
298 .func = fprobe_ftrace_entry,
299 .flags = FTRACE_OPS_FL_SAVE_ARGS,
300};
301static int fprobe_ftrace_active;
302
303static int fprobe_ftrace_add_ips(unsigned long *addrs, int num)
304{
305 int ret;
306
307 lockdep_assert_held(&fprobe_mutex);
308
309 ret = ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 0, 0);
310 if (ret)
311 return ret;
312
313 if (!fprobe_ftrace_active) {
314 ret = register_ftrace_function(&fprobe_ftrace_ops);
315 if (ret) {
316 ftrace_free_filter(&fprobe_ftrace_ops);
317 return ret;
318 }
319 }
320 fprobe_ftrace_active++;
321 return 0;
322}
323
324static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num)
325{
326 lockdep_assert_held(&fprobe_mutex);
327
328 fprobe_ftrace_active--;
329 if (!fprobe_ftrace_active)
330 unregister_ftrace_function(&fprobe_ftrace_ops);
331 if (num)
332 ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0);
333}
334
335static bool fprobe_is_ftrace(struct fprobe *fp)
336{
337 return !fp->exit_handler;
338}
339
340#ifdef CONFIG_MODULES
341static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove,
342 int reset)
343{
344 ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset);
345 ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, remove, reset);
346}
347#endif
348#else
349static int fprobe_ftrace_add_ips(unsigned long *addrs, int num)
350{
351 return -ENOENT;
352}
353
354static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num)
355{
356}
357
358static bool fprobe_is_ftrace(struct fprobe *fp)
359{
360 return false;
361}
362
363#ifdef CONFIG_MODULES
364static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove,
365 int reset)
366{
367 ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset);
368}
369#endif
370#endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_REGS */
371
372/* fgraph_ops callback, this processes fprobes which have exit_handler. */
373static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
374 struct ftrace_regs *fregs)
375{
376 unsigned long *fgraph_data = NULL;
377 unsigned long func = trace->func;
378 struct fprobe_hlist_node *node;
379 struct rhlist_head *head, *pos;
380 unsigned long ret_ip;
381 int reserved_words;
382 struct fprobe *fp;
383 int used, ret;
384
385 if (WARN_ON_ONCE(!fregs))
386 return 0;
387
388 guard(rcu)();
389 head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params);
390 reserved_words = 0;
391 rhl_for_each_entry_rcu(node, pos, head, hlist) {
392 if (node->addr != func)
393 continue;
394 fp = READ_ONCE(node->fp);
395 if (!fp || !fp->exit_handler)
396 continue;
397 /*
398 * Since fprobe can be enabled until the next loop, we ignore the
399 * fprobe's disabled flag in this loop.
400 */
401 reserved_words +=
402 FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size);
403 }
404 if (reserved_words) {
405 fgraph_data = fgraph_reserve_data(gops->idx, reserved_words * sizeof(long));
406 if (unlikely(!fgraph_data)) {
407 rhl_for_each_entry_rcu(node, pos, head, hlist) {
408 if (node->addr != func)
409 continue;
410 fp = READ_ONCE(node->fp);
411 if (fp && !fprobe_disabled(fp) && !fprobe_is_ftrace(fp))
412 fp->nmissed++;
413 }
414 return 0;
415 }
416 }
417
418 /*
419 * TODO: recursion detection has been done in the fgraph. Thus we need
420 * to add a callback to increment missed counter.
421 */
422 ret_ip = ftrace_regs_get_return_address(fregs);
423 used = 0;
424 rhl_for_each_entry_rcu(node, pos, head, hlist) {
425 int data_size;
426 void *data;
427
428 if (node->addr != func)
429 continue;
430 fp = READ_ONCE(node->fp);
431 if (unlikely(!fp || fprobe_disabled(fp) || fprobe_is_ftrace(fp)))
432 continue;
433
434 data_size = fp->entry_data_size;
435 if (data_size && fp->exit_handler)
436 data = fgraph_data + used + FPROBE_HEADER_SIZE_IN_LONG;
437 else
438 data = NULL;
439
440 if (fprobe_shared_with_kprobes(fp))
441 ret = __fprobe_kprobe_handler(func, ret_ip, fp, fregs, data);
442 else
443 ret = __fprobe_handler(func, ret_ip, fp, fregs, data);
444
445 /* If entry_handler returns !0, nmissed is not counted but skips exit_handler. */
446 if (!ret && fp->exit_handler) {
447 int size_words = SIZE_IN_LONG(data_size);
448
449 if (write_fprobe_header(&fgraph_data[used], fp, size_words))
450 used += FPROBE_HEADER_SIZE_IN_LONG + size_words;
451 }
452 }
453 if (used < reserved_words)
454 memset(fgraph_data + used, 0, reserved_words - used);
455
456 /* If any exit_handler is set, data must be used. */
457 return used != 0;
458}
459NOKPROBE_SYMBOL(fprobe_fgraph_entry);
460
461static void fprobe_return(struct ftrace_graph_ret *trace,
462 struct fgraph_ops *gops,
463 struct ftrace_regs *fregs)
464{
465 unsigned long *fgraph_data = NULL;
466 unsigned long ret_ip;
467 struct fprobe *fp;
468 int size, curr;
469 int size_words;
470
471 fgraph_data = (unsigned long *)fgraph_retrieve_data(gops->idx, &size);
472 if (WARN_ON_ONCE(!fgraph_data))
473 return;
474 size_words = SIZE_IN_LONG(size);
475 ret_ip = ftrace_regs_get_instruction_pointer(fregs);
476
477 preempt_disable_notrace();
478
479 curr = 0;
480 while (size_words > curr) {
481 read_fprobe_header(&fgraph_data[curr], &fp, &size);
482 if (!fp)
483 break;
484 curr += FPROBE_HEADER_SIZE_IN_LONG;
485 if (is_fprobe_still_exist(fp) && !fprobe_disabled(fp)) {
486 if (WARN_ON_ONCE(curr + size > size_words))
487 break;
488 fp->exit_handler(fp, trace->func, ret_ip, fregs,
489 size ? fgraph_data + curr : NULL);
490 }
491 curr += size;
492 }
493 preempt_enable_notrace();
494}
495NOKPROBE_SYMBOL(fprobe_return);
496
497static struct fgraph_ops fprobe_graph_ops = {
498 .entryfunc = fprobe_fgraph_entry,
499 .retfunc = fprobe_return,
500};
501static int fprobe_graph_active;
502
503/* Add @addrs to the ftrace filter and register fgraph if needed. */
504static int fprobe_graph_add_ips(unsigned long *addrs, int num)
505{
506 int ret;
507
508 lockdep_assert_held(&fprobe_mutex);
509
510 ret = ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 0, 0);
511 if (ret)
512 return ret;
513
514 if (!fprobe_graph_active) {
515 ret = register_ftrace_graph(&fprobe_graph_ops);
516 if (WARN_ON_ONCE(ret)) {
517 ftrace_free_filter(&fprobe_graph_ops.ops);
518 return ret;
519 }
520 }
521 fprobe_graph_active++;
522 return 0;
523}
524
525/* Remove @addrs from the ftrace filter and unregister fgraph if possible. */
526static void fprobe_graph_remove_ips(unsigned long *addrs, int num)
527{
528 lockdep_assert_held(&fprobe_mutex);
529
530 fprobe_graph_active--;
531 /* Q: should we unregister it ? */
532 if (!fprobe_graph_active)
533 unregister_ftrace_graph(&fprobe_graph_ops);
534
535 if (num)
536 ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 1, 0);
537}
538
539#ifdef CONFIG_MODULES
540
541#define FPROBE_IPS_BATCH_INIT 8
542/* instruction pointer address list */
543struct fprobe_addr_list {
544 int index;
545 int size;
546 unsigned long *addrs;
547};
548
549static int fprobe_addr_list_add(struct fprobe_addr_list *alist, unsigned long addr)
550{
551 unsigned long *addrs;
552
553 /* Previously we failed to expand the list. */
554 if (alist->index == alist->size)
555 return -ENOSPC;
556
557 alist->addrs[alist->index++] = addr;
558 if (alist->index < alist->size)
559 return 0;
560
561 /* Expand the address list */
562 addrs = kcalloc(alist->size * 2, sizeof(*addrs), GFP_KERNEL);
563 if (!addrs)
564 return -ENOMEM;
565
566 memcpy(addrs, alist->addrs, alist->size * sizeof(*addrs));
567 alist->size *= 2;
568 kfree(alist->addrs);
569 alist->addrs = addrs;
570
571 return 0;
572}
573
574static void fprobe_remove_node_in_module(struct module *mod, struct fprobe_hlist_node *node,
575 struct fprobe_addr_list *alist)
576{
577 if (!within_module(node->addr, mod))
578 return;
579 if (delete_fprobe_node(node))
580 return;
581 /*
582 * If failed to update alist, just continue to update hlist.
583 * Therefore, at list user handler will not hit anymore.
584 */
585 fprobe_addr_list_add(alist, node->addr);
586}
587
588/* Handle module unloading to manage fprobe_ip_table. */
589static int fprobe_module_callback(struct notifier_block *nb,
590 unsigned long val, void *data)
591{
592 struct fprobe_addr_list alist = {.size = FPROBE_IPS_BATCH_INIT};
593 struct fprobe_hlist_node *node;
594 struct rhashtable_iter iter;
595 struct module *mod = data;
596
597 if (val != MODULE_STATE_GOING)
598 return NOTIFY_DONE;
599
600 alist.addrs = kcalloc(alist.size, sizeof(*alist.addrs), GFP_KERNEL);
601 /* If failed to alloc memory, we can not remove ips from hash. */
602 if (!alist.addrs)
603 return NOTIFY_DONE;
604
605 mutex_lock(&fprobe_mutex);
606 rhltable_walk_enter(&fprobe_ip_table, &iter);
607 do {
608 rhashtable_walk_start(&iter);
609
610 while ((node = rhashtable_walk_next(&iter)) && !IS_ERR(node))
611 fprobe_remove_node_in_module(mod, node, &alist);
612
613 rhashtable_walk_stop(&iter);
614 } while (node == ERR_PTR(-EAGAIN));
615 rhashtable_walk_exit(&iter);
616
617 if (alist.index > 0)
618 fprobe_set_ips(alist.addrs, alist.index, 1, 0);
619 mutex_unlock(&fprobe_mutex);
620
621 kfree(alist.addrs);
622
623 return NOTIFY_DONE;
624}
625
626static struct notifier_block fprobe_module_nb = {
627 .notifier_call = fprobe_module_callback,
628 .priority = 0,
629};
630
631static int __init init_fprobe_module(void)
632{
633 return register_module_notifier(&fprobe_module_nb);
634}
635early_initcall(init_fprobe_module);
636#endif
637
638static int symbols_cmp(const void *a, const void *b)
639{
640 const char **str_a = (const char **) a;
641 const char **str_b = (const char **) b;
642
643 return strcmp(*str_a, *str_b);
644}
645
646/* Convert ftrace location address from symbols */
647static unsigned long *get_ftrace_locations(const char **syms, int num)
648{
649 unsigned long *addrs;
650
651 /* Convert symbols to symbol address */
652 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
653 if (!addrs)
654 return ERR_PTR(-ENOMEM);
655
656 /* ftrace_lookup_symbols expects sorted symbols */
657 sort(syms, num, sizeof(*syms), symbols_cmp, NULL);
658
659 if (!ftrace_lookup_symbols(syms, num, addrs))
660 return addrs;
661
662 kfree(addrs);
663 return ERR_PTR(-ENOENT);
664}
665
666struct filter_match_data {
667 const char *filter;
668 const char *notfilter;
669 size_t index;
670 size_t size;
671 unsigned long *addrs;
672 struct module **mods;
673};
674
675static int filter_match_callback(void *data, const char *name, unsigned long addr)
676{
677 struct filter_match_data *match = data;
678
679 if (!glob_match(match->filter, name) ||
680 (match->notfilter && glob_match(match->notfilter, name)))
681 return 0;
682
683 if (!ftrace_location(addr))
684 return 0;
685
686 if (match->addrs) {
687 struct module *mod = __module_text_address(addr);
688
689 if (mod && !try_module_get(mod))
690 return 0;
691
692 match->mods[match->index] = mod;
693 match->addrs[match->index] = addr;
694 }
695 match->index++;
696 return match->index == match->size;
697}
698
699/*
700 * Make IP list from the filter/no-filter glob patterns.
701 * Return the number of matched symbols, or errno.
702 * If @addrs == NULL, this just counts the number of matched symbols. If @addrs
703 * is passed with an array, we need to pass the an @mods array of the same size
704 * to increment the module refcount for each symbol.
705 * This means we also need to call `module_put` for each element of @mods after
706 * using the @addrs.
707 */
708static int get_ips_from_filter(const char *filter, const char *notfilter,
709 unsigned long *addrs, struct module **mods,
710 size_t size)
711{
712 struct filter_match_data match = { .filter = filter, .notfilter = notfilter,
713 .index = 0, .size = size, .addrs = addrs, .mods = mods};
714 int ret;
715
716 if (addrs && !mods)
717 return -EINVAL;
718
719 ret = kallsyms_on_each_symbol(filter_match_callback, &match);
720 if (ret < 0)
721 return ret;
722 if (IS_ENABLED(CONFIG_MODULES)) {
723 ret = module_kallsyms_on_each_symbol(NULL, filter_match_callback, &match);
724 if (ret < 0)
725 return ret;
726 }
727
728 return match.index ?: -ENOENT;
729}
730
731static void fprobe_fail_cleanup(struct fprobe *fp)
732{
733 kfree(fp->hlist_array);
734 fp->hlist_array = NULL;
735}
736
737/* Initialize the fprobe data structure. */
738static int fprobe_init(struct fprobe *fp, unsigned long *addrs, int num)
739{
740 struct fprobe_hlist *hlist_array;
741 unsigned long addr;
742 int size, i;
743
744 if (!fp || !addrs || num <= 0)
745 return -EINVAL;
746
747 size = ALIGN(fp->entry_data_size, sizeof(long));
748 if (size > MAX_FPROBE_DATA_SIZE)
749 return -E2BIG;
750 fp->entry_data_size = size;
751
752 hlist_array = kzalloc_flex(*hlist_array, array, num);
753 if (!hlist_array)
754 return -ENOMEM;
755
756 fp->nmissed = 0;
757
758 hlist_array->size = num;
759 fp->hlist_array = hlist_array;
760 hlist_array->fp = fp;
761 for (i = 0; i < num; i++) {
762 hlist_array->array[i].fp = fp;
763 addr = ftrace_location(addrs[i]);
764 if (!addr) {
765 fprobe_fail_cleanup(fp);
766 return -ENOENT;
767 }
768 hlist_array->array[i].addr = addr;
769 }
770 return 0;
771}
772
773#define FPROBE_IPS_MAX INT_MAX
774
775int fprobe_count_ips_from_filter(const char *filter, const char *notfilter)
776{
777 return get_ips_from_filter(filter, notfilter, NULL, NULL, FPROBE_IPS_MAX);
778}
779
780/**
781 * register_fprobe() - Register fprobe to ftrace by pattern.
782 * @fp: A fprobe data structure to be registered.
783 * @filter: A wildcard pattern of probed symbols.
784 * @notfilter: A wildcard pattern of NOT probed symbols.
785 *
786 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
787 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
788 *
789 * Return 0 if @fp is registered successfully, -errno if not.
790 */
791int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
792{
793 unsigned long *addrs __free(kfree) = NULL;
794 struct module **mods __free(kfree) = NULL;
795 int ret, num;
796
797 if (!fp || !filter)
798 return -EINVAL;
799
800 num = get_ips_from_filter(filter, notfilter, NULL, NULL, FPROBE_IPS_MAX);
801 if (num < 0)
802 return num;
803
804 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
805 if (!addrs)
806 return -ENOMEM;
807
808 mods = kzalloc_objs(*mods, num);
809 if (!mods)
810 return -ENOMEM;
811
812 ret = get_ips_from_filter(filter, notfilter, addrs, mods, num);
813 if (ret < 0)
814 return ret;
815
816 ret = register_fprobe_ips(fp, addrs, ret);
817
818 for (int i = 0; i < num; i++) {
819 if (mods[i])
820 module_put(mods[i]);
821 }
822 return ret;
823}
824EXPORT_SYMBOL_GPL(register_fprobe);
825
826/**
827 * register_fprobe_ips() - Register fprobe to ftrace by address.
828 * @fp: A fprobe data structure to be registered.
829 * @addrs: An array of target function address.
830 * @num: The number of entries of @addrs.
831 *
832 * Register @fp to ftrace for enabling the probe on the address given by @addrs.
833 * The @addrs must be the addresses of ftrace location address, which may be
834 * the symbol address + arch-dependent offset.
835 * If you unsure what this mean, please use other registration functions.
836 *
837 * Return 0 if @fp is registered successfully, -errno if not.
838 */
839int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
840{
841 struct fprobe_hlist *hlist_array;
842 int ret, i;
843
844 ret = fprobe_init(fp, addrs, num);
845 if (ret)
846 return ret;
847
848 mutex_lock(&fprobe_mutex);
849
850 hlist_array = fp->hlist_array;
851 if (fprobe_is_ftrace(fp))
852 ret = fprobe_ftrace_add_ips(addrs, num);
853 else
854 ret = fprobe_graph_add_ips(addrs, num);
855
856 if (!ret) {
857 add_fprobe_hash(fp);
858 for (i = 0; i < hlist_array->size; i++) {
859 ret = insert_fprobe_node(&hlist_array->array[i]);
860 if (ret)
861 break;
862 }
863 /* fallback on insert error */
864 if (ret) {
865 for (i--; i >= 0; i--)
866 delete_fprobe_node(&hlist_array->array[i]);
867 }
868 }
869 mutex_unlock(&fprobe_mutex);
870
871 if (ret)
872 fprobe_fail_cleanup(fp);
873
874 return ret;
875}
876EXPORT_SYMBOL_GPL(register_fprobe_ips);
877
878/**
879 * register_fprobe_syms() - Register fprobe to ftrace by symbols.
880 * @fp: A fprobe data structure to be registered.
881 * @syms: An array of target symbols.
882 * @num: The number of entries of @syms.
883 *
884 * Register @fp to the symbols given by @syms array. This will be useful if
885 * you are sure the symbols exist in the kernel.
886 *
887 * Return 0 if @fp is registered successfully, -errno if not.
888 */
889int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
890{
891 unsigned long *addrs;
892 int ret;
893
894 if (!fp || !syms || num <= 0)
895 return -EINVAL;
896
897 addrs = get_ftrace_locations(syms, num);
898 if (IS_ERR(addrs))
899 return PTR_ERR(addrs);
900
901 ret = register_fprobe_ips(fp, addrs, num);
902
903 kfree(addrs);
904
905 return ret;
906}
907EXPORT_SYMBOL_GPL(register_fprobe_syms);
908
909bool fprobe_is_registered(struct fprobe *fp)
910{
911 if (!fp || !fp->hlist_array)
912 return false;
913 return true;
914}
915
916/**
917 * unregister_fprobe() - Unregister fprobe.
918 * @fp: A fprobe data structure to be unregistered.
919 *
920 * Unregister fprobe (and remove ftrace hooks from the function entries).
921 *
922 * Return 0 if @fp is unregistered successfully, -errno if not.
923 */
924int unregister_fprobe(struct fprobe *fp)
925{
926 struct fprobe_hlist *hlist_array;
927 unsigned long *addrs = NULL;
928 int ret = 0, i, count;
929
930 mutex_lock(&fprobe_mutex);
931 if (!fp || !is_fprobe_still_exist(fp)) {
932 ret = -EINVAL;
933 goto out;
934 }
935
936 hlist_array = fp->hlist_array;
937 addrs = kcalloc(hlist_array->size, sizeof(unsigned long), GFP_KERNEL);
938 if (!addrs) {
939 ret = -ENOMEM; /* TODO: Fallback to one-by-one loop */
940 goto out;
941 }
942
943 /* Remove non-synonim ips from table and hash */
944 count = 0;
945 for (i = 0; i < hlist_array->size; i++) {
946 if (!delete_fprobe_node(&hlist_array->array[i]))
947 addrs[count++] = hlist_array->array[i].addr;
948 }
949 del_fprobe_hash(fp);
950
951 if (fprobe_is_ftrace(fp))
952 fprobe_ftrace_remove_ips(addrs, count);
953 else
954 fprobe_graph_remove_ips(addrs, count);
955
956 kfree_rcu(hlist_array, rcu);
957 fp->hlist_array = NULL;
958
959out:
960 mutex_unlock(&fprobe_mutex);
961
962 kfree(addrs);
963 return ret;
964}
965EXPORT_SYMBOL_GPL(unregister_fprobe);
966
967static int __init fprobe_initcall(void)
968{
969 rhltable_init(&fprobe_ip_table, &fprobe_rht_params);
970 return 0;
971}
972core_initcall(fprobe_initcall);