projects/linux·blob master · refs

1159 lines · 28.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fprobe - Simple ftrace probe wrapper for function entry.
4 */
5#define pr_fmt(fmt) "fprobe: " fmt
6
7#include <linux/cleanup.h>
8#include <linux/err.h>
9#include <linux/fprobe.h>
10#include <linux/kallsyms.h>
11#include <linux/kprobes.h>
12#include <linux/list.h>
13#include <linux/mutex.h>
14#include <linux/rhashtable.h>
15#include <linux/slab.h>
16#include <linux/sort.h>
17
18#include <asm/fprobe.h>
19
20#include "trace.h"
21
22#define FPROBE_IP_HASH_BITS 8
23#define FPROBE_IP_TABLE_SIZE (1 << FPROBE_IP_HASH_BITS)
24
25#define FPROBE_HASH_BITS 6
26#define FPROBE_TABLE_SIZE (1 << FPROBE_HASH_BITS)
27
28#define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2))
29
30/*
31 * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still
32 * exists. The key is the address of fprobe instance.
33 * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe
34 * instance related to the function address. The key is the ftrace IP
35 * address.
36 *
37 * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp
38 * are set NULL and delete those from both hash tables (by hlist_del_rcu).
39 * After an RCU grace period, the fprobe_hlist itself will be released.
40 *
41 * fprobe_table and fprobe_ip_table can be accessed from either
42 * - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held.
43 * - RCU hlist traversal under disabling preempt
44 */
45static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE];
46static struct rhltable fprobe_ip_table;
47static DEFINE_MUTEX(fprobe_mutex);
48static struct fgraph_ops fprobe_graph_ops;
49
50DEFINE_LOCK_GUARD_0(rcu_sched_notrace,
51 rcu_read_lock_sched_notrace(),
52 rcu_read_unlock_sched_notrace())
53
54static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed)
55{
56 return hash_ptr(*(unsigned long **)data, 32);
57}
58
59static int fprobe_node_cmp(struct rhashtable_compare_arg *arg,
60 const void *ptr)
61{
62 unsigned long key = *(unsigned long *)arg->key;
63 const struct fprobe_hlist_node *n = ptr;
64
65 return n->addr != key;
66}
67
68static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed)
69{
70 const struct fprobe_hlist_node *n = data;
71
72 return hash_ptr((void *)n->addr, 32);
73}
74
75static const struct rhashtable_params fprobe_rht_params = {
76 .head_offset = offsetof(struct fprobe_hlist_node, hlist),
77 .key_offset = offsetof(struct fprobe_hlist_node, addr),
78 .key_len = sizeof_field(struct fprobe_hlist_node, addr),
79 .hashfn = fprobe_node_hashfn,
80 .obj_hashfn = fprobe_node_obj_hashfn,
81 .obj_cmpfn = fprobe_node_cmp,
82 .automatic_shrinking = true,
83};
84
85/* Node insertion and deletion requires the fprobe_mutex */
86static int __insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp)
87{
88 int ret;
89
90 lockdep_assert_held(&fprobe_mutex);
91
92 ret = rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params);
93 /* Set the fprobe pointer if insertion was successful. */
94 if (!ret)
95 WRITE_ONCE(node->fp, fp);
96 return ret;
97}
98
99static void __delete_fprobe_node(struct fprobe_hlist_node *node)
100{
101 lockdep_assert_held(&fprobe_mutex);
102
103 /* Avoid double deleting and non-inserted nodes */
104 if (READ_ONCE(node->fp) != NULL) {
105 WRITE_ONCE(node->fp, NULL);
106 rhltable_remove(&fprobe_ip_table, &node->hlist,
107 fprobe_rht_params);
108 }
109}
110
111/* Check existence of the fprobe */
112static bool fprobe_registered(struct fprobe *fp)
113{
114 struct hlist_head *head;
115 struct fprobe_hlist *fph;
116
117 head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];
118 hlist_for_each_entry_rcu(fph, head, hlist,
119 lockdep_is_held(&fprobe_mutex)) {
120 if (fph->fp == fp)
121 return true;
122 }
123 return false;
124}
125NOKPROBE_SYMBOL(fprobe_registered);
126
127static int add_fprobe_hash(struct fprobe *fp)
128{
129 struct fprobe_hlist *fph = fp->hlist_array;
130 struct hlist_head *head;
131
132 lockdep_assert_held(&fprobe_mutex);
133
134 if (WARN_ON_ONCE(!fph))
135 return -EINVAL;
136
137 head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];
138 hlist_add_head_rcu(&fp->hlist_array->hlist, head);
139 return 0;
140}
141
142static int del_fprobe_hash(struct fprobe *fp)
143{
144 struct fprobe_hlist *fph = fp->hlist_array;
145
146 lockdep_assert_held(&fprobe_mutex);
147
148 if (WARN_ON_ONCE(!fph))
149 return -EINVAL;
150
151 if (!fprobe_registered(fp))
152 return -ENOENT;
153
154 fph->fp = NULL;
155 hlist_del_rcu(&fph->hlist);
156 return 0;
157}
158
159#ifdef ARCH_DEFINE_ENCODE_FPROBE_HEADER
160
161/* The arch should encode fprobe_header info into one unsigned long */
162#define FPROBE_HEADER_SIZE_IN_LONG 1
163
164static inline bool write_fprobe_header(unsigned long *stack,
165 struct fprobe *fp, unsigned int size_words)
166{
167 if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD ||
168 !arch_fprobe_header_encodable(fp)))
169 return false;
170
171 *stack = arch_encode_fprobe_header(fp, size_words);
172 return true;
173}
174
175static inline void read_fprobe_header(unsigned long *stack,
176 struct fprobe **fp, unsigned int *size_words)
177{
178 if (!*stack) {
179 *fp = NULL;
180 *size_words = 0;
181 return;
182 }
183 *fp = arch_decode_fprobe_header_fp(*stack);
184 *size_words = arch_decode_fprobe_header_size(*stack);
185}
186
187#else
188
189/* Generic fprobe_header */
190struct __fprobe_header {
191 struct fprobe *fp;
192 unsigned long size_words;
193};
194
195#define FPROBE_HEADER_SIZE_IN_LONG SIZE_IN_LONG(sizeof(struct __fprobe_header))
196
197static inline bool write_fprobe_header(unsigned long *stack,
198 struct fprobe *fp, unsigned int size_words)
199{
200 struct __fprobe_header *fph = (struct __fprobe_header *)stack;
201
202 if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD))
203 return false;
204
205 fph->fp = fp;
206 fph->size_words = size_words;
207 return true;
208}
209
210static inline void read_fprobe_header(unsigned long *stack,
211 struct fprobe **fp, unsigned int *size_words)
212{
213 struct __fprobe_header *fph = (struct __fprobe_header *)stack;
214
215 if (!*stack) {
216 *fp = NULL;
217 *size_words = 0;
218 return;
219 }
220
221 *fp = fph->fp;
222 *size_words = fph->size_words;
223}
224
225#endif
226
227/*
228 * fprobe shadow stack management:
229 * Since fprobe shares a single fgraph_ops, it needs to share the stack entry
230 * among the probes on the same function exit. Note that a new probe can be
231 * registered before a target function is returning, we can not use the hash
232 * table to find the corresponding probes. Thus the probe address is stored on
233 * the shadow stack with its entry data size.
234 *
235 */
236static inline int __fprobe_handler(unsigned long ip, unsigned long parent_ip,
237 struct fprobe *fp, struct ftrace_regs *fregs,
238 void *data)
239{
240 if (!fp->entry_handler)
241 return 0;
242
243 return fp->entry_handler(fp, ip, parent_ip, fregs, data);
244}
245
246static inline int __fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
247 struct fprobe *fp, struct ftrace_regs *fregs,
248 void *data)
249{
250 int ret;
251 /*
252 * This user handler is shared with other kprobes and is not expected to be
253 * called recursively. So if any other kprobe handler is running, this will
254 * exit as kprobe does. See the section 'Share the callbacks with kprobes'
255 * in Documentation/trace/fprobe.rst for more information.
256 */
257 if (unlikely(kprobe_running())) {
258 fp->nmissed++;
259 return 0;
260 }
261
262 kprobe_busy_begin();
263 ret = __fprobe_handler(ip, parent_ip, fp, fregs, data);
264 kprobe_busy_end();
265 return ret;
266}
267
268static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
269 struct ftrace_regs *fregs);
270static void fprobe_return(struct ftrace_graph_ret *trace,
271 struct fgraph_ops *gops,
272 struct ftrace_regs *fregs);
273
274static struct fgraph_ops fprobe_graph_ops = {
275 .entryfunc = fprobe_fgraph_entry,
276 .retfunc = fprobe_return,
277};
278/* Number of fgraph fprobe nodes */
279static int nr_fgraph_fprobes;
280/* Is fprobe_graph_ops registered? */
281static bool fprobe_graph_registered;
282
283/* Add @addrs to the ftrace filter and register fgraph if needed. */
284static int fprobe_graph_add_ips(unsigned long *addrs, int num)
285{
286 int ret;
287
288 lockdep_assert_held(&fprobe_mutex);
289
290 ret = ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 0, 0);
291 if (ret)
292 return ret;
293
294 if (!fprobe_graph_registered) {
295 ret = register_ftrace_graph(&fprobe_graph_ops);
296 if (WARN_ON_ONCE(ret)) {
297 ftrace_free_filter(&fprobe_graph_ops.ops);
298 return ret;
299 }
300 fprobe_graph_registered = true;
301 }
302 return 0;
303}
304
305static void __fprobe_graph_unregister(void)
306{
307 if (fprobe_graph_registered) {
308 unregister_ftrace_graph(&fprobe_graph_ops);
309 ftrace_free_filter(&fprobe_graph_ops.ops);
310 fprobe_graph_registered = false;
311 }
312}
313
314/* Remove @addrs from the ftrace filter and unregister fgraph if possible. */
315static void fprobe_graph_remove_ips(unsigned long *addrs, int num)
316{
317 lockdep_assert_held(&fprobe_mutex);
318
319 if (!nr_fgraph_fprobes)
320 __fprobe_graph_unregister();
321 else if (num)
322 ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 1, 0);
323}
324
325#if defined(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) || defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS)
326
327/* ftrace_ops callback, this processes fprobes which have only entry_handler. */
328static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip,
329 struct ftrace_ops *ops, struct ftrace_regs *fregs)
330{
331 struct fprobe_hlist_node *node;
332 struct rhlist_head *head, *pos;
333 struct fprobe *fp;
334 int bit;
335
336 if (!rcu_is_watching())
337 return;
338
339 bit = ftrace_test_recursion_trylock(ip, parent_ip);
340 if (bit < 0)
341 return;
342
343 guard(rcu_sched_notrace)();
344 head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params);
345
346 rhl_for_each_entry_rcu(node, pos, head, hlist) {
347 if (node->addr != ip)
348 break;
349 fp = READ_ONCE(node->fp);
350 if (unlikely(!fp || fprobe_disabled(fp) || fp->exit_handler))
351 continue;
352
353 if (fprobe_shared_with_kprobes(fp))
354 __fprobe_kprobe_handler(ip, parent_ip, fp, fregs, NULL);
355 else
356 __fprobe_handler(ip, parent_ip, fp, fregs, NULL);
357 }
358 ftrace_test_recursion_unlock(bit);
359}
360NOKPROBE_SYMBOL(fprobe_ftrace_entry);
361
362static struct ftrace_ops fprobe_ftrace_ops = {
363 .func = fprobe_ftrace_entry,
364 .flags = FTRACE_OPS_FL_SAVE_ARGS,
365};
366/* Number of ftrace fprobe nodes */
367static int nr_ftrace_fprobes;
368/* Is fprobe_ftrace_ops registered? */
369static bool fprobe_ftrace_registered;
370
371static int fprobe_ftrace_add_ips(unsigned long *addrs, int num)
372{
373 int ret;
374
375 lockdep_assert_held(&fprobe_mutex);
376
377 ret = ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 0, 0);
378 if (ret)
379 return ret;
380
381 if (!fprobe_ftrace_registered) {
382 ret = register_ftrace_function(&fprobe_ftrace_ops);
383 if (ret) {
384 ftrace_free_filter(&fprobe_ftrace_ops);
385 return ret;
386 }
387 fprobe_ftrace_registered = true;
388 }
389 return 0;
390}
391
392static void __fprobe_ftrace_unregister(void)
393{
394 if (fprobe_ftrace_registered) {
395 unregister_ftrace_function(&fprobe_ftrace_ops);
396 ftrace_free_filter(&fprobe_ftrace_ops);
397 fprobe_ftrace_registered = false;
398 }
399}
400
401static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num)
402{
403 lockdep_assert_held(&fprobe_mutex);
404
405 if (!nr_ftrace_fprobes)
406 __fprobe_ftrace_unregister();
407 else if (num)
408 ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0);
409}
410
411static bool fprobe_is_ftrace(struct fprobe *fp)
412{
413 return !fp->exit_handler;
414}
415
416/* Node insertion and deletion requires the fprobe_mutex */
417static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp)
418{
419 int ret;
420
421 lockdep_assert_held(&fprobe_mutex);
422
423 ret = __insert_fprobe_node(node, fp);
424 if (!ret) {
425 if (fprobe_is_ftrace(fp))
426 nr_ftrace_fprobes++;
427 else
428 nr_fgraph_fprobes++;
429 }
430
431 return ret;
432}
433
434static void delete_fprobe_node(struct fprobe_hlist_node *node)
435{
436 struct fprobe *fp;
437
438 lockdep_assert_held(&fprobe_mutex);
439
440 fp = READ_ONCE(node->fp);
441 if (fp) {
442 if (fprobe_is_ftrace(fp))
443 nr_ftrace_fprobes--;
444 else
445 nr_fgraph_fprobes--;
446 }
447 __delete_fprobe_node(node);
448}
449
450static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace)
451{
452 struct rhlist_head *head, *pos;
453 struct fprobe_hlist_node *node;
454 struct fprobe *fp;
455
456 guard(rcu)();
457 head = rhltable_lookup(&fprobe_ip_table, &ip,
458 fprobe_rht_params);
459 if (!head)
460 return false;
461 /* We have to check the same type on the list. */
462 rhl_for_each_entry_rcu(node, pos, head, hlist) {
463 if (node->addr != ip)
464 break;
465 fp = READ_ONCE(node->fp);
466 if (likely(fp)) {
467 if ((!ftrace && fp->exit_handler) ||
468 (ftrace && !fp->exit_handler))
469 return true;
470 }
471 }
472
473 return false;
474}
475
476#ifdef CONFIG_MODULES
477static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt)
478{
479 fprobe_graph_remove_ips(ips, cnt);
480 fprobe_ftrace_remove_ips(ips, cnt);
481}
482#endif
483#else
484static int fprobe_ftrace_add_ips(unsigned long *addrs, int num)
485{
486 return -ENOENT;
487}
488
489static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num)
490{
491}
492
493static bool fprobe_is_ftrace(struct fprobe *fp)
494{
495 return false;
496}
497
498/* Node insertion and deletion requires the fprobe_mutex */
499static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp)
500{
501 int ret;
502
503 lockdep_assert_held(&fprobe_mutex);
504
505 ret = __insert_fprobe_node(node, fp);
506 if (!ret)
507 nr_fgraph_fprobes++;
508
509 return ret;
510}
511
512static void delete_fprobe_node(struct fprobe_hlist_node *node)
513{
514 struct fprobe *fp;
515
516 lockdep_assert_held(&fprobe_mutex);
517
518 fp = READ_ONCE(node->fp);
519 if (fp)
520 nr_fgraph_fprobes--;
521 __delete_fprobe_node(node);
522}
523
524static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace __maybe_unused)
525{
526 struct rhlist_head *head, *pos;
527 struct fprobe_hlist_node *node;
528 struct fprobe *fp;
529
530 guard(rcu)();
531 head = rhltable_lookup(&fprobe_ip_table, &ip,
532 fprobe_rht_params);
533 if (!head)
534 return false;
535 /* We only need to check fp is there. */
536 rhl_for_each_entry_rcu(node, pos, head, hlist) {
537 if (node->addr != ip)
538 break;
539 fp = READ_ONCE(node->fp);
540 if (likely(fp))
541 return true;
542 }
543
544 return false;
545}
546
547#ifdef CONFIG_MODULES
548static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt)
549{
550 if (!nr_fgraph_fprobes)
551 __fprobe_graph_unregister();
552 else if (cnt)
553 ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0);
554}
555#endif
556#endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_REGS */
557
558/* fgraph_ops callback, this processes fprobes which have exit_handler. */
559static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
560 struct ftrace_regs *fregs)
561{
562 unsigned long *fgraph_data = NULL;
563 unsigned long func = trace->func;
564 struct fprobe_hlist_node *node;
565 struct rhlist_head *head, *pos;
566 unsigned long ret_ip;
567 int reserved_words;
568 struct fprobe *fp;
569 int used, ret;
570
571 if (!rcu_is_watching())
572 return 0;
573
574 if (WARN_ON_ONCE(!fregs))
575 return 0;
576
577 guard(rcu_sched_notrace)();
578 head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params);
579 reserved_words = 0;
580 rhl_for_each_entry_rcu(node, pos, head, hlist) {
581 if (node->addr != func)
582 continue;
583 fp = READ_ONCE(node->fp);
584 if (!fp || !fp->exit_handler)
585 continue;
586 /*
587 * Since fprobe can be enabled until the next loop, we ignore the
588 * fprobe's disabled flag in this loop.
589 */
590 reserved_words +=
591 FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size);
592 }
593 if (reserved_words) {
594 fgraph_data = fgraph_reserve_data(gops->idx, reserved_words * sizeof(long));
595 if (unlikely(!fgraph_data)) {
596 rhl_for_each_entry_rcu(node, pos, head, hlist) {
597 if (node->addr != func)
598 continue;
599 fp = READ_ONCE(node->fp);
600 if (fp && !fprobe_disabled(fp) && !fprobe_is_ftrace(fp))
601 fp->nmissed++;
602 }
603 return 0;
604 }
605 }
606
607 /*
608 * TODO: recursion detection has been done in the fgraph. Thus we need
609 * to add a callback to increment missed counter.
610 */
611 ret_ip = ftrace_regs_get_return_address(fregs);
612 used = 0;
613 rhl_for_each_entry_rcu(node, pos, head, hlist) {
614 int data_size;
615 void *data;
616
617 if (node->addr != func)
618 continue;
619 fp = READ_ONCE(node->fp);
620 if (unlikely(!fp || fprobe_disabled(fp) || fprobe_is_ftrace(fp)))
621 continue;
622
623 data_size = fp->entry_data_size;
624 /*
625 * The list may have grown since it was sized, so this node
626 * may not fit. Skip it as missed rather than overrun the
627 * reservation.
628 */
629 if (fp->exit_handler &&
630 used + FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(data_size) > reserved_words) {
631 fp->nmissed++;
632 continue;
633 }
634 if (data_size && fp->exit_handler)
635 data = fgraph_data + used + FPROBE_HEADER_SIZE_IN_LONG;
636 else
637 data = NULL;
638
639 if (fprobe_shared_with_kprobes(fp))
640 ret = __fprobe_kprobe_handler(func, ret_ip, fp, fregs, data);
641 else
642 ret = __fprobe_handler(func, ret_ip, fp, fregs, data);
643
644 /* If entry_handler returns !0, nmissed is not counted but skips exit_handler. */
645 if (!ret && fp->exit_handler) {
646 int size_words = SIZE_IN_LONG(data_size);
647
648 if (write_fprobe_header(&fgraph_data[used], fp, size_words))
649 used += FPROBE_HEADER_SIZE_IN_LONG + size_words;
650 }
651 }
652
653 /* Terminate the list, fgraph_reserve_data() does not clear it. */
654 if (used && used < reserved_words)
655 fgraph_data[used] = 0;
656
657 /* If any exit_handler is set, data must be used. */
658 return used != 0;
659}
660NOKPROBE_SYMBOL(fprobe_fgraph_entry);
661
662static void fprobe_return(struct ftrace_graph_ret *trace,
663 struct fgraph_ops *gops,
664 struct ftrace_regs *fregs)
665{
666 unsigned long *fgraph_data = NULL;
667 unsigned long ret_ip;
668 struct fprobe *fp;
669 int size, curr;
670 int size_words;
671
672 if (!rcu_is_watching())
673 return;
674
675 fgraph_data = (unsigned long *)fgraph_retrieve_data(gops->idx, &size);
676 if (WARN_ON_ONCE(!fgraph_data))
677 return;
678 size_words = SIZE_IN_LONG(size);
679 ret_ip = ftrace_regs_get_instruction_pointer(fregs);
680
681 guard(rcu_sched_notrace)();
682
683 curr = 0;
684 while (size_words > curr) {
685 read_fprobe_header(&fgraph_data[curr], &fp, &size);
686 if (!fp)
687 break;
688 curr += FPROBE_HEADER_SIZE_IN_LONG;
689 if (fprobe_registered(fp) && !fprobe_disabled(fp)) {
690 if (WARN_ON_ONCE(curr + size > size_words))
691 break;
692 fp->exit_handler(fp, trace->func, ret_ip, fregs,
693 size ? fgraph_data + curr : NULL);
694 }
695 curr += size;
696 }
697}
698NOKPROBE_SYMBOL(fprobe_return);
699
700#ifdef CONFIG_MODULES
701
702#define FPROBE_IPS_BATCH_INIT 128
703/* instruction pointer address list */
704struct fprobe_addr_list {
705 int index;
706 int size;
707 unsigned long *addrs;
708};
709
710static int fprobe_remove_node_in_module(struct module *mod, struct fprobe_hlist_node *node,
711 struct fprobe_addr_list *alist)
712{
713 lockdep_assert_in_rcu_read_lock();
714
715 if (!within_module(node->addr, mod))
716 return 0;
717
718 delete_fprobe_node(node);
719 /* If no address list is available, we can't track this address. */
720 if (!alist->addrs)
721 return 0;
722 /*
723 * Don't care the type here, because all fprobes on the same
724 * address must be removed eventually.
725 */
726 if (!rhltable_lookup(&fprobe_ip_table, &node->addr, fprobe_rht_params)) {
727 alist->addrs[alist->index++] = node->addr;
728 if (alist->index == alist->size)
729 return -ENOSPC;
730 }
731
732 return 0;
733}
734
735/* Handle module unloading to manage fprobe_ip_table. */
736static int fprobe_module_callback(struct notifier_block *nb,
737 unsigned long val, void *data)
738{
739 struct fprobe_addr_list alist = {.size = FPROBE_IPS_BATCH_INIT};
740 struct fprobe_hlist_node *node;
741 struct rhashtable_iter iter;
742 struct module *mod = data;
743 bool retry;
744
745 if (val != MODULE_STATE_GOING)
746 return NOTIFY_DONE;
747
748 alist.addrs = kcalloc(alist.size, sizeof(*alist.addrs), GFP_KERNEL);
749 /*
750 * If failed to alloc memory, ftrace_ops will not be able to remove ips from
751 * hash, but we can still remove nodes from fprobe_ip_table, so we can avoid
752 * the potential wrong callback. So just print a warning here and try to
753 * continue without address list.
754 */
755 WARN_ONCE(!alist.addrs,
756 "Failed to allocate memory for fprobe_addr_list, ftrace_ops will not be updated");
757
758 mutex_lock(&fprobe_mutex);
759again:
760 retry = false;
761 alist.index = 0;
762 rhltable_walk_enter(&fprobe_ip_table, &iter);
763 do {
764 rhashtable_walk_start(&iter);
765
766 while ((node = rhashtable_walk_next(&iter)) && !IS_ERR(node))
767 if (fprobe_remove_node_in_module(mod, node, &alist) < 0) {
768 retry = true;
769 break;
770 }
771
772 rhashtable_walk_stop(&iter);
773 } while (node == ERR_PTR(-EAGAIN) && !retry);
774 rhashtable_walk_exit(&iter);
775 /* Remove any ips from hash table(s) */
776 fprobe_remove_ips(alist.addrs, alist.index);
777 /*
778 * If we break rhashtable walk loop except for -EAGAIN, we need
779 * to restart looping from start for safety. Anyway, this is
780 * not a hotpath.
781 */
782 if (retry)
783 goto again;
784
785 mutex_unlock(&fprobe_mutex);
786
787 kfree(alist.addrs);
788
789 return NOTIFY_DONE;
790}
791
792static struct notifier_block fprobe_module_nb = {
793 .notifier_call = fprobe_module_callback,
794 .priority = 0,
795};
796
797static int __init init_fprobe_module(void)
798{
799 return register_module_notifier(&fprobe_module_nb);
800}
801early_initcall(init_fprobe_module);
802#endif
803
804static int symbols_cmp(const void *a, const void *b)
805{
806 const char **str_a = (const char **) a;
807 const char **str_b = (const char **) b;
808
809 return strcmp(*str_a, *str_b);
810}
811
812/* Convert ftrace location address from symbols */
813static unsigned long *get_ftrace_locations(const char **syms, int num)
814{
815 unsigned long *addrs;
816
817 /* Convert symbols to symbol address */
818 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
819 if (!addrs)
820 return ERR_PTR(-ENOMEM);
821
822 /* ftrace_lookup_symbols expects sorted symbols */
823 sort(syms, num, sizeof(*syms), symbols_cmp, NULL);
824
825 if (!ftrace_lookup_symbols(syms, num, addrs))
826 return addrs;
827
828 kfree(addrs);
829 return ERR_PTR(-ENOENT);
830}
831
832struct filter_match_data {
833 const char *filter;
834 const char *notfilter;
835 size_t index;
836 size_t size;
837 unsigned long *addrs;
838 struct module **mods;
839};
840
841static int filter_match_callback(void *data, const char *name, unsigned long addr)
842{
843 struct filter_match_data *match = data;
844
845 if (!glob_match(match->filter, name) ||
846 (match->notfilter && glob_match(match->notfilter, name)))
847 return 0;
848
849 if (!ftrace_location(addr))
850 return 0;
851
852 if (match->addrs) {
853 struct module *mod = __module_text_address(addr);
854
855 if (mod && !try_module_get(mod))
856 return 0;
857
858 match->mods[match->index] = mod;
859 match->addrs[match->index] = addr;
860 }
861 match->index++;
862 return match->index == match->size;
863}
864
865/*
866 * Make IP list from the filter/no-filter glob patterns.
867 * Return the number of matched symbols, or errno.
868 * If @addrs == NULL, this just counts the number of matched symbols. If @addrs
869 * is passed with an array, we need to pass the an @mods array of the same size
870 * to increment the module refcount for each symbol.
871 * This means we also need to call `module_put` for each element of @mods after
872 * using the @addrs.
873 */
874static int get_ips_from_filter(const char *filter, const char *notfilter,
875 unsigned long *addrs, struct module **mods,
876 size_t size)
877{
878 struct filter_match_data match = { .filter = filter, .notfilter = notfilter,
879 .index = 0, .size = size, .addrs = addrs, .mods = mods};
880 int ret;
881
882 if (addrs && !mods)
883 return -EINVAL;
884
885 ret = kallsyms_on_each_symbol(filter_match_callback, &match);
886 if (ret < 0)
887 return ret;
888 if (IS_ENABLED(CONFIG_MODULES)) {
889 ret = module_kallsyms_on_each_symbol(NULL, filter_match_callback, &match);
890 if (ret < 0)
891 return ret;
892 }
893
894 return match.index ?: -ENOENT;
895}
896
897static void fprobe_fail_cleanup(struct fprobe *fp)
898{
899 kfree(fp->hlist_array);
900 fp->hlist_array = NULL;
901}
902
903/* Initialize the fprobe data structure. */
904static int fprobe_init(struct fprobe *fp, unsigned long *addrs, int num)
905{
906 struct fprobe_hlist *hlist_array;
907 unsigned long addr;
908 int size, i;
909
910 if (!fp || !addrs || num <= 0)
911 return -EINVAL;
912
913 size = ALIGN(fp->entry_data_size, sizeof(long));
914 if (size > MAX_FPROBE_DATA_SIZE)
915 return -E2BIG;
916 fp->entry_data_size = size;
917
918 hlist_array = kzalloc_flex(*hlist_array, array, num);
919 if (!hlist_array)
920 return -ENOMEM;
921
922 fp->nmissed = 0;
923
924 hlist_array->size = num;
925 fp->hlist_array = hlist_array;
926 hlist_array->fp = fp;
927 for (i = 0; i < num; i++) {
928 addr = ftrace_location(addrs[i]);
929 if (!addr) {
930 fprobe_fail_cleanup(fp);
931 return -ENOENT;
932 }
933 hlist_array->array[i].addr = addr;
934 }
935 return 0;
936}
937
938#define FPROBE_IPS_MAX INT_MAX
939
940int fprobe_count_ips_from_filter(const char *filter, const char *notfilter)
941{
942 return get_ips_from_filter(filter, notfilter, NULL, NULL, FPROBE_IPS_MAX);
943}
944
945/**
946 * register_fprobe() - Register fprobe to ftrace by pattern.
947 * @fp: A fprobe data structure to be registered.
948 * @filter: A wildcard pattern of probed symbols.
949 * @notfilter: A wildcard pattern of NOT probed symbols.
950 *
951 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
952 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
953 *
954 * Return 0 if @fp is registered successfully, -errno if not.
955 */
956int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
957{
958 unsigned long *addrs __free(kfree) = NULL;
959 struct module **mods __free(kfree) = NULL;
960 int ret, num;
961
962 if (!fp || !filter)
963 return -EINVAL;
964
965 num = get_ips_from_filter(filter, notfilter, NULL, NULL, FPROBE_IPS_MAX);
966 if (num < 0)
967 return num;
968
969 addrs = kzalloc_objs(*addrs, num);
970 if (!addrs)
971 return -ENOMEM;
972
973 mods = kzalloc_objs(*mods, num);
974 if (!mods)
975 return -ENOMEM;
976
977 ret = get_ips_from_filter(filter, notfilter, addrs, mods, num);
978 if (ret >= 0)
979 ret = register_fprobe_ips(fp, addrs, ret);
980
981 for (int i = 0; i < num; i++) {
982 if (mods[i])
983 module_put(mods[i]);
984 }
985 return ret;
986}
987EXPORT_SYMBOL_GPL(register_fprobe);
988
989static int unregister_fprobe_nolock(struct fprobe *fp);
990
991/**
992 * register_fprobe_ips() - Register fprobe to ftrace by address.
993 * @fp: A fprobe data structure to be registered.
994 * @addrs: An array of target function address.
995 * @num: The number of entries of @addrs.
996 *
997 * Register @fp to ftrace for enabling the probe on the address given by @addrs.
998 * The @addrs must be the addresses of ftrace location address, which may be
999 * the symbol address + arch-dependent offset.
1000 * If you unsure what this mean, please use other registration functions.
1001 *
1002 * Return 0 if @fp is registered successfully, -errno if not.
1003 */
1004int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
1005{
1006 struct fprobe_hlist *hlist_array;
1007 int ret, i;
1008
1009 guard(mutex)(&fprobe_mutex);
1010 if (fprobe_registered(fp))
1011 return -EEXIST;
1012
1013 ret = fprobe_init(fp, addrs, num);
1014 if (ret)
1015 return ret;
1016
1017 if (fprobe_is_ftrace(fp))
1018 ret = fprobe_ftrace_add_ips(addrs, num);
1019 else
1020 ret = fprobe_graph_add_ips(addrs, num);
1021 if (ret) {
1022 fprobe_fail_cleanup(fp);
1023 return ret;
1024 }
1025
1026 hlist_array = fp->hlist_array;
1027 ret = add_fprobe_hash(fp);
1028 for (i = 0; i < hlist_array->size && !ret; i++)
1029 ret = insert_fprobe_node(&hlist_array->array[i], fp);
1030
1031 if (ret) {
1032 unregister_fprobe_nolock(fp);
1033 /* In error case, wait for clean up safely. */
1034 synchronize_rcu();
1035 }
1036
1037 return ret;
1038}
1039EXPORT_SYMBOL_GPL(register_fprobe_ips);
1040
1041/**
1042 * register_fprobe_syms() - Register fprobe to ftrace by symbols.
1043 * @fp: A fprobe data structure to be registered.
1044 * @syms: An array of target symbols.
1045 * @num: The number of entries of @syms.
1046 *
1047 * Register @fp to the symbols given by @syms array. This will be useful if
1048 * you are sure the symbols exist in the kernel.
1049 *
1050 * Return 0 if @fp is registered successfully, -errno if not.
1051 */
1052int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
1053{
1054 unsigned long *addrs;
1055 int ret;
1056
1057 if (!fp || !syms || num <= 0)
1058 return -EINVAL;
1059
1060 addrs = get_ftrace_locations(syms, num);
1061 if (IS_ERR(addrs))
1062 return PTR_ERR(addrs);
1063
1064 ret = register_fprobe_ips(fp, addrs, num);
1065
1066 kfree(addrs);
1067
1068 return ret;
1069}
1070EXPORT_SYMBOL_GPL(register_fprobe_syms);
1071
1072bool fprobe_is_registered(struct fprobe *fp)
1073{
1074 if (!fp || !fp->hlist_array)
1075 return false;
1076 return true;
1077}
1078
1079static int unregister_fprobe_nolock(struct fprobe *fp)
1080{
1081 struct fprobe_hlist *hlist_array = fp->hlist_array;
1082 unsigned long *addrs = NULL;
1083 int i, count;
1084
1085 addrs = kcalloc(hlist_array->size, sizeof(unsigned long), GFP_KERNEL);
1086 /*
1087 * This will remove fprobe_hash_node from the hash table even if
1088 * memory allocation fails. However, ftrace_ops will not be updated.
1089 * Anyway, when the last fprobe is unregistered, ftrace_ops is also
1090 * unregistered.
1091 */
1092 if (!addrs)
1093 pr_warn("Failed to allocate working array. ftrace_ops may not sync.\n");
1094
1095 /* Remove non-synonim ips from table and hash */
1096 count = 0;
1097 for (i = 0; i < hlist_array->size; i++) {
1098 delete_fprobe_node(&hlist_array->array[i]);
1099 if (addrs && !fprobe_exists_on_hash(hlist_array->array[i].addr,
1100 fprobe_is_ftrace(fp)))
1101 addrs[count++] = hlist_array->array[i].addr;
1102 }
1103 del_fprobe_hash(fp);
1104
1105 if (fprobe_is_ftrace(fp))
1106 fprobe_ftrace_remove_ips(addrs, count);
1107 else
1108 fprobe_graph_remove_ips(addrs, count);
1109
1110 kfree_rcu(hlist_array, rcu);
1111 fp->hlist_array = NULL;
1112 kfree(addrs);
1113
1114 return 0;
1115}
1116
1117/**
1118 * unregister_fprobe_async() - Unregister fprobe without RCU GP wait
1119 * @fp: A fprobe data structure to be unregistered.
1120 *
1121 * Unregister fprobe (and remove ftrace hooks from the function entries).
1122 * This function will NOT wait until the fprobe is no longer used.
1123 *
1124 * Return 0 if @fp is unregistered successfully, -errno if not.
1125 */
1126int unregister_fprobe_async(struct fprobe *fp)
1127{
1128 guard(mutex)(&fprobe_mutex);
1129 if (!fp || !fprobe_registered(fp))
1130 return -EINVAL;
1131
1132 return unregister_fprobe_nolock(fp);
1133}
1134
1135/**
1136 * unregister_fprobe() - Unregister fprobe with RCU GP wait
1137 * @fp: A fprobe data structure to be unregistered.
1138 *
1139 * Unregister fprobe (and remove ftrace hooks from the function entries).
1140 * This function will block until the fprobe is no longer used.
1141 *
1142 * Return 0 if @fp is unregistered successfully, -errno if not.
1143 */
1144int unregister_fprobe(struct fprobe *fp)
1145{
1146 int ret = unregister_fprobe_async(fp);
1147
1148 if (!ret)
1149 synchronize_rcu();
1150 return ret;
1151}
1152EXPORT_SYMBOL_GPL(unregister_fprobe);
1153
1154static int __init fprobe_initcall(void)
1155{
1156 rhltable_init(&fprobe_ip_table, &fprobe_rht_params);
1157 return 0;
1158}
1159core_initcall(fprobe_initcall);