projects/linux·blob v6.0 · refs

324 lines · 7.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fprobe - Simple ftrace probe wrapper for function entry.
4 */
5#define pr_fmt(fmt) "fprobe: " fmt
6
7#include <linux/err.h>
8#include <linux/fprobe.h>
9#include <linux/kallsyms.h>
10#include <linux/kprobes.h>
11#include <linux/rethook.h>
12#include <linux/slab.h>
13#include <linux/sort.h>
14
15#include "trace.h"
16
17struct fprobe_rethook_node {
18 struct rethook_node node;
19 unsigned long entry_ip;
20};
21
22static void fprobe_handler(unsigned long ip, unsigned long parent_ip,
23 struct ftrace_ops *ops, struct ftrace_regs *fregs)
24{
25 struct fprobe_rethook_node *fpr;
26 struct rethook_node *rh;
27 struct fprobe *fp;
28 int bit;
29
30 fp = container_of(ops, struct fprobe, ops);
31 if (fprobe_disabled(fp))
32 return;
33
34 bit = ftrace_test_recursion_trylock(ip, parent_ip);
35 if (bit < 0) {
36 fp->nmissed++;
37 return;
38 }
39
40 if (fp->entry_handler)
41 fp->entry_handler(fp, ip, ftrace_get_regs(fregs));
42
43 if (fp->exit_handler) {
44 rh = rethook_try_get(fp->rethook);
45 if (!rh) {
46 fp->nmissed++;
47 goto out;
48 }
49 fpr = container_of(rh, struct fprobe_rethook_node, node);
50 fpr->entry_ip = ip;
51 rethook_hook(rh, ftrace_get_regs(fregs), true);
52 }
53
54out:
55 ftrace_test_recursion_unlock(bit);
56}
57NOKPROBE_SYMBOL(fprobe_handler);
58
59static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
60 struct ftrace_ops *ops, struct ftrace_regs *fregs)
61{
62 struct fprobe *fp = container_of(ops, struct fprobe, ops);
63
64 if (unlikely(kprobe_running())) {
65 fp->nmissed++;
66 return;
67 }
68 kprobe_busy_begin();
69 fprobe_handler(ip, parent_ip, ops, fregs);
70 kprobe_busy_end();
71}
72
73static void fprobe_exit_handler(struct rethook_node *rh, void *data,
74 struct pt_regs *regs)
75{
76 struct fprobe *fp = (struct fprobe *)data;
77 struct fprobe_rethook_node *fpr;
78
79 if (!fp || fprobe_disabled(fp))
80 return;
81
82 fpr = container_of(rh, struct fprobe_rethook_node, node);
83
84 fp->exit_handler(fp, fpr->entry_ip, regs);
85}
86NOKPROBE_SYMBOL(fprobe_exit_handler);
87
88static int symbols_cmp(const void *a, const void *b)
89{
90 const char **str_a = (const char **) a;
91 const char **str_b = (const char **) b;
92
93 return strcmp(*str_a, *str_b);
94}
95
96/* Convert ftrace location address from symbols */
97static unsigned long *get_ftrace_locations(const char **syms, int num)
98{
99 unsigned long *addrs;
100
101 /* Convert symbols to symbol address */
102 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
103 if (!addrs)
104 return ERR_PTR(-ENOMEM);
105
106 /* ftrace_lookup_symbols expects sorted symbols */
107 sort(syms, num, sizeof(*syms), symbols_cmp, NULL);
108
109 if (!ftrace_lookup_symbols(syms, num, addrs))
110 return addrs;
111
112 kfree(addrs);
113 return ERR_PTR(-ENOENT);
114}
115
116static void fprobe_init(struct fprobe *fp)
117{
118 fp->nmissed = 0;
119 if (fprobe_shared_with_kprobes(fp))
120 fp->ops.func = fprobe_kprobe_handler;
121 else
122 fp->ops.func = fprobe_handler;
123 fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS;
124}
125
126static int fprobe_init_rethook(struct fprobe *fp, int num)
127{
128 int i, size;
129
130 if (num < 0)
131 return -EINVAL;
132
133 if (!fp->exit_handler) {
134 fp->rethook = NULL;
135 return 0;
136 }
137
138 /* Initialize rethook if needed */
139 size = num * num_possible_cpus() * 2;
140 if (size < 0)
141 return -E2BIG;
142
143 fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler);
144 for (i = 0; i < size; i++) {
145 struct fprobe_rethook_node *node;
146
147 node = kzalloc(sizeof(*node), GFP_KERNEL);
148 if (!node) {
149 rethook_free(fp->rethook);
150 fp->rethook = NULL;
151 return -ENOMEM;
152 }
153 rethook_add_node(fp->rethook, &node->node);
154 }
155 return 0;
156}
157
158static void fprobe_fail_cleanup(struct fprobe *fp)
159{
160 if (fp->rethook) {
161 /* Don't need to cleanup rethook->handler because this is not used. */
162 rethook_free(fp->rethook);
163 fp->rethook = NULL;
164 }
165 ftrace_free_filter(&fp->ops);
166}
167
168/**
169 * register_fprobe() - Register fprobe to ftrace by pattern.
170 * @fp: A fprobe data structure to be registered.
171 * @filter: A wildcard pattern of probed symbols.
172 * @notfilter: A wildcard pattern of NOT probed symbols.
173 *
174 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
175 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
176 *
177 * Return 0 if @fp is registered successfully, -errno if not.
178 */
179int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
180{
181 struct ftrace_hash *hash;
182 unsigned char *str;
183 int ret, len;
184
185 if (!fp || !filter)
186 return -EINVAL;
187
188 fprobe_init(fp);
189
190 len = strlen(filter);
191 str = kstrdup(filter, GFP_KERNEL);
192 ret = ftrace_set_filter(&fp->ops, str, len, 0);
193 kfree(str);
194 if (ret)
195 return ret;
196
197 if (notfilter) {
198 len = strlen(notfilter);
199 str = kstrdup(notfilter, GFP_KERNEL);
200 ret = ftrace_set_notrace(&fp->ops, str, len, 0);
201 kfree(str);
202 if (ret)
203 goto out;
204 }
205
206 /* TODO:
207 * correctly calculate the total number of filtered symbols
208 * from both filter and notfilter.
209 */
210 hash = rcu_access_pointer(fp->ops.local_hash.filter_hash);
211 if (WARN_ON_ONCE(!hash))
212 goto out;
213
214 ret = fprobe_init_rethook(fp, (int)hash->count);
215 if (!ret)
216 ret = register_ftrace_function(&fp->ops);
217
218out:
219 if (ret)
220 fprobe_fail_cleanup(fp);
221 return ret;
222}
223EXPORT_SYMBOL_GPL(register_fprobe);
224
225/**
226 * register_fprobe_ips() - Register fprobe to ftrace by address.
227 * @fp: A fprobe data structure to be registered.
228 * @addrs: An array of target ftrace location addresses.
229 * @num: The number of entries of @addrs.
230 *
231 * Register @fp to ftrace for enabling the probe on the address given by @addrs.
232 * The @addrs must be the addresses of ftrace location address, which may be
233 * the symbol address + arch-dependent offset.
234 * If you unsure what this mean, please use other registration functions.
235 *
236 * Return 0 if @fp is registered successfully, -errno if not.
237 */
238int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
239{
240 int ret;
241
242 if (!fp || !addrs || num <= 0)
243 return -EINVAL;
244
245 fprobe_init(fp);
246
247 ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0);
248 if (ret)
249 return ret;
250
251 ret = fprobe_init_rethook(fp, num);
252 if (!ret)
253 ret = register_ftrace_function(&fp->ops);
254
255 if (ret)
256 fprobe_fail_cleanup(fp);
257 return ret;
258}
259EXPORT_SYMBOL_GPL(register_fprobe_ips);
260
261/**
262 * register_fprobe_syms() - Register fprobe to ftrace by symbols.
263 * @fp: A fprobe data structure to be registered.
264 * @syms: An array of target symbols.
265 * @num: The number of entries of @syms.
266 *
267 * Register @fp to the symbols given by @syms array. This will be useful if
268 * you are sure the symbols exist in the kernel.
269 *
270 * Return 0 if @fp is registered successfully, -errno if not.
271 */
272int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
273{
274 unsigned long *addrs;
275 int ret;
276
277 if (!fp || !syms || num <= 0)
278 return -EINVAL;
279
280 addrs = get_ftrace_locations(syms, num);
281 if (IS_ERR(addrs))
282 return PTR_ERR(addrs);
283
284 ret = register_fprobe_ips(fp, addrs, num);
285
286 kfree(addrs);
287
288 return ret;
289}
290EXPORT_SYMBOL_GPL(register_fprobe_syms);
291
292/**
293 * unregister_fprobe() - Unregister fprobe from ftrace
294 * @fp: A fprobe data structure to be unregistered.
295 *
296 * Unregister fprobe (and remove ftrace hooks from the function entries).
297 *
298 * Return 0 if @fp is unregistered successfully, -errno if not.
299 */
300int unregister_fprobe(struct fprobe *fp)
301{
302 int ret;
303
304 if (!fp || fp->ops.func != fprobe_handler)
305 return -EINVAL;
306
307 /*
308 * rethook_free() starts disabling the rethook, but the rethook handlers
309 * may be running on other processors at this point. To make sure that all
310 * current running handlers are finished, call unregister_ftrace_function()
311 * after this.
312 */
313 if (fp->rethook)
314 rethook_free(fp->rethook);
315
316 ret = unregister_ftrace_function(&fp->ops);
317 if (ret < 0)
318 return ret;
319
320 ftrace_free_filter(&fp->ops);
321
322 return ret;
323}
324EXPORT_SYMBOL_GPL(unregister_fprobe);