projects/linux·blob v6.5 · refs

342 lines · 9.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3#define pr_fmt(fmt) "rethook: " fmt
4
5#include <linux/bug.h>
6#include <linux/kallsyms.h>
7#include <linux/kprobes.h>
8#include <linux/preempt.h>
9#include <linux/rethook.h>
10#include <linux/slab.h>
11#include <linux/sort.h>
12
13/* Return hook list (shadow stack by list) */
14
15/*
16 * This function is called from delayed_put_task_struct() when a task is
17 * dead and cleaned up to recycle any kretprobe instances associated with
18 * this task. These left over instances represent probed functions that
19 * have been called but will never return.
20 */
21void rethook_flush_task(struct task_struct *tk)
22{
23 struct rethook_node *rhn;
24 struct llist_node *node;
25
26 node = __llist_del_all(&tk->rethooks);
27 while (node) {
28 rhn = container_of(node, struct rethook_node, llist);
29 node = node->next;
30 preempt_disable();
31 rethook_recycle(rhn);
32 preempt_enable();
33 }
34}
35
36static void rethook_free_rcu(struct rcu_head *head)
37{
38 struct rethook *rh = container_of(head, struct rethook, rcu);
39 struct rethook_node *rhn;
40 struct freelist_node *node;
41 int count = 1;
42
43 node = rh->pool.head;
44 while (node) {
45 rhn = container_of(node, struct rethook_node, freelist);
46 node = node->next;
47 kfree(rhn);
48 count++;
49 }
50
51 /* The rh->ref is the number of pooled node + 1 */
52 if (refcount_sub_and_test(count, &rh->ref))
53 kfree(rh);
54}
55
56/**
57 * rethook_stop() - Stop using a rethook.
58 * @rh: the struct rethook to stop.
59 *
60 * Stop using a rethook to prepare for freeing it. If you want to wait for
61 * all running rethook handler before calling rethook_free(), you need to
62 * call this first and wait RCU, and call rethook_free().
63 */
64void rethook_stop(struct rethook *rh)
65{
66 WRITE_ONCE(rh->handler, NULL);
67}
68
69/**
70 * rethook_free() - Free struct rethook.
71 * @rh: the struct rethook to be freed.
72 *
73 * Free the rethook. Before calling this function, user must ensure the
74 * @rh::data is cleaned if needed (or, the handler can access it after
75 * calling this function.) This function will set the @rh to be freed
76 * after all rethook_node are freed (not soon). And the caller must
77 * not touch @rh after calling this.
78 */
79void rethook_free(struct rethook *rh)
80{
81 WRITE_ONCE(rh->handler, NULL);
82
83 call_rcu(&rh->rcu, rethook_free_rcu);
84}
85
86/**
87 * rethook_alloc() - Allocate struct rethook.
88 * @data: a data to pass the @handler when hooking the return.
89 * @handler: the return hook callback function.
90 *
91 * Allocate and initialize a new rethook with @data and @handler.
92 * Return NULL if memory allocation fails or @handler is NULL.
93 * Note that @handler == NULL means this rethook is going to be freed.
94 */
95struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
96{
97 struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
98
99 if (!rh || !handler) {
100 kfree(rh);
101 return NULL;
102 }
103
104 rh->data = data;
105 rh->handler = handler;
106 rh->pool.head = NULL;
107 refcount_set(&rh->ref, 1);
108
109 return rh;
110}
111
112/**
113 * rethook_add_node() - Add a new node to the rethook.
114 * @rh: the struct rethook.
115 * @node: the struct rethook_node to be added.
116 *
117 * Add @node to @rh. User must allocate @node (as a part of user's
118 * data structure.) The @node fields are initialized in this function.
119 */
120void rethook_add_node(struct rethook *rh, struct rethook_node *node)
121{
122 node->rethook = rh;
123 freelist_add(&node->freelist, &rh->pool);
124 refcount_inc(&rh->ref);
125}
126
127static void free_rethook_node_rcu(struct rcu_head *head)
128{
129 struct rethook_node *node = container_of(head, struct rethook_node, rcu);
130
131 if (refcount_dec_and_test(&node->rethook->ref))
132 kfree(node->rethook);
133 kfree(node);
134}
135
136/**
137 * rethook_recycle() - return the node to rethook.
138 * @node: The struct rethook_node to be returned.
139 *
140 * Return back the @node to @node::rethook. If the @node::rethook is already
141 * marked as freed, this will free the @node.
142 */
143void rethook_recycle(struct rethook_node *node)
144{
145 lockdep_assert_preemption_disabled();
146
147 if (likely(READ_ONCE(node->rethook->handler)))
148 freelist_add(&node->freelist, &node->rethook->pool);
149 else
150 call_rcu(&node->rcu, free_rethook_node_rcu);
151}
152NOKPROBE_SYMBOL(rethook_recycle);
153
154/**
155 * rethook_try_get() - get an unused rethook node.
156 * @rh: The struct rethook which pools the nodes.
157 *
158 * Get an unused rethook node from @rh. If the node pool is empty, this
159 * will return NULL. Caller must disable preemption.
160 */
161struct rethook_node *rethook_try_get(struct rethook *rh)
162{
163 rethook_handler_t handler = READ_ONCE(rh->handler);
164 struct freelist_node *fn;
165
166 lockdep_assert_preemption_disabled();
167
168 /* Check whether @rh is going to be freed. */
169 if (unlikely(!handler))
170 return NULL;
171
172 /*
173 * This expects the caller will set up a rethook on a function entry.
174 * When the function returns, the rethook will eventually be reclaimed
175 * or released in the rethook_recycle() with call_rcu().
176 * This means the caller must be run in the RCU-availabe context.
177 */
178 if (unlikely(!rcu_is_watching()))
179 return NULL;
180
181 fn = freelist_try_get(&rh->pool);
182 if (!fn)
183 return NULL;
184
185 return container_of(fn, struct rethook_node, freelist);
186}
187NOKPROBE_SYMBOL(rethook_try_get);
188
189/**
190 * rethook_hook() - Hook the current function return.
191 * @node: The struct rethook node to hook the function return.
192 * @regs: The struct pt_regs for the function entry.
193 * @mcount: True if this is called from mcount(ftrace) context.
194 *
195 * Hook the current running function return. This must be called when the
196 * function entry (or at least @regs must be the registers of the function
197 * entry.) @mcount is used for identifying the context. If this is called
198 * from ftrace (mcount) callback, @mcount must be set true. If this is called
199 * from the real function entry (e.g. kprobes) @mcount must be set false.
200 * This is because the way to hook the function return depends on the context.
201 */
202void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
203{
204 arch_rethook_prepare(node, regs, mcount);
205 __llist_add(&node->llist, &current->rethooks);
206}
207NOKPROBE_SYMBOL(rethook_hook);
208
209/* This assumes the 'tsk' is the current task or is not running. */
210static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
211 struct llist_node **cur)
212{
213 struct rethook_node *rh = NULL;
214 struct llist_node *node = *cur;
215
216 if (!node)
217 node = tsk->rethooks.first;
218 else
219 node = node->next;
220
221 while (node) {
222 rh = container_of(node, struct rethook_node, llist);
223 if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
224 *cur = node;
225 return rh->ret_addr;
226 }
227 node = node->next;
228 }
229 return 0;
230}
231NOKPROBE_SYMBOL(__rethook_find_ret_addr);
232
233/**
234 * rethook_find_ret_addr -- Find correct return address modified by rethook
235 * @tsk: Target task
236 * @frame: A frame pointer
237 * @cur: a storage of the loop cursor llist_node pointer for next call
238 *
239 * Find the correct return address modified by a rethook on @tsk in unsigned
240 * long type.
241 * The @tsk must be 'current' or a task which is not running. @frame is a hint
242 * to get the currect return address - which is compared with the
243 * rethook::frame field. The @cur is a loop cursor for searching the
244 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
245 * first call, but '@cur' itself must NOT NULL.
246 *
247 * Returns found address value or zero if not found.
248 */
249unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
250 struct llist_node **cur)
251{
252 struct rethook_node *rhn = NULL;
253 unsigned long ret;
254
255 if (WARN_ON_ONCE(!cur))
256 return 0;
257
258 if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
259 return 0;
260
261 do {
262 ret = __rethook_find_ret_addr(tsk, cur);
263 if (!ret)
264 break;
265 rhn = container_of(*cur, struct rethook_node, llist);
266 } while (rhn->frame != frame);
267
268 return ret;
269}
270NOKPROBE_SYMBOL(rethook_find_ret_addr);
271
272void __weak arch_rethook_fixup_return(struct pt_regs *regs,
273 unsigned long correct_ret_addr)
274{
275 /*
276 * Do nothing by default. If the architecture which uses a
277 * frame pointer to record real return address on the stack,
278 * it should fill this function to fixup the return address
279 * so that stacktrace works from the rethook handler.
280 */
281}
282
283/* This function will be called from each arch-defined trampoline. */
284unsigned long rethook_trampoline_handler(struct pt_regs *regs,
285 unsigned long frame)
286{
287 struct llist_node *first, *node = NULL;
288 unsigned long correct_ret_addr;
289 rethook_handler_t handler;
290 struct rethook_node *rhn;
291
292 correct_ret_addr = __rethook_find_ret_addr(current, &node);
293 if (!correct_ret_addr) {
294 pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
295 BUG_ON(1);
296 }
297
298 instruction_pointer_set(regs, correct_ret_addr);
299
300 /*
301 * These loops must be protected from rethook_free_rcu() because those
302 * are accessing 'rhn->rethook'.
303 */
304 preempt_disable_notrace();
305
306 /*
307 * Run the handler on the shadow stack. Do not unlink the list here because
308 * stackdump inside the handlers needs to decode it.
309 */
310 first = current->rethooks.first;
311 while (first) {
312 rhn = container_of(first, struct rethook_node, llist);
313 if (WARN_ON_ONCE(rhn->frame != frame))
314 break;
315 handler = READ_ONCE(rhn->rethook->handler);
316 if (handler)
317 handler(rhn, rhn->rethook->data,
318 correct_ret_addr, regs);
319
320 if (first == node)
321 break;
322 first = first->next;
323 }
324
325 /* Fixup registers for returning to correct address. */
326 arch_rethook_fixup_return(regs, correct_ret_addr);
327
328 /* Unlink used shadow stack */
329 first = current->rethooks.first;
330 current->rethooks.first = node->next;
331 node->next = NULL;
332
333 while (first) {
334 rhn = container_of(first, struct rethook_node, llist);
335 first = first->next;
336 rethook_recycle(rhn);
337 }
338 preempt_enable_notrace();
339
340 return correct_ret_addr;
341}
342NOKPROBE_SYMBOL(rethook_trampoline_handler);