| 1 | |
| 2 |
|
| 3 | #define pr_fmt(fmt) "rethook: " fmt
|
| 4 |
|
| 5 | #include <linux/bug.h>
|
| 6 | #include <linux/kallsyms.h>
|
| 7 | #include <linux/kprobes.h>
|
| 8 | #include <linux/preempt.h>
|
| 9 | #include <linux/rethook.h>
|
| 10 | #include <linux/slab.h>
|
| 11 | #include <linux/sort.h>
|
| 12 |
|
| 13 |
|
| 14 |
|
| 15 | |
| 16 | * This function is called from delayed_put_task_struct() when a task is
|
| 17 | * dead and cleaned up to recycle any kretprobe instances associated with
|
| 18 | * this task. These left over instances represent probed functions that
|
| 19 | * have been called but will never return.
|
| 20 |
|
| 21 | void rethook_flush_task(struct task_struct *tk)
|
| 22 | {
|
| 23 | struct rethook_node *rhn;
|
| 24 | struct llist_node *node;
|
| 25 |
|
| 26 | node = __llist_del_all(&tk->rethooks);
|
| 27 | while (node) {
|
| 28 | rhn = container_of(node, struct rethook_node, llist);
|
| 29 | node = node->next;
|
| 30 | preempt_disable();
|
| 31 | rethook_recycle(rhn);
|
| 32 | preempt_enable();
|
| 33 | }
|
| 34 | }
|
| 35 |
|
| 36 | static void rethook_free_rcu(struct rcu_head *head)
|
| 37 | {
|
| 38 | struct rethook *rh = container_of(head, struct rethook, rcu);
|
| 39 | struct rethook_node *rhn;
|
| 40 | struct freelist_node *node;
|
| 41 | int count = 1;
|
| 42 |
|
| 43 | node = rh->pool.head;
|
| 44 | while (node) {
|
| 45 | rhn = container_of(node, struct rethook_node, freelist);
|
| 46 | node = node->next;
|
| 47 | kfree(rhn);
|
| 48 | count++;
|
| 49 | }
|
| 50 |
|
| 51 |
|
| 52 | if (refcount_sub_and_test(count, &rh->ref))
|
| 53 | kfree(rh);
|
| 54 | }
|
| 55 |
|
| 56 | |
| 57 | * rethook_free() - Free struct rethook.
|
| 58 | * @rh: the struct rethook to be freed.
|
| 59 | *
|
| 60 | * Free the rethook. Before calling this function, user must ensure the
|
| 61 | * @rh::data is cleaned if needed (or, the handler can access it after
|
| 62 | * calling this function.) This function will set the @rh to be freed
|
| 63 | * after all rethook_node are freed (not soon). And the caller must
|
| 64 | * not touch @rh after calling this.
|
| 65 |
|
| 66 | void rethook_free(struct rethook *rh)
|
| 67 | {
|
| 68 | WRITE_ONCE(rh->handler, NULL);
|
| 69 |
|
| 70 | call_rcu(&rh->rcu, rethook_free_rcu);
|
| 71 | }
|
| 72 |
|
| 73 | |
| 74 | * rethook_alloc() - Allocate struct rethook.
|
| 75 | * @data: a data to pass the @handler when hooking the return.
|
| 76 | * @handler: the return hook callback function.
|
| 77 | *
|
| 78 | * Allocate and initialize a new rethook with @data and @handler.
|
| 79 | * Return NULL if memory allocation fails or @handler is NULL.
|
| 80 | * Note that @handler == NULL means this rethook is going to be freed.
|
| 81 |
|
| 82 | struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
|
| 83 | {
|
| 84 | struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
|
| 85 |
|
| 86 | if (!rh || !handler)
|
| 87 | return NULL;
|
| 88 |
|
| 89 | rh->data = data;
|
| 90 | rh->handler = handler;
|
| 91 | rh->pool.head = NULL;
|
| 92 | refcount_set(&rh->ref, 1);
|
| 93 |
|
| 94 | return rh;
|
| 95 | }
|
| 96 |
|
| 97 | |
| 98 | * rethook_add_node() - Add a new node to the rethook.
|
| 99 | * @rh: the struct rethook.
|
| 100 | * @node: the struct rethook_node to be added.
|
| 101 | *
|
| 102 | * Add @node to @rh. User must allocate @node (as a part of user's
|
| 103 | * data structure.) The @node fields are initialized in this function.
|
| 104 |
|
| 105 | void rethook_add_node(struct rethook *rh, struct rethook_node *node)
|
| 106 | {
|
| 107 | node->rethook = rh;
|
| 108 | freelist_add(&node->freelist, &rh->pool);
|
| 109 | refcount_inc(&rh->ref);
|
| 110 | }
|
| 111 |
|
| 112 | static void free_rethook_node_rcu(struct rcu_head *head)
|
| 113 | {
|
| 114 | struct rethook_node *node = container_of(head, struct rethook_node, rcu);
|
| 115 |
|
| 116 | if (refcount_dec_and_test(&node->rethook->ref))
|
| 117 | kfree(node->rethook);
|
| 118 | kfree(node);
|
| 119 | }
|
| 120 |
|
| 121 | |
| 122 | * rethook_recycle() - return the node to rethook.
|
| 123 | * @node: The struct rethook_node to be returned.
|
| 124 | *
|
| 125 | * Return back the @node to @node::rethook. If the @node::rethook is already
|
| 126 | * marked as freed, this will free the @node.
|
| 127 |
|
| 128 | void rethook_recycle(struct rethook_node *node)
|
| 129 | {
|
| 130 | lockdep_assert_preemption_disabled();
|
| 131 |
|
| 132 | if (likely(READ_ONCE(node->rethook->handler)))
|
| 133 | freelist_add(&node->freelist, &node->rethook->pool);
|
| 134 | else
|
| 135 | call_rcu(&node->rcu, free_rethook_node_rcu);
|
| 136 | }
|
| 137 | NOKPROBE_SYMBOL(rethook_recycle);
|
| 138 |
|
| 139 | |
| 140 | * rethook_try_get() - get an unused rethook node.
|
| 141 | * @rh: The struct rethook which pools the nodes.
|
| 142 | *
|
| 143 | * Get an unused rethook node from @rh. If the node pool is empty, this
|
| 144 | * will return NULL. Caller must disable preemption.
|
| 145 |
|
| 146 | struct rethook_node *rethook_try_get(struct rethook *rh)
|
| 147 | {
|
| 148 | rethook_handler_t handler = READ_ONCE(rh->handler);
|
| 149 | struct freelist_node *fn;
|
| 150 |
|
| 151 | lockdep_assert_preemption_disabled();
|
| 152 |
|
| 153 |
|
| 154 | if (unlikely(!handler))
|
| 155 | return NULL;
|
| 156 |
|
| 157 | |
| 158 | * This expects the caller will set up a rethook on a function entry.
|
| 159 | * When the function returns, the rethook will eventually be reclaimed
|
| 160 | * or released in the rethook_recycle() with call_rcu().
|
| 161 | * This means the caller must be run in the RCU-availabe context.
|
| 162 |
|
| 163 | if (unlikely(!rcu_is_watching()))
|
| 164 | return NULL;
|
| 165 |
|
| 166 | fn = freelist_try_get(&rh->pool);
|
| 167 | if (!fn)
|
| 168 | return NULL;
|
| 169 |
|
| 170 | return container_of(fn, struct rethook_node, freelist);
|
| 171 | }
|
| 172 | NOKPROBE_SYMBOL(rethook_try_get);
|
| 173 |
|
| 174 | |
| 175 | * rethook_hook() - Hook the current function return.
|
| 176 | * @node: The struct rethook node to hook the function return.
|
| 177 | * @regs: The struct pt_regs for the function entry.
|
| 178 | * @mcount: True if this is called from mcount(ftrace) context.
|
| 179 | *
|
| 180 | * Hook the current running function return. This must be called when the
|
| 181 | * function entry (or at least @regs must be the registers of the function
|
| 182 | * entry.) @mcount is used for identifying the context. If this is called
|
| 183 | * from ftrace (mcount) callback, @mcount must be set true. If this is called
|
| 184 | * from the real function entry (e.g. kprobes) @mcount must be set false.
|
| 185 | * This is because the way to hook the function return depends on the context.
|
| 186 |
|
| 187 | void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
|
| 188 | {
|
| 189 | arch_rethook_prepare(node, regs, mcount);
|
| 190 | __llist_add(&node->llist, ¤t->rethooks);
|
| 191 | }
|
| 192 | NOKPROBE_SYMBOL(rethook_hook);
|
| 193 |
|
| 194 |
|
| 195 | static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
|
| 196 | struct llist_node **cur)
|
| 197 | {
|
| 198 | struct rethook_node *rh = NULL;
|
| 199 | struct llist_node *node = *cur;
|
| 200 |
|
| 201 | if (!node)
|
| 202 | node = tsk->rethooks.first;
|
| 203 | else
|
| 204 | node = node->next;
|
| 205 |
|
| 206 | while (node) {
|
| 207 | rh = container_of(node, struct rethook_node, llist);
|
| 208 | if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
|
| 209 | *cur = node;
|
| 210 | return rh->ret_addr;
|
| 211 | }
|
| 212 | node = node->next;
|
| 213 | }
|
| 214 | return 0;
|
| 215 | }
|
| 216 | NOKPROBE_SYMBOL(__rethook_find_ret_addr);
|
| 217 |
|
| 218 | |
| 219 | * rethook_find_ret_addr -- Find correct return address modified by rethook
|
| 220 | * @tsk: Target task
|
| 221 | * @frame: A frame pointer
|
| 222 | * @cur: a storage of the loop cursor llist_node pointer for next call
|
| 223 | *
|
| 224 | * Find the correct return address modified by a rethook on @tsk in unsigned
|
| 225 | * long type.
|
| 226 | * The @tsk must be 'current' or a task which is not running. @frame is a hint
|
| 227 | * to get the currect return address - which is compared with the
|
| 228 | * rethook::frame field. The @cur is a loop cursor for searching the
|
| 229 | * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
|
| 230 | * first call, but '@cur' itself must NOT NULL.
|
| 231 | *
|
| 232 | * Returns found address value or zero if not found.
|
| 233 |
|
| 234 | unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
|
| 235 | struct llist_node **cur)
|
| 236 | {
|
| 237 | struct rethook_node *rhn = NULL;
|
| 238 | unsigned long ret;
|
| 239 |
|
| 240 | if (WARN_ON_ONCE(!cur))
|
| 241 | return 0;
|
| 242 |
|
| 243 | if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
|
| 244 | return 0;
|
| 245 |
|
| 246 | do {
|
| 247 | ret = __rethook_find_ret_addr(tsk, cur);
|
| 248 | if (!ret)
|
| 249 | break;
|
| 250 | rhn = container_of(*cur, struct rethook_node, llist);
|
| 251 | } while (rhn->frame != frame);
|
| 252 |
|
| 253 | return ret;
|
| 254 | }
|
| 255 | NOKPROBE_SYMBOL(rethook_find_ret_addr);
|
| 256 |
|
| 257 | void __weak arch_rethook_fixup_return(struct pt_regs *regs,
|
| 258 | unsigned long correct_ret_addr)
|
| 259 | {
|
| 260 | |
| 261 | * Do nothing by default. If the architecture which uses a
|
| 262 | * frame pointer to record real return address on the stack,
|
| 263 | * it should fill this function to fixup the return address
|
| 264 | * so that stacktrace works from the rethook handler.
|
| 265 |
|
| 266 | }
|
| 267 |
|
| 268 |
|
| 269 | unsigned long rethook_trampoline_handler(struct pt_regs *regs,
|
| 270 | unsigned long frame)
|
| 271 | {
|
| 272 | struct llist_node *first, *node = NULL;
|
| 273 | unsigned long correct_ret_addr;
|
| 274 | rethook_handler_t handler;
|
| 275 | struct rethook_node *rhn;
|
| 276 |
|
| 277 | correct_ret_addr = __rethook_find_ret_addr(current, &node);
|
| 278 | if (!correct_ret_addr) {
|
| 279 | pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
|
| 280 | BUG_ON(1);
|
| 281 | }
|
| 282 |
|
| 283 | instruction_pointer_set(regs, correct_ret_addr);
|
| 284 |
|
| 285 | |
| 286 | * These loops must be protected from rethook_free_rcu() because those
|
| 287 | * are accessing 'rhn->rethook'.
|
| 288 |
|
| 289 | preempt_disable();
|
| 290 |
|
| 291 | |
| 292 | * Run the handler on the shadow stack. Do not unlink the list here because
|
| 293 | * stackdump inside the handlers needs to decode it.
|
| 294 |
|
| 295 | first = current->rethooks.first;
|
| 296 | while (first) {
|
| 297 | rhn = container_of(first, struct rethook_node, llist);
|
| 298 | if (WARN_ON_ONCE(rhn->frame != frame))
|
| 299 | break;
|
| 300 | handler = READ_ONCE(rhn->rethook->handler);
|
| 301 | if (handler)
|
| 302 | handler(rhn, rhn->rethook->data, regs);
|
| 303 |
|
| 304 | if (first == node)
|
| 305 | break;
|
| 306 | first = first->next;
|
| 307 | }
|
| 308 |
|
| 309 |
|
| 310 | arch_rethook_fixup_return(regs, correct_ret_addr);
|
| 311 |
|
| 312 |
|
| 313 | first = current->rethooks.first;
|
| 314 | current->rethooks.first = node->next;
|
| 315 | node->next = NULL;
|
| 316 |
|
| 317 | while (first) {
|
| 318 | rhn = container_of(first, struct rethook_node, llist);
|
| 319 | first = first->next;
|
| 320 | rethook_recycle(rhn);
|
| 321 | }
|
| 322 | preempt_enable();
|
| 323 |
|
| 324 | return correct_ret_addr;
|
| 325 | }
|
| 326 | NOKPROBE_SYMBOL(rethook_trampoline_handler);
|