| 1 | |
| 2 | |
| 3 | * Kernel unwinding support
|
| 4 | *
|
| 5 | * (c) 2002-2004 Randolph Chung <[email protected]>
|
| 6 | *
|
| 7 | * Derived partially from the IA64 implementation. The PA-RISC
|
| 8 | * Runtime Architecture Document is also a useful reference to
|
| 9 | * understand what is happening here
|
| 10 |
|
| 11 |
|
| 12 | #include <linux/kernel.h>
|
| 13 | #include <linux/init.h>
|
| 14 | #include <linux/sched.h>
|
| 15 | #include <linux/slab.h>
|
| 16 | #include <linux/sort.h>
|
| 17 | #include <linux/bsearch.h>
|
| 18 | #include <linux/sched/task_stack.h>
|
| 19 |
|
| 20 | #include <linux/uaccess.h>
|
| 21 | #include <asm/assembly.h>
|
| 22 | #include <asm/asm-offsets.h>
|
| 23 | #include <asm/ptrace.h>
|
| 24 |
|
| 25 | #include <asm/unwind.h>
|
| 26 | #include <asm/switch_to.h>
|
| 27 | #include <asm/sections.h>
|
| 28 | #include <asm/ftrace.h>
|
| 29 |
|
| 30 |
|
| 31 | #ifdef DEBUG
|
| 32 | #define dbg(x...) pr_debug(x)
|
| 33 | #else
|
| 34 | #define dbg(x...) do { } while (0)
|
| 35 | #endif
|
| 36 |
|
| 37 | #define KERNEL_START (KERNEL_BINARY_TEXT_START)
|
| 38 |
|
| 39 | #define ALIGNMENT_OK(ptr, type) (((ptr) & (sizeof(type) - 1)) == 0)
|
| 40 |
|
| 41 | extern struct unwind_table_entry __start___unwind[];
|
| 42 | extern struct unwind_table_entry __stop___unwind[];
|
| 43 |
|
| 44 | static DEFINE_SPINLOCK(unwind_lock);
|
| 45 | |
| 46 | * the kernel unwind block is not dynamically allocated so that
|
| 47 | * we can call unwind_init as early in the bootup process as
|
| 48 | * possible (before the slab allocator is initialized)
|
| 49 |
|
| 50 | static struct unwind_table kernel_unwind_table __ro_after_init;
|
| 51 | static LIST_HEAD(unwind_tables);
|
| 52 |
|
| 53 | static int cmp_unwind_entry(const void *key, const void *elt)
|
| 54 | {
|
| 55 | unsigned long addr = (unsigned long)key;
|
| 56 | const struct unwind_table_entry *e = elt;
|
| 57 |
|
| 58 | if (addr < e->region_start)
|
| 59 | return -1;
|
| 60 | if (addr > e->region_end)
|
| 61 | return 1;
|
| 62 | return 0;
|
| 63 | }
|
| 64 |
|
| 65 | static inline const struct unwind_table_entry *
|
| 66 | find_unwind_entry_in_table(const struct unwind_table *table, unsigned long addr)
|
| 67 | {
|
| 68 | return bsearch((void *)addr, table->table, table->length,
|
| 69 | sizeof(*table->table), cmp_unwind_entry);
|
| 70 | }
|
| 71 |
|
| 72 | static const struct unwind_table_entry *
|
| 73 | find_unwind_entry(unsigned long addr)
|
| 74 | {
|
| 75 | struct unwind_table *table;
|
| 76 | const struct unwind_table_entry *e = NULL;
|
| 77 |
|
| 78 | if (addr >= kernel_unwind_table.start &&
|
| 79 | addr <= kernel_unwind_table.end)
|
| 80 | e = find_unwind_entry_in_table(&kernel_unwind_table, addr);
|
| 81 | else {
|
| 82 | unsigned long flags;
|
| 83 |
|
| 84 | spin_lock_irqsave(&unwind_lock, flags);
|
| 85 | list_for_each_entry(table, &unwind_tables, list) {
|
| 86 | if (addr >= table->start &&
|
| 87 | addr <= table->end)
|
| 88 | e = find_unwind_entry_in_table(table, addr);
|
| 89 | if (e) {
|
| 90 |
|
| 91 | list_move(&table->list, &unwind_tables);
|
| 92 | break;
|
| 93 | }
|
| 94 | }
|
| 95 | spin_unlock_irqrestore(&unwind_lock, flags);
|
| 96 | }
|
| 97 |
|
| 98 | return e;
|
| 99 | }
|
| 100 |
|
| 101 | static void
|
| 102 | unwind_table_init(struct unwind_table *table, const char *name,
|
| 103 | unsigned long base_addr, unsigned long gp,
|
| 104 | void *table_start, void *table_end)
|
| 105 | {
|
| 106 | struct unwind_table_entry *start = table_start;
|
| 107 | struct unwind_table_entry *end =
|
| 108 | (struct unwind_table_entry *)table_end - 1;
|
| 109 |
|
| 110 | table->name = name;
|
| 111 | table->base_addr = base_addr;
|
| 112 | table->gp = gp;
|
| 113 | table->start = base_addr + start->region_start;
|
| 114 | table->end = base_addr + end->region_end;
|
| 115 | table->table = (struct unwind_table_entry *)table_start;
|
| 116 | table->length = end - start + 1;
|
| 117 | INIT_LIST_HEAD(&table->list);
|
| 118 |
|
| 119 | for (; start <= end; start++) {
|
| 120 | if (start < end &&
|
| 121 | start->region_end > (start+1)->region_start) {
|
| 122 | pr_warn("Out of order unwind entry! %px and %px\n",
|
| 123 | start, start+1);
|
| 124 | }
|
| 125 |
|
| 126 | start->region_start += base_addr;
|
| 127 | start->region_end += base_addr;
|
| 128 | }
|
| 129 | }
|
| 130 |
|
| 131 | static int cmp_unwind_table_entry(const void *a, const void *b)
|
| 132 | {
|
| 133 | return ((const struct unwind_table_entry *)a)->region_start
|
| 134 | - ((const struct unwind_table_entry *)b)->region_start;
|
| 135 | }
|
| 136 |
|
| 137 | static void
|
| 138 | unwind_table_sort(struct unwind_table_entry *start,
|
| 139 | struct unwind_table_entry *finish)
|
| 140 | {
|
| 141 | sort(start, finish - start, sizeof(struct unwind_table_entry),
|
| 142 | cmp_unwind_table_entry, NULL);
|
| 143 | }
|
| 144 |
|
| 145 | struct unwind_table *
|
| 146 | unwind_table_add(const char *name, unsigned long base_addr,
|
| 147 | unsigned long gp,
|
| 148 | void *start, void *end)
|
| 149 | {
|
| 150 | struct unwind_table *table;
|
| 151 | unsigned long flags;
|
| 152 | struct unwind_table_entry *s = (struct unwind_table_entry *)start;
|
| 153 | struct unwind_table_entry *e = (struct unwind_table_entry *)end;
|
| 154 |
|
| 155 | unwind_table_sort(s, e);
|
| 156 |
|
| 157 | table = kmalloc_obj(struct unwind_table, GFP_USER);
|
| 158 | if (table == NULL)
|
| 159 | return NULL;
|
| 160 | unwind_table_init(table, name, base_addr, gp, start, end);
|
| 161 | spin_lock_irqsave(&unwind_lock, flags);
|
| 162 | list_add_tail(&table->list, &unwind_tables);
|
| 163 | spin_unlock_irqrestore(&unwind_lock, flags);
|
| 164 |
|
| 165 | return table;
|
| 166 | }
|
| 167 |
|
| 168 | void unwind_table_remove(struct unwind_table *table)
|
| 169 | {
|
| 170 | unsigned long flags;
|
| 171 |
|
| 172 | spin_lock_irqsave(&unwind_lock, flags);
|
| 173 | list_del(&table->list);
|
| 174 | spin_unlock_irqrestore(&unwind_lock, flags);
|
| 175 |
|
| 176 | kfree(table);
|
| 177 | }
|
| 178 |
|
| 179 |
|
| 180 | int __init unwind_init(void)
|
| 181 | {
|
| 182 | long start __maybe_unused, stop __maybe_unused;
|
| 183 | register unsigned long gp __asm__ ("r27");
|
| 184 |
|
| 185 | start = (long)&__start___unwind[0];
|
| 186 | stop = (long)&__stop___unwind[0];
|
| 187 |
|
| 188 | dbg("unwind_init: start = 0x%lx, end = 0x%lx, entries = %lu\n",
|
| 189 | start, stop,
|
| 190 | (stop - start) / sizeof(struct unwind_table_entry));
|
| 191 |
|
| 192 | unwind_table_init(&kernel_unwind_table, "kernel", KERNEL_START,
|
| 193 | gp,
|
| 194 | &__start___unwind[0], &__stop___unwind[0]);
|
| 195 | #if 0 |
| 196 | {
|
| 197 | int i;
|
| 198 | for (i = 0; i < 10; i++)
|
| 199 | {
|
| 200 | printk("region 0x%x-0x%x\n",
|
| 201 | __start___unwind[i].region_start,
|
| 202 | __start___unwind[i].region_end);
|
| 203 | }
|
| 204 | }
|
| 205 | #endif
|
| 206 | return 0;
|
| 207 | }
|
| 208 |
|
| 209 | static bool pc_is_kernel_fn(unsigned long pc, void *fn)
|
| 210 | {
|
| 211 | return (unsigned long)dereference_kernel_function_descriptor(fn) == pc;
|
| 212 | }
|
| 213 |
|
| 214 | static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int frame_size)
|
| 215 | {
|
| 216 | |
| 217 | * We have to use void * instead of a function pointer, because
|
| 218 | * function pointers aren't a pointer to the function on 64-bit.
|
| 219 | * Make them const so the compiler knows they live in .text
|
| 220 | * Note: We could use dereference_kernel_function_descriptor()
|
| 221 | * instead but we want to keep it simple here.
|
| 222 |
|
| 223 | extern void * const ret_from_kernel_thread;
|
| 224 | extern void * const syscall_exit;
|
| 225 | extern void * const intr_return;
|
| 226 | extern void * const _switch_to_ret;
|
| 227 | #ifdef CONFIG_IRQSTACKS
|
| 228 | extern void * const _call_on_stack;
|
| 229 | #endif
|
| 230 |
|
| 231 | if (pc_is_kernel_fn(pc, handle_interruption)) {
|
| 232 | struct pt_regs *regs = (struct pt_regs *)(info->sp - frame_size - PT_SZ_ALGN);
|
| 233 | dbg("Unwinding through handle_interruption()\n");
|
| 234 | info->prev_sp = regs->gr[30];
|
| 235 | info->prev_ip = regs->iaoq[0];
|
| 236 | return 1;
|
| 237 | }
|
| 238 |
|
| 239 | if (pc == (unsigned long)&ret_from_kernel_thread ||
|
| 240 | pc == (unsigned long)&syscall_exit) {
|
| 241 | info->prev_sp = info->prev_ip = 0;
|
| 242 | return 1;
|
| 243 | }
|
| 244 |
|
| 245 | if (pc == (unsigned long)&intr_return) {
|
| 246 | struct pt_regs *regs;
|
| 247 |
|
| 248 | dbg("Found intr_return()\n");
|
| 249 | regs = (struct pt_regs *)(info->sp - PT_SZ_ALGN);
|
| 250 | info->prev_sp = regs->gr[30];
|
| 251 | info->prev_ip = regs->iaoq[0];
|
| 252 | info->rp = regs->gr[2];
|
| 253 | return 1;
|
| 254 | }
|
| 255 |
|
| 256 | if (pc_is_kernel_fn(pc, _switch_to) ||
|
| 257 | pc == (unsigned long)&_switch_to_ret) {
|
| 258 | info->prev_sp = info->sp - CALLEE_SAVE_FRAME_SIZE;
|
| 259 | if (ALIGNMENT_OK(info->prev_sp, long))
|
| 260 | info->prev_ip = *(unsigned long *)(info->prev_sp - RP_OFFSET);
|
| 261 | else
|
| 262 | info->prev_ip = info->prev_sp = 0;
|
| 263 | return 1;
|
| 264 | }
|
| 265 |
|
| 266 | #ifdef CONFIG_IRQSTACKS
|
| 267 | if (pc == (unsigned long)&_call_on_stack && ALIGNMENT_OK(info->sp, long)) {
|
| 268 | info->prev_sp = *(unsigned long *)(info->sp - FRAME_SIZE - REG_SZ);
|
| 269 | info->prev_ip = *(unsigned long *)(info->sp - FRAME_SIZE - RP_OFFSET);
|
| 270 | return 1;
|
| 271 | }
|
| 272 | #endif
|
| 273 | return 0;
|
| 274 | }
|
| 275 |
|
| 276 | static void unwind_frame_regs(struct unwind_frame_info *info)
|
| 277 | {
|
| 278 | const struct unwind_table_entry *e;
|
| 279 | unsigned long npc;
|
| 280 | unsigned int insn;
|
| 281 | long frame_size = 0;
|
| 282 | int looking_for_rp, rpoffset = 0;
|
| 283 |
|
| 284 | e = find_unwind_entry(info->ip);
|
| 285 | if (e == NULL) {
|
| 286 | unsigned long sp;
|
| 287 |
|
| 288 | dbg("Cannot find unwind entry for %pS; forced unwinding\n",
|
| 289 | (void *) info->ip);
|
| 290 |
|
| 291 | |
| 292 | we are adjusting the stack correctly or extracting the rp
|
| 293 | correctly. The rp is checked to see if it belongs to the
|
| 294 | kernel text section, if not we assume we don't have a
|
| 295 | correct stack frame and we continue to unwind the stack.
|
| 296 | This is not quite correct, and will fail for loadable
|
| 297 | modules.
|
| 298 | sp = info->sp & ~63;
|
| 299 | do {
|
| 300 | unsigned long tmp;
|
| 301 |
|
| 302 | info->prev_sp = sp - 64;
|
| 303 | info->prev_ip = 0;
|
| 304 |
|
| 305 |
|
| 306 | if ((info->prev_sp - (unsigned long) task_stack_page(info->t))
|
| 307 | >= THREAD_SIZE) {
|
| 308 | info->prev_sp = 0;
|
| 309 | break;
|
| 310 | }
|
| 311 |
|
| 312 | if (copy_from_kernel_nofault(&tmp,
|
| 313 | (void *)info->prev_sp - RP_OFFSET, sizeof(tmp)))
|
| 314 | break;
|
| 315 | info->prev_ip = tmp;
|
| 316 | sp = info->prev_sp;
|
| 317 | } while (!kernel_text_address(info->prev_ip));
|
| 318 |
|
| 319 | info->rp = 0;
|
| 320 |
|
| 321 | dbg("analyzing func @ %lx with no unwind info, setting "
|
| 322 | "prev_sp=%lx prev_ip=%lx\n", info->ip,
|
| 323 | info->prev_sp, info->prev_ip);
|
| 324 | } else {
|
| 325 | dbg("e->start = 0x%x, e->end = 0x%x, Save_SP = %d, "
|
| 326 | "Save_RP = %d, Millicode = %d size = %u\n",
|
| 327 | e->region_start, e->region_end, e->Save_SP, e->Save_RP,
|
| 328 | e->Millicode, e->Total_frame_size);
|
| 329 |
|
| 330 | looking_for_rp = e->Save_RP;
|
| 331 |
|
| 332 | for (npc = e->region_start;
|
| 333 | (frame_size < (e->Total_frame_size << 3) ||
|
| 334 | looking_for_rp) &&
|
| 335 | npc < info->ip;
|
| 336 | npc += 4) {
|
| 337 |
|
| 338 | insn = *(unsigned int *)npc;
|
| 339 |
|
| 340 | if ((insn & 0xffffc001) == 0x37de0000 ||
|
| 341 | (insn & 0xffe00001) == 0x6fc00000) {
|
| 342 |
|
| 343 | frame_size += (insn & 0x3fff) >> 1;
|
| 344 | dbg("analyzing func @ %lx, insn=%08x @ "
|
| 345 | "%lx, frame_size = %ld\n", info->ip,
|
| 346 | insn, npc, frame_size);
|
| 347 | } else if ((insn & 0xffe00009) == 0x73c00008) {
|
| 348 |
|
| 349 | frame_size += ((insn >> 4) & 0x3ff) << 3;
|
| 350 | dbg("analyzing func @ %lx, insn=%08x @ "
|
| 351 | "%lx, frame_size = %ld\n", info->ip,
|
| 352 | insn, npc, frame_size);
|
| 353 | } else if (insn == 0x6bc23fd9) {
|
| 354 |
|
| 355 | rpoffset = 20;
|
| 356 | looking_for_rp = 0;
|
| 357 | dbg("analyzing func @ %lx, insn=stw rp,"
|
| 358 | "-20(sp) @ %lx\n", info->ip, npc);
|
| 359 | } else if (insn == 0x0fc212c1) {
|
| 360 |
|
| 361 | rpoffset = 16;
|
| 362 | looking_for_rp = 0;
|
| 363 | dbg("analyzing func @ %lx, insn=std rp,"
|
| 364 | "-16(sp) @ %lx\n", info->ip, npc);
|
| 365 | }
|
| 366 | }
|
| 367 |
|
| 368 | if (frame_size > e->Total_frame_size << 3)
|
| 369 | frame_size = e->Total_frame_size << 3;
|
| 370 |
|
| 371 | if (!unwind_special(info, e->region_start, frame_size)) {
|
| 372 | info->prev_sp = info->sp - frame_size;
|
| 373 | if (e->Millicode)
|
| 374 | info->rp = info->r31;
|
| 375 | else if (rpoffset && ALIGNMENT_OK(info->prev_sp, long))
|
| 376 | info->rp = *(unsigned long *)(info->prev_sp - rpoffset);
|
| 377 | else
|
| 378 | info->rp = 0;
|
| 379 | info->prev_ip = info->rp;
|
| 380 | info->rp = 0;
|
| 381 | }
|
| 382 |
|
| 383 | dbg("analyzing func @ %lx, setting prev_sp=%lx "
|
| 384 | "prev_ip=%lx npc=%lx\n", info->ip, info->prev_sp,
|
| 385 | info->prev_ip, npc);
|
| 386 | }
|
| 387 | }
|
| 388 |
|
| 389 | void unwind_frame_init(struct unwind_frame_info *info, struct task_struct *t,
|
| 390 | struct pt_regs *regs)
|
| 391 | {
|
| 392 | memset(info, 0, sizeof(struct unwind_frame_info));
|
| 393 | info->t = t;
|
| 394 | info->sp = regs->gr[30];
|
| 395 | info->ip = regs->iaoq[0];
|
| 396 | info->rp = regs->gr[2];
|
| 397 | info->r31 = regs->gr[31];
|
| 398 |
|
| 399 | dbg("(%d) Start unwind from sp=%08lx ip=%08lx\n",
|
| 400 | t ? (int)t->pid : -1, info->sp, info->ip);
|
| 401 | }
|
| 402 |
|
| 403 | void unwind_frame_init_from_blocked_task(struct unwind_frame_info *info, struct task_struct *t)
|
| 404 | {
|
| 405 | struct pt_regs *r = &t->thread.regs;
|
| 406 | struct pt_regs *r2;
|
| 407 |
|
| 408 | r2 = kmalloc_obj(struct pt_regs, GFP_ATOMIC);
|
| 409 | if (!r2)
|
| 410 | return;
|
| 411 | *r2 = *r;
|
| 412 | r2->gr[30] = r->ksp;
|
| 413 | r2->iaoq[0] = r->kpc;
|
| 414 | unwind_frame_init(info, t, r2);
|
| 415 | kfree(r2);
|
| 416 | }
|
| 417 |
|
| 418 | #define get_parisc_stackpointer() ({ \
|
| 419 | unsigned long sp; \
|
| 420 | __asm__("copy %%r30, %0" : "=r"(sp)); \
|
| 421 | (sp); \
|
| 422 | })
|
| 423 |
|
| 424 | void unwind_frame_init_task(struct unwind_frame_info *info,
|
| 425 | struct task_struct *task, struct pt_regs *regs)
|
| 426 | {
|
| 427 | task = task ? task : current;
|
| 428 |
|
| 429 | if (task == current) {
|
| 430 | struct pt_regs r;
|
| 431 |
|
| 432 | if (!regs) {
|
| 433 | memset(&r, 0, sizeof(r));
|
| 434 | r.iaoq[0] = _THIS_IP_;
|
| 435 | r.gr[2] = _RET_IP_;
|
| 436 | r.gr[30] = get_parisc_stackpointer();
|
| 437 | regs = &r;
|
| 438 | }
|
| 439 | unwind_frame_init(info, task, regs);
|
| 440 | } else {
|
| 441 | unwind_frame_init_from_blocked_task(info, task);
|
| 442 | }
|
| 443 | }
|
| 444 |
|
| 445 | int unwind_once(struct unwind_frame_info *next_frame)
|
| 446 | {
|
| 447 | unwind_frame_regs(next_frame);
|
| 448 |
|
| 449 | if (next_frame->prev_sp == 0 ||
|
| 450 | next_frame->prev_ip == 0)
|
| 451 | return -1;
|
| 452 |
|
| 453 | next_frame->sp = next_frame->prev_sp;
|
| 454 | next_frame->ip = next_frame->prev_ip;
|
| 455 | next_frame->prev_sp = 0;
|
| 456 | next_frame->prev_ip = 0;
|
| 457 |
|
| 458 | dbg("(%d) Continue unwind to sp=%08lx ip=%08lx\n",
|
| 459 | next_frame->t ? (int)next_frame->t->pid : -1,
|
| 460 | next_frame->sp, next_frame->ip);
|
| 461 |
|
| 462 | return 0;
|
| 463 | }
|
| 464 |
|
| 465 | int unwind_to_user(struct unwind_frame_info *info)
|
| 466 | {
|
| 467 | int ret;
|
| 468 |
|
| 469 | do {
|
| 470 | ret = unwind_once(info);
|
| 471 | } while (!ret && !(info->ip & 3));
|
| 472 |
|
| 473 | return ret;
|
| 474 | }
|
| 475 |
|
| 476 | unsigned long return_address(unsigned int level)
|
| 477 | {
|
| 478 | struct unwind_frame_info info;
|
| 479 |
|
| 480 |
|
| 481 | unwind_frame_init_task(&info, current, NULL);
|
| 482 |
|
| 483 |
|
| 484 | level += 2;
|
| 485 | do {
|
| 486 | if (unwind_once(&info) < 0 || info.ip == 0)
|
| 487 | return 0;
|
| 488 | if (!kernel_text_address(info.ip))
|
| 489 | return 0;
|
| 490 | } while (info.ip && level--);
|
| 491 |
|
| 492 | return info.ip;
|
| 493 | }
|