| 1 | |
| 2 | |
| 3 | * Kernel Probes (KProbes)
|
| 4 | *
|
| 5 | * Copyright (C) IBM Corporation, 2002, 2004
|
| 6 | *
|
| 7 | * 2002-Oct Created by Vamsi Krishna S <[email protected]> Kernel
|
| 8 | * Probes initial implementation ( includes contributions from
|
| 9 | * Rusty Russell).
|
| 10 | * 2004-July Suparna Bhattacharya <[email protected]> added jumper probes
|
| 11 | * interface to access function arguments.
|
| 12 | * 2004-Nov Ananth N Mavinakayanahalli <[email protected]> kprobes port
|
| 13 | * for PPC64
|
| 14 |
|
| 15 |
|
| 16 | #include <linux/kprobes.h>
|
| 17 | #include <linux/ptrace.h>
|
| 18 | #include <linux/preempt.h>
|
| 19 | #include <linux/extable.h>
|
| 20 | #include <linux/kdebug.h>
|
| 21 | #include <linux/slab.h>
|
| 22 | #include <linux/set_memory.h>
|
| 23 | #include <linux/execmem.h>
|
| 24 | #include <asm/text-patching.h>
|
| 25 | #include <asm/cacheflush.h>
|
| 26 | #include <asm/sstep.h>
|
| 27 | #include <asm/sections.h>
|
| 28 | #include <asm/inst.h>
|
| 29 | #include <linux/uaccess.h>
|
| 30 |
|
| 31 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
|
| 32 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
|
| 33 |
|
| 34 | struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
|
| 35 |
|
| 36 | bool arch_within_kprobe_blacklist(unsigned long addr)
|
| 37 | {
|
| 38 | return (addr >= (unsigned long)__kprobes_text_start &&
|
| 39 | addr < (unsigned long)__kprobes_text_end) ||
|
| 40 | (addr >= (unsigned long)_stext &&
|
| 41 | addr < (unsigned long)__head_end);
|
| 42 | }
|
| 43 |
|
| 44 | kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)
|
| 45 | {
|
| 46 | kprobe_opcode_t *addr = NULL;
|
| 47 |
|
| 48 | #ifdef CONFIG_PPC64_ELF_ABI_V2
|
| 49 |
|
| 50 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
|
| 51 | if (addr && !offset) {
|
| 52 | #ifdef CONFIG_KPROBES_ON_FTRACE
|
| 53 | unsigned long faddr;
|
| 54 | |
| 55 | * Per livepatch.h, ftrace location is always within the first
|
| 56 | * 16 bytes of a function on powerpc with -mprofile-kernel.
|
| 57 |
|
| 58 | faddr = ftrace_location_range((unsigned long)addr,
|
| 59 | (unsigned long)addr + 16);
|
| 60 | if (faddr)
|
| 61 | addr = (kprobe_opcode_t *)faddr;
|
| 62 | else
|
| 63 | #endif
|
| 64 | addr = (kprobe_opcode_t *)ppc_function_entry(addr);
|
| 65 | }
|
| 66 | #elif defined(CONFIG_PPC64_ELF_ABI_V1)
|
| 67 | |
| 68 | * 64bit powerpc ABIv1 uses function descriptors:
|
| 69 | * - Check for the dot variant of the symbol first.
|
| 70 | * - If that fails, try looking up the symbol provided.
|
| 71 | *
|
| 72 | * This ensures we always get to the actual symbol and not
|
| 73 | * the descriptor.
|
| 74 | *
|
| 75 | * Also handle <module:symbol> format.
|
| 76 |
|
| 77 | char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN];
|
| 78 | bool dot_appended = false;
|
| 79 | const char *c;
|
| 80 | ssize_t ret = 0;
|
| 81 | int len = 0;
|
| 82 |
|
| 83 | if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
|
| 84 | c++;
|
| 85 | len = c - name;
|
| 86 | memcpy(dot_name, name, len);
|
| 87 | } else
|
| 88 | c = name;
|
| 89 |
|
| 90 | if (*c != '\0' && *c != '.') {
|
| 91 | dot_name[len++] = '.';
|
| 92 | dot_appended = true;
|
| 93 | }
|
| 94 | ret = strscpy(dot_name + len, c, KSYM_NAME_LEN);
|
| 95 | if (ret > 0)
|
| 96 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name);
|
| 97 |
|
| 98 |
|
| 99 | if (!addr && dot_appended)
|
| 100 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
|
| 101 | #else
|
| 102 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
|
| 103 | #endif
|
| 104 |
|
| 105 | return addr;
|
| 106 | }
|
| 107 |
|
| 108 | static bool arch_kprobe_on_func_entry(unsigned long addr, unsigned long offset)
|
| 109 | {
|
| 110 | unsigned long ip = ftrace_location(addr);
|
| 111 |
|
| 112 | if (ip)
|
| 113 | return offset <= (ip - addr);
|
| 114 | if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
|
| 115 | return offset <= 8;
|
| 116 | return !offset;
|
| 117 | }
|
| 118 |
|
| 119 |
|
| 120 | kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset,
|
| 121 | bool *on_func_entry)
|
| 122 | {
|
| 123 | *on_func_entry = arch_kprobe_on_func_entry(addr, offset);
|
| 124 | return (kprobe_opcode_t *)(addr + offset);
|
| 125 | }
|
| 126 |
|
| 127 | int arch_prepare_kprobe(struct kprobe *p)
|
| 128 | {
|
| 129 | int ret = 0;
|
| 130 | struct kprobe *prev;
|
| 131 | ppc_inst_t insn = ppc_inst_read(p->addr);
|
| 132 |
|
| 133 | if ((unsigned long)p->addr & 0x03) {
|
| 134 | printk("Attempt to register kprobe at an unaligned address\n");
|
| 135 | ret = -EINVAL;
|
| 136 | } else if (!can_single_step(ppc_inst_val(insn))) {
|
| 137 | printk("Cannot register a kprobe on instructions that can't be single stepped\n");
|
| 138 | ret = -EINVAL;
|
| 139 | } else if ((unsigned long)p->addr & ~PAGE_MASK &&
|
| 140 | ppc_inst_prefixed(ppc_inst_read(p->addr - 1))) {
|
| 141 | printk("Cannot register a kprobe on the second word of prefixed instruction\n");
|
| 142 | ret = -EINVAL;
|
| 143 | }
|
| 144 | prev = get_kprobe(p->addr - 1);
|
| 145 |
|
| 146 | |
| 147 | * When prev is a ftrace-based kprobe, we don't have an insn, and it
|
| 148 | * doesn't probe for prefixed instruction.
|
| 149 |
|
| 150 | if (prev && !kprobe_ftrace(prev) &&
|
| 151 | ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) {
|
| 152 | printk("Cannot register a kprobe on the second word of prefixed instruction\n");
|
| 153 | ret = -EINVAL;
|
| 154 | }
|
| 155 |
|
| 156 | |
| 157 | * not explicitly required on ppc32 (right now), but it doesn't hurt
|
| 158 | if (!ret) {
|
| 159 | p->ainsn.insn = get_insn_slot();
|
| 160 | if (!p->ainsn.insn)
|
| 161 | ret = -ENOMEM;
|
| 162 | }
|
| 163 |
|
| 164 | if (!ret) {
|
| 165 | patch_instruction(p->ainsn.insn, insn);
|
| 166 | p->opcode = ppc_inst_val(insn);
|
| 167 | }
|
| 168 |
|
| 169 | p->ainsn.boostable = 0;
|
| 170 | return ret;
|
| 171 | }
|
| 172 | NOKPROBE_SYMBOL(arch_prepare_kprobe);
|
| 173 |
|
| 174 | void arch_arm_kprobe(struct kprobe *p)
|
| 175 | {
|
| 176 | WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(BREAKPOINT_INSTRUCTION)));
|
| 177 | }
|
| 178 | NOKPROBE_SYMBOL(arch_arm_kprobe);
|
| 179 |
|
| 180 | void arch_disarm_kprobe(struct kprobe *p)
|
| 181 | {
|
| 182 | WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(p->opcode)));
|
| 183 | }
|
| 184 | NOKPROBE_SYMBOL(arch_disarm_kprobe);
|
| 185 |
|
| 186 | void arch_remove_kprobe(struct kprobe *p)
|
| 187 | {
|
| 188 | if (p->ainsn.insn) {
|
| 189 | free_insn_slot(p->ainsn.insn, 0);
|
| 190 | p->ainsn.insn = NULL;
|
| 191 | }
|
| 192 | }
|
| 193 | NOKPROBE_SYMBOL(arch_remove_kprobe);
|
| 194 |
|
| 195 | static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
|
| 196 | {
|
| 197 | enable_single_step(regs);
|
| 198 |
|
| 199 | |
| 200 | * On powerpc we should single step on the original
|
| 201 | * instruction even if the probed insn is a trap
|
| 202 | * variant as values in regs could play a part in
|
| 203 | * if the trap is taken or not
|
| 204 |
|
| 205 | regs_set_return_ip(regs, (unsigned long)p->ainsn.insn);
|
| 206 | }
|
| 207 |
|
| 208 | static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
|
| 209 | {
|
| 210 | kcb->prev_kprobe.kp = kprobe_running();
|
| 211 | kcb->prev_kprobe.status = kcb->kprobe_status;
|
| 212 | kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
|
| 213 | }
|
| 214 |
|
| 215 | static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
|
| 216 | {
|
| 217 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
|
| 218 | kcb->kprobe_status = kcb->prev_kprobe.status;
|
| 219 | kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
|
| 220 | }
|
| 221 |
|
| 222 | static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
|
| 223 | struct kprobe_ctlblk *kcb)
|
| 224 | {
|
| 225 | __this_cpu_write(current_kprobe, p);
|
| 226 | kcb->kprobe_saved_msr = regs->msr;
|
| 227 | }
|
| 228 |
|
| 229 | static int try_to_emulate(struct kprobe *p, struct pt_regs *regs)
|
| 230 | {
|
| 231 | int ret;
|
| 232 | ppc_inst_t insn = ppc_inst_read(p->ainsn.insn);
|
| 233 |
|
| 234 |
|
| 235 | ret = emulate_step(regs, insn);
|
| 236 | if (ret > 0) {
|
| 237 | |
| 238 | * Once this instruction has been boosted
|
| 239 | * successfully, set the boostable flag
|
| 240 |
|
| 241 | if (unlikely(p->ainsn.boostable == 0))
|
| 242 | p->ainsn.boostable = 1;
|
| 243 | } else if (ret < 0) {
|
| 244 | |
| 245 | * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
|
| 246 | * So, we should never get here... but, its still
|
| 247 | * good to catch them, just in case...
|
| 248 |
|
| 249 | printk("Can't step on instruction %08lx\n", ppc_inst_as_ulong(insn));
|
| 250 | BUG();
|
| 251 | } else {
|
| 252 | |
| 253 | * If we haven't previously emulated this instruction, then it
|
| 254 | * can't be boosted. Note it down so we don't try to do so again.
|
| 255 | *
|
| 256 | * If, however, we had emulated this instruction in the past,
|
| 257 | * then this is just an error with the current run (for
|
| 258 | * instance, exceptions due to a load/store). We return 0 so
|
| 259 | * that this is now single-stepped, but continue to try
|
| 260 | * emulating it in subsequent probe hits.
|
| 261 |
|
| 262 | if (unlikely(p->ainsn.boostable != 1))
|
| 263 | p->ainsn.boostable = -1;
|
| 264 | }
|
| 265 |
|
| 266 | return ret;
|
| 267 | }
|
| 268 | NOKPROBE_SYMBOL(try_to_emulate);
|
| 269 |
|
| 270 | int kprobe_handler(struct pt_regs *regs)
|
| 271 | {
|
| 272 | struct kprobe *p;
|
| 273 | int ret = 0;
|
| 274 | unsigned int *addr = (unsigned int *)regs->nip;
|
| 275 | struct kprobe_ctlblk *kcb;
|
| 276 |
|
| 277 | if (user_mode(regs))
|
| 278 | return 0;
|
| 279 |
|
| 280 | if (!IS_ENABLED(CONFIG_BOOKE) &&
|
| 281 | (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR)))
|
| 282 | return 0;
|
| 283 |
|
| 284 | |
| 285 | * We don't want to be preempted for the entire
|
| 286 | * duration of kprobe processing
|
| 287 |
|
| 288 | preempt_disable();
|
| 289 | kcb = get_kprobe_ctlblk();
|
| 290 |
|
| 291 | p = get_kprobe(addr);
|
| 292 | if (!p) {
|
| 293 | unsigned int instr;
|
| 294 |
|
| 295 | if (get_kernel_nofault(instr, addr))
|
| 296 | goto no_kprobe;
|
| 297 |
|
| 298 | if (instr != BREAKPOINT_INSTRUCTION) {
|
| 299 | |
| 300 | * PowerPC has multiple variants of the "trap"
|
| 301 | * instruction. If the current instruction is a
|
| 302 | * trap variant, it could belong to someone else
|
| 303 |
|
| 304 | if (is_trap(instr))
|
| 305 | goto no_kprobe;
|
| 306 | |
| 307 | * The breakpoint instruction was removed right
|
| 308 | * after we hit it. Another cpu has removed
|
| 309 | * either a probepoint or a debugger breakpoint
|
| 310 | * at this address. In either case, no further
|
| 311 | * handling of this interrupt is appropriate.
|
| 312 |
|
| 313 | ret = 1;
|
| 314 | }
|
| 315 |
|
| 316 | goto no_kprobe;
|
| 317 | }
|
| 318 |
|
| 319 |
|
| 320 | if (kprobe_running()) {
|
| 321 | kprobe_opcode_t insn = *p->ainsn.insn;
|
| 322 | if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) {
|
| 323 |
|
| 324 | regs_set_return_msr(regs,
|
| 325 | (regs->msr & ~MSR_SINGLESTEP) |
|
| 326 | kcb->kprobe_saved_msr);
|
| 327 | goto no_kprobe;
|
| 328 | }
|
| 329 |
|
| 330 | |
| 331 | * We have reentered the kprobe_handler(), since another probe
|
| 332 | * was hit while within the handler. We here save the original
|
| 333 | * kprobes variables and just single step on the instruction of
|
| 334 | * the new probe without calling any user handlers.
|
| 335 |
|
| 336 | save_previous_kprobe(kcb);
|
| 337 | set_current_kprobe(p, regs, kcb);
|
| 338 | kprobes_inc_nmissed_count(p);
|
| 339 | kcb->kprobe_status = KPROBE_REENTER;
|
| 340 | if (p->ainsn.boostable >= 0) {
|
| 341 | ret = try_to_emulate(p, regs);
|
| 342 |
|
| 343 | if (ret > 0) {
|
| 344 | restore_previous_kprobe(kcb);
|
| 345 | preempt_enable();
|
| 346 | return 1;
|
| 347 | }
|
| 348 | }
|
| 349 | prepare_singlestep(p, regs);
|
| 350 | return 1;
|
| 351 | }
|
| 352 |
|
| 353 | kcb->kprobe_status = KPROBE_HIT_ACTIVE;
|
| 354 | set_current_kprobe(p, regs, kcb);
|
| 355 | if (p->pre_handler && p->pre_handler(p, regs)) {
|
| 356 |
|
| 357 | reset_current_kprobe();
|
| 358 | preempt_enable();
|
| 359 | return 1;
|
| 360 | }
|
| 361 |
|
| 362 | if (p->ainsn.boostable >= 0) {
|
| 363 | ret = try_to_emulate(p, regs);
|
| 364 |
|
| 365 | if (ret > 0) {
|
| 366 | if (p->post_handler)
|
| 367 | p->post_handler(p, regs, 0);
|
| 368 |
|
| 369 | kcb->kprobe_status = KPROBE_HIT_SSDONE;
|
| 370 | reset_current_kprobe();
|
| 371 | preempt_enable();
|
| 372 | return 1;
|
| 373 | }
|
| 374 | }
|
| 375 | prepare_singlestep(p, regs);
|
| 376 | kcb->kprobe_status = KPROBE_HIT_SS;
|
| 377 | return 1;
|
| 378 |
|
| 379 | no_kprobe:
|
| 380 | preempt_enable();
|
| 381 | return ret;
|
| 382 | }
|
| 383 | NOKPROBE_SYMBOL(kprobe_handler);
|
| 384 |
|
| 385 | |
| 386 | * Called after single-stepping. p->addr is the address of the
|
| 387 | * instruction whose first byte has been replaced by the "breakpoint"
|
| 388 | * instruction. To avoid the SMP problems that can occur when we
|
| 389 | * temporarily put back the original opcode to single-step, we
|
| 390 | * single-stepped a copy of the instruction. The address of this
|
| 391 | * copy is p->ainsn.insn.
|
| 392 |
|
| 393 | int kprobe_post_handler(struct pt_regs *regs)
|
| 394 | {
|
| 395 | int len;
|
| 396 | struct kprobe *cur = kprobe_running();
|
| 397 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
|
| 398 |
|
| 399 | if (!cur || user_mode(regs))
|
| 400 | return 0;
|
| 401 |
|
| 402 | len = ppc_inst_len(ppc_inst_read(cur->ainsn.insn));
|
| 403 |
|
| 404 | if (((unsigned long)cur->ainsn.insn + len) != regs->nip)
|
| 405 | return 0;
|
| 406 |
|
| 407 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
|
| 408 | kcb->kprobe_status = KPROBE_HIT_SSDONE;
|
| 409 | cur->post_handler(cur, regs, 0);
|
| 410 | }
|
| 411 |
|
| 412 |
|
| 413 | regs_set_return_ip(regs, (unsigned long)cur->addr + len);
|
| 414 | regs_set_return_msr(regs, regs->msr | kcb->kprobe_saved_msr);
|
| 415 |
|
| 416 |
|
| 417 | if (kcb->kprobe_status == KPROBE_REENTER) {
|
| 418 | restore_previous_kprobe(kcb);
|
| 419 | goto out;
|
| 420 | }
|
| 421 | reset_current_kprobe();
|
| 422 | out:
|
| 423 | preempt_enable();
|
| 424 |
|
| 425 | |
| 426 | * if somebody else is singlestepping across a probe point, msr
|
| 427 | * will have DE/SE set, in which case, continue the remaining processing
|
| 428 | * of do_debug, as if this is not a probe hit.
|
| 429 |
|
| 430 | if (regs->msr & MSR_SINGLESTEP)
|
| 431 | return 0;
|
| 432 |
|
| 433 | return 1;
|
| 434 | }
|
| 435 | NOKPROBE_SYMBOL(kprobe_post_handler);
|
| 436 |
|
| 437 | int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
|
| 438 | {
|
| 439 | struct kprobe *cur = kprobe_running();
|
| 440 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
|
| 441 | const struct exception_table_entry *entry;
|
| 442 |
|
| 443 | switch(kcb->kprobe_status) {
|
| 444 | case KPROBE_HIT_SS:
|
| 445 | case KPROBE_REENTER:
|
| 446 | |
| 447 | * We are here because the instruction being single
|
| 448 | * stepped caused a page fault. We reset the current
|
| 449 | * kprobe and the nip points back to the probe address
|
| 450 | * and allow the page fault handler to continue as a
|
| 451 | * normal page fault.
|
| 452 |
|
| 453 | regs_set_return_ip(regs, (unsigned long)cur->addr);
|
| 454 |
|
| 455 | regs_set_return_msr(regs,
|
| 456 | (regs->msr & ~MSR_SINGLESTEP) |
|
| 457 | kcb->kprobe_saved_msr);
|
| 458 | if (kcb->kprobe_status == KPROBE_REENTER)
|
| 459 | restore_previous_kprobe(kcb);
|
| 460 | else
|
| 461 | reset_current_kprobe();
|
| 462 | preempt_enable();
|
| 463 | break;
|
| 464 | case KPROBE_HIT_ACTIVE:
|
| 465 | case KPROBE_HIT_SSDONE:
|
| 466 | |
| 467 | * In case the user-specified fault handler returned
|
| 468 | * zero, try to fix up.
|
| 469 |
|
| 470 | if ((entry = search_exception_tables(regs->nip)) != NULL) {
|
| 471 | regs_set_return_ip(regs, extable_fixup(entry));
|
| 472 | return 1;
|
| 473 | }
|
| 474 |
|
| 475 | |
| 476 | * fixup_exception() could not handle it,
|
| 477 | * Let do_page_fault() fix it.
|
| 478 |
|
| 479 | break;
|
| 480 | default:
|
| 481 | break;
|
| 482 | }
|
| 483 | return 0;
|
| 484 | }
|
| 485 | NOKPROBE_SYMBOL(kprobe_fault_handler);
|
| 486 |
|
| 487 | int arch_trampoline_kprobe(struct kprobe *p)
|
| 488 | {
|
| 489 | if (p->addr == (kprobe_opcode_t *)&arch_rethook_trampoline)
|
| 490 | return 1;
|
| 491 |
|
| 492 | return 0;
|
| 493 | }
|
| 494 | NOKPROBE_SYMBOL(arch_trampoline_kprobe);
|