| 1 | |
| 2 | #include <linux/kernel.h>
|
| 3 | #include <linux/spinlock.h>
|
| 4 | #include <linux/kprobes.h>
|
| 5 | #include <linux/mm.h>
|
| 6 | #include <linux/stop_machine.h>
|
| 7 |
|
| 8 | #include <asm/cacheflush.h>
|
| 9 | #include <asm/fixmap.h>
|
| 10 | #include <asm/smp_plat.h>
|
| 11 | #include <asm/opcodes.h>
|
| 12 | #include <asm/text-patching.h>
|
| 13 |
|
| 14 | struct patch {
|
| 15 | void *addr;
|
| 16 | unsigned int insn;
|
| 17 | };
|
| 18 |
|
| 19 | #ifdef CONFIG_MMU
|
| 20 | static DEFINE_RAW_SPINLOCK(patch_lock);
|
| 21 |
|
| 22 | static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags)
|
| 23 | {
|
| 24 | unsigned int uintaddr = (uintptr_t) addr;
|
| 25 | bool module = !core_kernel_text(uintaddr);
|
| 26 | struct page *page;
|
| 27 |
|
| 28 | if (module && IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
|
| 29 | page = vmalloc_to_page(addr);
|
| 30 | else if (!module && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX))
|
| 31 | page = virt_to_page(addr);
|
| 32 | else
|
| 33 | return addr;
|
| 34 |
|
| 35 | if (flags)
|
| 36 | raw_spin_lock_irqsave(&patch_lock, *flags);
|
| 37 |
|
| 38 | set_fixmap(fixmap, page_to_phys(page));
|
| 39 |
|
| 40 | return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
|
| 41 | }
|
| 42 |
|
| 43 | static void __kprobes patch_unmap(int fixmap, unsigned long *flags)
|
| 44 | {
|
| 45 | clear_fixmap(fixmap);
|
| 46 |
|
| 47 | if (flags)
|
| 48 | raw_spin_unlock_irqrestore(&patch_lock, *flags);
|
| 49 | }
|
| 50 | #else
|
| 51 | static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags)
|
| 52 | {
|
| 53 | return addr;
|
| 54 | }
|
| 55 | static void __kprobes patch_unmap(int fixmap, unsigned long *flags) { }
|
| 56 | #endif
|
| 57 |
|
| 58 | void __kprobes __patch_text_real(void *addr, unsigned int insn, bool remap)
|
| 59 | {
|
| 60 | bool thumb2 = IS_ENABLED(CONFIG_THUMB2_KERNEL);
|
| 61 | unsigned int uintaddr = (uintptr_t) addr;
|
| 62 | bool twopage = false;
|
| 63 | unsigned long flags;
|
| 64 | void *waddr = addr;
|
| 65 | int size;
|
| 66 |
|
| 67 | if (remap)
|
| 68 | waddr = patch_map(addr, FIX_TEXT_POKE0, &flags);
|
| 69 |
|
| 70 | if (thumb2 && __opcode_is_thumb16(insn)) {
|
| 71 | *(u16 *)waddr = __opcode_to_mem_thumb16(insn);
|
| 72 | size = sizeof(u16);
|
| 73 | } else if (thumb2 && (uintaddr & 2)) {
|
| 74 | u16 first = __opcode_thumb32_first(insn);
|
| 75 | u16 second = __opcode_thumb32_second(insn);
|
| 76 | u16 *addrh0 = waddr;
|
| 77 | u16 *addrh1 = waddr + 2;
|
| 78 |
|
| 79 | twopage = (uintaddr & ~PAGE_MASK) == PAGE_SIZE - 2;
|
| 80 | if (twopage && remap)
|
| 81 | addrh1 = patch_map(addr + 2, FIX_TEXT_POKE1, NULL);
|
| 82 |
|
| 83 | *addrh0 = __opcode_to_mem_thumb16(first);
|
| 84 | *addrh1 = __opcode_to_mem_thumb16(second);
|
| 85 |
|
| 86 | if (twopage && addrh1 != addr + 2) {
|
| 87 | flush_kernel_vmap_range(addrh1, 2);
|
| 88 | patch_unmap(FIX_TEXT_POKE1, NULL);
|
| 89 | }
|
| 90 |
|
| 91 | size = sizeof(u32);
|
| 92 | } else {
|
| 93 | if (thumb2)
|
| 94 | insn = __opcode_to_mem_thumb32(insn);
|
| 95 | else
|
| 96 | insn = __opcode_to_mem_arm(insn);
|
| 97 |
|
| 98 | *(u32 *)waddr = insn;
|
| 99 | size = sizeof(u32);
|
| 100 | }
|
| 101 |
|
| 102 | if (waddr != addr) {
|
| 103 | flush_kernel_vmap_range(waddr, twopage ? size / 2 : size);
|
| 104 | patch_unmap(FIX_TEXT_POKE0, &flags);
|
| 105 | }
|
| 106 |
|
| 107 | flush_icache_range((uintptr_t)(addr),
|
| 108 | (uintptr_t)(addr) + size);
|
| 109 | }
|
| 110 |
|
| 111 | static int __kprobes patch_text_stop_machine(void *data)
|
| 112 | {
|
| 113 | struct patch *patch = data;
|
| 114 |
|
| 115 | __patch_text(patch->addr, patch->insn);
|
| 116 |
|
| 117 | return 0;
|
| 118 | }
|
| 119 |
|
| 120 | void __kprobes patch_text(void *addr, unsigned int insn)
|
| 121 | {
|
| 122 | struct patch patch = {
|
| 123 | .addr = addr,
|
| 124 | .insn = insn,
|
| 125 | };
|
| 126 |
|
| 127 | stop_machine_cpuslocked(patch_text_stop_machine, &patch, NULL);
|
| 128 | }
|