| 1 | |
| 2 | |
| 3 |
|
| 4 | #include <linux/cpu.h>
|
| 5 | #include <linux/jump_label.h>
|
| 6 | #include <linux/kernel.h>
|
| 7 | #include <linux/memory.h>
|
| 8 | #include <linux/stop_machine.h>
|
| 9 | #include <linux/types.h>
|
| 10 |
|
| 11 | #include <asm/cacheflush.h>
|
| 12 |
|
| 13 | #define J_OFFSET_MASK 0x0003ffff
|
| 14 | #define J_SIGN_MASK (~(J_OFFSET_MASK >> 1))
|
| 15 |
|
| 16 | #if defined(__XTENSA_EL__)
|
| 17 | #define J_INSN 0x6
|
| 18 | #define NOP_INSN 0x0020f0
|
| 19 | #elif defined(__XTENSA_EB__)
|
| 20 | #define J_INSN 0x60000000
|
| 21 | #define NOP_INSN 0x0f020000
|
| 22 | #else
|
| 23 | #error Unsupported endianness.
|
| 24 | #endif
|
| 25 |
|
| 26 | struct patch {
|
| 27 | atomic_t cpu_count;
|
| 28 | unsigned long addr;
|
| 29 | size_t sz;
|
| 30 | const void *data;
|
| 31 | };
|
| 32 |
|
| 33 | static void local_patch_text(unsigned long addr, const void *data, size_t sz)
|
| 34 | {
|
| 35 | memcpy((void *)addr, data, sz);
|
| 36 | local_flush_icache_range(addr, addr + sz);
|
| 37 | }
|
| 38 |
|
| 39 | static int patch_text_stop_machine(void *data)
|
| 40 | {
|
| 41 | struct patch *patch = data;
|
| 42 |
|
| 43 | if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) {
|
| 44 | local_patch_text(patch->addr, patch->data, patch->sz);
|
| 45 | atomic_inc(&patch->cpu_count);
|
| 46 | } else {
|
| 47 | while (atomic_read(&patch->cpu_count) <= num_online_cpus())
|
| 48 | cpu_relax();
|
| 49 | __invalidate_icache_range(patch->addr, patch->sz);
|
| 50 | }
|
| 51 | return 0;
|
| 52 | }
|
| 53 |
|
| 54 | static void patch_text(unsigned long addr, const void *data, size_t sz)
|
| 55 | {
|
| 56 | if (IS_ENABLED(CONFIG_SMP)) {
|
| 57 | struct patch patch = {
|
| 58 | .cpu_count = ATOMIC_INIT(0),
|
| 59 | .addr = addr,
|
| 60 | .sz = sz,
|
| 61 | .data = data,
|
| 62 | };
|
| 63 | stop_machine_cpuslocked(patch_text_stop_machine,
|
| 64 | &patch, cpu_online_mask);
|
| 65 | } else {
|
| 66 | unsigned long flags;
|
| 67 |
|
| 68 | local_irq_save(flags);
|
| 69 | local_patch_text(addr, data, sz);
|
| 70 | local_irq_restore(flags);
|
| 71 | }
|
| 72 | }
|
| 73 |
|
| 74 | void arch_jump_label_transform(struct jump_entry *e,
|
| 75 | enum jump_label_type type)
|
| 76 | {
|
| 77 | u32 d = (jump_entry_target(e) - (jump_entry_code(e) + 4));
|
| 78 | u32 insn;
|
| 79 |
|
| 80 |
|
| 81 | BUG_ON(!((d & J_SIGN_MASK) == 0 ||
|
| 82 | (d & J_SIGN_MASK) == J_SIGN_MASK));
|
| 83 |
|
| 84 | if (type == JUMP_LABEL_JMP) {
|
| 85 | #if defined(__XTENSA_EL__)
|
| 86 | insn = ((d & J_OFFSET_MASK) << 6) | J_INSN;
|
| 87 | #elif defined(__XTENSA_EB__)
|
| 88 | insn = ((d & J_OFFSET_MASK) << 8) | J_INSN;
|
| 89 | #endif
|
| 90 | } else {
|
| 91 | insn = NOP_INSN;
|
| 92 | }
|
| 93 |
|
| 94 | patch_text(jump_entry_code(e), &insn, JUMP_LABEL_NOP_SIZE);
|
| 95 | }
|