projects/linux·blob uaccess-cleanups · refs

649 lines · 16.1 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2008 Michael Ellerman, IBM Corporation.
4 */
5
6#include <linux/kprobes.h>
7#include <linux/mmu_context.h>
8#include <linux/random.h>
9#include <linux/vmalloc.h>
10#include <linux/init.h>
11#include <linux/cpuhotplug.h>
12#include <linux/uaccess.h>
13#include <linux/jump_label.h>
14
15#include <asm/debug.h>
16#include <asm/pgalloc.h>
17#include <asm/tlb.h>
18#include <asm/tlbflush.h>
19#include <asm/page.h>
20#include <asm/text-patching.h>
21#include <asm/inst.h>
22
23static int __patch_mem(void *exec_addr, unsigned long val, void *patch_addr, bool is_dword)
24{
25 if (!IS_ENABLED(CONFIG_PPC64) || likely(!is_dword)) {
26 /* For big endian correctness: plain address would use the wrong half */
27 u32 val32 = val;
28
29 __put_kernel_nofault(patch_addr, &val32, u32, failed);
30 } else {
31 __put_kernel_nofault(patch_addr, &val, u64, failed);
32 }
33
34 asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
35 "r" (exec_addr));
36
37 return 0;
38
39failed:
40 mb(); /* sync */
41 return -EPERM;
42}
43
44int raw_patch_instruction(u32 *addr, ppc_inst_t instr)
45{
46 if (ppc_inst_prefixed(instr))
47 return __patch_mem(addr, ppc_inst_as_ulong(instr), addr, true);
48 else
49 return __patch_mem(addr, ppc_inst_val(instr), addr, false);
50}
51
52struct patch_context {
53 union {
54 struct vm_struct *area;
55 struct mm_struct *mm;
56 };
57 unsigned long addr;
58 pte_t *pte;
59};
60
61static DEFINE_PER_CPU(struct patch_context, cpu_patching_context);
62
63static bool mm_patch_enabled(void)
64{
65 return IS_ENABLED(CONFIG_SMP) && radix_enabled();
66}
67
68/*
69 * The following applies for Radix MMU. Hash MMU has different requirements,
70 * and so is not supported.
71 *
72 * Changing mm requires context synchronising instructions on both sides of
73 * the context switch, as well as a hwsync between the last instruction for
74 * which the address of an associated storage access was translated using
75 * the current context.
76 *
77 * switch_mm_irqs_off() performs an isync after the context switch. It is
78 * the responsibility of the caller to perform the CSI and hwsync before
79 * starting/stopping the temp mm.
80 */
81static struct mm_struct *start_using_temp_mm(struct mm_struct *temp_mm)
82{
83 struct mm_struct *orig_mm = current->active_mm;
84
85 lockdep_assert_irqs_disabled();
86 switch_mm_irqs_off(orig_mm, temp_mm, current);
87
88 WARN_ON(!mm_is_thread_local(temp_mm));
89
90 suspend_breakpoints();
91 return orig_mm;
92}
93
94static void stop_using_temp_mm(struct mm_struct *temp_mm,
95 struct mm_struct *orig_mm)
96{
97 lockdep_assert_irqs_disabled();
98 switch_mm_irqs_off(temp_mm, orig_mm, current);
99 restore_breakpoints();
100}
101
102static int text_area_cpu_up(unsigned int cpu)
103{
104 struct vm_struct *area;
105 unsigned long addr;
106 int err;
107
108 area = get_vm_area(PAGE_SIZE, 0);
109 if (!area) {
110 WARN_ONCE(1, "Failed to create text area for cpu %d\n",
111 cpu);
112 return -1;
113 }
114
115 // Map/unmap the area to ensure all page tables are pre-allocated
116 addr = (unsigned long)area->addr;
117 err = map_kernel_page(addr, __pa_symbol(empty_zero_page), PAGE_KERNEL_RO);
118 if (err)
119 return err;
120
121 unmap_kernel_page(addr);
122
123 this_cpu_write(cpu_patching_context.area, area);
124 this_cpu_write(cpu_patching_context.addr, addr);
125 this_cpu_write(cpu_patching_context.pte, virt_to_kpte(addr));
126
127 return 0;
128}
129
130static int text_area_cpu_down(unsigned int cpu)
131{
132 free_vm_area(this_cpu_read(cpu_patching_context.area));
133 this_cpu_write(cpu_patching_context.area, NULL);
134 this_cpu_write(cpu_patching_context.addr, 0);
135 this_cpu_write(cpu_patching_context.pte, NULL);
136 return 0;
137}
138
139static void put_patching_mm(struct mm_struct *mm, unsigned long patching_addr)
140{
141 struct mmu_gather tlb;
142
143 tlb_gather_mmu(&tlb, mm);
144 free_pgd_range(&tlb, patching_addr, patching_addr + PAGE_SIZE, 0, 0);
145 mmput(mm);
146}
147
148static int text_area_cpu_up_mm(unsigned int cpu)
149{
150 struct mm_struct *mm;
151 unsigned long addr;
152 pte_t *pte;
153 spinlock_t *ptl;
154
155 mm = mm_alloc();
156 if (WARN_ON(!mm))
157 goto fail_no_mm;
158
159 /*
160 * Choose a random page-aligned address from the interval
161 * [PAGE_SIZE .. DEFAULT_MAP_WINDOW - PAGE_SIZE].
162 * The lower address bound is PAGE_SIZE to avoid the zero-page.
163 */
164 addr = (1 + (get_random_long() % (DEFAULT_MAP_WINDOW / PAGE_SIZE - 2))) << PAGE_SHIFT;
165
166 /*
167 * PTE allocation uses GFP_KERNEL which means we need to
168 * pre-allocate the PTE here because we cannot do the
169 * allocation during patching when IRQs are disabled.
170 *
171 * Using get_locked_pte() to avoid open coding, the lock
172 * is unnecessary.
173 */
174 pte = get_locked_pte(mm, addr, &ptl);
175 if (!pte)
176 goto fail_no_pte;
177 pte_unmap_unlock(pte, ptl);
178
179 this_cpu_write(cpu_patching_context.mm, mm);
180 this_cpu_write(cpu_patching_context.addr, addr);
181
182 return 0;
183
184fail_no_pte:
185 put_patching_mm(mm, addr);
186fail_no_mm:
187 return -ENOMEM;
188}
189
190static int text_area_cpu_down_mm(unsigned int cpu)
191{
192 put_patching_mm(this_cpu_read(cpu_patching_context.mm),
193 this_cpu_read(cpu_patching_context.addr));
194
195 this_cpu_write(cpu_patching_context.mm, NULL);
196 this_cpu_write(cpu_patching_context.addr, 0);
197
198 return 0;
199}
200
201static __ro_after_init DEFINE_STATIC_KEY_FALSE(poking_init_done);
202
203void __init poking_init(void)
204{
205 int ret;
206
207 if (mm_patch_enabled())
208 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
209 "powerpc/text_poke_mm:online",
210 text_area_cpu_up_mm,
211 text_area_cpu_down_mm);
212 else
213 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
214 "powerpc/text_poke:online",
215 text_area_cpu_up,
216 text_area_cpu_down);
217
218 /* cpuhp_setup_state returns >= 0 on success */
219 if (WARN_ON(ret < 0))
220 return;
221
222 static_branch_enable(&poking_init_done);
223}
224
225static unsigned long get_patch_pfn(void *addr)
226{
227 if (IS_ENABLED(CONFIG_EXECMEM) && is_vmalloc_or_module_addr(addr))
228 return vmalloc_to_pfn(addr);
229 else
230 return __pa_symbol(addr) >> PAGE_SHIFT;
231}
232
233static int __do_patch_mem_mm(void *addr, unsigned long val, bool is_dword)
234{
235 int err;
236 u32 *patch_addr;
237 unsigned long text_poke_addr;
238 pte_t *pte;
239 unsigned long pfn = get_patch_pfn(addr);
240 struct mm_struct *patching_mm;
241 struct mm_struct *orig_mm;
242 spinlock_t *ptl;
243
244 patching_mm = __this_cpu_read(cpu_patching_context.mm);
245 text_poke_addr = __this_cpu_read(cpu_patching_context.addr);
246 patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
247
248 pte = get_locked_pte(patching_mm, text_poke_addr, &ptl);
249 if (!pte)
250 return -ENOMEM;
251
252 __set_pte_at(patching_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0);
253
254 /* order PTE update before use, also serves as the hwsync */
255 asm volatile("ptesync": : :"memory");
256
257 /* order context switch after arbitrary prior code */
258 isync();
259
260 orig_mm = start_using_temp_mm(patching_mm);
261
262 err = __patch_mem(addr, val, patch_addr, is_dword);
263
264 /* context synchronisation performed by __patch_instruction (isync or exception) */
265 stop_using_temp_mm(patching_mm, orig_mm);
266
267 pte_clear(patching_mm, text_poke_addr, pte);
268 /*
269 * ptesync to order PTE update before TLB invalidation done
270 * by radix__local_flush_tlb_page_psize (in _tlbiel_va)
271 */
272 local_flush_tlb_page_psize(patching_mm, text_poke_addr, mmu_virtual_psize);
273
274 pte_unmap_unlock(pte, ptl);
275
276 return err;
277}
278
279static int __do_patch_mem(void *addr, unsigned long val, bool is_dword)
280{
281 int err;
282 u32 *patch_addr;
283 unsigned long text_poke_addr;
284 pte_t *pte;
285 unsigned long pfn = get_patch_pfn(addr);
286
287 text_poke_addr = (unsigned long)__this_cpu_read(cpu_patching_context.addr) & PAGE_MASK;
288 patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
289
290 pte = __this_cpu_read(cpu_patching_context.pte);
291 __set_pte_at(&init_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0);
292 /* See ptesync comment in radix__set_pte_at() */
293 if (radix_enabled())
294 asm volatile("ptesync": : :"memory");
295
296 err = __patch_mem(addr, val, patch_addr, is_dword);
297
298 pte_clear(&init_mm, text_poke_addr, pte);
299 flush_tlb_kernel_range(text_poke_addr, text_poke_addr + PAGE_SIZE);
300
301 return err;
302}
303
304static int patch_mem(void *addr, unsigned long val, bool is_dword)
305{
306 int err;
307 unsigned long flags;
308
309 /*
310 * During early early boot patch_instruction is called
311 * when text_poke_area is not ready, but we still need
312 * to allow patching. We just do the plain old patching
313 */
314 if (!IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) ||
315 !static_branch_likely(&poking_init_done))
316 return __patch_mem(addr, val, addr, is_dword);
317
318 local_irq_save(flags);
319 if (mm_patch_enabled())
320 err = __do_patch_mem_mm(addr, val, is_dword);
321 else
322 err = __do_patch_mem(addr, val, is_dword);
323 local_irq_restore(flags);
324
325 return err;
326}
327
328#ifdef CONFIG_PPC64
329
330int patch_instruction(u32 *addr, ppc_inst_t instr)
331{
332 if (ppc_inst_prefixed(instr))
333 return patch_mem(addr, ppc_inst_as_ulong(instr), true);
334 else
335 return patch_mem(addr, ppc_inst_val(instr), false);
336}
337NOKPROBE_SYMBOL(patch_instruction);
338
339int patch_uint(void *addr, unsigned int val)
340{
341 if (!IS_ALIGNED((unsigned long)addr, sizeof(unsigned int)))
342 return -EINVAL;
343
344 return patch_mem(addr, val, false);
345}
346NOKPROBE_SYMBOL(patch_uint);
347
348int patch_ulong(void *addr, unsigned long val)
349{
350 if (!IS_ALIGNED((unsigned long)addr, sizeof(unsigned long)))
351 return -EINVAL;
352
353 return patch_mem(addr, val, true);
354}
355NOKPROBE_SYMBOL(patch_ulong);
356
357#else
358
359int patch_instruction(u32 *addr, ppc_inst_t instr)
360{
361 return patch_mem(addr, ppc_inst_val(instr), false);
362}
363NOKPROBE_SYMBOL(patch_instruction)
364
365#endif
366
367static int patch_memset64(u64 *addr, u64 val, size_t count)
368{
369 for (u64 *end = addr + count; addr < end; addr++)
370 __put_kernel_nofault(addr, &val, u64, failed);
371
372 return 0;
373
374failed:
375 return -EPERM;
376}
377
378static int patch_memset32(u32 *addr, u32 val, size_t count)
379{
380 for (u32 *end = addr + count; addr < end; addr++)
381 __put_kernel_nofault(addr, &val, u32, failed);
382
383 return 0;
384
385failed:
386 return -EPERM;
387}
388
389static int __patch_instructions(u32 *patch_addr, u32 *code, size_t len, bool repeat_instr)
390{
391 unsigned long start = (unsigned long)patch_addr;
392 int err;
393
394 /* Repeat instruction */
395 if (repeat_instr) {
396 ppc_inst_t instr = ppc_inst_read(code);
397
398 if (ppc_inst_prefixed(instr)) {
399 u64 val = ppc_inst_as_ulong(instr);
400
401 err = patch_memset64((u64 *)patch_addr, val, len / 8);
402 } else {
403 u32 val = ppc_inst_val(instr);
404
405 err = patch_memset32(patch_addr, val, len / 4);
406 }
407 } else {
408 err = copy_to_kernel_nofault(patch_addr, code, len);
409 }
410
411 smp_wmb(); /* smp write barrier */
412 flush_icache_range(start, start + len);
413 return err;
414}
415
416/*
417 * A page is mapped and instructions that fit the page are patched.
418 * Assumes 'len' to be (PAGE_SIZE - offset_in_page(addr)) or below.
419 */
420static int __do_patch_instructions_mm(u32 *addr, u32 *code, size_t len, bool repeat_instr)
421{
422 struct mm_struct *patching_mm, *orig_mm;
423 unsigned long pfn = get_patch_pfn(addr);
424 unsigned long text_poke_addr;
425 spinlock_t *ptl;
426 u32 *patch_addr;
427 pte_t *pte;
428 int err;
429
430 patching_mm = __this_cpu_read(cpu_patching_context.mm);
431 text_poke_addr = __this_cpu_read(cpu_patching_context.addr);
432 patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
433
434 pte = get_locked_pte(patching_mm, text_poke_addr, &ptl);
435 if (!pte)
436 return -ENOMEM;
437
438 __set_pte_at(patching_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0);
439
440 /* order PTE update before use, also serves as the hwsync */
441 asm volatile("ptesync" ::: "memory");
442
443 /* order context switch after arbitrary prior code */
444 isync();
445
446 orig_mm = start_using_temp_mm(patching_mm);
447
448 kasan_disable_current();
449 err = __patch_instructions(patch_addr, code, len, repeat_instr);
450 kasan_enable_current();
451
452 /* context synchronisation performed by __patch_instructions */
453 stop_using_temp_mm(patching_mm, orig_mm);
454
455 pte_clear(patching_mm, text_poke_addr, pte);
456 /*
457 * ptesync to order PTE update before TLB invalidation done
458 * by radix__local_flush_tlb_page_psize (in _tlbiel_va)
459 */
460 local_flush_tlb_page_psize(patching_mm, text_poke_addr, mmu_virtual_psize);
461
462 pte_unmap_unlock(pte, ptl);
463
464 return err;
465}
466
467/*
468 * A page is mapped and instructions that fit the page are patched.
469 * Assumes 'len' to be (PAGE_SIZE - offset_in_page(addr)) or below.
470 */
471static int __do_patch_instructions(u32 *addr, u32 *code, size_t len, bool repeat_instr)
472{
473 unsigned long pfn = get_patch_pfn(addr);
474 unsigned long text_poke_addr;
475 u32 *patch_addr;
476 pte_t *pte;
477 int err;
478
479 text_poke_addr = (unsigned long)__this_cpu_read(cpu_patching_context.addr) & PAGE_MASK;
480 patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
481
482 pte = __this_cpu_read(cpu_patching_context.pte);
483 __set_pte_at(&init_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0);
484 /* See ptesync comment in radix__set_pte_at() */
485 if (radix_enabled())
486 asm volatile("ptesync" ::: "memory");
487
488 err = __patch_instructions(patch_addr, code, len, repeat_instr);
489
490 pte_clear(&init_mm, text_poke_addr, pte);
491 flush_tlb_kernel_range(text_poke_addr, text_poke_addr + PAGE_SIZE);
492
493 return err;
494}
495
496/*
497 * Patch 'addr' with 'len' bytes of instructions from 'code'.
498 *
499 * If repeat_instr is true, the same instruction is filled for
500 * 'len' bytes.
501 */
502int patch_instructions(u32 *addr, u32 *code, size_t len, bool repeat_instr)
503{
504 while (len > 0) {
505 unsigned long flags;
506 size_t plen;
507 int err;
508
509 plen = min_t(size_t, PAGE_SIZE - offset_in_page(addr), len);
510
511 local_irq_save(flags);
512 if (mm_patch_enabled())
513 err = __do_patch_instructions_mm(addr, code, plen, repeat_instr);
514 else
515 err = __do_patch_instructions(addr, code, plen, repeat_instr);
516 local_irq_restore(flags);
517 if (err)
518 return err;
519
520 len -= plen;
521 addr = (u32 *)((unsigned long)addr + plen);
522 if (!repeat_instr)
523 code = (u32 *)((unsigned long)code + plen);
524 }
525
526 return 0;
527}
528NOKPROBE_SYMBOL(patch_instructions);
529
530int patch_branch(u32 *addr, unsigned long target, int flags)
531{
532 ppc_inst_t instr;
533
534 if (create_branch(&instr, addr, target, flags))
535 return -ERANGE;
536
537 return patch_instruction(addr, instr);
538}
539
540/*
541 * Helper to check if a given instruction is a conditional branch
542 * Derived from the conditional checks in analyse_instr()
543 */
544bool is_conditional_branch(ppc_inst_t instr)
545{
546 unsigned int opcode = ppc_inst_primary_opcode(instr);
547
548 if (opcode == 16) /* bc, bca, bcl, bcla */
549 return true;
550 if (opcode == 19) {
551 switch ((ppc_inst_val(instr) >> 1) & 0x3ff) {
552 case 16: /* bclr, bclrl */
553 case 528: /* bcctr, bcctrl */
554 case 560: /* bctar, bctarl */
555 return true;
556 }
557 }
558 return false;
559}
560NOKPROBE_SYMBOL(is_conditional_branch);
561
562int create_cond_branch(ppc_inst_t *instr, const u32 *addr,
563 unsigned long target, int flags)
564{
565 long offset;
566
567 offset = target;
568 if (! (flags & BRANCH_ABSOLUTE))
569 offset = offset - (unsigned long)addr;
570
571 /* Check we can represent the target in the instruction format */
572 if (!is_offset_in_cond_branch_range(offset))
573 return 1;
574
575 /* Mask out the flags and target, so they don't step on each other. */
576 *instr = ppc_inst(0x40000000 | (flags & 0x3FF0003) | (offset & 0xFFFC));
577
578 return 0;
579}
580
581int instr_is_relative_branch(ppc_inst_t instr)
582{
583 if (ppc_inst_val(instr) & BRANCH_ABSOLUTE)
584 return 0;
585
586 return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
587}
588
589int instr_is_relative_link_branch(ppc_inst_t instr)
590{
591 return instr_is_relative_branch(instr) && (ppc_inst_val(instr) & BRANCH_SET_LINK);
592}
593
594static unsigned long branch_iform_target(const u32 *instr)
595{
596 signed long imm;
597
598 imm = ppc_inst_val(ppc_inst_read(instr)) & 0x3FFFFFC;
599
600 /* If the top bit of the immediate value is set this is negative */
601 if (imm & 0x2000000)
602 imm -= 0x4000000;
603
604 if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
605 imm += (unsigned long)instr;
606
607 return (unsigned long)imm;
608}
609
610static unsigned long branch_bform_target(const u32 *instr)
611{
612 signed long imm;
613
614 imm = ppc_inst_val(ppc_inst_read(instr)) & 0xFFFC;
615
616 /* If the top bit of the immediate value is set this is negative */
617 if (imm & 0x8000)
618 imm -= 0x10000;
619
620 if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
621 imm += (unsigned long)instr;
622
623 return (unsigned long)imm;
624}
625
626unsigned long branch_target(const u32 *instr)
627{
628 if (instr_is_branch_iform(ppc_inst_read(instr)))
629 return branch_iform_target(instr);
630 else if (instr_is_branch_bform(ppc_inst_read(instr)))
631 return branch_bform_target(instr);
632
633 return 0;
634}
635
636int translate_branch(ppc_inst_t *instr, const u32 *dest, const u32 *src)
637{
638 unsigned long target;
639 target = branch_target(src);
640
641 if (instr_is_branch_iform(ppc_inst_read(src)))
642 return create_branch(instr, dest, target,
643 ppc_inst_val(ppc_inst_read(src)));
644 else if (instr_is_branch_bform(ppc_inst_read(src)))
645 return create_cond_branch(instr, dest, target,
646 ppc_inst_val(ppc_inst_read(src)));
647
648 return 1;
649}