projects/linux·blob master · refs

696 lines · 18.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Code for replacing ftrace calls with jumps.
4 *
5 * Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
6 *
7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
8 *
9 * Added function graph tracer code, taken from x86 that was written
10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
11 *
12 */
13
14#define pr_fmt(fmt) "ftrace-powerpc: " fmt
15
16#include <linux/spinlock.h>
17#include <linux/hardirq.h>
18#include <linux/uaccess.h>
19#include <linux/module.h>
20#include <linux/ftrace.h>
21#include <linux/percpu.h>
22#include <linux/init.h>
23#include <linux/list.h>
24
25#include <asm/cacheflush.h>
26#include <asm/text-patching.h>
27#include <asm/ftrace.h>
28#include <asm/syscall.h>
29#include <asm/inst.h>
30#include <asm/sections.h>
31
32#define NUM_FTRACE_TRAMPS 2
33static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS];
34
35unsigned long ftrace_call_adjust(unsigned long addr)
36{
37 if (addr >= (unsigned long)__exittext_begin && addr < (unsigned long)__exittext_end)
38 return 0;
39
40 if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)) {
41 if (!IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
42 addr += MCOUNT_INSN_SIZE;
43 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
44 addr += MCOUNT_INSN_SIZE;
45 } else if (IS_ENABLED(CONFIG_CC_IS_CLANG) && IS_ENABLED(CONFIG_PPC64)) {
46 /*
47 * addr points to global entry point though the NOP was emitted at local
48 * entry point due to https://github.com/llvm/llvm-project/issues/163706
49 * Handle that here with ppc_function_entry() for kernel symbols while
50 * adjusting module addresses in the else case, by looking for the below
51 * module global entry point sequence:
52 * ld r2, -8(r12)
53 * add r2, r2, r12
54 */
55 if (is_kernel_text(addr) || is_kernel_inittext(addr))
56 addr = ppc_function_entry((void *)addr);
57 else if ((ppc_inst_val(ppc_inst_read((u32 *)addr)) ==
58 PPC_RAW_LD(_R2, _R12, -8)) &&
59 (ppc_inst_val(ppc_inst_read((u32 *)(addr+4))) ==
60 PPC_RAW_ADD(_R2, _R2, _R12)))
61 addr += 8;
62 }
63 }
64
65 return addr;
66}
67
68static ppc_inst_t ftrace_create_branch_inst(unsigned long ip, unsigned long addr, int link)
69{
70 ppc_inst_t op;
71
72 WARN_ON(!is_offset_in_branch_range(addr - ip));
73 create_branch(&op, (u32 *)ip, addr, link ? BRANCH_SET_LINK : 0);
74
75 return op;
76}
77
78static inline int ftrace_read_inst(unsigned long ip, ppc_inst_t *op)
79{
80 if (copy_inst_from_kernel_nofault(op, (void *)ip)) {
81 pr_err("0x%lx: fetching instruction failed\n", ip);
82 return -EFAULT;
83 }
84
85 return 0;
86}
87
88static inline int ftrace_validate_inst(unsigned long ip, ppc_inst_t inst)
89{
90 ppc_inst_t op;
91 int ret;
92
93 ret = ftrace_read_inst(ip, &op);
94 if (!ret && !ppc_inst_equal(op, inst)) {
95 pr_err("0x%lx: expected (%08lx) != found (%08lx)\n",
96 ip, ppc_inst_as_ulong(inst), ppc_inst_as_ulong(op));
97 ret = -EINVAL;
98 }
99
100 return ret;
101}
102
103static inline int ftrace_modify_code(unsigned long ip, ppc_inst_t old, ppc_inst_t new)
104{
105 int ret = ftrace_validate_inst(ip, old);
106
107 if (!ret && !ppc_inst_equal(old, new))
108 ret = patch_instruction((u32 *)ip, new);
109
110 return ret;
111}
112
113static int is_bl_op(ppc_inst_t op)
114{
115 return (ppc_inst_val(op) & ~PPC_LI_MASK) == PPC_RAW_BL(0);
116}
117
118static unsigned long find_ftrace_tramp(unsigned long ip)
119{
120 int i;
121
122 for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
123 if (!ftrace_tramps[i])
124 continue;
125 else if (is_offset_in_branch_range(ftrace_tramps[i] - ip))
126 return ftrace_tramps[i];
127
128 return 0;
129}
130
131#ifdef CONFIG_MODULES
132static unsigned long ftrace_lookup_module_stub(unsigned long ip, unsigned long addr)
133{
134 struct module *mod = NULL;
135
136 scoped_guard(rcu)
137 mod = __module_text_address(ip);
138 if (!mod)
139 pr_err("No module loaded at addr=%lx\n", ip);
140
141 return (addr == (unsigned long)ftrace_caller ? mod->arch.tramp : mod->arch.tramp_regs);
142}
143#else
144static unsigned long ftrace_lookup_module_stub(unsigned long ip, unsigned long addr)
145{
146 return 0;
147}
148#endif
149
150static unsigned long ftrace_get_ool_stub(struct dyn_ftrace *rec)
151{
152#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
153 return rec->arch.ool_stub;
154#else
155 BUILD_BUG();
156#endif
157}
158
159static int ftrace_get_call_inst(struct dyn_ftrace *rec, unsigned long addr, ppc_inst_t *call_inst)
160{
161 unsigned long ip;
162 unsigned long stub;
163
164 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
165 ip = ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE; /* second instruction in stub */
166 else
167 ip = rec->ip;
168
169 if (!is_offset_in_branch_range(addr - ip) && addr != FTRACE_ADDR &&
170 addr != FTRACE_REGS_ADDR) {
171 /* This can only happen with ftrace direct */
172 if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS)) {
173 pr_err("0x%lx (0x%lx): Unexpected target address 0x%lx\n",
174 ip, rec->ip, addr);
175 return -EINVAL;
176 }
177 addr = FTRACE_ADDR;
178 }
179
180 if (is_offset_in_branch_range(addr - ip))
181 /* Within range */
182 stub = addr;
183 else if (core_kernel_text(ip))
184 /* We would be branching to one of our ftrace stubs */
185 stub = find_ftrace_tramp(ip);
186 else
187 stub = ftrace_lookup_module_stub(ip, addr);
188
189 if (!stub) {
190 pr_err("0x%lx (0x%lx): No ftrace stubs reachable\n", ip, rec->ip);
191 return -EINVAL;
192 }
193
194 *call_inst = ftrace_create_branch_inst(ip, stub, 1);
195 return 0;
196}
197
198static int ftrace_init_ool_stub(struct module *mod, struct dyn_ftrace *rec)
199{
200#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
201 static int ool_stub_text_index, ool_stub_text_end_index, ool_stub_inittext_index;
202 int ret = 0, ool_stub_count, *ool_stub_index;
203 ppc_inst_t inst;
204 /*
205 * See ftrace_entry.S if changing the below instruction sequence, as we rely on
206 * decoding the last branch instruction here to recover the correct function ip.
207 */
208 struct ftrace_ool_stub *ool_stub, ool_stub_template = {
209 .insn = {
210 PPC_RAW_MFLR(_R0),
211 PPC_RAW_NOP(), /* bl ftrace_caller */
212 PPC_RAW_MTLR(_R0),
213 PPC_RAW_NOP() /* b rec->ip + 4 */
214 }
215 };
216
217 WARN_ON(rec->arch.ool_stub);
218
219 if (is_kernel_inittext(rec->ip)) {
220 ool_stub = ftrace_ool_stub_inittext;
221 ool_stub_index = &ool_stub_inittext_index;
222 ool_stub_count = ftrace_ool_stub_inittext_count;
223 } else if (is_kernel_text(rec->ip)) {
224 /*
225 * ftrace records are sorted, so we first use up the stub area within .text
226 * (ftrace_ool_stub_text) before using the area at the end of .text
227 * (ftrace_ool_stub_text_end), unless the stub is out of range of the record.
228 */
229 if (ool_stub_text_index >= ftrace_ool_stub_text_count ||
230 !is_offset_in_branch_range((long)rec->ip -
231 (long)&ftrace_ool_stub_text[ool_stub_text_index])) {
232 ool_stub = ftrace_ool_stub_text_end;
233 ool_stub_index = &ool_stub_text_end_index;
234 ool_stub_count = ftrace_ool_stub_text_end_count;
235 } else {
236 ool_stub = ftrace_ool_stub_text;
237 ool_stub_index = &ool_stub_text_index;
238 ool_stub_count = ftrace_ool_stub_text_count;
239 }
240#ifdef CONFIG_MODULES
241 } else if (mod) {
242 ool_stub = mod->arch.ool_stubs;
243 ool_stub_index = &mod->arch.ool_stub_index;
244 ool_stub_count = mod->arch.ool_stub_count;
245#endif
246 } else {
247 return -EINVAL;
248 }
249
250 ool_stub += (*ool_stub_index)++;
251
252 if (WARN_ON(*ool_stub_index > ool_stub_count))
253 return -EINVAL;
254
255 if (!is_offset_in_branch_range((long)rec->ip - (long)&ool_stub->insn[0]) ||
256 !is_offset_in_branch_range((long)(rec->ip + MCOUNT_INSN_SIZE) -
257 (long)&ool_stub->insn[3])) {
258 pr_err("%s: ftrace ool stub out of range (%p -> %p).\n",
259 __func__, (void *)rec->ip, (void *)&ool_stub->insn[0]);
260 return -EINVAL;
261 }
262
263 rec->arch.ool_stub = (unsigned long)&ool_stub->insn[0];
264
265 /* bl ftrace_caller */
266 if (!mod)
267 ret = ftrace_get_call_inst(rec, (unsigned long)ftrace_caller, &inst);
268#ifdef CONFIG_MODULES
269 else
270 /*
271 * We can't use ftrace_get_call_inst() since that uses
272 * __module_text_address(rec->ip) to look up the module.
273 * But, since the module is not fully formed at this stage,
274 * the lookup fails. We know the target though, so generate
275 * the branch inst directly.
276 */
277 inst = ftrace_create_branch_inst(ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE,
278 mod->arch.tramp, 1);
279#endif
280 ool_stub_template.insn[1] = ppc_inst_val(inst);
281
282 /* b rec->ip + 4 */
283 if (!ret && create_branch(&inst, &ool_stub->insn[3], rec->ip + MCOUNT_INSN_SIZE, 0))
284 return -EINVAL;
285 ool_stub_template.insn[3] = ppc_inst_val(inst);
286
287 if (!ret)
288 ret = patch_instructions((u32 *)ool_stub, (u32 *)&ool_stub_template,
289 sizeof(ool_stub_template), false);
290
291 return ret;
292#else /* !CONFIG_PPC_FTRACE_OUT_OF_LINE */
293 BUILD_BUG();
294#endif
295}
296
297#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
298static const struct ftrace_ops *powerpc_rec_get_ops(struct dyn_ftrace *rec)
299{
300 const struct ftrace_ops *ops = NULL;
301
302 if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
303 ops = ftrace_find_unique_ops(rec);
304 WARN_ON_ONCE(!ops);
305 }
306
307 if (!ops)
308 ops = &ftrace_list_ops;
309
310 return ops;
311}
312
313static int ftrace_rec_set_ops(struct dyn_ftrace *rec, const struct ftrace_ops *ops)
314{
315 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
316 return patch_ulong((void *)(ftrace_get_ool_stub(rec) - sizeof(unsigned long)),
317 (unsigned long)ops);
318 else
319 return patch_ulong((void *)(rec->ip - MCOUNT_INSN_SIZE - sizeof(unsigned long)),
320 (unsigned long)ops);
321}
322
323static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec)
324{
325 return ftrace_rec_set_ops(rec, &ftrace_nop_ops);
326}
327
328static int ftrace_rec_update_ops(struct dyn_ftrace *rec)
329{
330 return ftrace_rec_set_ops(rec, powerpc_rec_get_ops(rec));
331}
332#else
333static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; }
334static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; }
335#endif
336
337#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
338int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr)
339{
340 /* This should never be called since we override ftrace_replace_code() */
341 WARN_ON(1);
342 return -EINVAL;
343}
344#endif
345
346int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
347{
348 ppc_inst_t old, new;
349 unsigned long ip = rec->ip;
350 int ret = 0;
351
352 /* This can only ever be called during module load */
353 if (WARN_ON(!IS_ENABLED(CONFIG_MODULES) || core_kernel_text(ip)))
354 return -EINVAL;
355
356 old = ppc_inst(PPC_RAW_NOP());
357 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
358 ip = ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE; /* second instruction in stub */
359 ret = ftrace_get_call_inst(rec, (unsigned long)ftrace_caller, &old);
360 }
361
362 ret |= ftrace_get_call_inst(rec, addr, &new);
363
364 if (!ret)
365 ret = ftrace_modify_code(ip, old, new);
366
367 ret = ftrace_rec_update_ops(rec);
368 if (ret)
369 return ret;
370
371 if (!ret && IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
372 ret = ftrace_modify_code(rec->ip, ppc_inst(PPC_RAW_NOP()),
373 ppc_inst(PPC_RAW_BRANCH((long)ftrace_get_ool_stub(rec) - (long)rec->ip)));
374
375 return ret;
376}
377
378int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr)
379{
380 /*
381 * This should never be called since we override ftrace_replace_code(),
382 * as well as ftrace_init_nop()
383 */
384 WARN_ON(1);
385 return -EINVAL;
386}
387
388void ftrace_replace_code(int enable)
389{
390 ppc_inst_t old, new, call_inst, new_call_inst;
391 ppc_inst_t nop_inst = ppc_inst(PPC_RAW_NOP());
392 unsigned long ip, new_addr, addr;
393 struct ftrace_rec_iter *iter;
394 struct dyn_ftrace *rec;
395 int ret = 0, update;
396
397 for_ftrace_rec_iter(iter) {
398 rec = ftrace_rec_iter_record(iter);
399 ip = rec->ip;
400
401 if (rec->flags & FTRACE_FL_DISABLED && !(rec->flags & FTRACE_FL_ENABLED))
402 continue;
403
404 addr = ftrace_get_addr_curr(rec);
405 new_addr = ftrace_get_addr_new(rec);
406 update = ftrace_update_record(rec, enable);
407
408 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) && update != FTRACE_UPDATE_IGNORE) {
409 ip = ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE;
410 ret = ftrace_get_call_inst(rec, (unsigned long)ftrace_caller, &nop_inst);
411 if (ret)
412 goto out;
413 }
414
415 switch (update) {
416 case FTRACE_UPDATE_IGNORE:
417 default:
418 continue;
419 case FTRACE_UPDATE_MODIFY_CALL:
420 ret = ftrace_get_call_inst(rec, new_addr, &new_call_inst);
421 ret |= ftrace_get_call_inst(rec, addr, &call_inst);
422 ret |= ftrace_rec_update_ops(rec);
423 old = call_inst;
424 new = new_call_inst;
425 break;
426 case FTRACE_UPDATE_MAKE_NOP:
427 ret = ftrace_get_call_inst(rec, addr, &call_inst);
428 ret |= ftrace_rec_set_nop_ops(rec);
429 old = call_inst;
430 new = nop_inst;
431 break;
432 case FTRACE_UPDATE_MAKE_CALL:
433 ret = ftrace_get_call_inst(rec, new_addr, &call_inst);
434 ret |= ftrace_rec_update_ops(rec);
435 old = nop_inst;
436 new = call_inst;
437 break;
438 }
439
440 if (!ret)
441 ret = ftrace_modify_code(ip, old, new);
442
443 if (!ret && IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) &&
444 (update == FTRACE_UPDATE_MAKE_NOP || update == FTRACE_UPDATE_MAKE_CALL)) {
445 /* Update the actual ftrace location */
446 call_inst = ppc_inst(PPC_RAW_BRANCH((long)ftrace_get_ool_stub(rec) -
447 (long)rec->ip));
448 nop_inst = ppc_inst(PPC_RAW_NOP());
449 ip = rec->ip;
450
451 if (update == FTRACE_UPDATE_MAKE_NOP)
452 ret = ftrace_modify_code(ip, call_inst, nop_inst);
453 else
454 ret = ftrace_modify_code(ip, nop_inst, call_inst);
455
456 if (ret)
457 goto out;
458 }
459
460 if (ret)
461 goto out;
462 }
463
464out:
465 if (ret)
466 ftrace_bug(ret, rec);
467 return;
468}
469
470int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
471{
472 unsigned long addr, ip = rec->ip;
473 ppc_inst_t old, new;
474 int ret = 0;
475
476 /* Verify instructions surrounding the ftrace location */
477 if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)) {
478 /* Expect nops */
479 if (!IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
480 ret = ftrace_validate_inst(ip - 4, ppc_inst(PPC_RAW_NOP()));
481 if (!ret)
482 ret = ftrace_validate_inst(ip, ppc_inst(PPC_RAW_NOP()));
483 } else if (IS_ENABLED(CONFIG_PPC32)) {
484 /* Expected sequence: 'mflr r0', 'stw r0,4(r1)', 'bl _mcount' */
485 ret = ftrace_validate_inst(ip - 8, ppc_inst(PPC_RAW_MFLR(_R0)));
486 if (ret)
487 return ret;
488 ret = ftrace_modify_code(ip - 4, ppc_inst(PPC_RAW_STW(_R0, _R1, 4)),
489 ppc_inst(PPC_RAW_NOP()));
490 } else if (IS_ENABLED(CONFIG_MPROFILE_KERNEL)) {
491 /* Expected sequence: 'mflr r0', ['std r0,16(r1)'], 'bl _mcount' */
492 ret = ftrace_read_inst(ip - 4, &old);
493 if (!ret && !ppc_inst_equal(old, ppc_inst(PPC_RAW_MFLR(_R0)))) {
494 /* Gcc v5.x emit the additional 'std' instruction, gcc v6.x don't */
495 ret = ftrace_validate_inst(ip - 8, ppc_inst(PPC_RAW_MFLR(_R0)));
496 if (ret)
497 return ret;
498 ret = ftrace_modify_code(ip - 4, ppc_inst(PPC_RAW_STD(_R0, _R1, 16)),
499 ppc_inst(PPC_RAW_NOP()));
500 }
501 } else {
502 return -EINVAL;
503 }
504
505 if (ret)
506 return ret;
507
508 /* Set up out-of-line stub */
509 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
510 ret = ftrace_init_ool_stub(mod, rec);
511 goto out;
512 }
513
514 /* Nop-out the ftrace location */
515 new = ppc_inst(PPC_RAW_NOP());
516 addr = MCOUNT_ADDR;
517 if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)) {
518 /* we instead patch-in the 'mflr r0' */
519 old = ppc_inst(PPC_RAW_NOP());
520 new = ppc_inst(PPC_RAW_MFLR(_R0));
521 ret = ftrace_modify_code(ip - 4, old, new);
522 } else if (is_offset_in_branch_range(addr - ip)) {
523 /* Within range */
524 old = ftrace_create_branch_inst(ip, addr, 1);
525 ret = ftrace_modify_code(ip, old, new);
526 } else if (core_kernel_text(ip) || (IS_ENABLED(CONFIG_MODULES) && mod)) {
527 /*
528 * We would be branching to a linker-generated stub, or to the module _mcount
529 * stub. Let's just confirm we have a 'bl' here.
530 */
531 ret = ftrace_read_inst(ip, &old);
532 if (ret)
533 return ret;
534 if (!is_bl_op(old)) {
535 pr_err("0x%lx: expected (bl) != found (%08lx)\n", ip, ppc_inst_as_ulong(old));
536 return -EINVAL;
537 }
538 ret = patch_instruction((u32 *)ip, new);
539 } else {
540 return -EINVAL;
541 }
542
543out:
544 if (!ret)
545 ret = ftrace_rec_set_nop_ops(rec);
546
547 return ret;
548}
549
550int ftrace_update_ftrace_func(ftrace_func_t func)
551{
552 unsigned long ip = (unsigned long)(&ftrace_call);
553 ppc_inst_t old, new;
554 int ret;
555
556 /*
557 * When using CALL_OPS, the function to call is associated with the
558 * call site, and we don't have a global function pointer to update.
559 */
560 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
561 return 0;
562
563 old = ppc_inst_read((u32 *)&ftrace_call);
564 new = ftrace_create_branch_inst(ip, ppc_function_entry(func), 1);
565 ret = ftrace_modify_code(ip, old, new);
566
567 /* Also update the regs callback function */
568 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && !ret) {
569 ip = (unsigned long)(&ftrace_regs_call);
570 old = ppc_inst_read((u32 *)&ftrace_regs_call);
571 new = ftrace_create_branch_inst(ip, ppc_function_entry(func), 1);
572 ret = ftrace_modify_code(ip, old, new);
573 }
574
575 return ret;
576}
577
578/*
579 * Use the default ftrace_modify_all_code, but without
580 * stop_machine().
581 */
582void arch_ftrace_update_code(int command)
583{
584 ftrace_modify_all_code(command);
585}
586
587void ftrace_free_init_tramp(void)
588{
589 int i;
590
591 for (i = 0; i < NUM_FTRACE_TRAMPS && ftrace_tramps[i]; i++)
592 if (ftrace_tramps[i] == (unsigned long)ftrace_tramp_init) {
593 ftrace_tramps[i] = 0;
594 return;
595 }
596}
597
598static void __init add_ftrace_tramp(unsigned long tramp)
599{
600 int i;
601
602 for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
603 if (!ftrace_tramps[i]) {
604 ftrace_tramps[i] = tramp;
605 return;
606 }
607}
608
609int __init ftrace_dyn_arch_init(void)
610{
611 unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init };
612 unsigned long addr = FTRACE_REGS_ADDR;
613 long reladdr;
614 int i;
615 u32 stub_insns[] = {
616#ifdef CONFIG_PPC_KERNEL_PCREL
617 /* pla r12,addr */
618 PPC_PREFIX_MLS | __PPC_PRFX_R(1),
619 PPC_INST_PADDI | ___PPC_RT(_R12),
620 PPC_RAW_MTCTR(_R12),
621 PPC_RAW_BCTR()
622#elif defined(CONFIG_PPC64)
623 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)),
624 PPC_RAW_ADDIS(_R12, _R12, 0),
625 PPC_RAW_ADDI(_R12, _R12, 0),
626 PPC_RAW_MTCTR(_R12),
627 PPC_RAW_BCTR()
628#else
629 PPC_RAW_LIS(_R12, 0),
630 PPC_RAW_ADDI(_R12, _R12, 0),
631 PPC_RAW_MTCTR(_R12),
632 PPC_RAW_BCTR()
633#endif
634 };
635
636 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
637 for (i = 0; i < 2; i++) {
638 reladdr = addr - (unsigned long)tramp[i];
639
640 if (reladdr >= (long)SZ_8G || reladdr < -(long)SZ_8G) {
641 pr_err("Address of %ps out of range of pcrel address.\n",
642 (void *)addr);
643 return -1;
644 }
645
646 memcpy(tramp[i], stub_insns, sizeof(stub_insns));
647 tramp[i][0] |= IMM_H18(reladdr);
648 tramp[i][1] |= IMM_L(reladdr);
649 add_ftrace_tramp((unsigned long)tramp[i]);
650 }
651 } else if (IS_ENABLED(CONFIG_PPC64)) {
652 reladdr = addr - kernel_toc_addr();
653
654 if (reladdr >= (long)SZ_2G || reladdr < -(long long)SZ_2G) {
655 pr_err("Address of %ps out of range of kernel_toc.\n",
656 (void *)addr);
657 return -1;
658 }
659
660 for (i = 0; i < 2; i++) {
661 memcpy(tramp[i], stub_insns, sizeof(stub_insns));
662 tramp[i][1] |= PPC_HA(reladdr);
663 tramp[i][2] |= PPC_LO(reladdr);
664 add_ftrace_tramp((unsigned long)tramp[i]);
665 }
666 } else {
667 for (i = 0; i < 2; i++) {
668 memcpy(tramp[i], stub_insns, sizeof(stub_insns));
669 tramp[i][0] |= PPC_HA(addr);
670 tramp[i][1] |= PPC_LO(addr);
671 add_ftrace_tramp((unsigned long)tramp[i]);
672 }
673 }
674
675 return 0;
676}
677
678#ifdef CONFIG_FUNCTION_GRAPH_TRACER
679void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
680 struct ftrace_ops *op, struct ftrace_regs *fregs)
681{
682 unsigned long sp = arch_ftrace_regs(fregs)->regs.gpr[1];
683
684 if (unlikely(ftrace_graph_is_dead()))
685 goto out;
686
687 if (unlikely(atomic_read(&current->tracing_graph_pause)))
688 goto out;
689
690 if (!function_graph_enter_regs(parent_ip, ip, 0, (unsigned long *)sp, fregs))
691 parent_ip = ppc_function_entry(return_to_handler);
692
693out:
694 arch_ftrace_regs(fregs)->regs.link = parent_ip;
695}
696#endif /* CONFIG_FUNCTION_GRAPH_TRACER */