projects/linux·blob v6.17 · refs

1461 lines · 40.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Infrastructure to took into function calls and returns.
4 * Copyright (c) 2008-2009 Frederic Weisbecker <[email protected]>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <[email protected]>
7 *
8 * Highly modified by Steven Rostedt (VMware).
9 */
10#include <linux/bits.h>
11#include <linux/jump_label.h>
12#include <linux/suspend.h>
13#include <linux/ftrace.h>
14#include <linux/static_call.h>
15#include <linux/slab.h>
16
17#include <trace/events/sched.h>
18
19#include "ftrace_internal.h"
20#include "trace.h"
21
22/*
23 * FGRAPH_FRAME_SIZE: Size in bytes of the meta data on the shadow stack
24 * FGRAPH_FRAME_OFFSET: Size in long words of the meta data frame
25 */
26#define FGRAPH_FRAME_SIZE sizeof(struct ftrace_ret_stack)
27#define FGRAPH_FRAME_OFFSET DIV_ROUND_UP(FGRAPH_FRAME_SIZE, sizeof(long))
28
29/*
30 * On entry to a function (via function_graph_enter()), a new fgraph frame
31 * (ftrace_ret_stack) is pushed onto the stack as well as a word that
32 * holds a bitmask and a type (called "bitmap"). The bitmap is defined as:
33 *
34 * bits: 0 - 9 offset in words from the previous ftrace_ret_stack
35 *
36 * bits: 10 - 11 Type of storage
37 * 0 - reserved
38 * 1 - bitmap of fgraph_array index
39 * 2 - reserved data
40 *
41 * For type with "bitmap of fgraph_array index" (FGRAPH_TYPE_BITMAP):
42 * bits: 12 - 27 The bitmap of fgraph_ops fgraph_array index
43 * That is, it's a bitmask of 0-15 (16 bits)
44 * where if a corresponding ops in the fgraph_array[]
45 * expects a callback from the return of the function
46 * it's corresponding bit will be set.
47 *
48 *
49 * The top of the ret_stack (when not empty) will always have a reference
50 * word that points to the last fgraph frame that was saved.
51 *
52 * For reserved data:
53 * bits: 12 - 17 The size in words that is stored
54 * bits: 18 - 23 The index of fgraph_array, which shows who is stored
55 *
56 * That is, at the end of function_graph_enter, if the first and forth
57 * fgraph_ops on the fgraph_array[] (index 0 and 3) needs their retfunc called
58 * on the return of the function being traced, and the forth fgraph_ops
59 * stored two words of data, this is what will be on the task's shadow
60 * ret_stack: (the stack grows upward)
61 *
62 * ret_stack[SHADOW_STACK_OFFSET]
63 * | SHADOW_STACK_TASK_VARS(ret_stack)[15] |
64 * ...
65 * | SHADOW_STACK_TASK_VARS(ret_stack)[0] |
66 * ret_stack[SHADOW_STACK_MAX_OFFSET]
67 * ...
68 * | | <- task->curr_ret_stack
69 * +--------------------------------------------+
70 * | (3 << 12) | (3 << 10) | FGRAPH_FRAME_OFFSET|
71 * | *or put another way* |
72 * | (3 << FGRAPH_DATA_INDEX_SHIFT)| \ | This is for fgraph_ops[3].
73 * | ((2 - 1) << FGRAPH_DATA_SHIFT)| \ | The data size is 2 words.
74 * | (FGRAPH_TYPE_DATA << FGRAPH_TYPE_SHIFT)| \ |
75 * | (offset2:FGRAPH_FRAME_OFFSET+3) | <- the offset2 is from here
76 * +--------------------------------------------+ ( It is 4 words from the ret_stack)
77 * | STORED DATA WORD 2 |
78 * | STORED DATA WORD 1 |
79 * +--------------------------------------------+
80 * | (9 << 12) | (1 << 10) | FGRAPH_FRAME_OFFSET|
81 * | *or put another way* |
82 * | (BIT(3)|BIT(0)) << FGRAPH_INDEX_SHIFT | \ |
83 * | FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT| \ |
84 * | (offset1:FGRAPH_FRAME_OFFSET) | <- the offset1 is from here
85 * +--------------------------------------------+
86 * | struct ftrace_ret_stack |
87 * | (stores the saved ret pointer) | <- the offset points here
88 * +--------------------------------------------+
89 * | (X) | (N) | ( N words away from
90 * | | previous ret_stack)
91 * ...
92 * ret_stack[0]
93 *
94 * If a backtrace is required, and the real return pointer needs to be
95 * fetched, then it looks at the task's curr_ret_stack offset, if it
96 * is greater than zero (reserved, or right before popped), it would mask
97 * the value by FGRAPH_FRAME_OFFSET_MASK to get the offset of the
98 * ftrace_ret_stack structure stored on the shadow stack.
99 */
100
101/*
102 * The following is for the top word on the stack:
103 *
104 * FGRAPH_FRAME_OFFSET (0-9) holds the offset delta to the fgraph frame
105 * FGRAPH_TYPE (10-11) holds the type of word this is.
106 * (RESERVED or BITMAP)
107 */
108#define FGRAPH_FRAME_OFFSET_BITS 10
109#define FGRAPH_FRAME_OFFSET_MASK GENMASK(FGRAPH_FRAME_OFFSET_BITS - 1, 0)
110
111#define FGRAPH_TYPE_BITS 2
112#define FGRAPH_TYPE_MASK GENMASK(FGRAPH_TYPE_BITS - 1, 0)
113#define FGRAPH_TYPE_SHIFT FGRAPH_FRAME_OFFSET_BITS
114
115enum {
116 FGRAPH_TYPE_RESERVED = 0,
117 FGRAPH_TYPE_BITMAP = 1,
118 FGRAPH_TYPE_DATA = 2,
119};
120
121/*
122 * For BITMAP type:
123 * FGRAPH_INDEX (12-27) bits holding the gops index wanting return callback called
124 */
125#define FGRAPH_INDEX_BITS 16
126#define FGRAPH_INDEX_MASK GENMASK(FGRAPH_INDEX_BITS - 1, 0)
127#define FGRAPH_INDEX_SHIFT (FGRAPH_TYPE_SHIFT + FGRAPH_TYPE_BITS)
128
129/*
130 * For DATA type:
131 * FGRAPH_DATA (12-17) bits hold the size of data (in words)
132 * FGRAPH_INDEX (18-23) bits hold the index for which gops->idx the data is for
133 *
134 * Note:
135 * data_size == 0 means 1 word, and 31 (=2^5 - 1) means 32 words.
136 */
137#define FGRAPH_DATA_BITS 5
138#define FGRAPH_DATA_MASK GENMASK(FGRAPH_DATA_BITS - 1, 0)
139#define FGRAPH_DATA_SHIFT (FGRAPH_TYPE_SHIFT + FGRAPH_TYPE_BITS)
140#define FGRAPH_MAX_DATA_SIZE (sizeof(long) * (1 << FGRAPH_DATA_BITS))
141
142#define FGRAPH_DATA_INDEX_BITS 4
143#define FGRAPH_DATA_INDEX_MASK GENMASK(FGRAPH_DATA_INDEX_BITS - 1, 0)
144#define FGRAPH_DATA_INDEX_SHIFT (FGRAPH_DATA_SHIFT + FGRAPH_DATA_BITS)
145
146#define FGRAPH_MAX_INDEX \
147 ((FGRAPH_INDEX_SIZE << FGRAPH_DATA_BITS) + FGRAPH_RET_INDEX)
148
149#define FGRAPH_ARRAY_SIZE FGRAPH_INDEX_BITS
150
151/*
152 * SHADOW_STACK_SIZE: The size in bytes of the entire shadow stack
153 * SHADOW_STACK_OFFSET: The size in long words of the shadow stack
154 * SHADOW_STACK_MAX_OFFSET: The max offset of the stack for a new frame to be added
155 */
156#define SHADOW_STACK_SIZE (4096)
157#define SHADOW_STACK_OFFSET (SHADOW_STACK_SIZE / sizeof(long))
158/* Leave on a buffer at the end */
159#define SHADOW_STACK_MAX_OFFSET \
160 (SHADOW_STACK_OFFSET - (FGRAPH_FRAME_OFFSET + 1 + FGRAPH_ARRAY_SIZE))
161
162/* RET_STACK(): Return the frame from a given @offset from task @t */
163#define RET_STACK(t, offset) ((struct ftrace_ret_stack *)(&(t)->ret_stack[offset]))
164
165/*
166 * Each fgraph_ops has a reservered unsigned long at the end (top) of the
167 * ret_stack to store task specific state.
168 */
169#define SHADOW_STACK_TASK_VARS(ret_stack) \
170 ((unsigned long *)(&(ret_stack)[SHADOW_STACK_OFFSET - FGRAPH_ARRAY_SIZE]))
171
172DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph);
173int ftrace_graph_active;
174
175static struct kmem_cache *fgraph_stack_cachep;
176
177static struct fgraph_ops *fgraph_array[FGRAPH_ARRAY_SIZE];
178static unsigned long fgraph_array_bitmask;
179
180/* LRU index table for fgraph_array */
181static int fgraph_lru_table[FGRAPH_ARRAY_SIZE];
182static int fgraph_lru_next;
183static int fgraph_lru_last;
184
185/* Initialize fgraph_lru_table with unused index */
186static void fgraph_lru_init(void)
187{
188 int i;
189
190 for (i = 0; i < FGRAPH_ARRAY_SIZE; i++)
191 fgraph_lru_table[i] = i;
192}
193
194/* Release the used index to the LRU table */
195static int fgraph_lru_release_index(int idx)
196{
197 if (idx < 0 || idx >= FGRAPH_ARRAY_SIZE ||
198 WARN_ON_ONCE(fgraph_lru_table[fgraph_lru_last] != -1))
199 return -1;
200
201 fgraph_lru_table[fgraph_lru_last] = idx;
202 fgraph_lru_last = (fgraph_lru_last + 1) % FGRAPH_ARRAY_SIZE;
203
204 clear_bit(idx, &fgraph_array_bitmask);
205 return 0;
206}
207
208/* Allocate a new index from LRU table */
209static int fgraph_lru_alloc_index(void)
210{
211 int idx = fgraph_lru_table[fgraph_lru_next];
212
213 /* No id is available */
214 if (idx == -1)
215 return -1;
216
217 fgraph_lru_table[fgraph_lru_next] = -1;
218 fgraph_lru_next = (fgraph_lru_next + 1) % FGRAPH_ARRAY_SIZE;
219
220 set_bit(idx, &fgraph_array_bitmask);
221 return idx;
222}
223
224/* Get the offset to the fgraph frame from a ret_stack value */
225static inline int __get_offset(unsigned long val)
226{
227 return val & FGRAPH_FRAME_OFFSET_MASK;
228}
229
230/* Get the type of word from a ret_stack value */
231static inline int __get_type(unsigned long val)
232{
233 return (val >> FGRAPH_TYPE_SHIFT) & FGRAPH_TYPE_MASK;
234}
235
236/* Get the data_index for a DATA type ret_stack word */
237static inline int __get_data_index(unsigned long val)
238{
239 return (val >> FGRAPH_DATA_INDEX_SHIFT) & FGRAPH_DATA_INDEX_MASK;
240}
241
242/* Get the data_size for a DATA type ret_stack word */
243static inline int __get_data_size(unsigned long val)
244{
245 return ((val >> FGRAPH_DATA_SHIFT) & FGRAPH_DATA_MASK) + 1;
246}
247
248/* Get the word from the ret_stack at @offset */
249static inline unsigned long get_fgraph_entry(struct task_struct *t, int offset)
250{
251 return t->ret_stack[offset];
252}
253
254/* Get the FRAME_OFFSET from the word from the @offset on ret_stack */
255static inline int get_frame_offset(struct task_struct *t, int offset)
256{
257 return __get_offset(t->ret_stack[offset]);
258}
259
260/* For BITMAP type: get the bitmask from the @offset at ret_stack */
261static inline unsigned long
262get_bitmap_bits(struct task_struct *t, int offset)
263{
264 return (t->ret_stack[offset] >> FGRAPH_INDEX_SHIFT) & FGRAPH_INDEX_MASK;
265}
266
267/* Write the bitmap to the ret_stack at @offset (does index, offset and bitmask) */
268static inline void
269set_bitmap(struct task_struct *t, int offset, unsigned long bitmap)
270{
271 t->ret_stack[offset] = (bitmap << FGRAPH_INDEX_SHIFT) |
272 (FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET;
273}
274
275/* For DATA type: get the data saved under the ret_stack word at @offset */
276static inline void *get_data_type_data(struct task_struct *t, int offset)
277{
278 unsigned long val = t->ret_stack[offset];
279
280 if (__get_type(val) != FGRAPH_TYPE_DATA)
281 return NULL;
282 offset -= __get_data_size(val);
283 return (void *)&t->ret_stack[offset];
284}
285
286/* Create the ret_stack word for a DATA type */
287static inline unsigned long make_data_type_val(int idx, int size, int offset)
288{
289 return (idx << FGRAPH_DATA_INDEX_SHIFT) |
290 ((size - 1) << FGRAPH_DATA_SHIFT) |
291 (FGRAPH_TYPE_DATA << FGRAPH_TYPE_SHIFT) | offset;
292}
293
294/* ftrace_graph_entry set to this to tell some archs to run function graph */
295static int entry_run(struct ftrace_graph_ent *trace, struct fgraph_ops *ops,
296 struct ftrace_regs *fregs)
297{
298 return 0;
299}
300
301/* ftrace_graph_return set to this to tell some archs to run function graph */
302static void return_run(struct ftrace_graph_ret *trace, struct fgraph_ops *ops,
303 struct ftrace_regs *fregs)
304{
305}
306
307static void ret_stack_set_task_var(struct task_struct *t, int idx, long val)
308{
309 unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack);
310
311 gvals[idx] = val;
312}
313
314static unsigned long *
315ret_stack_get_task_var(struct task_struct *t, int idx)
316{
317 unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack);
318
319 return &gvals[idx];
320}
321
322static void ret_stack_init_task_vars(unsigned long *ret_stack)
323{
324 unsigned long *gvals = SHADOW_STACK_TASK_VARS(ret_stack);
325
326 memset(gvals, 0, sizeof(*gvals) * FGRAPH_ARRAY_SIZE);
327}
328
329/**
330 * fgraph_reserve_data - Reserve storage on the task's ret_stack
331 * @idx: The index of fgraph_array
332 * @size_bytes: The size in bytes to reserve
333 *
334 * Reserves space of up to FGRAPH_MAX_DATA_SIZE bytes on the
335 * task's ret_stack shadow stack, for a given fgraph_ops during
336 * the entryfunc() call. If entryfunc() returns zero, the storage
337 * is discarded. An entryfunc() can only call this once per iteration.
338 * The fgraph_ops retfunc() can retrieve this stored data with
339 * fgraph_retrieve_data().
340 *
341 * Returns: On success, a pointer to the data on the stack.
342 * Otherwise, NULL if there's not enough space left on the
343 * ret_stack for the data, or if fgraph_reserve_data() was called
344 * more than once for a single entryfunc() call.
345 */
346void *fgraph_reserve_data(int idx, int size_bytes)
347{
348 unsigned long val;
349 void *data;
350 int curr_ret_stack = current->curr_ret_stack;
351 int data_size;
352
353 if (size_bytes > FGRAPH_MAX_DATA_SIZE)
354 return NULL;
355
356 /* Convert the data size to number of longs. */
357 data_size = (size_bytes + sizeof(long) - 1) >> (sizeof(long) == 4 ? 2 : 3);
358
359 val = get_fgraph_entry(current, curr_ret_stack - 1);
360 data = &current->ret_stack[curr_ret_stack];
361
362 curr_ret_stack += data_size + 1;
363 if (unlikely(curr_ret_stack >= SHADOW_STACK_MAX_OFFSET))
364 return NULL;
365
366 val = make_data_type_val(idx, data_size, __get_offset(val) + data_size + 1);
367
368 /* Set the last word to be reserved */
369 current->ret_stack[curr_ret_stack - 1] = val;
370
371 /* Make sure interrupts see this */
372 barrier();
373 current->curr_ret_stack = curr_ret_stack;
374 /* Again sync with interrupts, and reset reserve */
375 current->ret_stack[curr_ret_stack - 1] = val;
376
377 return data;
378}
379
380/**
381 * fgraph_retrieve_data - Retrieve stored data from fgraph_reserve_data()
382 * @idx: the index of fgraph_array (fgraph_ops::idx)
383 * @size_bytes: pointer to retrieved data size.
384 *
385 * This is to be called by a fgraph_ops retfunc(), to retrieve data that
386 * was stored by the fgraph_ops entryfunc() on the function entry.
387 * That is, this will retrieve the data that was reserved on the
388 * entry of the function that corresponds to the exit of the function
389 * that the fgraph_ops retfunc() is called on.
390 *
391 * Returns: The stored data from fgraph_reserve_data() called by the
392 * matching entryfunc() for the retfunc() this is called from.
393 * Or NULL if there was nothing stored.
394 */
395void *fgraph_retrieve_data(int idx, int *size_bytes)
396{
397 return fgraph_retrieve_parent_data(idx, size_bytes, 0);
398}
399
400/**
401 * fgraph_get_task_var - retrieve a task specific state variable
402 * @gops: The ftrace_ops that owns the task specific variable
403 *
404 * Every registered fgraph_ops has a task state variable
405 * reserved on the task's ret_stack. This function returns the
406 * address to that variable.
407 *
408 * Returns the address to the fgraph_ops @gops tasks specific
409 * unsigned long variable.
410 */
411unsigned long *fgraph_get_task_var(struct fgraph_ops *gops)
412{
413 return ret_stack_get_task_var(current, gops->idx);
414}
415
416/*
417 * @offset: The offset into @t->ret_stack to find the ret_stack entry
418 * @frame_offset: Where to place the offset into @t->ret_stack of that entry
419 *
420 * Returns a pointer to the previous ret_stack below @offset or NULL
421 * when it reaches the bottom of the stack.
422 *
423 * Calling this with:
424 *
425 * offset = task->curr_ret_stack;
426 * do {
427 * ret_stack = get_ret_stack(task, offset, &offset);
428 * } while (ret_stack);
429 *
430 * Will iterate through all the ret_stack entries from curr_ret_stack
431 * down to the first one.
432 */
433static inline struct ftrace_ret_stack *
434get_ret_stack(struct task_struct *t, int offset, int *frame_offset)
435{
436 int offs;
437
438 BUILD_BUG_ON(FGRAPH_FRAME_SIZE % sizeof(long));
439
440 if (unlikely(offset <= 0))
441 return NULL;
442
443 offs = get_frame_offset(t, --offset);
444 if (WARN_ON_ONCE(offs <= 0 || offs > offset))
445 return NULL;
446
447 offset -= offs;
448
449 *frame_offset = offset;
450 return RET_STACK(t, offset);
451}
452
453/**
454 * fgraph_retrieve_parent_data - get data from a parent function
455 * @idx: The index into the fgraph_array (fgraph_ops::idx)
456 * @size_bytes: A pointer to retrieved data size
457 * @depth: The depth to find the parent (0 is the current function)
458 *
459 * This is similar to fgraph_retrieve_data() but can be used to retrieve
460 * data from a parent caller function.
461 *
462 * Return: a pointer to the specified parent data or NULL if not found
463 */
464void *fgraph_retrieve_parent_data(int idx, int *size_bytes, int depth)
465{
466 struct ftrace_ret_stack *ret_stack = NULL;
467 int offset = current->curr_ret_stack;
468 unsigned long val;
469
470 if (offset <= 0)
471 return NULL;
472
473 for (;;) {
474 int next_offset;
475
476 ret_stack = get_ret_stack(current, offset, &next_offset);
477 if (!ret_stack || --depth < 0)
478 break;
479 offset = next_offset;
480 }
481
482 if (!ret_stack)
483 return NULL;
484
485 offset--;
486
487 val = get_fgraph_entry(current, offset);
488 while (__get_type(val) == FGRAPH_TYPE_DATA) {
489 if (__get_data_index(val) == idx)
490 goto found;
491 offset -= __get_data_size(val) + 1;
492 val = get_fgraph_entry(current, offset);
493 }
494 return NULL;
495found:
496 if (size_bytes)
497 *size_bytes = __get_data_size(val) * sizeof(long);
498 return get_data_type_data(current, offset);
499}
500
501/* Both enabled by default (can be cleared by function_graph tracer flags */
502bool fgraph_sleep_time = true;
503
504#ifdef CONFIG_DYNAMIC_FTRACE
505/*
506 * archs can override this function if they must do something
507 * to enable hook for graph tracer.
508 */
509int __weak ftrace_enable_ftrace_graph_caller(void)
510{
511 return 0;
512}
513
514/*
515 * archs can override this function if they must do something
516 * to disable hook for graph tracer.
517 */
518int __weak ftrace_disable_ftrace_graph_caller(void)
519{
520 return 0;
521}
522#endif
523
524int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace,
525 struct fgraph_ops *gops,
526 struct ftrace_regs *fregs)
527{
528 return 0;
529}
530
531static void ftrace_graph_ret_stub(struct ftrace_graph_ret *trace,
532 struct fgraph_ops *gops,
533 struct ftrace_regs *fregs)
534{
535}
536
537static struct fgraph_ops fgraph_stub = {
538 .entryfunc = ftrace_graph_entry_stub,
539 .retfunc = ftrace_graph_ret_stub,
540};
541
542static struct fgraph_ops *fgraph_direct_gops = &fgraph_stub;
543DEFINE_STATIC_CALL(fgraph_func, ftrace_graph_entry_stub);
544DEFINE_STATIC_CALL(fgraph_retfunc, ftrace_graph_ret_stub);
545static DEFINE_STATIC_KEY_TRUE(fgraph_do_direct);
546
547/**
548 * ftrace_graph_stop - set to permanently disable function graph tracing
549 *
550 * In case of an error int function graph tracing, this is called
551 * to try to keep function graph tracing from causing any more harm.
552 * Usually this is pretty severe and this is called to try to at least
553 * get a warning out to the user.
554 */
555void ftrace_graph_stop(void)
556{
557 static_branch_enable(&kill_ftrace_graph);
558}
559
560/* Add a function return address to the trace stack on thread info.*/
561static int
562ftrace_push_return_trace(unsigned long ret, unsigned long func,
563 unsigned long frame_pointer, unsigned long *retp,
564 int fgraph_idx)
565{
566 struct ftrace_ret_stack *ret_stack;
567 unsigned long val;
568 int offset;
569
570 if (unlikely(ftrace_graph_is_dead()))
571 return -EBUSY;
572
573 if (!current->ret_stack)
574 return -EBUSY;
575
576 BUILD_BUG_ON(SHADOW_STACK_SIZE % sizeof(long));
577
578 /* Set val to "reserved" with the delta to the new fgraph frame */
579 val = (FGRAPH_TYPE_RESERVED << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET;
580
581 /*
582 * We must make sure the ret_stack is tested before we read
583 * anything else.
584 */
585 smp_rmb();
586
587 /*
588 * Check if there's room on the shadow stack to fit a fraph frame
589 * and a bitmap word.
590 */
591 if (current->curr_ret_stack + FGRAPH_FRAME_OFFSET + 1 >= SHADOW_STACK_MAX_OFFSET) {
592 atomic_inc(&current->trace_overrun);
593 return -EBUSY;
594 }
595
596 offset = READ_ONCE(current->curr_ret_stack);
597 ret_stack = RET_STACK(current, offset);
598 offset += FGRAPH_FRAME_OFFSET;
599
600 /* ret offset = FGRAPH_FRAME_OFFSET ; type = reserved */
601 current->ret_stack[offset] = val;
602 ret_stack->ret = ret;
603 /*
604 * The unwinders expect curr_ret_stack to point to either zero
605 * or an offset where to find the next ret_stack. Even though the
606 * ret stack might be bogus, we want to write the ret and the
607 * offset to find the ret_stack before we increment the stack point.
608 * If an interrupt comes in now before we increment the curr_ret_stack
609 * it may blow away what we wrote. But that's fine, because the
610 * offset will still be correct (even though the 'ret' won't be).
611 * What we worry about is the offset being correct after we increment
612 * the curr_ret_stack and before we update that offset, as if an
613 * interrupt comes in and does an unwind stack dump, it will need
614 * at least a correct offset!
615 */
616 barrier();
617 WRITE_ONCE(current->curr_ret_stack, offset + 1);
618 /*
619 * This next barrier is to ensure that an interrupt coming in
620 * will not corrupt what we are about to write.
621 */
622 barrier();
623
624 /* Still keep it reserved even if an interrupt came in */
625 current->ret_stack[offset] = val;
626
627 ret_stack->ret = ret;
628 ret_stack->func = func;
629#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
630 ret_stack->fp = frame_pointer;
631#endif
632 ret_stack->retp = retp;
633 return offset;
634}
635
636/*
637 * Not all archs define MCOUNT_INSN_SIZE which is used to look for direct
638 * functions. But those archs currently don't support direct functions
639 * anyway, and ftrace_find_rec_direct() is just a stub for them.
640 * Define MCOUNT_INSN_SIZE to keep those archs compiling.
641 */
642#ifndef MCOUNT_INSN_SIZE
643/* Make sure this only works without direct calls */
644# ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
645# error MCOUNT_INSN_SIZE not defined with direct calls enabled
646# endif
647# define MCOUNT_INSN_SIZE 0
648#endif
649
650/* If the caller does not use ftrace, call this function. */
651int function_graph_enter_regs(unsigned long ret, unsigned long func,
652 unsigned long frame_pointer, unsigned long *retp,
653 struct ftrace_regs *fregs)
654{
655 struct ftrace_graph_ent trace;
656 unsigned long bitmap = 0;
657 int offset;
658 int bit;
659 int i;
660
661 bit = ftrace_test_recursion_trylock(func, ret);
662 if (bit < 0)
663 return -EBUSY;
664
665 trace.func = func;
666 trace.depth = ++current->curr_ret_depth;
667
668 offset = ftrace_push_return_trace(ret, func, frame_pointer, retp, 0);
669 if (offset < 0)
670 goto out;
671
672#ifdef CONFIG_HAVE_STATIC_CALL
673 if (static_branch_likely(&fgraph_do_direct)) {
674 int save_curr_ret_stack = current->curr_ret_stack;
675
676 if (static_call(fgraph_func)(&trace, fgraph_direct_gops, fregs))
677 bitmap |= BIT(fgraph_direct_gops->idx);
678 else
679 /* Clear out any saved storage */
680 current->curr_ret_stack = save_curr_ret_stack;
681 } else
682#endif
683 {
684 for_each_set_bit(i, &fgraph_array_bitmask,
685 sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) {
686 struct fgraph_ops *gops = READ_ONCE(fgraph_array[i]);
687 int save_curr_ret_stack;
688
689 if (gops == &fgraph_stub)
690 continue;
691
692 save_curr_ret_stack = current->curr_ret_stack;
693 if (ftrace_ops_test(&gops->ops, func, NULL) &&
694 gops->entryfunc(&trace, gops, fregs))
695 bitmap |= BIT(i);
696 else
697 /* Clear out any saved storage */
698 current->curr_ret_stack = save_curr_ret_stack;
699 }
700 }
701
702 if (!bitmap)
703 goto out_ret;
704
705 /*
706 * Since this function uses fgraph_idx = 0 as a tail-call checking
707 * flag, set that bit always.
708 */
709 set_bitmap(current, offset, bitmap | BIT(0));
710 ftrace_test_recursion_unlock(bit);
711 return 0;
712 out_ret:
713 current->curr_ret_stack -= FGRAPH_FRAME_OFFSET + 1;
714 out:
715 current->curr_ret_depth--;
716 ftrace_test_recursion_unlock(bit);
717 return -EBUSY;
718}
719
720/* Retrieve a function return address to the trace stack on thread info.*/
721static struct ftrace_ret_stack *
722ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
723 unsigned long frame_pointer, int *offset)
724{
725 struct ftrace_ret_stack *ret_stack;
726
727 ret_stack = get_ret_stack(current, current->curr_ret_stack, offset);
728
729 if (unlikely(!ret_stack)) {
730 ftrace_graph_stop();
731 WARN(1, "Bad function graph ret_stack pointer: %d",
732 current->curr_ret_stack);
733 /* Might as well panic, otherwise we have no where to go */
734 *ret = (unsigned long)panic;
735 return NULL;
736 }
737
738#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
739 /*
740 * The arch may choose to record the frame pointer used
741 * and check it here to make sure that it is what we expect it
742 * to be. If gcc does not set the place holder of the return
743 * address in the frame pointer, and does a copy instead, then
744 * the function graph trace will fail. This test detects this
745 * case.
746 *
747 * Currently, x86_32 with optimize for size (-Os) makes the latest
748 * gcc do the above.
749 *
750 * Note, -mfentry does not use frame pointers, and this test
751 * is not needed if CC_USING_FENTRY is set.
752 */
753 if (unlikely(ret_stack->fp != frame_pointer)) {
754 ftrace_graph_stop();
755 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
756 " from func %ps return to %lx\n",
757 ret_stack->fp,
758 frame_pointer,
759 (void *)ret_stack->func,
760 ret_stack->ret);
761 *ret = (unsigned long)panic;
762 return NULL;
763 }
764#endif
765
766 *offset += FGRAPH_FRAME_OFFSET;
767 *ret = ret_stack->ret;
768 trace->func = ret_stack->func;
769 trace->overrun = atomic_read(&current->trace_overrun);
770 trace->depth = current->curr_ret_depth;
771 /*
772 * We still want to trace interrupts coming in if
773 * max_depth is set to 1. Make sure the decrement is
774 * seen before ftrace_graph_return.
775 */
776 barrier();
777
778 return ret_stack;
779}
780
781/*
782 * Hibernation protection.
783 * The state of the current task is too much unstable during
784 * suspend/restore to disk. We want to protect against that.
785 */
786static int
787ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
788 void *unused)
789{
790 switch (state) {
791 case PM_HIBERNATION_PREPARE:
792 pause_graph_tracing();
793 break;
794
795 case PM_POST_HIBERNATION:
796 unpause_graph_tracing();
797 break;
798 }
799 return NOTIFY_DONE;
800}
801
802static struct notifier_block ftrace_suspend_notifier = {
803 .notifier_call = ftrace_suspend_notifier_call,
804};
805
806/*
807 * Send the trace to the ring-buffer.
808 * @return the original return address.
809 */
810static inline unsigned long
811__ftrace_return_to_handler(struct ftrace_regs *fregs, unsigned long frame_pointer)
812{
813 struct ftrace_ret_stack *ret_stack;
814 struct ftrace_graph_ret trace;
815 unsigned long bitmap;
816 unsigned long ret;
817 int offset;
818 int bit;
819 int i;
820
821 ret_stack = ftrace_pop_return_trace(&trace, &ret, frame_pointer, &offset);
822
823 if (unlikely(!ret_stack)) {
824 ftrace_graph_stop();
825 WARN_ON(1);
826 /* Might as well panic. What else to do? */
827 return (unsigned long)panic;
828 }
829
830 if (fregs)
831 ftrace_regs_set_instruction_pointer(fregs, ret);
832
833 bit = ftrace_test_recursion_trylock(trace.func, ret);
834 /*
835 * This can fail because ftrace_test_recursion_trylock() allows one nest
836 * call. If we are already in a nested call, then we don't probe this and
837 * just return the original return address.
838 */
839 if (unlikely(bit < 0))
840 goto out;
841
842#ifdef CONFIG_FUNCTION_GRAPH_RETVAL
843 trace.retval = ftrace_regs_get_return_value(fregs);
844#endif
845
846 bitmap = get_bitmap_bits(current, offset);
847
848#ifdef CONFIG_HAVE_STATIC_CALL
849 if (static_branch_likely(&fgraph_do_direct)) {
850 if (test_bit(fgraph_direct_gops->idx, &bitmap))
851 static_call(fgraph_retfunc)(&trace, fgraph_direct_gops, fregs);
852 } else
853#endif
854 {
855 for_each_set_bit(i, &bitmap, sizeof(bitmap) * BITS_PER_BYTE) {
856 struct fgraph_ops *gops = READ_ONCE(fgraph_array[i]);
857
858 if (gops == &fgraph_stub)
859 continue;
860
861 gops->retfunc(&trace, gops, fregs);
862 }
863 }
864
865 ftrace_test_recursion_unlock(bit);
866out:
867 /*
868 * The ftrace_graph_return() may still access the current
869 * ret_stack structure, we need to make sure the update of
870 * curr_ret_stack is after that.
871 */
872 barrier();
873 current->curr_ret_stack = offset - FGRAPH_FRAME_OFFSET;
874
875 current->curr_ret_depth--;
876 return ret;
877}
878
879/*
880 * After all architectures have selected HAVE_FUNCTION_GRAPH_FREGS, we can
881 * leave only ftrace_return_to_handler(fregs).
882 */
883#ifdef CONFIG_HAVE_FUNCTION_GRAPH_FREGS
884unsigned long ftrace_return_to_handler(struct ftrace_regs *fregs)
885{
886 return __ftrace_return_to_handler(fregs,
887 ftrace_regs_get_frame_pointer(fregs));
888}
889#else
890unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
891{
892 return __ftrace_return_to_handler(NULL, frame_pointer);
893}
894#endif
895
896/**
897 * ftrace_graph_get_ret_stack - return the entry of the shadow stack
898 * @task: The task to read the shadow stack from.
899 * @idx: Index down the shadow stack
900 *
901 * Return the ret_struct on the shadow stack of the @task at the
902 * call graph at @idx starting with zero. If @idx is zero, it
903 * will return the last saved ret_stack entry. If it is greater than
904 * zero, it will return the corresponding ret_stack for the depth
905 * of saved return addresses.
906 */
907struct ftrace_ret_stack *
908ftrace_graph_get_ret_stack(struct task_struct *task, int idx)
909{
910 struct ftrace_ret_stack *ret_stack = NULL;
911 int offset = task->curr_ret_stack;
912
913 if (offset < 0)
914 return NULL;
915
916 do {
917 ret_stack = get_ret_stack(task, offset, &offset);
918 } while (ret_stack && --idx >= 0);
919
920 return ret_stack;
921}
922
923/**
924 * ftrace_graph_top_ret_addr - return the top return address in the shadow stack
925 * @task: The task to read the shadow stack from.
926 *
927 * Return the first return address on the shadow stack of the @task, which is
928 * not the fgraph's return_to_handler.
929 */
930unsigned long ftrace_graph_top_ret_addr(struct task_struct *task)
931{
932 unsigned long return_handler = (unsigned long)dereference_kernel_function_descriptor(return_to_handler);
933 struct ftrace_ret_stack *ret_stack = NULL;
934 int offset = task->curr_ret_stack;
935
936 if (offset < 0)
937 return 0;
938
939 do {
940 ret_stack = get_ret_stack(task, offset, &offset);
941 } while (ret_stack && ret_stack->ret == return_handler);
942
943 return ret_stack ? ret_stack->ret : 0;
944}
945
946/**
947 * ftrace_graph_ret_addr - return the original value of the return address
948 * @task: The task the unwinder is being executed on
949 * @idx: An initialized pointer to the next stack index to use
950 * @ret: The current return address (likely pointing to return_handler)
951 * @retp: The address on the stack of the current return location
952 *
953 * This function can be called by stack unwinding code to convert a found stack
954 * return address (@ret) to its original value, in case the function graph
955 * tracer has modified it to be 'return_to_handler'. If the address hasn't
956 * been modified, the unchanged value of @ret is returned.
957 *
958 * @idx holds the last index used to know where to start from. It should be
959 * initialized to zero for the first iteration as that will mean to start
960 * at the top of the shadow stack. If the location is found, this pointer
961 * will be assigned that location so that if called again, it will continue
962 * where it left off.
963 *
964 * @retp is a pointer to the return address on the stack.
965 */
966unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
967 unsigned long ret, unsigned long *retp)
968{
969 struct ftrace_ret_stack *ret_stack;
970 unsigned long return_handler = (unsigned long)dereference_kernel_function_descriptor(return_to_handler);
971 int i;
972
973 if (ret != return_handler)
974 return ret;
975
976 if (!idx)
977 return ret;
978
979 i = *idx ? : task->curr_ret_stack;
980 while (i > 0) {
981 ret_stack = get_ret_stack(task, i, &i);
982 if (!ret_stack)
983 break;
984 /*
985 * For the tail-call, there would be 2 or more ftrace_ret_stacks on
986 * the ret_stack, which records "return_to_handler" as the return
987 * address except for the last one.
988 * But on the real stack, there should be 1 entry because tail-call
989 * reuses the return address on the stack and jump to the next function.
990 * Thus we will continue to find real return address.
991 */
992 if (ret_stack->retp == retp &&
993 ret_stack->ret != return_handler) {
994 *idx = i;
995 return ret_stack->ret;
996 }
997 }
998
999 return ret;
1000}
1001
1002static struct ftrace_ops graph_ops = {
1003 .func = ftrace_graph_func,
1004 .flags = FTRACE_OPS_GRAPH_STUB,
1005#ifdef FTRACE_GRAPH_TRAMP_ADDR
1006 .trampoline = FTRACE_GRAPH_TRAMP_ADDR,
1007 /* trampoline_size is only needed for dynamically allocated tramps */
1008#endif
1009};
1010
1011void fgraph_init_ops(struct ftrace_ops *dst_ops,
1012 struct ftrace_ops *src_ops)
1013{
1014 dst_ops->flags = FTRACE_OPS_FL_PID | FTRACE_OPS_GRAPH_STUB;
1015
1016#ifdef CONFIG_DYNAMIC_FTRACE
1017 if (src_ops) {
1018 dst_ops->func_hash = &src_ops->local_hash;
1019 mutex_init(&dst_ops->local_hash.regex_lock);
1020 INIT_LIST_HEAD(&dst_ops->subop_list);
1021 dst_ops->flags |= FTRACE_OPS_FL_INITIALIZED;
1022 }
1023#endif
1024}
1025
1026void ftrace_graph_sleep_time_control(bool enable)
1027{
1028 fgraph_sleep_time = enable;
1029}
1030
1031/*
1032 * Simply points to ftrace_stub, but with the proper protocol.
1033 * Defined by the linker script in linux/vmlinux.lds.h
1034 */
1035void ftrace_stub_graph(struct ftrace_graph_ret *trace, struct fgraph_ops *gops,
1036 struct ftrace_regs *fregs);
1037
1038/* The callbacks that hook a function */
1039trace_func_graph_ret_t ftrace_graph_return = ftrace_stub_graph;
1040trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
1041
1042/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
1043static int alloc_retstack_tasklist(unsigned long **ret_stack_list)
1044{
1045 int i;
1046 int ret = 0;
1047 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
1048 struct task_struct *g, *t;
1049
1050 if (WARN_ON_ONCE(!fgraph_stack_cachep))
1051 return -ENOMEM;
1052
1053 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
1054 ret_stack_list[i] = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);
1055 if (!ret_stack_list[i]) {
1056 start = 0;
1057 end = i;
1058 ret = -ENOMEM;
1059 goto free;
1060 }
1061 }
1062
1063 rcu_read_lock();
1064 for_each_process_thread(g, t) {
1065 if (start == end) {
1066 ret = -EAGAIN;
1067 goto unlock;
1068 }
1069
1070 if (t->ret_stack == NULL) {
1071 atomic_set(&t->trace_overrun, 0);
1072 ret_stack_init_task_vars(ret_stack_list[start]);
1073 t->curr_ret_stack = 0;
1074 t->curr_ret_depth = -1;
1075 /* Make sure the tasks see the 0 first: */
1076 smp_wmb();
1077 t->ret_stack = ret_stack_list[start++];
1078 }
1079 }
1080
1081unlock:
1082 rcu_read_unlock();
1083free:
1084 for (i = start; i < end; i++)
1085 kmem_cache_free(fgraph_stack_cachep, ret_stack_list[i]);
1086 return ret;
1087}
1088
1089static void
1090ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
1091 struct task_struct *prev,
1092 struct task_struct *next,
1093 unsigned int prev_state)
1094{
1095 unsigned long long timestamp;
1096
1097 /*
1098 * Does the user want to count the time a function was asleep.
1099 * If so, do not update the time stamps.
1100 */
1101 if (fgraph_sleep_time)
1102 return;
1103
1104 timestamp = trace_clock_local();
1105
1106 prev->ftrace_timestamp = timestamp;
1107
1108 /* only process tasks that we timestamped */
1109 if (!next->ftrace_timestamp)
1110 return;
1111
1112 next->ftrace_sleeptime += timestamp - next->ftrace_timestamp;
1113}
1114
1115static DEFINE_PER_CPU(unsigned long *, idle_ret_stack);
1116
1117static void
1118graph_init_task(struct task_struct *t, unsigned long *ret_stack)
1119{
1120 atomic_set(&t->trace_overrun, 0);
1121 ret_stack_init_task_vars(ret_stack);
1122 t->ftrace_timestamp = 0;
1123 t->curr_ret_stack = 0;
1124 t->curr_ret_depth = -1;
1125 /* make curr_ret_stack visible before we add the ret_stack */
1126 smp_wmb();
1127 t->ret_stack = ret_stack;
1128}
1129
1130/*
1131 * Allocate a return stack for the idle task. May be the first
1132 * time through, or it may be done by CPU hotplug online.
1133 */
1134void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
1135{
1136 t->curr_ret_stack = 0;
1137 t->curr_ret_depth = -1;
1138 /*
1139 * The idle task has no parent, it either has its own
1140 * stack or no stack at all.
1141 */
1142 if (t->ret_stack)
1143 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
1144
1145 if (ftrace_graph_active) {
1146 unsigned long *ret_stack;
1147
1148 if (WARN_ON_ONCE(!fgraph_stack_cachep))
1149 return;
1150
1151 ret_stack = per_cpu(idle_ret_stack, cpu);
1152 if (!ret_stack) {
1153 ret_stack = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);
1154 if (!ret_stack)
1155 return;
1156 per_cpu(idle_ret_stack, cpu) = ret_stack;
1157 }
1158 graph_init_task(t, ret_stack);
1159 }
1160}
1161
1162/* Allocate a return stack for newly created task */
1163void ftrace_graph_init_task(struct task_struct *t)
1164{
1165 /* Make sure we do not use the parent ret_stack */
1166 t->ret_stack = NULL;
1167 t->curr_ret_stack = 0;
1168 t->curr_ret_depth = -1;
1169
1170 if (ftrace_graph_active) {
1171 unsigned long *ret_stack;
1172
1173 if (WARN_ON_ONCE(!fgraph_stack_cachep))
1174 return;
1175
1176 ret_stack = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);
1177 if (!ret_stack)
1178 return;
1179 graph_init_task(t, ret_stack);
1180 }
1181}
1182
1183void ftrace_graph_exit_task(struct task_struct *t)
1184{
1185 unsigned long *ret_stack = t->ret_stack;
1186
1187 t->ret_stack = NULL;
1188 /* NULL must become visible to IRQs before we free it: */
1189 barrier();
1190
1191 if (ret_stack) {
1192 if (WARN_ON_ONCE(!fgraph_stack_cachep))
1193 return;
1194 kmem_cache_free(fgraph_stack_cachep, ret_stack);
1195 }
1196}
1197
1198#ifdef CONFIG_DYNAMIC_FTRACE
1199static int fgraph_pid_func(struct ftrace_graph_ent *trace,
1200 struct fgraph_ops *gops,
1201 struct ftrace_regs *fregs)
1202{
1203 struct trace_array *tr = gops->ops.private;
1204 int pid;
1205
1206 if (tr) {
1207 pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);
1208 if (pid == FTRACE_PID_IGNORE)
1209 return 0;
1210 if (pid != FTRACE_PID_TRACE &&
1211 pid != current->pid)
1212 return 0;
1213 }
1214
1215 return gops->saved_func(trace, gops, fregs);
1216}
1217
1218void fgraph_update_pid_func(void)
1219{
1220 struct fgraph_ops *gops;
1221 struct ftrace_ops *op;
1222
1223 if (!(graph_ops.flags & FTRACE_OPS_FL_INITIALIZED))
1224 return;
1225
1226 list_for_each_entry(op, &graph_ops.subop_list, list) {
1227 if (op->flags & FTRACE_OPS_FL_PID) {
1228 gops = container_of(op, struct fgraph_ops, ops);
1229 gops->entryfunc = ftrace_pids_enabled(op) ?
1230 fgraph_pid_func : gops->saved_func;
1231 if (ftrace_graph_active == 1)
1232 static_call_update(fgraph_func, gops->entryfunc);
1233 }
1234 }
1235}
1236#endif
1237
1238/* Allocate a return stack for each task */
1239static int start_graph_tracing(void)
1240{
1241 unsigned long **ret_stack_list;
1242 int ret, cpu;
1243
1244 ret_stack_list = kcalloc(FTRACE_RETSTACK_ALLOC_SIZE,
1245 sizeof(*ret_stack_list), GFP_KERNEL);
1246
1247 if (!ret_stack_list)
1248 return -ENOMEM;
1249
1250 /* The cpu_boot init_task->ret_stack will never be freed */
1251 for_each_online_cpu(cpu) {
1252 if (!idle_task(cpu)->ret_stack)
1253 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
1254 }
1255
1256 do {
1257 ret = alloc_retstack_tasklist(ret_stack_list);
1258 } while (ret == -EAGAIN);
1259
1260 if (!ret) {
1261 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
1262 if (ret)
1263 pr_info("ftrace_graph: Couldn't activate tracepoint"
1264 " probe to kernel_sched_switch\n");
1265 }
1266
1267 kfree(ret_stack_list);
1268 return ret;
1269}
1270
1271static void init_task_vars(int idx)
1272{
1273 struct task_struct *g, *t;
1274 int cpu;
1275
1276 for_each_online_cpu(cpu) {
1277 if (idle_task(cpu)->ret_stack)
1278 ret_stack_set_task_var(idle_task(cpu), idx, 0);
1279 }
1280
1281 read_lock(&tasklist_lock);
1282 for_each_process_thread(g, t) {
1283 if (t->ret_stack)
1284 ret_stack_set_task_var(t, idx, 0);
1285 }
1286 read_unlock(&tasklist_lock);
1287}
1288
1289static void ftrace_graph_enable_direct(bool enable_branch, struct fgraph_ops *gops)
1290{
1291 trace_func_graph_ent_t func = NULL;
1292 trace_func_graph_ret_t retfunc = NULL;
1293 int i;
1294
1295 if (gops) {
1296 func = gops->entryfunc;
1297 retfunc = gops->retfunc;
1298 fgraph_direct_gops = gops;
1299 } else {
1300 for_each_set_bit(i, &fgraph_array_bitmask,
1301 sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) {
1302 func = fgraph_array[i]->entryfunc;
1303 retfunc = fgraph_array[i]->retfunc;
1304 fgraph_direct_gops = fgraph_array[i];
1305 }
1306 }
1307 if (WARN_ON_ONCE(!func))
1308 return;
1309
1310 static_call_update(fgraph_func, func);
1311 static_call_update(fgraph_retfunc, retfunc);
1312 if (enable_branch)
1313 static_branch_disable(&fgraph_do_direct);
1314}
1315
1316static void ftrace_graph_disable_direct(bool disable_branch)
1317{
1318 if (disable_branch)
1319 static_branch_disable(&fgraph_do_direct);
1320 static_call_update(fgraph_func, ftrace_graph_entry_stub);
1321 static_call_update(fgraph_retfunc, ftrace_graph_ret_stub);
1322 fgraph_direct_gops = &fgraph_stub;
1323}
1324
1325/* The cpu_boot init_task->ret_stack will never be freed */
1326static int fgraph_cpu_init(unsigned int cpu)
1327{
1328 if (!idle_task(cpu)->ret_stack)
1329 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
1330 return 0;
1331}
1332
1333int register_ftrace_graph(struct fgraph_ops *gops)
1334{
1335 static bool fgraph_initialized;
1336 int command = 0;
1337 int ret = 0;
1338 int i = -1;
1339
1340 if (WARN_ONCE(gops->ops.flags & FTRACE_OPS_FL_GRAPH,
1341 "function graph ops registered again"))
1342 return -EBUSY;
1343
1344 guard(mutex)(&ftrace_lock);
1345
1346 if (!fgraph_stack_cachep) {
1347 fgraph_stack_cachep = kmem_cache_create("fgraph_stack",
1348 SHADOW_STACK_SIZE,
1349 SHADOW_STACK_SIZE, 0, NULL);
1350 if (!fgraph_stack_cachep)
1351 return -ENOMEM;
1352 }
1353
1354 if (!fgraph_initialized) {
1355 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "fgraph:online",
1356 fgraph_cpu_init, NULL);
1357 if (ret < 0) {
1358 pr_warn("fgraph: Error to init cpu hotplug support\n");
1359 return ret;
1360 }
1361 fgraph_initialized = true;
1362 ret = 0;
1363 }
1364
1365 if (!fgraph_array[0]) {
1366 /* The array must always have real data on it */
1367 for (i = 0; i < FGRAPH_ARRAY_SIZE; i++)
1368 fgraph_array[i] = &fgraph_stub;
1369 fgraph_lru_init();
1370 }
1371
1372 i = fgraph_lru_alloc_index();
1373 if (i < 0 || WARN_ON_ONCE(fgraph_array[i] != &fgraph_stub))
1374 return -ENOSPC;
1375 gops->idx = i;
1376
1377 ftrace_graph_active++;
1378
1379 if (ftrace_graph_active == 2)
1380 ftrace_graph_disable_direct(true);
1381
1382 if (ftrace_graph_active == 1) {
1383 ftrace_graph_enable_direct(false, gops);
1384 register_pm_notifier(&ftrace_suspend_notifier);
1385 ret = start_graph_tracing();
1386 if (ret)
1387 goto error;
1388 /*
1389 * Some archs just test to see if these are not
1390 * the default function
1391 */
1392 ftrace_graph_return = return_run;
1393 ftrace_graph_entry = entry_run;
1394 command = FTRACE_START_FUNC_RET;
1395 } else {
1396 init_task_vars(gops->idx);
1397 }
1398 /* Always save the function, and reset at unregistering */
1399 gops->saved_func = gops->entryfunc;
1400
1401 gops->ops.flags |= FTRACE_OPS_FL_GRAPH;
1402
1403 ret = ftrace_startup_subops(&graph_ops, &gops->ops, command);
1404 if (!ret)
1405 fgraph_array[i] = gops;
1406
1407error:
1408 if (ret) {
1409 ftrace_graph_active--;
1410 gops->saved_func = NULL;
1411 fgraph_lru_release_index(i);
1412 if (!ftrace_graph_active)
1413 unregister_pm_notifier(&ftrace_suspend_notifier);
1414 }
1415 return ret;
1416}
1417
1418void unregister_ftrace_graph(struct fgraph_ops *gops)
1419{
1420 int command = 0;
1421
1422 if (WARN_ONCE(!(gops->ops.flags & FTRACE_OPS_FL_GRAPH),
1423 "function graph ops unregistered without registering"))
1424 return;
1425
1426 guard(mutex)(&ftrace_lock);
1427
1428 if (unlikely(!ftrace_graph_active))
1429 goto out;
1430
1431 if (unlikely(gops->idx < 0 || gops->idx >= FGRAPH_ARRAY_SIZE ||
1432 fgraph_array[gops->idx] != gops))
1433 goto out;
1434
1435 if (fgraph_lru_release_index(gops->idx) < 0)
1436 goto out;
1437
1438 fgraph_array[gops->idx] = &fgraph_stub;
1439
1440 ftrace_graph_active--;
1441
1442 if (!ftrace_graph_active)
1443 command = FTRACE_STOP_FUNC_RET;
1444
1445 ftrace_shutdown_subops(&graph_ops, &gops->ops, command);
1446
1447 if (ftrace_graph_active == 1)
1448 ftrace_graph_enable_direct(true, NULL);
1449 else if (!ftrace_graph_active)
1450 ftrace_graph_disable_direct(false);
1451
1452 if (!ftrace_graph_active) {
1453 ftrace_graph_return = ftrace_stub_graph;
1454 ftrace_graph_entry = ftrace_graph_entry_stub;
1455 unregister_pm_notifier(&ftrace_suspend_notifier);
1456 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
1457 }
1458 gops->saved_func = NULL;
1459 out:
1460 gops->ops.flags &= ~FTRACE_OPS_FL_GRAPH;
1461}