projects/linux·blob assoc-array-fixes-v2 · refs

1466 lines · 40.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Infrastructure to took into function calls and returns.
4 * Copyright (c) 2008-2009 Frederic Weisbecker <[email protected]>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <[email protected]>
7 *
8 * Highly modified by Steven Rostedt (VMware).
9 */
10#include <linux/bits.h>
11#include <linux/jump_label.h>
12#include <linux/suspend.h>
13#include <linux/ftrace.h>
14#include <linux/static_call.h>
15#include <linux/slab.h>
16
17#include <trace/events/sched.h>
18
19#include "ftrace_internal.h"
20#include "trace.h"
21
22/*
23 * FGRAPH_FRAME_SIZE: Size in bytes of the meta data on the shadow stack
24 * FGRAPH_FRAME_OFFSET: Size in long words of the meta data frame
25 */
26#define FGRAPH_FRAME_SIZE sizeof(struct ftrace_ret_stack)
27#define FGRAPH_FRAME_OFFSET DIV_ROUND_UP(FGRAPH_FRAME_SIZE, sizeof(long))
28
29/*
30 * On entry to a function (via function_graph_enter()), a new fgraph frame
31 * (ftrace_ret_stack) is pushed onto the stack as well as a word that
32 * holds a bitmask and a type (called "bitmap"). The bitmap is defined as:
33 *
34 * bits: 0 - 9 offset in words from the previous ftrace_ret_stack
35 *
36 * bits: 10 - 11 Type of storage
37 * 0 - reserved
38 * 1 - bitmap of fgraph_array index
39 * 2 - reserved data
40 *
41 * For type with "bitmap of fgraph_array index" (FGRAPH_TYPE_BITMAP):
42 * bits: 12 - 27 The bitmap of fgraph_ops fgraph_array index
43 * That is, it's a bitmask of 0-15 (16 bits)
44 * where if a corresponding ops in the fgraph_array[]
45 * expects a callback from the return of the function
46 * it's corresponding bit will be set.
47 *
48 *
49 * The top of the ret_stack (when not empty) will always have a reference
50 * word that points to the last fgraph frame that was saved.
51 *
52 * For reserved data:
53 * bits: 12 - 17 The size in words that is stored
54 * bits: 18 - 23 The index of fgraph_array, which shows who is stored
55 *
56 * That is, at the end of function_graph_enter, if the first and forth
57 * fgraph_ops on the fgraph_array[] (index 0 and 3) needs their retfunc called
58 * on the return of the function being traced, and the forth fgraph_ops
59 * stored two words of data, this is what will be on the task's shadow
60 * ret_stack: (the stack grows upward)
61 *
62 * ret_stack[SHADOW_STACK_OFFSET]
63 * | SHADOW_STACK_TASK_VARS(ret_stack)[15] |
64 * ...
65 * | SHADOW_STACK_TASK_VARS(ret_stack)[0] |
66 * ret_stack[SHADOW_STACK_MAX_OFFSET]
67 * ...
68 * | | <- task->curr_ret_stack
69 * +--------------------------------------------+
70 * | (3 << 12) | (3 << 10) | FGRAPH_FRAME_OFFSET|
71 * | *or put another way* |
72 * | (3 << FGRAPH_DATA_INDEX_SHIFT)| \ | This is for fgraph_ops[3].
73 * | ((2 - 1) << FGRAPH_DATA_SHIFT)| \ | The data size is 2 words.
74 * | (FGRAPH_TYPE_DATA << FGRAPH_TYPE_SHIFT)| \ |
75 * | (offset2:FGRAPH_FRAME_OFFSET+3) | <- the offset2 is from here
76 * +--------------------------------------------+ ( It is 4 words from the ret_stack)
77 * | STORED DATA WORD 2 |
78 * | STORED DATA WORD 1 |
79 * +--------------------------------------------+
80 * | (9 << 12) | (1 << 10) | FGRAPH_FRAME_OFFSET|
81 * | *or put another way* |
82 * | (BIT(3)|BIT(0)) << FGRAPH_INDEX_SHIFT | \ |
83 * | FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT| \ |
84 * | (offset1:FGRAPH_FRAME_OFFSET) | <- the offset1 is from here
85 * +--------------------------------------------+
86 * | struct ftrace_ret_stack |
87 * | (stores the saved ret pointer) | <- the offset points here
88 * +--------------------------------------------+
89 * | (X) | (N) | ( N words away from
90 * | | previous ret_stack)
91 * ...
92 * ret_stack[0]
93 *
94 * If a backtrace is required, and the real return pointer needs to be
95 * fetched, then it looks at the task's curr_ret_stack offset, if it
96 * is greater than zero (reserved, or right before popped), it would mask
97 * the value by FGRAPH_FRAME_OFFSET_MASK to get the offset of the
98 * ftrace_ret_stack structure stored on the shadow stack.
99 */
100
101/*
102 * The following is for the top word on the stack:
103 *
104 * FGRAPH_FRAME_OFFSET (0-9) holds the offset delta to the fgraph frame
105 * FGRAPH_TYPE (10-11) holds the type of word this is.
106 * (RESERVED or BITMAP)
107 */
108#define FGRAPH_FRAME_OFFSET_BITS 10
109#define FGRAPH_FRAME_OFFSET_MASK GENMASK(FGRAPH_FRAME_OFFSET_BITS - 1, 0)
110
111#define FGRAPH_TYPE_BITS 2
112#define FGRAPH_TYPE_MASK GENMASK(FGRAPH_TYPE_BITS - 1, 0)
113#define FGRAPH_TYPE_SHIFT FGRAPH_FRAME_OFFSET_BITS
114
115enum {
116 FGRAPH_TYPE_RESERVED = 0,
117 FGRAPH_TYPE_BITMAP = 1,
118 FGRAPH_TYPE_DATA = 2,
119};
120
121/*
122 * For BITMAP type:
123 * FGRAPH_INDEX (12-27) bits holding the gops index wanting return callback called
124 */
125#define FGRAPH_INDEX_BITS 16
126#define FGRAPH_INDEX_MASK GENMASK(FGRAPH_INDEX_BITS - 1, 0)
127#define FGRAPH_INDEX_SHIFT (FGRAPH_TYPE_SHIFT + FGRAPH_TYPE_BITS)
128
129/*
130 * For DATA type:
131 * FGRAPH_DATA (12-17) bits hold the size of data (in words)
132 * FGRAPH_INDEX (18-23) bits hold the index for which gops->idx the data is for
133 *
134 * Note:
135 * data_size == 0 means 1 word, and 31 (=2^5 - 1) means 32 words.
136 */
137#define FGRAPH_DATA_BITS 5
138#define FGRAPH_DATA_MASK GENMASK(FGRAPH_DATA_BITS - 1, 0)
139#define FGRAPH_DATA_SHIFT (FGRAPH_TYPE_SHIFT + FGRAPH_TYPE_BITS)
140#define FGRAPH_MAX_DATA_SIZE (sizeof(long) * (1 << FGRAPH_DATA_BITS))
141
142#define FGRAPH_DATA_INDEX_BITS 4
143#define FGRAPH_DATA_INDEX_MASK GENMASK(FGRAPH_DATA_INDEX_BITS - 1, 0)
144#define FGRAPH_DATA_INDEX_SHIFT (FGRAPH_DATA_SHIFT + FGRAPH_DATA_BITS)
145
146#define FGRAPH_ARRAY_SIZE FGRAPH_INDEX_BITS
147
148/*
149 * SHADOW_STACK_SIZE: The size in bytes of the entire shadow stack
150 * SHADOW_STACK_OFFSET: The size in long words of the shadow stack
151 * SHADOW_STACK_MAX_OFFSET: The max offset of the stack for a new frame to be added
152 */
153#define SHADOW_STACK_SIZE (4096)
154#define SHADOW_STACK_OFFSET (SHADOW_STACK_SIZE / sizeof(long))
155/* Leave on a buffer at the end */
156#define SHADOW_STACK_MAX_OFFSET \
157 (SHADOW_STACK_OFFSET - (FGRAPH_FRAME_OFFSET + 1 + FGRAPH_ARRAY_SIZE))
158
159/* RET_STACK(): Return the frame from a given @offset from task @t */
160#define RET_STACK(t, offset) ((struct ftrace_ret_stack *)(&(t)->ret_stack[offset]))
161
162/*
163 * Each fgraph_ops has a reserved unsigned long at the end (top) of the
164 * ret_stack to store task specific state.
165 */
166#define SHADOW_STACK_TASK_VARS(ret_stack) \
167 ((unsigned long *)(&(ret_stack)[SHADOW_STACK_OFFSET - FGRAPH_ARRAY_SIZE]))
168
169DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph);
170int ftrace_graph_active;
171
172static struct kmem_cache *fgraph_stack_cachep;
173
174static struct fgraph_ops *fgraph_array[FGRAPH_ARRAY_SIZE];
175static unsigned long fgraph_array_bitmask;
176
177/* LRU index table for fgraph_array */
178static int fgraph_lru_table[FGRAPH_ARRAY_SIZE];
179static int fgraph_lru_next;
180static int fgraph_lru_last;
181
182/* Initialize fgraph_lru_table with unused index */
183static void fgraph_lru_init(void)
184{
185 int i;
186
187 for (i = 0; i < FGRAPH_ARRAY_SIZE; i++)
188 fgraph_lru_table[i] = i;
189}
190
191/* Release the used index to the LRU table */
192static int fgraph_lru_release_index(int idx)
193{
194 if (idx < 0 || idx >= FGRAPH_ARRAY_SIZE ||
195 WARN_ON_ONCE(fgraph_lru_table[fgraph_lru_last] != -1))
196 return -1;
197
198 fgraph_lru_table[fgraph_lru_last] = idx;
199 fgraph_lru_last = (fgraph_lru_last + 1) % FGRAPH_ARRAY_SIZE;
200
201 clear_bit(idx, &fgraph_array_bitmask);
202 return 0;
203}
204
205/* Allocate a new index from LRU table */
206static int fgraph_lru_alloc_index(void)
207{
208 int idx = fgraph_lru_table[fgraph_lru_next];
209
210 /* No id is available */
211 if (idx == -1)
212 return -1;
213
214 fgraph_lru_table[fgraph_lru_next] = -1;
215 fgraph_lru_next = (fgraph_lru_next + 1) % FGRAPH_ARRAY_SIZE;
216
217 set_bit(idx, &fgraph_array_bitmask);
218 return idx;
219}
220
221/* Get the offset to the fgraph frame from a ret_stack value */
222static inline int __get_offset(unsigned long val)
223{
224 return val & FGRAPH_FRAME_OFFSET_MASK;
225}
226
227/* Get the type of word from a ret_stack value */
228static inline int __get_type(unsigned long val)
229{
230 return (val >> FGRAPH_TYPE_SHIFT) & FGRAPH_TYPE_MASK;
231}
232
233/* Get the data_index for a DATA type ret_stack word */
234static inline int __get_data_index(unsigned long val)
235{
236 return (val >> FGRAPH_DATA_INDEX_SHIFT) & FGRAPH_DATA_INDEX_MASK;
237}
238
239/* Get the data_size for a DATA type ret_stack word */
240static inline int __get_data_size(unsigned long val)
241{
242 return ((val >> FGRAPH_DATA_SHIFT) & FGRAPH_DATA_MASK) + 1;
243}
244
245/* Get the word from the ret_stack at @offset */
246static inline unsigned long get_fgraph_entry(struct task_struct *t, int offset)
247{
248 return t->ret_stack[offset];
249}
250
251/* Get the FRAME_OFFSET from the word from the @offset on ret_stack */
252static inline int get_frame_offset(struct task_struct *t, int offset)
253{
254 return __get_offset(t->ret_stack[offset]);
255}
256
257/* For BITMAP type: get the bitmask from the @offset at ret_stack */
258static inline unsigned long
259get_bitmap_bits(struct task_struct *t, int offset)
260{
261 return (t->ret_stack[offset] >> FGRAPH_INDEX_SHIFT) & FGRAPH_INDEX_MASK;
262}
263
264/* Write the bitmap to the ret_stack at @offset (does index, offset and bitmask) */
265static inline void
266set_bitmap(struct task_struct *t, int offset, unsigned long bitmap)
267{
268 t->ret_stack[offset] = (bitmap << FGRAPH_INDEX_SHIFT) |
269 (FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET;
270}
271
272/* For DATA type: get the data saved under the ret_stack word at @offset */
273static inline void *get_data_type_data(struct task_struct *t, int offset)
274{
275 unsigned long val = t->ret_stack[offset];
276
277 if (__get_type(val) != FGRAPH_TYPE_DATA)
278 return NULL;
279 offset -= __get_data_size(val);
280 return (void *)&t->ret_stack[offset];
281}
282
283/* Create the ret_stack word for a DATA type */
284static inline unsigned long make_data_type_val(int idx, int size, int offset)
285{
286 return (idx << FGRAPH_DATA_INDEX_SHIFT) |
287 ((size - 1) << FGRAPH_DATA_SHIFT) |
288 (FGRAPH_TYPE_DATA << FGRAPH_TYPE_SHIFT) | offset;
289}
290
291/* ftrace_graph_entry set to this to tell some archs to run function graph */
292static int entry_run(struct ftrace_graph_ent *trace, struct fgraph_ops *ops,
293 struct ftrace_regs *fregs)
294{
295 return 0;
296}
297
298/* ftrace_graph_return set to this to tell some archs to run function graph */
299static void return_run(struct ftrace_graph_ret *trace, struct fgraph_ops *ops,
300 struct ftrace_regs *fregs)
301{
302}
303
304static void ret_stack_set_task_var(struct task_struct *t, int idx, long val)
305{
306 unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack);
307
308 gvals[idx] = val;
309}
310
311static unsigned long *
312ret_stack_get_task_var(struct task_struct *t, int idx)
313{
314 unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack);
315
316 return &gvals[idx];
317}
318
319static void ret_stack_init_task_vars(unsigned long *ret_stack)
320{
321 unsigned long *gvals = SHADOW_STACK_TASK_VARS(ret_stack);
322
323 memset(gvals, 0, sizeof(*gvals) * FGRAPH_ARRAY_SIZE);
324}
325
326/**
327 * fgraph_reserve_data - Reserve storage on the task's ret_stack
328 * @idx: The index of fgraph_array
329 * @size_bytes: The size in bytes to reserve
330 *
331 * Reserves space of up to FGRAPH_MAX_DATA_SIZE bytes on the
332 * task's ret_stack shadow stack, for a given fgraph_ops during
333 * the entryfunc() call. If entryfunc() returns zero, the storage
334 * is discarded. An entryfunc() can only call this once per iteration.
335 * The fgraph_ops retfunc() can retrieve this stored data with
336 * fgraph_retrieve_data().
337 *
338 * Returns: On success, a pointer to the data on the stack.
339 * Otherwise, NULL if there's not enough space left on the
340 * ret_stack for the data, or if fgraph_reserve_data() was called
341 * more than once for a single entryfunc() call.
342 */
343void *fgraph_reserve_data(int idx, int size_bytes)
344{
345 unsigned long val;
346 void *data;
347 int curr_ret_stack = current->curr_ret_stack;
348 int data_size;
349
350 if (size_bytes > FGRAPH_MAX_DATA_SIZE)
351 return NULL;
352
353 /* Convert the data size to number of longs. */
354 data_size = (size_bytes + sizeof(long) - 1) >> (sizeof(long) == 4 ? 2 : 3);
355
356 val = get_fgraph_entry(current, curr_ret_stack - 1);
357 data = &current->ret_stack[curr_ret_stack];
358
359 curr_ret_stack += data_size + 1;
360 if (unlikely(curr_ret_stack >= SHADOW_STACK_MAX_OFFSET))
361 return NULL;
362
363 val = make_data_type_val(idx, data_size, __get_offset(val) + data_size + 1);
364
365 /* Set the last word to be reserved */
366 current->ret_stack[curr_ret_stack - 1] = val;
367
368 /* Make sure interrupts see this */
369 barrier();
370 current->curr_ret_stack = curr_ret_stack;
371 /* Again sync with interrupts, and reset reserve */
372 current->ret_stack[curr_ret_stack - 1] = val;
373
374 return data;
375}
376
377/**
378 * fgraph_retrieve_data - Retrieve stored data from fgraph_reserve_data()
379 * @idx: the index of fgraph_array (fgraph_ops::idx)
380 * @size_bytes: pointer to retrieved data size.
381 *
382 * This is to be called by a fgraph_ops retfunc(), to retrieve data that
383 * was stored by the fgraph_ops entryfunc() on the function entry.
384 * That is, this will retrieve the data that was reserved on the
385 * entry of the function that corresponds to the exit of the function
386 * that the fgraph_ops retfunc() is called on.
387 *
388 * Returns: The stored data from fgraph_reserve_data() called by the
389 * matching entryfunc() for the retfunc() this is called from.
390 * Or NULL if there was nothing stored.
391 */
392void *fgraph_retrieve_data(int idx, int *size_bytes)
393{
394 return fgraph_retrieve_parent_data(idx, size_bytes, 0);
395}
396
397/**
398 * fgraph_get_task_var - retrieve a task specific state variable
399 * @gops: The ftrace_ops that owns the task specific variable
400 *
401 * Every registered fgraph_ops has a task state variable
402 * reserved on the task's ret_stack. This function returns the
403 * address to that variable.
404 *
405 * Returns the address to the fgraph_ops @gops tasks specific
406 * unsigned long variable.
407 */
408unsigned long *fgraph_get_task_var(struct fgraph_ops *gops)
409{
410 return ret_stack_get_task_var(current, gops->idx);
411}
412
413/*
414 * @offset: The offset into @t->ret_stack to find the ret_stack entry
415 * @frame_offset: Where to place the offset into @t->ret_stack of that entry
416 *
417 * Returns a pointer to the previous ret_stack below @offset or NULL
418 * when it reaches the bottom of the stack.
419 *
420 * Calling this with:
421 *
422 * offset = task->curr_ret_stack;
423 * do {
424 * ret_stack = get_ret_stack(task, offset, &offset);
425 * } while (ret_stack);
426 *
427 * Will iterate through all the ret_stack entries from curr_ret_stack
428 * down to the first one.
429 */
430static inline struct ftrace_ret_stack *
431get_ret_stack(struct task_struct *t, int offset, int *frame_offset)
432{
433 int offs;
434
435 BUILD_BUG_ON(FGRAPH_FRAME_SIZE % sizeof(long));
436
437 if (unlikely(offset <= 0))
438 return NULL;
439
440 offs = get_frame_offset(t, --offset);
441 if (WARN_ON_ONCE(offs <= 0 || offs > offset))
442 return NULL;
443
444 offset -= offs;
445
446 *frame_offset = offset;
447 return RET_STACK(t, offset);
448}
449
450/**
451 * fgraph_retrieve_parent_data - get data from a parent function
452 * @idx: The index into the fgraph_array (fgraph_ops::idx)
453 * @size_bytes: A pointer to retrieved data size
454 * @depth: The depth to find the parent (0 is the current function)
455 *
456 * This is similar to fgraph_retrieve_data() but can be used to retrieve
457 * data from a parent caller function.
458 *
459 * Return: a pointer to the specified parent data or NULL if not found
460 */
461void *fgraph_retrieve_parent_data(int idx, int *size_bytes, int depth)
462{
463 struct ftrace_ret_stack *ret_stack = NULL;
464 int offset = current->curr_ret_stack;
465 unsigned long val;
466
467 if (offset <= 0)
468 return NULL;
469
470 for (;;) {
471 int next_offset;
472
473 ret_stack = get_ret_stack(current, offset, &next_offset);
474 if (!ret_stack || --depth < 0)
475 break;
476 offset = next_offset;
477 }
478
479 if (!ret_stack)
480 return NULL;
481
482 offset--;
483
484 val = get_fgraph_entry(current, offset);
485 while (__get_type(val) == FGRAPH_TYPE_DATA) {
486 if (__get_data_index(val) == idx)
487 goto found;
488 offset -= __get_data_size(val) + 1;
489 val = get_fgraph_entry(current, offset);
490 }
491 return NULL;
492found:
493 if (size_bytes)
494 *size_bytes = __get_data_size(val) * sizeof(long);
495 return get_data_type_data(current, offset);
496}
497
498#ifdef CONFIG_DYNAMIC_FTRACE
499/*
500 * archs can override this function if they must do something
501 * to enable hook for graph tracer.
502 */
503int __weak ftrace_enable_ftrace_graph_caller(void)
504{
505 return 0;
506}
507
508/*
509 * archs can override this function if they must do something
510 * to disable hook for graph tracer.
511 */
512int __weak ftrace_disable_ftrace_graph_caller(void)
513{
514 return 0;
515}
516#endif
517
518int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace,
519 struct fgraph_ops *gops,
520 struct ftrace_regs *fregs)
521{
522 return 0;
523}
524
525static void ftrace_graph_ret_stub(struct ftrace_graph_ret *trace,
526 struct fgraph_ops *gops,
527 struct ftrace_regs *fregs)
528{
529}
530
531static struct fgraph_ops fgraph_stub = {
532 .entryfunc = ftrace_graph_entry_stub,
533 .retfunc = ftrace_graph_ret_stub,
534};
535
536static struct fgraph_ops *fgraph_direct_gops = &fgraph_stub;
537DEFINE_STATIC_CALL(fgraph_func, ftrace_graph_entry_stub);
538DEFINE_STATIC_CALL(fgraph_retfunc, ftrace_graph_ret_stub);
539#if FGRAPH_NO_DIRECT
540static DEFINE_STATIC_KEY_FALSE(fgraph_do_direct);
541#else
542static DEFINE_STATIC_KEY_TRUE(fgraph_do_direct);
543#endif
544
545/**
546 * ftrace_graph_stop - set to permanently disable function graph tracing
547 *
548 * In case of an error int function graph tracing, this is called
549 * to try to keep function graph tracing from causing any more harm.
550 * Usually this is pretty severe and this is called to try to at least
551 * get a warning out to the user.
552 */
553void ftrace_graph_stop(void)
554{
555 static_branch_enable(&kill_ftrace_graph);
556}
557
558/* Add a function return address to the trace stack on thread info.*/
559static int
560ftrace_push_return_trace(unsigned long ret, unsigned long func,
561 unsigned long frame_pointer, unsigned long *retp,
562 int fgraph_idx)
563{
564 struct ftrace_ret_stack *ret_stack;
565 unsigned long val;
566 int offset;
567
568 if (unlikely(ftrace_graph_is_dead()))
569 return -EBUSY;
570
571 if (!current->ret_stack)
572 return -EBUSY;
573
574 BUILD_BUG_ON(SHADOW_STACK_SIZE % sizeof(long));
575
576 /* Set val to "reserved" with the delta to the new fgraph frame */
577 val = (FGRAPH_TYPE_RESERVED << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET;
578
579 /*
580 * We must make sure the ret_stack is tested before we read
581 * anything else.
582 */
583 smp_rmb();
584
585 /*
586 * Check if there's room on the shadow stack to fit a fraph frame
587 * and a bitmap word.
588 */
589 if (current->curr_ret_stack + FGRAPH_FRAME_OFFSET + 1 >= SHADOW_STACK_MAX_OFFSET) {
590 atomic_inc(&current->trace_overrun);
591 return -EBUSY;
592 }
593
594 offset = READ_ONCE(current->curr_ret_stack);
595 ret_stack = RET_STACK(current, offset);
596 offset += FGRAPH_FRAME_OFFSET;
597
598 /* ret offset = FGRAPH_FRAME_OFFSET ; type = reserved */
599 current->ret_stack[offset] = val;
600 ret_stack->ret = ret;
601 /*
602 * The unwinders expect curr_ret_stack to point to either zero
603 * or an offset where to find the next ret_stack. Even though the
604 * ret stack might be bogus, we want to write the ret and the
605 * offset to find the ret_stack before we increment the stack point.
606 * If an interrupt comes in now before we increment the curr_ret_stack
607 * it may blow away what we wrote. But that's fine, because the
608 * offset will still be correct (even though the 'ret' won't be).
609 * What we worry about is the offset being correct after we increment
610 * the curr_ret_stack and before we update that offset, as if an
611 * interrupt comes in and does an unwind stack dump, it will need
612 * at least a correct offset!
613 */
614 barrier();
615 WRITE_ONCE(current->curr_ret_stack, offset + 1);
616 /*
617 * This next barrier is to ensure that an interrupt coming in
618 * will not corrupt what we are about to write.
619 */
620 barrier();
621
622 /* Still keep it reserved even if an interrupt came in */
623 current->ret_stack[offset] = val;
624
625 ret_stack->ret = ret;
626 ret_stack->func = func;
627#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
628 ret_stack->fp = frame_pointer;
629#endif
630 ret_stack->retp = retp;
631 return offset;
632}
633
634/*
635 * Not all archs define MCOUNT_INSN_SIZE which is used to look for direct
636 * functions. But those archs currently don't support direct functions
637 * anyway, and ftrace_find_rec_direct() is just a stub for them.
638 * Define MCOUNT_INSN_SIZE to keep those archs compiling.
639 */
640#ifndef MCOUNT_INSN_SIZE
641/* Make sure this only works without direct calls */
642# ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
643# error MCOUNT_INSN_SIZE not defined with direct calls enabled
644# endif
645# define MCOUNT_INSN_SIZE 0
646#endif
647
648/* If the caller does not use ftrace, call this function. */
649int function_graph_enter_regs(unsigned long ret, unsigned long func,
650 unsigned long frame_pointer, unsigned long *retp,
651 struct ftrace_regs *fregs)
652{
653 struct ftrace_graph_ent trace;
654 unsigned long bitmap = 0;
655 int offset;
656 int bit;
657 int i;
658
659 bit = ftrace_test_recursion_trylock(func, ret);
660 if (bit < 0)
661 return -EBUSY;
662
663 trace.func = func;
664 trace.depth = ++current->curr_ret_depth;
665
666 offset = ftrace_push_return_trace(ret, func, frame_pointer, retp, 0);
667 if (offset < 0)
668 goto out;
669
670#ifdef CONFIG_HAVE_STATIC_CALL
671 if (static_branch_likely(&fgraph_do_direct)) {
672 int save_curr_ret_stack = current->curr_ret_stack;
673
674 if (static_call(fgraph_func)(&trace, fgraph_direct_gops, fregs))
675 bitmap |= BIT(fgraph_direct_gops->idx);
676 else
677 /* Clear out any saved storage */
678 current->curr_ret_stack = save_curr_ret_stack;
679 } else
680#endif
681 {
682 for_each_set_bit(i, &fgraph_array_bitmask,
683 sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) {
684 struct fgraph_ops *gops = READ_ONCE(fgraph_array[i]);
685 int save_curr_ret_stack;
686
687 if (gops == &fgraph_stub)
688 continue;
689
690 save_curr_ret_stack = current->curr_ret_stack;
691 if (ftrace_ops_test(&gops->ops, func, NULL) &&
692 gops->entryfunc(&trace, gops, fregs))
693 bitmap |= BIT(i);
694 else
695 /* Clear out any saved storage */
696 current->curr_ret_stack = save_curr_ret_stack;
697 }
698 }
699
700 if (!bitmap)
701 goto out_ret;
702
703 /*
704 * Since this function uses fgraph_idx = 0 as a tail-call checking
705 * flag, set that bit always.
706 */
707 set_bitmap(current, offset, bitmap | BIT(0));
708 ftrace_test_recursion_unlock(bit);
709 return 0;
710 out_ret:
711 current->curr_ret_stack -= FGRAPH_FRAME_OFFSET + 1;
712 out:
713 current->curr_ret_depth--;
714 ftrace_test_recursion_unlock(bit);
715 return -EBUSY;
716}
717
718/* Retrieve a function return address to the trace stack on thread info.*/
719static struct ftrace_ret_stack *
720ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
721 unsigned long frame_pointer, int *offset)
722{
723 struct ftrace_ret_stack *ret_stack;
724
725 ret_stack = get_ret_stack(current, current->curr_ret_stack, offset);
726
727 if (unlikely(!ret_stack)) {
728 ftrace_graph_stop();
729 WARN(1, "Bad function graph ret_stack pointer: %d",
730 current->curr_ret_stack);
731 /* Might as well panic, otherwise we have no where to go */
732 *ret = (unsigned long)panic;
733 return NULL;
734 }
735
736#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
737 /*
738 * The arch may choose to record the frame pointer used
739 * and check it here to make sure that it is what we expect it
740 * to be. If gcc does not set the place holder of the return
741 * address in the frame pointer, and does a copy instead, then
742 * the function graph trace will fail. This test detects this
743 * case.
744 *
745 * Currently, x86_32 with optimize for size (-Os) makes the latest
746 * gcc do the above.
747 *
748 * Note, -mfentry does not use frame pointers, and this test
749 * is not needed if CC_USING_FENTRY is set.
750 */
751 if (unlikely(ret_stack->fp != frame_pointer)) {
752 ftrace_graph_stop();
753 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
754 " from func %ps return to %lx\n",
755 ret_stack->fp,
756 frame_pointer,
757 (void *)ret_stack->func,
758 ret_stack->ret);
759 *ret = (unsigned long)panic;
760 return NULL;
761 }
762#endif
763
764 *offset += FGRAPH_FRAME_OFFSET;
765 *ret = ret_stack->ret;
766 trace->func = ret_stack->func;
767 trace->overrun = atomic_read(&current->trace_overrun);
768 trace->depth = current->curr_ret_depth;
769 /*
770 * We still want to trace interrupts coming in if
771 * max_depth is set to 1. Make sure the decrement is
772 * seen before ftrace_graph_return.
773 */
774 barrier();
775
776 return ret_stack;
777}
778
779/*
780 * Hibernation protection.
781 * The state of the current task is too much unstable during
782 * suspend/restore to disk. We want to protect against that.
783 */
784static int
785ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
786 void *unused)
787{
788 switch (state) {
789 case PM_HIBERNATION_PREPARE:
790 pause_graph_tracing();
791 break;
792
793 case PM_POST_HIBERNATION:
794 unpause_graph_tracing();
795 break;
796 }
797 return NOTIFY_DONE;
798}
799
800static struct notifier_block ftrace_suspend_notifier = {
801 .notifier_call = ftrace_suspend_notifier_call,
802};
803
804/*
805 * Send the trace to the ring-buffer.
806 * @return the original return address.
807 */
808static inline unsigned long
809__ftrace_return_to_handler(struct ftrace_regs *fregs, unsigned long frame_pointer)
810{
811 struct ftrace_ret_stack *ret_stack;
812 struct ftrace_graph_ret trace;
813 unsigned long bitmap;
814 unsigned long ret;
815 int offset;
816 int bit;
817 int i;
818
819 ret_stack = ftrace_pop_return_trace(&trace, &ret, frame_pointer, &offset);
820
821 if (unlikely(!ret_stack)) {
822 ftrace_graph_stop();
823 WARN_ON(1);
824 /* Might as well panic. What else to do? */
825 return (unsigned long)panic;
826 }
827
828 if (fregs)
829 ftrace_regs_set_instruction_pointer(fregs, ret);
830
831 bit = ftrace_test_recursion_trylock(trace.func, ret);
832 /*
833 * This can fail because ftrace_test_recursion_trylock() allows one nest
834 * call. If we are already in a nested call, then we don't probe this and
835 * just return the original return address.
836 */
837 if (unlikely(bit < 0))
838 goto out;
839
840#ifdef CONFIG_FUNCTION_GRAPH_RETVAL
841 trace.retval = ftrace_regs_get_return_value(fregs);
842#endif
843
844 bitmap = get_bitmap_bits(current, offset);
845
846#ifdef CONFIG_HAVE_STATIC_CALL
847 if (!FGRAPH_NO_DIRECT && static_branch_likely(&fgraph_do_direct)) {
848 if (test_bit(fgraph_direct_gops->idx, &bitmap))
849 static_call(fgraph_retfunc)(&trace, fgraph_direct_gops, fregs);
850 } else
851#endif
852 {
853 for_each_set_bit(i, &bitmap, sizeof(bitmap) * BITS_PER_BYTE) {
854 struct fgraph_ops *gops = READ_ONCE(fgraph_array[i]);
855
856 if (gops == &fgraph_stub)
857 continue;
858
859 gops->retfunc(&trace, gops, fregs);
860 }
861 }
862
863 ftrace_test_recursion_unlock(bit);
864out:
865 /*
866 * The ftrace_graph_return() may still access the current
867 * ret_stack structure, we need to make sure the update of
868 * curr_ret_stack is after that.
869 */
870 barrier();
871 current->curr_ret_stack = offset - FGRAPH_FRAME_OFFSET;
872
873 current->curr_ret_depth--;
874 return ret;
875}
876
877/*
878 * After all architectures have selected HAVE_FUNCTION_GRAPH_FREGS, we can
879 * leave only ftrace_return_to_handler(fregs).
880 */
881#ifdef CONFIG_HAVE_FUNCTION_GRAPH_FREGS
882unsigned long ftrace_return_to_handler(struct ftrace_regs *fregs)
883{
884 return __ftrace_return_to_handler(fregs,
885 ftrace_regs_get_frame_pointer(fregs));
886}
887#else
888unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
889{
890 return __ftrace_return_to_handler(NULL, frame_pointer);
891}
892#endif
893
894/**
895 * ftrace_graph_get_ret_stack - return the entry of the shadow stack
896 * @task: The task to read the shadow stack from.
897 * @idx: Index down the shadow stack
898 *
899 * Return the ret_struct on the shadow stack of the @task at the
900 * call graph at @idx starting with zero. If @idx is zero, it
901 * will return the last saved ret_stack entry. If it is greater than
902 * zero, it will return the corresponding ret_stack for the depth
903 * of saved return addresses.
904 */
905struct ftrace_ret_stack *
906ftrace_graph_get_ret_stack(struct task_struct *task, int idx)
907{
908 struct ftrace_ret_stack *ret_stack = NULL;
909 int offset = task->curr_ret_stack;
910
911 if (offset < 0)
912 return NULL;
913
914 do {
915 ret_stack = get_ret_stack(task, offset, &offset);
916 } while (ret_stack && --idx >= 0);
917
918 return ret_stack;
919}
920
921/**
922 * ftrace_graph_top_ret_addr - return the top return address in the shadow stack
923 * @task: The task to read the shadow stack from.
924 *
925 * Return the first return address on the shadow stack of the @task, which is
926 * not the fgraph's return_to_handler.
927 */
928unsigned long ftrace_graph_top_ret_addr(struct task_struct *task)
929{
930 unsigned long return_handler = (unsigned long)dereference_kernel_function_descriptor(return_to_handler);
931 struct ftrace_ret_stack *ret_stack = NULL;
932 int offset = task->curr_ret_stack;
933
934 if (offset < 0)
935 return 0;
936
937 do {
938 ret_stack = get_ret_stack(task, offset, &offset);
939 } while (ret_stack && ret_stack->ret == return_handler);
940
941 return ret_stack ? ret_stack->ret : 0;
942}
943
944/**
945 * ftrace_graph_ret_addr - return the original value of the return address
946 * @task: The task the unwinder is being executed on
947 * @idx: An initialized pointer to the next stack index to use
948 * @ret: The current return address (likely pointing to return_handler)
949 * @retp: The address on the stack of the current return location
950 *
951 * This function can be called by stack unwinding code to convert a found stack
952 * return address (@ret) to its original value, in case the function graph
953 * tracer has modified it to be 'return_to_handler'. If the address hasn't
954 * been modified, the unchanged value of @ret is returned.
955 *
956 * @idx holds the last index used to know where to start from. It should be
957 * initialized to zero for the first iteration as that will mean to start
958 * at the top of the shadow stack. If the location is found, this pointer
959 * will be assigned that location so that if called again, it will continue
960 * where it left off.
961 *
962 * @retp is a pointer to the return address on the stack.
963 */
964unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
965 unsigned long ret, unsigned long *retp)
966{
967 struct ftrace_ret_stack *ret_stack;
968 unsigned long return_handler = (unsigned long)dereference_kernel_function_descriptor(return_to_handler);
969 int i;
970
971 if (ret != return_handler)
972 return ret;
973
974 if (!idx)
975 return ret;
976
977 i = *idx ? : task->curr_ret_stack;
978 while (i > 0) {
979 ret_stack = get_ret_stack(task, i, &i);
980 if (!ret_stack)
981 break;
982 /*
983 * For the tail-call, there would be 2 or more ftrace_ret_stacks on
984 * the ret_stack, which records "return_to_handler" as the return
985 * address except for the last one.
986 * But on the real stack, there should be 1 entry because tail-call
987 * reuses the return address on the stack and jump to the next function.
988 * Thus we will continue to find real return address.
989 */
990 if (ret_stack->retp == retp &&
991 ret_stack->ret != return_handler) {
992 *idx = i;
993 return ret_stack->ret;
994 }
995 }
996
997 return ret;
998}
999
1000static struct ftrace_ops graph_ops = {
1001 .func = ftrace_graph_func,
1002 .flags = FTRACE_OPS_GRAPH_STUB,
1003#ifdef FTRACE_GRAPH_TRAMP_ADDR
1004 .trampoline = FTRACE_GRAPH_TRAMP_ADDR,
1005 /* trampoline_size is only needed for dynamically allocated tramps */
1006#endif
1007};
1008
1009void fgraph_init_ops(struct ftrace_ops *dst_ops,
1010 struct ftrace_ops *src_ops)
1011{
1012 dst_ops->flags = FTRACE_OPS_FL_PID | FTRACE_OPS_GRAPH_STUB;
1013
1014#ifdef CONFIG_DYNAMIC_FTRACE
1015 if (src_ops) {
1016 dst_ops->func_hash = &src_ops->local_hash;
1017 mutex_init(&dst_ops->local_hash.regex_lock);
1018 INIT_LIST_HEAD(&dst_ops->subop_list);
1019 dst_ops->flags |= FTRACE_OPS_FL_INITIALIZED;
1020 dst_ops->private = src_ops->private;
1021 }
1022#endif
1023}
1024
1025/*
1026 * Simply points to ftrace_stub, but with the proper protocol.
1027 * Defined by the linker script in linux/vmlinux.lds.h
1028 */
1029void ftrace_stub_graph(struct ftrace_graph_ret *trace, struct fgraph_ops *gops,
1030 struct ftrace_regs *fregs);
1031
1032/* The callbacks that hook a function */
1033trace_func_graph_ret_t ftrace_graph_return = ftrace_stub_graph;
1034trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
1035
1036/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
1037static int alloc_retstack_tasklist(unsigned long **ret_stack_list)
1038{
1039 int i;
1040 int ret = 0;
1041 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
1042 struct task_struct *g, *t;
1043
1044 if (WARN_ON_ONCE(!fgraph_stack_cachep))
1045 return -ENOMEM;
1046
1047 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
1048 ret_stack_list[i] = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);
1049 if (!ret_stack_list[i]) {
1050 start = 0;
1051 end = i;
1052 ret = -ENOMEM;
1053 goto free;
1054 }
1055 }
1056
1057 rcu_read_lock();
1058 for_each_process_thread(g, t) {
1059 if (start == end) {
1060 ret = -EAGAIN;
1061 goto unlock;
1062 }
1063
1064 if (t->ret_stack == NULL) {
1065 atomic_set(&t->trace_overrun, 0);
1066 ret_stack_init_task_vars(ret_stack_list[start]);
1067 t->curr_ret_stack = 0;
1068 t->curr_ret_depth = -1;
1069 /* Make sure the tasks see the 0 first: */
1070 smp_wmb();
1071 t->ret_stack = ret_stack_list[start++];
1072 }
1073 }
1074
1075unlock:
1076 rcu_read_unlock();
1077free:
1078 for (i = start; i < end; i++)
1079 kmem_cache_free(fgraph_stack_cachep, ret_stack_list[i]);
1080 return ret;
1081}
1082
1083static void
1084ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
1085 struct task_struct *prev,
1086 struct task_struct *next,
1087 unsigned int prev_state)
1088{
1089 unsigned long long timestamp;
1090
1091 /*
1092 * Does the user want to count the time a function was asleep.
1093 * If so, do not update the time stamps.
1094 */
1095 if (!fgraph_no_sleep_time)
1096 return;
1097
1098 timestamp = trace_clock_local();
1099
1100 prev->ftrace_timestamp = timestamp;
1101
1102 /* only process tasks that we timestamped */
1103 if (!next->ftrace_timestamp)
1104 return;
1105
1106 next->ftrace_sleeptime += timestamp - next->ftrace_timestamp;
1107}
1108
1109static DEFINE_PER_CPU(unsigned long *, idle_ret_stack);
1110
1111static void
1112graph_init_task(struct task_struct *t, unsigned long *ret_stack)
1113{
1114 atomic_set(&t->trace_overrun, 0);
1115 ret_stack_init_task_vars(ret_stack);
1116 t->ftrace_timestamp = 0;
1117 t->curr_ret_stack = 0;
1118 t->curr_ret_depth = -1;
1119 /* make curr_ret_stack visible before we add the ret_stack */
1120 smp_wmb();
1121 t->ret_stack = ret_stack;
1122}
1123
1124/*
1125 * Allocate a return stack for the idle task. May be the first
1126 * time through, or it may be done by CPU hotplug online.
1127 */
1128void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
1129{
1130 t->curr_ret_stack = 0;
1131 t->curr_ret_depth = -1;
1132 /*
1133 * The idle task has no parent, it either has its own
1134 * stack or no stack at all.
1135 */
1136 if (t->ret_stack)
1137 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
1138
1139 if (ftrace_graph_active) {
1140 unsigned long *ret_stack;
1141
1142 if (WARN_ON_ONCE(!fgraph_stack_cachep))
1143 return;
1144
1145 ret_stack = per_cpu(idle_ret_stack, cpu);
1146 if (!ret_stack) {
1147 ret_stack = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);
1148 if (!ret_stack)
1149 return;
1150 per_cpu(idle_ret_stack, cpu) = ret_stack;
1151 }
1152 graph_init_task(t, ret_stack);
1153 }
1154}
1155
1156/* Allocate a return stack for newly created task */
1157void ftrace_graph_init_task(struct task_struct *t)
1158{
1159 /* Make sure we do not use the parent ret_stack */
1160 t->ret_stack = NULL;
1161 t->curr_ret_stack = 0;
1162 t->curr_ret_depth = -1;
1163
1164 if (ftrace_graph_active) {
1165 unsigned long *ret_stack;
1166
1167 if (WARN_ON_ONCE(!fgraph_stack_cachep))
1168 return;
1169
1170 ret_stack = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);
1171 if (!ret_stack)
1172 return;
1173 graph_init_task(t, ret_stack);
1174 }
1175}
1176
1177void ftrace_graph_exit_task(struct task_struct *t)
1178{
1179 unsigned long *ret_stack = t->ret_stack;
1180
1181 t->ret_stack = NULL;
1182 /* NULL must become visible to IRQs before we free it: */
1183 barrier();
1184
1185 if (ret_stack) {
1186 if (WARN_ON_ONCE(!fgraph_stack_cachep))
1187 return;
1188 kmem_cache_free(fgraph_stack_cachep, ret_stack);
1189 }
1190}
1191
1192#ifdef CONFIG_DYNAMIC_FTRACE
1193static int fgraph_pid_func(struct ftrace_graph_ent *trace,
1194 struct fgraph_ops *gops,
1195 struct ftrace_regs *fregs)
1196{
1197 struct trace_array *tr = gops->ops.private;
1198 int pid;
1199
1200 if (tr) {
1201 pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);
1202 if (pid == FTRACE_PID_IGNORE)
1203 return 0;
1204 if (pid != FTRACE_PID_TRACE &&
1205 pid != current->pid)
1206 return 0;
1207 }
1208
1209 return gops->saved_func(trace, gops, fregs);
1210}
1211
1212void fgraph_update_pid_func(void)
1213{
1214 struct fgraph_ops *gops;
1215 struct ftrace_ops *op;
1216
1217 if (!(graph_ops.flags & FTRACE_OPS_FL_INITIALIZED))
1218 return;
1219
1220 list_for_each_entry(op, &graph_ops.subop_list, list) {
1221 if (op->flags & FTRACE_OPS_FL_PID) {
1222 gops = container_of(op, struct fgraph_ops, ops);
1223 gops->entryfunc = ftrace_pids_enabled(op) ?
1224 fgraph_pid_func : gops->saved_func;
1225 if (ftrace_graph_active == 1)
1226 static_call_update(fgraph_func, gops->entryfunc);
1227 }
1228 }
1229}
1230#endif
1231
1232/* Allocate a return stack for each task */
1233static int start_graph_tracing(void)
1234{
1235 unsigned long **ret_stack_list;
1236 int ret, cpu;
1237
1238 ret_stack_list = kcalloc(FTRACE_RETSTACK_ALLOC_SIZE,
1239 sizeof(*ret_stack_list), GFP_KERNEL);
1240
1241 if (!ret_stack_list)
1242 return -ENOMEM;
1243
1244 /* The cpu_boot init_task->ret_stack will never be freed */
1245 for_each_online_cpu(cpu) {
1246 if (!idle_task(cpu)->ret_stack)
1247 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
1248 }
1249
1250 do {
1251 ret = alloc_retstack_tasklist(ret_stack_list);
1252 } while (ret == -EAGAIN);
1253
1254 if (!ret) {
1255 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
1256 if (ret)
1257 pr_info("ftrace_graph: Couldn't activate tracepoint"
1258 " probe to kernel_sched_switch\n");
1259 }
1260
1261 kfree(ret_stack_list);
1262 return ret;
1263}
1264
1265static void init_task_vars(int idx)
1266{
1267 struct task_struct *g, *t;
1268 int cpu;
1269
1270 for_each_online_cpu(cpu) {
1271 if (idle_task(cpu)->ret_stack)
1272 ret_stack_set_task_var(idle_task(cpu), idx, 0);
1273 }
1274
1275 read_lock(&tasklist_lock);
1276 for_each_process_thread(g, t) {
1277 if (t->ret_stack)
1278 ret_stack_set_task_var(t, idx, 0);
1279 }
1280 read_unlock(&tasklist_lock);
1281}
1282
1283static void ftrace_graph_enable_direct(bool enable_branch, struct fgraph_ops *gops)
1284{
1285 trace_func_graph_ent_t func = NULL;
1286 trace_func_graph_ret_t retfunc = NULL;
1287 int i;
1288
1289 if (FGRAPH_NO_DIRECT)
1290 return;
1291
1292 if (gops) {
1293 func = gops->entryfunc;
1294 retfunc = gops->retfunc;
1295 fgraph_direct_gops = gops;
1296 } else {
1297 for_each_set_bit(i, &fgraph_array_bitmask,
1298 sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) {
1299 func = fgraph_array[i]->entryfunc;
1300 retfunc = fgraph_array[i]->retfunc;
1301 fgraph_direct_gops = fgraph_array[i];
1302 }
1303 }
1304 if (WARN_ON_ONCE(!func))
1305 return;
1306
1307 static_call_update(fgraph_func, func);
1308 static_call_update(fgraph_retfunc, retfunc);
1309 if (enable_branch)
1310 static_branch_enable(&fgraph_do_direct);
1311}
1312
1313static void ftrace_graph_disable_direct(bool disable_branch)
1314{
1315 if (FGRAPH_NO_DIRECT)
1316 return;
1317
1318 if (disable_branch)
1319 static_branch_disable(&fgraph_do_direct);
1320 static_call_update(fgraph_func, ftrace_graph_entry_stub);
1321 static_call_update(fgraph_retfunc, ftrace_graph_ret_stub);
1322 fgraph_direct_gops = &fgraph_stub;
1323}
1324
1325/* The cpu_boot init_task->ret_stack will never be freed */
1326static int fgraph_cpu_init(unsigned int cpu)
1327{
1328 if (!idle_task(cpu)->ret_stack)
1329 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
1330 return 0;
1331}
1332
1333int register_ftrace_graph(struct fgraph_ops *gops)
1334{
1335 static bool fgraph_initialized;
1336 int command = 0;
1337 int ret = 0;
1338 int i = -1;
1339
1340 if (WARN_ONCE(gops->ops.flags & FTRACE_OPS_FL_GRAPH,
1341 "function graph ops registered again"))
1342 return -EBUSY;
1343
1344 guard(mutex)(&ftrace_lock);
1345
1346 if (!fgraph_stack_cachep) {
1347 fgraph_stack_cachep = kmem_cache_create("fgraph_stack",
1348 SHADOW_STACK_SIZE,
1349 SHADOW_STACK_SIZE, 0, NULL);
1350 if (!fgraph_stack_cachep)
1351 return -ENOMEM;
1352 }
1353
1354 if (!fgraph_initialized) {
1355 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "fgraph:online",
1356 fgraph_cpu_init, NULL);
1357 if (ret < 0) {
1358 pr_warn("fgraph: Error to init cpu hotplug support\n");
1359 return ret;
1360 }
1361 fgraph_initialized = true;
1362 ret = 0;
1363 }
1364
1365 if (!fgraph_array[0]) {
1366 /* The array must always have real data on it */
1367 for (i = 0; i < FGRAPH_ARRAY_SIZE; i++)
1368 fgraph_array[i] = &fgraph_stub;
1369 fgraph_lru_init();
1370 }
1371
1372 i = fgraph_lru_alloc_index();
1373 if (i < 0 || WARN_ON_ONCE(fgraph_array[i] != &fgraph_stub))
1374 return -ENOSPC;
1375 gops->idx = i;
1376
1377 ftrace_graph_active++;
1378
1379 /* Always save the function, and reset at unregistering */
1380 gops->saved_func = gops->entryfunc;
1381#ifdef CONFIG_DYNAMIC_FTRACE
1382 if (ftrace_pids_enabled(&gops->ops))
1383 gops->entryfunc = fgraph_pid_func;
1384#endif
1385
1386 if (ftrace_graph_active == 2)
1387 ftrace_graph_disable_direct(true);
1388
1389 if (ftrace_graph_active == 1) {
1390 ftrace_graph_enable_direct(false, gops);
1391 register_pm_notifier(&ftrace_suspend_notifier);
1392 ret = start_graph_tracing();
1393 if (ret)
1394 goto error;
1395 /*
1396 * Some archs just test to see if these are not
1397 * the default function
1398 */
1399 ftrace_graph_return = return_run;
1400 ftrace_graph_entry = entry_run;
1401 command = FTRACE_START_FUNC_RET;
1402 } else {
1403 init_task_vars(gops->idx);
1404 }
1405
1406 gops->ops.flags |= FTRACE_OPS_FL_GRAPH;
1407
1408 ret = ftrace_startup_subops(&graph_ops, &gops->ops, command);
1409 if (!ret)
1410 fgraph_array[i] = gops;
1411
1412error:
1413 if (ret) {
1414 ftrace_graph_active--;
1415 gops->saved_func = NULL;
1416 fgraph_lru_release_index(i);
1417 if (!ftrace_graph_active)
1418 unregister_pm_notifier(&ftrace_suspend_notifier);
1419 }
1420 return ret;
1421}
1422
1423void unregister_ftrace_graph(struct fgraph_ops *gops)
1424{
1425 int command = 0;
1426
1427 if (WARN_ONCE(!(gops->ops.flags & FTRACE_OPS_FL_GRAPH),
1428 "function graph ops unregistered without registering"))
1429 return;
1430
1431 guard(mutex)(&ftrace_lock);
1432
1433 if (unlikely(!ftrace_graph_active))
1434 goto out;
1435
1436 if (unlikely(gops->idx < 0 || gops->idx >= FGRAPH_ARRAY_SIZE ||
1437 fgraph_array[gops->idx] != gops))
1438 goto out;
1439
1440 if (fgraph_lru_release_index(gops->idx) < 0)
1441 goto out;
1442
1443 fgraph_array[gops->idx] = &fgraph_stub;
1444
1445 ftrace_graph_active--;
1446
1447 if (!ftrace_graph_active)
1448 command = FTRACE_STOP_FUNC_RET;
1449
1450 ftrace_shutdown_subops(&graph_ops, &gops->ops, command);
1451
1452 if (ftrace_graph_active == 1)
1453 ftrace_graph_enable_direct(true, NULL);
1454 else if (!ftrace_graph_active)
1455 ftrace_graph_disable_direct(false);
1456
1457 if (!ftrace_graph_active) {
1458 ftrace_graph_return = ftrace_stub_graph;
1459 ftrace_graph_entry = ftrace_graph_entry_stub;
1460 unregister_pm_notifier(&ftrace_suspend_notifier);
1461 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
1462 }
1463 gops->saved_func = NULL;
1464 out:
1465 gops->ops.flags &= ~FTRACE_OPS_FL_GRAPH;
1466}