projects/linux·blob master · refs

330 lines · 8.1 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Performance events callchain code, extracted from core.c:
4 *
5 * Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <[email protected]>
6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8 * Copyright © 2009 Paul Mackerras, IBM Corp. <[email protected]>
9 */
10
11#include <linux/perf_event.h>
12#include <linux/slab.h>
13#include <linux/sched/task_stack.h>
14#include <linux/uprobes.h>
15
16#include "internal.h"
17
18struct callchain_cpus_entries {
19 struct rcu_head rcu_head;
20 struct perf_callchain_entry *cpu_entries[];
21};
22
23int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
24int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;
25static const int six_hundred_forty_kb = 640 * 1024;
26
27static inline size_t perf_callchain_entry__sizeof(void)
28{
29 return (sizeof(struct perf_callchain_entry) +
30 sizeof(__u64) * (sysctl_perf_event_max_stack +
31 sysctl_perf_event_max_contexts_per_stack));
32}
33
34static DEFINE_PER_CPU(u8, callchain_recursion[PERF_NR_CONTEXTS]);
35static atomic_t nr_callchain_events;
36static DEFINE_MUTEX(callchain_mutex);
37static struct callchain_cpus_entries *callchain_cpus_entries;
38
39
40__weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
41 struct pt_regs *regs)
42{
43}
44
45__weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
46 struct pt_regs *regs)
47{
48}
49
50static void release_callchain_buffers_rcu(struct rcu_head *head)
51{
52 struct callchain_cpus_entries *entries;
53 int cpu;
54
55 entries = container_of(head, struct callchain_cpus_entries, rcu_head);
56
57 for_each_possible_cpu(cpu)
58 kfree(entries->cpu_entries[cpu]);
59
60 kfree(entries);
61}
62
63static void release_callchain_buffers(void)
64{
65 struct callchain_cpus_entries *entries;
66
67 entries = callchain_cpus_entries;
68 RCU_INIT_POINTER(callchain_cpus_entries, NULL);
69 call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
70}
71
72static int alloc_callchain_buffers(void)
73{
74 int cpu;
75 int size;
76 struct callchain_cpus_entries *entries;
77
78 /*
79 * We can't use the percpu allocation API for data that can be
80 * accessed from NMI. Use a temporary manual per cpu allocation
81 * until that gets sorted out.
82 */
83 size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
84
85 entries = kzalloc(size, GFP_KERNEL);
86 if (!entries)
87 return -ENOMEM;
88
89 size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;
90
91 for_each_possible_cpu(cpu) {
92 entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
93 cpu_to_node(cpu));
94 if (!entries->cpu_entries[cpu])
95 goto fail;
96 }
97
98 rcu_assign_pointer(callchain_cpus_entries, entries);
99
100 return 0;
101
102fail:
103 for_each_possible_cpu(cpu)
104 kfree(entries->cpu_entries[cpu]);
105 kfree(entries);
106
107 return -ENOMEM;
108}
109
110int get_callchain_buffers(int event_max_stack)
111{
112 int err = 0;
113 int count;
114
115 mutex_lock(&callchain_mutex);
116
117 count = atomic_inc_return(&nr_callchain_events);
118 if (WARN_ON_ONCE(count < 1)) {
119 err = -EINVAL;
120 goto exit;
121 }
122
123 /*
124 * If requesting per event more than the global cap,
125 * return a different error to help userspace figure
126 * this out.
127 *
128 * And also do it here so that we have &callchain_mutex held.
129 */
130 if (event_max_stack > sysctl_perf_event_max_stack) {
131 err = -EOVERFLOW;
132 goto exit;
133 }
134
135 if (count == 1)
136 err = alloc_callchain_buffers();
137exit:
138 if (err)
139 atomic_dec(&nr_callchain_events);
140
141 mutex_unlock(&callchain_mutex);
142
143 return err;
144}
145
146void put_callchain_buffers(void)
147{
148 if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
149 release_callchain_buffers();
150 mutex_unlock(&callchain_mutex);
151 }
152}
153
154struct perf_callchain_entry *get_callchain_entry(int *rctx)
155{
156 int cpu;
157 struct callchain_cpus_entries *entries;
158
159 *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
160 if (*rctx == -1)
161 return NULL;
162
163 entries = rcu_dereference(callchain_cpus_entries);
164 if (!entries) {
165 put_recursion_context(this_cpu_ptr(callchain_recursion), *rctx);
166 return NULL;
167 }
168
169 cpu = smp_processor_id();
170
171 return (((void *)entries->cpu_entries[cpu]) +
172 (*rctx * perf_callchain_entry__sizeof()));
173}
174
175void
176put_callchain_entry(int rctx)
177{
178 put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
179}
180
181static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entry,
182 int start_entry_idx)
183{
184#ifdef CONFIG_UPROBES
185 struct uprobe_task *utask = current->utask;
186 struct return_instance *ri;
187 __u64 *cur_ip, *last_ip, tramp_addr;
188
189 if (likely(!utask || !utask->return_instances))
190 return;
191
192 cur_ip = &entry->ip[start_entry_idx];
193 last_ip = &entry->ip[entry->nr - 1];
194 ri = utask->return_instances;
195 tramp_addr = uprobe_get_trampoline_vaddr();
196
197 /*
198 * If there are pending uretprobes for the current thread, they are
199 * recorded in a list inside utask->return_instances; each such
200 * pending uretprobe replaces traced user function's return address on
201 * the stack, so when stack trace is captured, instead of seeing
202 * actual function's return address, we'll have one or many uretprobe
203 * trampoline addresses in the stack trace, which are not helpful and
204 * misleading to users.
205 * So here we go over the pending list of uretprobes, and each
206 * encountered trampoline address is replaced with actual return
207 * address.
208 */
209 while (ri && cur_ip <= last_ip) {
210 if (*cur_ip == tramp_addr) {
211 *cur_ip = ri->orig_ret_vaddr;
212 ri = ri->next;
213 }
214 cur_ip++;
215 }
216#endif
217}
218
219struct perf_callchain_entry *
220get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
221 u32 max_stack, bool crosstask, bool add_mark, u64 defer_cookie)
222{
223 struct perf_callchain_entry *entry;
224 struct perf_callchain_entry_ctx ctx;
225 int rctx, start_entry_idx;
226
227 /* crosstask is not supported for user stacks */
228 if (crosstask && user && !kernel)
229 return NULL;
230
231 entry = get_callchain_entry(&rctx);
232 if (!entry)
233 return NULL;
234
235 ctx.entry = entry;
236 ctx.max_stack = max_stack;
237 ctx.nr = entry->nr = 0;
238 ctx.contexts = 0;
239 ctx.contexts_maxed = false;
240
241 if (kernel && !user_mode(regs)) {
242 if (add_mark)
243 perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
244 perf_callchain_kernel(&ctx, regs);
245 }
246
247 if (user && !crosstask) {
248 if (!user_mode(regs)) {
249 if (!is_user_task(current))
250 goto exit_put;
251 regs = task_pt_regs(current);
252 }
253
254 if (defer_cookie) {
255 /*
256 * Foretell the coming of PERF_RECORD_CALLCHAIN_DEFERRED
257 * which can be stitched to this one, and add
258 * the cookie after it (it will be cut off when the
259 * user stack is copied to the callchain).
260 */
261 perf_callchain_store_context(&ctx, PERF_CONTEXT_USER_DEFERRED);
262 perf_callchain_store_context(&ctx, defer_cookie);
263 goto exit_put;
264 }
265
266 if (add_mark)
267 perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
268
269 start_entry_idx = entry->nr;
270 perf_callchain_user(&ctx, regs);
271 fixup_uretprobe_trampoline_entries(entry, start_entry_idx);
272 }
273
274exit_put:
275 put_callchain_entry(rctx);
276
277 return entry;
278}
279
280static int perf_event_max_stack_handler(const struct ctl_table *table, int write,
281 void *buffer, size_t *lenp, loff_t *ppos)
282{
283 int *value = table->data;
284 int new_value = *value, ret;
285 struct ctl_table new_table = *table;
286
287 new_table.data = &new_value;
288 ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
289 if (ret || !write)
290 return ret;
291
292 mutex_lock(&callchain_mutex);
293 if (atomic_read(&nr_callchain_events))
294 ret = -EBUSY;
295 else
296 *value = new_value;
297
298 mutex_unlock(&callchain_mutex);
299
300 return ret;
301}
302
303static const struct ctl_table callchain_sysctl_table[] = {
304 {
305 .procname = "perf_event_max_stack",
306 .data = &sysctl_perf_event_max_stack,
307 .maxlen = sizeof(sysctl_perf_event_max_stack),
308 .mode = 0644,
309 .proc_handler = perf_event_max_stack_handler,
310 .extra1 = SYSCTL_ZERO,
311 .extra2 = (void *)&six_hundred_forty_kb,
312 },
313 {
314 .procname = "perf_event_max_contexts_per_stack",
315 .data = &sysctl_perf_event_max_contexts_per_stack,
316 .maxlen = sizeof(sysctl_perf_event_max_contexts_per_stack),
317 .mode = 0644,
318 .proc_handler = perf_event_max_stack_handler,
319 .extra1 = SYSCTL_ZERO,
320 .extra2 = SYSCTL_ONE_THOUSAND,
321 },
322};
323
324static int __init init_callchain_sysctls(void)
325{
326 register_sysctl_init("kernel", callchain_sysctl_table);
327 return 0;
328}
329core_initcall(init_callchain_sysctls);
330