projects/linux·blob master · refs

446 lines · 9.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * unlikely profiler
4 *
5 * Copyright (C) 2008 Steven Rostedt <[email protected]>
6 */
7#include <linux/kallsyms.h>
8#include <linux/seq_file.h>
9#include <linux/spinlock.h>
10#include <linux/irqflags.h>
11#include <linux/uaccess.h>
12#include <linux/module.h>
13#include <linux/ftrace.h>
14#include <linux/hash.h>
15#include <linux/fs.h>
16#include <asm/local.h>
17
18#include "trace.h"
19#include "trace_stat.h"
20#include "trace_output.h"
21
22#ifdef CONFIG_BRANCH_TRACER
23
24static struct tracer branch_trace;
25static int branch_tracing_enabled __read_mostly;
26static DEFINE_MUTEX(branch_tracing_mutex);
27
28static struct trace_array *branch_tracer;
29
30static void
31probe_likely_condition(struct ftrace_likely_data *f, int val, int expect)
32{
33 struct trace_array *tr = branch_tracer;
34 struct trace_buffer *buffer;
35 struct ring_buffer_event *event;
36 struct trace_branch *entry;
37 unsigned long flags;
38 unsigned int trace_ctx;
39 const char *p;
40
41 if (current->trace_recursion & TRACE_BRANCH_BIT)
42 return;
43
44 /*
45 * I would love to save just the ftrace_likely_data pointer, but
46 * this code can also be used by modules. Ugly things can happen
47 * if the module is unloaded, and then we go and read the
48 * pointer. This is slower, but much safer.
49 */
50
51 if (unlikely(!tr))
52 return;
53
54 raw_local_irq_save(flags);
55 current->trace_recursion |= TRACE_BRANCH_BIT;
56 if (!tracer_tracing_is_on_cpu(tr, raw_smp_processor_id()))
57 goto out;
58
59 trace_ctx = tracing_gen_ctx_flags(flags);
60 buffer = tr->array_buffer.buffer;
61 event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH,
62 sizeof(*entry), trace_ctx);
63 if (!event)
64 goto out;
65
66 entry = ring_buffer_event_data(event);
67
68 /* Strip off the path, only save the file */
69 p = f->data.file + strlen(f->data.file);
70 while (p >= f->data.file && *p != '/')
71 p--;
72 p++;
73
74 strscpy(entry->func, f->data.func);
75 strscpy(entry->file, p);
76 entry->constant = f->constant;
77 entry->line = f->data.line;
78 entry->correct = val == expect;
79
80 trace_buffer_unlock_commit_nostack(buffer, event);
81
82 out:
83 current->trace_recursion &= ~TRACE_BRANCH_BIT;
84 raw_local_irq_restore(flags);
85}
86
87static inline
88void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
89{
90 if (!branch_tracing_enabled)
91 return;
92
93 probe_likely_condition(f, val, expect);
94}
95
96int enable_branch_tracing(struct trace_array *tr)
97{
98 mutex_lock(&branch_tracing_mutex);
99 branch_tracer = tr;
100 /*
101 * Must be seen before enabling. The reader is a condition
102 * where we do not need a matching rmb()
103 */
104 smp_wmb();
105 branch_tracing_enabled++;
106 mutex_unlock(&branch_tracing_mutex);
107
108 return 0;
109}
110
111void disable_branch_tracing(void)
112{
113 mutex_lock(&branch_tracing_mutex);
114
115 if (!branch_tracing_enabled)
116 goto out_unlock;
117
118 branch_tracing_enabled--;
119
120 out_unlock:
121 mutex_unlock(&branch_tracing_mutex);
122}
123
124static int branch_trace_init(struct trace_array *tr)
125{
126 return enable_branch_tracing(tr);
127}
128
129static void branch_trace_reset(struct trace_array *tr)
130{
131 disable_branch_tracing();
132}
133
134static enum print_line_t trace_branch_print(struct trace_iterator *iter,
135 int flags, struct trace_event *event)
136{
137 struct trace_branch *field;
138
139 trace_assign_type(field, iter->ent);
140
141 trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
142 field->correct ? " ok " : " MISS ",
143 field->func,
144 field->file,
145 field->line);
146
147 return trace_handle_return(&iter->seq);
148}
149
150static void branch_print_header(struct seq_file *s)
151{
152 seq_puts(s, "# TASK-PID CPU# TIMESTAMP CORRECT"
153 " FUNC:FILE:LINE\n"
154 "# | | | | | "
155 " |\n");
156}
157
158static struct trace_event_functions trace_branch_funcs = {
159 .trace = trace_branch_print,
160};
161
162static struct trace_event trace_branch_event = {
163 .type = TRACE_BRANCH,
164 .funcs = &trace_branch_funcs,
165};
166
167static struct tracer branch_trace __read_mostly =
168{
169 .name = "branch",
170 .init = branch_trace_init,
171 .reset = branch_trace_reset,
172#ifdef CONFIG_FTRACE_SELFTEST
173 .selftest = trace_selftest_startup_branch,
174#endif /* CONFIG_FTRACE_SELFTEST */
175 .print_header = branch_print_header,
176};
177
178__init static int init_branch_tracer(void)
179{
180 int ret;
181
182 ret = register_trace_event(&trace_branch_event);
183 if (!ret) {
184 pr_warn("Warning: could not register branch events\n");
185 return 1;
186 }
187 return register_tracer(&branch_trace);
188}
189core_initcall(init_branch_tracer);
190
191#else
192static inline
193void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
194{
195}
196#endif /* CONFIG_BRANCH_TRACER */
197
198void ftrace_likely_update(struct ftrace_likely_data *f, int val,
199 int expect, int is_constant)
200{
201 unsigned long flags = user_access_save();
202
203 /* A constant is always correct */
204 if (is_constant) {
205 f->constant++;
206 val = expect;
207 }
208 /*
209 * I would love to have a trace point here instead, but the
210 * trace point code is so inundated with unlikely and likely
211 * conditions that the recursive nightmare that exists is too
212 * much to try to get working. At least for now.
213 */
214 trace_likely_condition(f, val, expect);
215
216 /* FIXME: Make this atomic! */
217 if (val == expect)
218 f->data.correct++;
219 else
220 f->data.incorrect++;
221
222 user_access_restore(flags);
223}
224EXPORT_SYMBOL(ftrace_likely_update);
225
226extern unsigned long __start_annotated_branch_profile[];
227extern unsigned long __stop_annotated_branch_profile[];
228
229static int annotated_branch_stat_headers(struct seq_file *m)
230{
231 seq_puts(m, " correct incorrect % "
232 " Function "
233 " File Line\n"
234 " ------- --------- - "
235 " -------- "
236 " ---- ----\n");
237 return 0;
238}
239
240static inline long get_incorrect_percent(const struct ftrace_branch_data *p)
241{
242 long percent;
243
244 if (p->correct) {
245 percent = p->incorrect * 100;
246 percent /= p->correct + p->incorrect;
247 } else
248 percent = p->incorrect ? 100 : -1;
249
250 return percent;
251}
252
253static const char *branch_stat_process_file(struct ftrace_branch_data *p)
254{
255 const char *f;
256
257 /* Only print the file, not the path */
258 f = p->file + strlen(p->file);
259 while (f >= p->file && *f != '/')
260 f--;
261 return ++f;
262}
263
264static void branch_stat_show(struct seq_file *m,
265 struct ftrace_branch_data *p, const char *f)
266{
267 long percent;
268
269 /*
270 * The miss is overlayed on correct, and hit on incorrect.
271 */
272 percent = get_incorrect_percent(p);
273
274 if (percent < 0)
275 seq_puts(m, " X ");
276 else
277 seq_printf(m, "%3ld ", percent);
278
279 seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
280}
281
282static int branch_stat_show_normal(struct seq_file *m,
283 struct ftrace_branch_data *p, const char *f)
284{
285 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect);
286 branch_stat_show(m, p, f);
287 return 0;
288}
289
290static int annotate_branch_stat_show(struct seq_file *m, void *v)
291{
292 struct ftrace_likely_data *p = v;
293 const char *f;
294 int l;
295
296 f = branch_stat_process_file(&p->data);
297
298 if (!p->constant)
299 return branch_stat_show_normal(m, &p->data, f);
300
301 l = snprintf(NULL, 0, "/%lu", p->constant);
302 l = l > 8 ? 0 : 8 - l;
303
304 seq_printf(m, "%8lu/%lu %*lu ",
305 p->data.correct, p->constant, l, p->data.incorrect);
306 branch_stat_show(m, &p->data, f);
307 return 0;
308}
309
310static void *annotated_branch_stat_start(struct tracer_stat *trace)
311{
312 return __start_annotated_branch_profile;
313}
314
315static void *
316annotated_branch_stat_next(void *v, int idx)
317{
318 struct ftrace_likely_data *p = v;
319
320 ++p;
321
322 if ((void *)p >= (void *)__stop_annotated_branch_profile)
323 return NULL;
324
325 return p;
326}
327
328static int annotated_branch_stat_cmp(const void *p1, const void *p2)
329{
330 const struct ftrace_branch_data *a = p1;
331 const struct ftrace_branch_data *b = p2;
332
333 long percent_a, percent_b;
334
335 percent_a = get_incorrect_percent(a);
336 percent_b = get_incorrect_percent(b);
337
338 if (percent_a < percent_b)
339 return -1;
340 if (percent_a > percent_b)
341 return 1;
342
343 if (a->incorrect < b->incorrect)
344 return -1;
345 if (a->incorrect > b->incorrect)
346 return 1;
347
348 /*
349 * Since the above shows worse (incorrect) cases
350 * first, we continue that by showing best (correct)
351 * cases last.
352 */
353 if (a->correct > b->correct)
354 return -1;
355 if (a->correct < b->correct)
356 return 1;
357
358 return 0;
359}
360
361static struct tracer_stat annotated_branch_stats = {
362 .name = "branch_annotated",
363 .stat_start = annotated_branch_stat_start,
364 .stat_next = annotated_branch_stat_next,
365 .stat_cmp = annotated_branch_stat_cmp,
366 .stat_headers = annotated_branch_stat_headers,
367 .stat_show = annotate_branch_stat_show
368};
369
370__init static int init_annotated_branch_stats(void)
371{
372 int ret;
373
374 ret = register_stat_tracer(&annotated_branch_stats);
375 if (ret) {
376 pr_warn("Warning: could not register annotated branches stats\n");
377 return ret;
378 }
379 return 0;
380}
381fs_initcall(init_annotated_branch_stats);
382
383#ifdef CONFIG_PROFILE_ALL_BRANCHES
384
385extern unsigned long __start_branch_profile[];
386extern unsigned long __stop_branch_profile[];
387
388static int all_branch_stat_headers(struct seq_file *m)
389{
390 seq_puts(m, " miss hit % "
391 " Function "
392 " File Line\n"
393 " ------- --------- - "
394 " -------- "
395 " ---- ----\n");
396 return 0;
397}
398
399static void *all_branch_stat_start(struct tracer_stat *trace)
400{
401 return __start_branch_profile;
402}
403
404static void *
405all_branch_stat_next(void *v, int idx)
406{
407 struct ftrace_branch_data *p = v;
408
409 ++p;
410
411 if ((void *)p >= (void *)__stop_branch_profile)
412 return NULL;
413
414 return p;
415}
416
417static int all_branch_stat_show(struct seq_file *m, void *v)
418{
419 struct ftrace_branch_data *p = v;
420 const char *f;
421
422 f = branch_stat_process_file(p);
423 return branch_stat_show_normal(m, p, f);
424}
425
426static struct tracer_stat all_branch_stats = {
427 .name = "branch_all",
428 .stat_start = all_branch_stat_start,
429 .stat_next = all_branch_stat_next,
430 .stat_headers = all_branch_stat_headers,
431 .stat_show = all_branch_stat_show
432};
433
434__init static int all_annotated_branch_stats(void)
435{
436 int ret;
437
438 ret = register_stat_tracer(&all_branch_stats);
439 if (ret) {
440 pr_warn("Warning: could not register all branches stats\n");
441 return ret;
442 }
443 return 0;
444}
445fs_initcall(all_annotated_branch_stats);
446#endif /* CONFIG_PROFILE_ALL_BRANCHES */