projects/linux·blob assoc-array-fixes-v2 · refs

10120 lines · 249.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * ring buffer based function tracer
4 *
5 * Copyright (C) 2007-2012 Steven Rostedt <[email protected]>
6 * Copyright (C) 2008 Ingo Molnar <[email protected]>
7 *
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <[email protected]>
10 *
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
14 */
15#include <linux/ring_buffer.h>
16#include <linux/utsname.h>
17#include <linux/stacktrace.h>
18#include <linux/writeback.h>
19#include <linux/kallsyms.h>
20#include <linux/security.h>
21#include <linux/seq_file.h>
22#include <linux/irqflags.h>
23#include <linux/syscalls.h>
24#include <linux/debugfs.h>
25#include <linux/tracefs.h>
26#include <linux/pagemap.h>
27#include <linux/hardirq.h>
28#include <linux/linkage.h>
29#include <linux/uaccess.h>
30#include <linux/cleanup.h>
31#include <linux/vmalloc.h>
32#include <linux/ftrace.h>
33#include <linux/module.h>
34#include <linux/percpu.h>
35#include <linux/splice.h>
36#include <linux/kdebug.h>
37#include <linux/string.h>
38#include <linux/mount.h>
39#include <linux/rwsem.h>
40#include <linux/slab.h>
41#include <linux/ctype.h>
42#include <linux/init.h>
43#include <linux/panic_notifier.h>
44#include <linux/poll.h>
45#include <linux/nmi.h>
46#include <linux/fs.h>
47#include <linux/trace.h>
48#include <linux/sched/clock.h>
49#include <linux/sched/rt.h>
50#include <linux/irq_work.h>
51#include <linux/workqueue.h>
52#include <linux/sort.h>
53#include <linux/io.h> /* vmap_page_range() */
54#include <linux/fs_context.h>
55
56#include <asm/setup.h> /* COMMAND_LINE_SIZE */
57
58#include "trace.h"
59#include "trace_output.h"
60
61#ifdef CONFIG_FTRACE_STARTUP_TEST
62/*
63 * We need to change this state when a selftest is running.
64 * A selftest will lurk into the ring-buffer to count the
65 * entries inserted during the selftest although some concurrent
66 * insertions into the ring-buffer such as trace_printk could occurred
67 * at the same time, giving false positive or negative results.
68 */
69bool __read_mostly tracing_selftest_running;
70
71/*
72 * If boot-time tracing including tracers/events via kernel cmdline
73 * is running, we do not want to run SELFTEST.
74 */
75bool __read_mostly tracing_selftest_disabled;
76
77void __init disable_tracing_selftest(const char *reason)
78{
79 if (!tracing_selftest_disabled) {
80 tracing_selftest_disabled = true;
81 pr_info("Ftrace startup test is disabled due to %s\n", reason);
82 }
83}
84#else
85#define tracing_selftest_disabled 0
86#endif
87
88/* Pipe tracepoints to printk */
89static struct trace_iterator *tracepoint_print_iter;
90static int tracepoint_printk;
91static bool tracepoint_printk_stop_on_boot __initdata;
92static bool traceoff_after_boot __initdata;
93static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
94
95/* Store tracers and their flags per instance */
96struct tracers {
97 struct list_head list;
98 struct tracer *tracer;
99 struct tracer_flags *flags;
100};
101
102/*
103 * To prevent the comm cache from being overwritten when no
104 * tracing is active, only save the comm when a trace event
105 * occurred.
106 */
107DEFINE_PER_CPU(bool, trace_taskinfo_save);
108
109/*
110 * Kill all tracing for good (never come back).
111 * It is initialized to 1 but will turn to zero if the initialization
112 * of the tracer is successful. But that is the only place that sets
113 * this back to zero.
114 */
115int tracing_disabled = 1;
116
117cpumask_var_t __read_mostly tracing_buffer_mask;
118
119#define MAX_TRACER_SIZE 100
120/*
121 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
122 *
123 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
124 * is set, then ftrace_dump is called. This will output the contents
125 * of the ftrace buffers to the console. This is very useful for
126 * capturing traces that lead to crashes and outputting it to a
127 * serial console.
128 *
129 * It is default off, but you can enable it with either specifying
130 * "ftrace_dump_on_oops" in the kernel command line, or setting
131 * /proc/sys/kernel/ftrace_dump_on_oops
132 * Set 1 if you want to dump buffers of all CPUs
133 * Set 2 if you want to dump the buffer of the CPU that triggered oops
134 * Set instance name if you want to dump the specific trace instance
135 * Multiple instance dump is also supported, and instances are separated
136 * by commas.
137 */
138/* Set to string format zero to disable by default */
139static char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0";
140
141/* When set, tracing will stop when a WARN*() is hit */
142static int __disable_trace_on_warning;
143
144int tracepoint_printk_sysctl(const struct ctl_table *table, int write,
145 void *buffer, size_t *lenp, loff_t *ppos);
146static const struct ctl_table trace_sysctl_table[] = {
147 {
148 .procname = "ftrace_dump_on_oops",
149 .data = &ftrace_dump_on_oops,
150 .maxlen = MAX_TRACER_SIZE,
151 .mode = 0644,
152 .proc_handler = proc_dostring,
153 },
154 {
155 .procname = "traceoff_on_warning",
156 .data = &__disable_trace_on_warning,
157 .maxlen = sizeof(__disable_trace_on_warning),
158 .mode = 0644,
159 .proc_handler = proc_dointvec,
160 },
161 {
162 .procname = "tracepoint_printk",
163 .data = &tracepoint_printk,
164 .maxlen = sizeof(tracepoint_printk),
165 .mode = 0644,
166 .proc_handler = tracepoint_printk_sysctl,
167 },
168};
169
170static int __init init_trace_sysctls(void)
171{
172 register_sysctl_init("kernel", trace_sysctl_table);
173 return 0;
174}
175subsys_initcall(init_trace_sysctls);
176
177#ifdef CONFIG_TRACE_EVAL_MAP_FILE
178/* Map of enums to their values, for "eval_map" file */
179struct trace_eval_map_head {
180 struct module *mod;
181 unsigned long length;
182};
183
184union trace_eval_map_item;
185
186struct trace_eval_map_tail {
187 /*
188 * "end" is first and points to NULL as it must be different
189 * than "mod" or "eval_string"
190 */
191 union trace_eval_map_item *next;
192 const char *end; /* points to NULL */
193};
194
195static DEFINE_MUTEX(trace_eval_mutex);
196
197/*
198 * The trace_eval_maps are saved in an array with two extra elements,
199 * one at the beginning, and one at the end. The beginning item contains
200 * the count of the saved maps (head.length), and the module they
201 * belong to if not built in (head.mod). The ending item contains a
202 * pointer to the next array of saved eval_map items.
203 */
204union trace_eval_map_item {
205 struct trace_eval_map map;
206 struct trace_eval_map_head head;
207 struct trace_eval_map_tail tail;
208};
209
210static union trace_eval_map_item *trace_eval_maps;
211#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
212
213int tracing_set_tracer(struct trace_array *tr, const char *buf);
214static void ftrace_trace_userstack(struct trace_array *tr,
215 struct trace_buffer *buffer,
216 unsigned int trace_ctx);
217
218static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
219static char *default_bootup_tracer;
220
221static char boot_instance_info[COMMAND_LINE_SIZE] __initdata;
222static int boot_instance_index;
223
224/*
225 * Repeated boot parameters, including Bootconfig array expansions, need
226 * to stay in the delimiter form that the existing parser consumes.
227 */
228void __init trace_append_boot_param(char *buf, const char *str, char sep,
229 int size)
230{
231 int len, needed, str_len;
232
233 if (!*str)
234 return;
235
236 len = strlen(buf);
237 str_len = strlen(str);
238 needed = len + str_len + 1;
239
240 /* For continuation, account for the separator. */
241 if (len)
242 needed++;
243 if (needed > size)
244 return;
245
246 if (len)
247 buf[len++] = sep;
248
249 strscpy(buf + len, str, size - len);
250}
251
252static int __init set_cmdline_ftrace(char *str)
253{
254 strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
255 default_bootup_tracer = bootup_tracer_buf;
256 /* We are using ftrace early, expand it */
257 trace_set_ring_buffer_expanded(NULL);
258 return 1;
259}
260__setup("ftrace=", set_cmdline_ftrace);
261
262int ftrace_dump_on_oops_enabled(void)
263{
264 if (!strcmp("0", ftrace_dump_on_oops))
265 return 0;
266 else
267 return 1;
268}
269
270static int __init set_ftrace_dump_on_oops(char *str)
271{
272 if (!*str) {
273 strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
274 return 1;
275 }
276
277 if (*str == ',') {
278 strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
279 strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1);
280 return 1;
281 }
282
283 if (*str++ == '=') {
284 strscpy(ftrace_dump_on_oops, str, MAX_TRACER_SIZE);
285 return 1;
286 }
287
288 return 0;
289}
290__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
291
292static int __init stop_trace_on_warning(char *str)
293{
294 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
295 __disable_trace_on_warning = 1;
296 return 1;
297}
298__setup("traceoff_on_warning", stop_trace_on_warning);
299
300static int __init boot_instance(char *str)
301{
302 char *slot = boot_instance_info + boot_instance_index;
303 int left = sizeof(boot_instance_info) - boot_instance_index;
304 int ret;
305
306 if (strlen(str) >= left)
307 return -1;
308
309 ret = snprintf(slot, left, "%s\t", str);
310 boot_instance_index += ret;
311
312 return 1;
313}
314__setup("trace_instance=", boot_instance);
315
316
317static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
318
319static int __init set_trace_boot_options(char *str)
320{
321 trace_append_boot_param(trace_boot_options_buf, str, ',',
322 MAX_TRACER_SIZE);
323 return 1;
324}
325__setup("trace_options=", set_trace_boot_options);
326
327static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
328static char *trace_boot_clock __initdata;
329
330static int __init set_trace_boot_clock(char *str)
331{
332 strscpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
333 trace_boot_clock = trace_boot_clock_buf;
334 return 1;
335}
336__setup("trace_clock=", set_trace_boot_clock);
337
338static int __init set_tracepoint_printk(char *str)
339{
340 /* Ignore the "tp_printk_stop_on_boot" param */
341 if (*str == '_')
342 return 0;
343
344 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
345 tracepoint_printk = 1;
346 return 1;
347}
348__setup("tp_printk", set_tracepoint_printk);
349
350static int __init set_tracepoint_printk_stop(char *str)
351{
352 tracepoint_printk_stop_on_boot = true;
353 return 1;
354}
355__setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);
356
357static int __init set_traceoff_after_boot(char *str)
358{
359 traceoff_after_boot = true;
360 return 1;
361}
362__setup("traceoff_after_boot", set_traceoff_after_boot);
363
364unsigned long long ns2usecs(u64 nsec)
365{
366 nsec += 500;
367 do_div(nsec, 1000);
368 return nsec;
369}
370
371static void
372trace_process_export(struct trace_export *export,
373 struct ring_buffer_event *event, int flag)
374{
375 struct trace_entry *entry;
376 unsigned int size = 0;
377
378 if (export->flags & flag) {
379 entry = ring_buffer_event_data(event);
380 size = ring_buffer_event_length(event);
381 export->write(export, entry, size);
382 }
383}
384
385static DEFINE_MUTEX(ftrace_export_lock);
386
387static struct trace_export __rcu *ftrace_exports_list __read_mostly;
388
389static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
390static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
391static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
392
393static inline void ftrace_exports_enable(struct trace_export *export)
394{
395 if (export->flags & TRACE_EXPORT_FUNCTION)
396 static_branch_inc(&trace_function_exports_enabled);
397
398 if (export->flags & TRACE_EXPORT_EVENT)
399 static_branch_inc(&trace_event_exports_enabled);
400
401 if (export->flags & TRACE_EXPORT_MARKER)
402 static_branch_inc(&trace_marker_exports_enabled);
403}
404
405static inline void ftrace_exports_disable(struct trace_export *export)
406{
407 if (export->flags & TRACE_EXPORT_FUNCTION)
408 static_branch_dec(&trace_function_exports_enabled);
409
410 if (export->flags & TRACE_EXPORT_EVENT)
411 static_branch_dec(&trace_event_exports_enabled);
412
413 if (export->flags & TRACE_EXPORT_MARKER)
414 static_branch_dec(&trace_marker_exports_enabled);
415}
416
417static void ftrace_exports(struct ring_buffer_event *event, int flag)
418{
419 struct trace_export *export;
420
421 guard(preempt_notrace)();
422
423 export = rcu_dereference_raw_check(ftrace_exports_list);
424 while (export) {
425 trace_process_export(export, event, flag);
426 export = rcu_dereference_raw_check(export->next);
427 }
428}
429
430static inline void
431add_trace_export(struct trace_export **list, struct trace_export *export)
432{
433 rcu_assign_pointer(export->next, *list);
434 /*
435 * We are entering export into the list but another
436 * CPU might be walking that list. We need to make sure
437 * the export->next pointer is valid before another CPU sees
438 * the export pointer included into the list.
439 */
440 rcu_assign_pointer(*list, export);
441}
442
443static inline int
444rm_trace_export(struct trace_export **list, struct trace_export *export)
445{
446 struct trace_export **p;
447
448 for (p = list; *p != NULL; p = &(*p)->next)
449 if (*p == export)
450 break;
451
452 if (*p != export)
453 return -1;
454
455 rcu_assign_pointer(*p, (*p)->next);
456
457 return 0;
458}
459
460static inline void
461add_ftrace_export(struct trace_export **list, struct trace_export *export)
462{
463 ftrace_exports_enable(export);
464
465 add_trace_export(list, export);
466}
467
468static inline int
469rm_ftrace_export(struct trace_export **list, struct trace_export *export)
470{
471 int ret;
472
473 ret = rm_trace_export(list, export);
474 ftrace_exports_disable(export);
475
476 return ret;
477}
478
479int register_ftrace_export(struct trace_export *export)
480{
481 if (WARN_ON_ONCE(!export->write))
482 return -1;
483
484 guard(mutex)(&ftrace_export_lock);
485
486 add_ftrace_export(&ftrace_exports_list, export);
487
488 return 0;
489}
490EXPORT_SYMBOL_GPL(register_ftrace_export);
491
492int unregister_ftrace_export(struct trace_export *export)
493{
494 guard(mutex)(&ftrace_export_lock);
495 return rm_ftrace_export(&ftrace_exports_list, export);
496}
497EXPORT_SYMBOL_GPL(unregister_ftrace_export);
498
499/* trace_flags holds trace_options default values */
500#define TRACE_DEFAULT_FLAGS \
501 (FUNCTION_DEFAULT_FLAGS | FPROFILE_DEFAULT_FLAGS | \
502 TRACE_ITER(PRINT_PARENT) | TRACE_ITER(PRINTK) | \
503 TRACE_ITER(ANNOTATE) | TRACE_ITER(CONTEXT_INFO) | \
504 TRACE_ITER(RECORD_CMD) | TRACE_ITER(OVERWRITE) | \
505 TRACE_ITER(IRQ_INFO) | TRACE_ITER(MARKERS) | \
506 TRACE_ITER(HASH_PTR) | TRACE_ITER(TRACE_PRINTK) | \
507 TRACE_ITER(COPY_MARKER))
508
509/* trace_options that are only supported by global_trace */
510#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER(PRINTK) | \
511 TRACE_ITER(PRINTK_MSGONLY) | TRACE_ITER(RECORD_CMD) | \
512 TRACE_ITER(PROF_TEXT_OFFSET) | FPROFILE_DEFAULT_FLAGS)
513
514/* trace_flags that are default zero for instances */
515#define ZEROED_TRACE_FLAGS \
516 (TRACE_ITER(EVENT_FORK) | TRACE_ITER(FUNC_FORK) | TRACE_ITER(TRACE_PRINTK) | \
517 TRACE_ITER(COPY_MARKER))
518
519/*
520 * The global_trace is the descriptor that holds the top-level tracing
521 * buffers for the live tracing.
522 */
523static struct trace_array global_trace = {
524 .trace_flags = TRACE_DEFAULT_FLAGS,
525};
526
527struct trace_array *printk_trace = &global_trace;
528
529/* List of trace_arrays interested in the top level trace_marker */
530static LIST_HEAD(marker_copies);
531
532static void update_printk_trace(struct trace_array *tr)
533{
534 if (printk_trace == tr)
535 return;
536
537 printk_trace->trace_flags &= ~TRACE_ITER(TRACE_PRINTK);
538 printk_trace = tr;
539 tr->trace_flags |= TRACE_ITER(TRACE_PRINTK);
540}
541
542/* Returns true if the status of tr changed */
543static bool update_marker_trace(struct trace_array *tr, int enabled)
544{
545 lockdep_assert_held(&event_mutex);
546
547 if (enabled) {
548 if (tr->trace_flags & TRACE_ITER(COPY_MARKER))
549 return false;
550
551 list_add_rcu(&tr->marker_list, &marker_copies);
552 tr->trace_flags |= TRACE_ITER(COPY_MARKER);
553 return true;
554 }
555
556 if (!(tr->trace_flags & TRACE_ITER(COPY_MARKER)))
557 return false;
558
559 list_del_rcu(&tr->marker_list);
560 tr->trace_flags &= ~TRACE_ITER(COPY_MARKER);
561 return true;
562}
563
564void trace_set_ring_buffer_expanded(struct trace_array *tr)
565{
566 if (!tr)
567 tr = &global_trace;
568 tr->ring_buffer_expanded = true;
569}
570
571static void trace_array_autoremove(struct work_struct *work)
572{
573 struct trace_array *tr = container_of(work, struct trace_array, autoremove_work);
574
575 trace_array_destroy(tr);
576}
577
578static struct workqueue_struct *autoremove_wq;
579
580static void trace_array_kick_autoremove(struct trace_array *tr)
581{
582 if (autoremove_wq)
583 queue_work(autoremove_wq, &tr->autoremove_work);
584}
585
586static void trace_array_cancel_autoremove(struct trace_array *tr)
587{
588 /*
589 * Since this can be called inside trace_array_autoremove(),
590 * it has to avoid deadlock of the workqueue.
591 */
592 if (work_pending(&tr->autoremove_work))
593 cancel_work_sync(&tr->autoremove_work);
594}
595
596static void trace_array_init_autoremove(struct trace_array *tr)
597{
598 INIT_WORK(&tr->autoremove_work, trace_array_autoremove);
599}
600
601static void trace_array_start_autoremove(void)
602{
603 if (autoremove_wq)
604 return;
605
606 autoremove_wq = alloc_workqueue("tr_autoremove_wq",
607 WQ_UNBOUND | WQ_HIGHPRI, 0);
608 if (!autoremove_wq)
609 pr_warn("Unable to allocate tr_autoremove_wq. autoremove disabled.\n");
610}
611
612LIST_HEAD(ftrace_trace_arrays);
613
614static int __trace_array_get(struct trace_array *this_tr)
615{
616 /* When free_on_close is set, this is not available anymore. */
617 if (autoremove_wq && this_tr->free_on_close)
618 return -ENODEV;
619
620 this_tr->ref++;
621 return 0;
622}
623
624int trace_array_get(struct trace_array *this_tr)
625{
626 struct trace_array *tr;
627
628 guard(mutex)(&trace_types_lock);
629 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
630 if (tr == this_tr) {
631 return __trace_array_get(tr);
632 }
633 }
634
635 return -ENODEV;
636}
637
638static void __trace_array_put(struct trace_array *this_tr)
639{
640 WARN_ON(!this_tr->ref);
641 this_tr->ref--;
642 /*
643 * When free_on_close is set, prepare removing the array
644 * when the last reference is released.
645 */
646 if (this_tr->ref == 1 && this_tr->free_on_close)
647 trace_array_kick_autoremove(this_tr);
648}
649
650/**
651 * trace_array_put - Decrement the reference counter for this trace array.
652 * @this_tr : pointer to the trace array
653 *
654 * NOTE: Use this when we no longer need the trace array returned by
655 * trace_array_get_by_name(). This ensures the trace array can be later
656 * destroyed.
657 *
658 */
659void trace_array_put(struct trace_array *this_tr)
660{
661 if (!this_tr)
662 return;
663
664 guard(mutex)(&trace_types_lock);
665 __trace_array_put(this_tr);
666}
667EXPORT_SYMBOL_GPL(trace_array_put);
668
669int tracing_check_open_get_tr(struct trace_array *tr)
670{
671 int ret;
672
673 ret = security_locked_down(LOCKDOWN_TRACEFS);
674 if (ret)
675 return ret;
676
677 if (tracing_disabled)
678 return -ENODEV;
679
680 if (tr && trace_array_get(tr) < 0)
681 return -ENODEV;
682
683 return 0;
684}
685
686static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
687{
688 u64 ts;
689
690 /* Early boot up does not have a buffer yet */
691 if (!buf->buffer)
692 return trace_clock_local();
693
694 ts = ring_buffer_time_stamp(buf->buffer);
695 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
696
697 return ts;
698}
699
700u64 ftrace_now(int cpu)
701{
702 return buffer_ftrace_now(&global_trace.array_buffer, cpu);
703}
704
705/**
706 * tracing_is_enabled - Show if global_trace has been enabled
707 *
708 * Shows if the global trace has been enabled or not. It uses the
709 * mirror flag "buffer_disabled" to be used in fast paths such as for
710 * the irqsoff tracer. But it may be inaccurate due to races. If you
711 * need to know the accurate state, use tracing_is_on() which is a little
712 * slower, but accurate.
713 */
714int tracing_is_enabled(void)
715{
716 /*
717 * For quick access (irqsoff uses this in fast path), just
718 * return the mirror variable of the state of the ring buffer.
719 * It's a little racy, but we don't really care.
720 */
721 return !global_trace.buffer_disabled;
722}
723
724/*
725 * trace_buf_size is the size in bytes that is allocated
726 * for a buffer. Note, the number of bytes is always rounded
727 * to page size.
728 *
729 * This number is purposely set to a low number of 16384.
730 * If the dump on oops happens, it will be much appreciated
731 * to not have to wait for all that output. Anyway this can be
732 * boot time and run time configurable.
733 */
734#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
735
736static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
737
738/* trace_types holds a link list of available tracers. */
739static struct tracer *trace_types __read_mostly;
740
741/*
742 * trace_types_lock is used to protect the trace_types list.
743 */
744DEFINE_MUTEX(trace_types_lock);
745
746/*
747 * serialize the access of the ring buffer
748 *
749 * ring buffer serializes readers, but it is low level protection.
750 * The validity of the events (which returns by ring_buffer_peek() ..etc)
751 * are not protected by ring buffer.
752 *
753 * The content of events may become garbage if we allow other process consumes
754 * these events concurrently:
755 * A) the page of the consumed events may become a normal page
756 * (not reader page) in ring buffer, and this page will be rewritten
757 * by events producer.
758 * B) The page of the consumed events may become a page for splice_read,
759 * and this page will be returned to system.
760 *
761 * These primitives allow multi process access to different cpu ring buffer
762 * concurrently.
763 *
764 * These primitives don't distinguish read-only and read-consume access.
765 * Multi read-only access are also serialized.
766 */
767
768#ifdef CONFIG_SMP
769static DECLARE_RWSEM(all_cpu_access_lock);
770static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
771
772static inline void trace_access_lock(int cpu)
773{
774 if (cpu == RING_BUFFER_ALL_CPUS) {
775 /* gain it for accessing the whole ring buffer. */
776 down_write(&all_cpu_access_lock);
777 } else {
778 /* gain it for accessing a cpu ring buffer. */
779
780 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
781 down_read(&all_cpu_access_lock);
782
783 /* Secondly block other access to this @cpu ring buffer. */
784 mutex_lock(&per_cpu(cpu_access_lock, cpu));
785 }
786}
787
788static inline void trace_access_unlock(int cpu)
789{
790 if (cpu == RING_BUFFER_ALL_CPUS) {
791 up_write(&all_cpu_access_lock);
792 } else {
793 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
794 up_read(&all_cpu_access_lock);
795 }
796}
797
798static inline void trace_access_lock_init(void)
799{
800 int cpu;
801
802 for_each_possible_cpu(cpu)
803 mutex_init(&per_cpu(cpu_access_lock, cpu));
804}
805
806#else
807
808static DEFINE_MUTEX(access_lock);
809
810static inline void trace_access_lock(int cpu)
811{
812 (void)cpu;
813 mutex_lock(&access_lock);
814}
815
816static inline void trace_access_unlock(int cpu)
817{
818 (void)cpu;
819 mutex_unlock(&access_lock);
820}
821
822static inline void trace_access_lock_init(void)
823{
824}
825
826#endif
827
828void tracer_tracing_on(struct trace_array *tr)
829{
830 if (tr->array_buffer.buffer)
831 ring_buffer_record_on(tr->array_buffer.buffer);
832 /*
833 * This flag is looked at when buffers haven't been allocated
834 * yet, or by some tracers (like irqsoff), that just want to
835 * know if the ring buffer has been disabled, but it can handle
836 * races of where it gets disabled but we still do a record.
837 * As the check is in the fast path of the tracers, it is more
838 * important to be fast than accurate.
839 */
840 tr->buffer_disabled = 0;
841}
842
843/**
844 * tracing_on - enable tracing buffers
845 *
846 * This function enables tracing buffers that may have been
847 * disabled with tracing_off.
848 */
849void tracing_on(void)
850{
851 tracer_tracing_on(&global_trace);
852}
853EXPORT_SYMBOL_GPL(tracing_on);
854
855#ifdef CONFIG_TRACER_SNAPSHOT
856/**
857 * tracing_snapshot - take a snapshot of the current buffer.
858 *
859 * This causes a swap between the snapshot buffer and the current live
860 * tracing buffer. You can use this to take snapshots of the live
861 * trace when some condition is triggered, but continue to trace.
862 *
863 * Note, make sure to allocate the snapshot with either
864 * a tracing_snapshot_alloc(), or by doing it manually
865 * with: echo 1 > /sys/kernel/tracing/snapshot
866 *
867 * If the snapshot buffer is not allocated, it will stop tracing.
868 * Basically making a permanent snapshot.
869 */
870void tracing_snapshot(void)
871{
872 struct trace_array *tr = &global_trace;
873
874 tracing_snapshot_instance(tr);
875}
876EXPORT_SYMBOL_GPL(tracing_snapshot);
877
878/**
879 * tracing_alloc_snapshot - allocate snapshot buffer.
880 *
881 * This only allocates the snapshot buffer if it isn't already
882 * allocated - it doesn't also take a snapshot.
883 *
884 * This is meant to be used in cases where the snapshot buffer needs
885 * to be set up for events that can't sleep but need to be able to
886 * trigger a snapshot.
887 */
888int tracing_alloc_snapshot(void)
889{
890 struct trace_array *tr = &global_trace;
891 int ret;
892
893 ret = tracing_alloc_snapshot_instance(tr);
894 WARN_ON(ret < 0);
895
896 return ret;
897}
898#else
899void tracing_snapshot(void)
900{
901 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
902}
903EXPORT_SYMBOL_GPL(tracing_snapshot);
904void tracing_snapshot_alloc(void)
905{
906 /* Give warning */
907 tracing_snapshot();
908}
909EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
910#endif /* CONFIG_TRACER_SNAPSHOT */
911
912void tracer_tracing_off(struct trace_array *tr)
913{
914 if (tr->array_buffer.buffer)
915 ring_buffer_record_off(tr->array_buffer.buffer);
916 /*
917 * This flag is looked at when buffers haven't been allocated
918 * yet, or by some tracers (like irqsoff), that just want to
919 * know if the ring buffer has been disabled, but it can handle
920 * races of where it gets disabled but we still do a record.
921 * As the check is in the fast path of the tracers, it is more
922 * important to be fast than accurate.
923 */
924 tr->buffer_disabled = 1;
925}
926
927/**
928 * tracer_tracing_disable() - temporary disable the buffer from write
929 * @tr: The trace array to disable its buffer for
930 *
931 * Expects trace_tracing_enable() to re-enable tracing.
932 * The difference between this and tracer_tracing_off() is that this
933 * is a counter and can nest, whereas, tracer_tracing_off() can
934 * be called multiple times and a single trace_tracing_on() will
935 * enable it.
936 */
937void tracer_tracing_disable(struct trace_array *tr)
938{
939 if (WARN_ON_ONCE(!tr->array_buffer.buffer))
940 return;
941
942 ring_buffer_record_disable(tr->array_buffer.buffer);
943}
944
945/**
946 * tracer_tracing_enable() - counter part of tracer_tracing_disable()
947 * @tr: The trace array that had tracer_tracincg_disable() called on it
948 *
949 * This is called after tracer_tracing_disable() has been called on @tr,
950 * when it's safe to re-enable tracing.
951 */
952void tracer_tracing_enable(struct trace_array *tr)
953{
954 if (WARN_ON_ONCE(!tr->array_buffer.buffer))
955 return;
956
957 ring_buffer_record_enable(tr->array_buffer.buffer);
958}
959
960/**
961 * tracing_off - turn off tracing buffers
962 *
963 * This function stops the tracing buffers from recording data.
964 * It does not disable any overhead the tracers themselves may
965 * be causing. This function simply causes all recording to
966 * the ring buffers to fail.
967 */
968void tracing_off(void)
969{
970 tracer_tracing_off(&global_trace);
971}
972EXPORT_SYMBOL_GPL(tracing_off);
973
974void disable_trace_on_warning(void)
975{
976 if (__disable_trace_on_warning) {
977 struct trace_array *tr = READ_ONCE(printk_trace);
978
979 trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
980 "Disabling tracing due to warning\n");
981 tracing_off();
982
983 /* Disable trace_printk() buffer too */
984 if (tr != &global_trace) {
985 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
986 "Disabling tracing due to warning\n");
987 tracer_tracing_off(tr);
988 }
989 }
990}
991
992/**
993 * tracer_tracing_is_on - show real state of ring buffer enabled
994 * @tr : the trace array to know if ring buffer is enabled
995 *
996 * Shows real state of the ring buffer if it is enabled or not.
997 */
998bool tracer_tracing_is_on(struct trace_array *tr)
999{
1000 if (tr->array_buffer.buffer)
1001 return ring_buffer_record_is_set_on(tr->array_buffer.buffer);
1002 return !tr->buffer_disabled;
1003}
1004
1005/**
1006 * tracing_is_on - show state of ring buffers enabled
1007 */
1008int tracing_is_on(void)
1009{
1010 return tracer_tracing_is_on(&global_trace);
1011}
1012EXPORT_SYMBOL_GPL(tracing_is_on);
1013
1014static int __init set_buf_size(char *str)
1015{
1016 unsigned long buf_size;
1017
1018 if (!str)
1019 return 0;
1020 buf_size = memparse(str, &str);
1021 /*
1022 * nr_entries can not be zero and the startup
1023 * tests require some buffer space. Therefore
1024 * ensure we have at least 4096 bytes of buffer.
1025 */
1026 trace_buf_size = max(4096UL, buf_size);
1027 return 1;
1028}
1029__setup("trace_buf_size=", set_buf_size);
1030
1031static int __init set_tracing_thresh(char *str)
1032{
1033 unsigned long threshold;
1034 int ret;
1035
1036 if (!str)
1037 return 0;
1038 ret = kstrtoul(str, 0, &threshold);
1039 if (ret < 0)
1040 return 0;
1041 tracing_thresh = threshold * 1000;
1042 return 1;
1043}
1044__setup("tracing_thresh=", set_tracing_thresh);
1045
1046unsigned long nsecs_to_usecs(unsigned long nsecs)
1047{
1048 return nsecs / 1000;
1049}
1050
1051/*
1052 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1053 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1054 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1055 * of strings in the order that the evals (enum) were defined.
1056 */
1057#undef C
1058#define C(a, b) b
1059
1060/* These must match the bit positions in trace_iterator_flags */
1061static const char *trace_options[] = {
1062 TRACE_FLAGS
1063 NULL
1064};
1065
1066static struct {
1067 u64 (*func)(void);
1068 const char *name;
1069 int in_ns; /* is this clock in nanoseconds? */
1070} trace_clocks[] = {
1071 { trace_clock_local, "local", 1 },
1072 { trace_clock_global, "global", 1 },
1073 { trace_clock_counter, "counter", 0 },
1074 { trace_clock_jiffies, "uptime", 0 },
1075 { trace_clock, "perf", 1 },
1076 { ktime_get_mono_fast_ns, "mono", 1 },
1077 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1078 { ktime_get_boot_fast_ns, "boot", 1 },
1079 { ktime_get_tai_fast_ns, "tai", 1 },
1080 ARCH_TRACE_CLOCKS
1081};
1082
1083bool trace_clock_in_ns(struct trace_array *tr)
1084{
1085 if (trace_clocks[tr->clock_id].in_ns)
1086 return true;
1087
1088 return false;
1089}
1090
1091/*
1092 * trace_parser_get_init - gets the buffer for trace parser
1093 */
1094int trace_parser_get_init(struct trace_parser *parser, int size)
1095{
1096 memset(parser, 0, sizeof(*parser));
1097
1098 parser->buffer = kmalloc(size, GFP_KERNEL);
1099 if (!parser->buffer)
1100 return 1;
1101
1102 parser->size = size;
1103 return 0;
1104}
1105
1106/*
1107 * trace_parser_put - frees the buffer for trace parser
1108 */
1109void trace_parser_put(struct trace_parser *parser)
1110{
1111 kfree(parser->buffer);
1112 parser->buffer = NULL;
1113}
1114
1115/*
1116 * trace_get_user - reads the user input string separated by space
1117 * (matched by isspace(ch))
1118 *
1119 * For each string found the 'struct trace_parser' is updated,
1120 * and the function returns.
1121 *
1122 * Returns number of bytes read.
1123 *
1124 * See kernel/trace/trace.h for 'struct trace_parser' details.
1125 */
1126int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1127 size_t cnt, loff_t *ppos)
1128{
1129 char ch;
1130 size_t read = 0;
1131 ssize_t ret;
1132
1133 if (!*ppos)
1134 trace_parser_clear(parser);
1135
1136 ret = get_user(ch, ubuf++);
1137 if (ret)
1138 goto fail;
1139
1140 read++;
1141 cnt--;
1142
1143 /*
1144 * The parser is not finished with the last write,
1145 * continue reading the user input without skipping spaces.
1146 */
1147 if (!parser->cont) {
1148 /* skip white space */
1149 while (cnt && isspace(ch)) {
1150 ret = get_user(ch, ubuf++);
1151 if (ret)
1152 goto fail;
1153 read++;
1154 cnt--;
1155 }
1156
1157 parser->idx = 0;
1158
1159 /* only spaces were written */
1160 if (isspace(ch) || !ch) {
1161 *ppos += read;
1162 return read;
1163 }
1164 }
1165
1166 /* read the non-space input */
1167 while (cnt && !isspace(ch) && ch) {
1168 if (parser->idx < parser->size - 1)
1169 parser->buffer[parser->idx++] = ch;
1170 else {
1171 ret = -EINVAL;
1172 goto fail;
1173 }
1174
1175 ret = get_user(ch, ubuf++);
1176 if (ret)
1177 goto fail;
1178 read++;
1179 cnt--;
1180 }
1181
1182 /* We either got finished input or we have to wait for another call. */
1183 if (isspace(ch) || !ch) {
1184 parser->buffer[parser->idx] = 0;
1185 parser->cont = false;
1186 } else if (parser->idx < parser->size - 1) {
1187 parser->cont = true;
1188 parser->buffer[parser->idx++] = ch;
1189 /* Make sure the parsed string always terminates with '\0'. */
1190 parser->buffer[parser->idx] = 0;
1191 } else {
1192 ret = -EINVAL;
1193 goto fail;
1194 }
1195
1196 *ppos += read;
1197 return read;
1198fail:
1199 trace_parser_fail(parser);
1200 return ret;
1201}
1202
1203/* TODO add a seq_buf_to_buffer() */
1204static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1205{
1206 int len;
1207
1208 if (trace_seq_used(s) <= s->readpos)
1209 return -EBUSY;
1210
1211 len = trace_seq_used(s) - s->readpos;
1212 if (cnt > len)
1213 cnt = len;
1214 memcpy(buf, s->buffer + s->readpos, cnt);
1215
1216 s->readpos += cnt;
1217 return cnt;
1218}
1219
1220unsigned long __read_mostly tracing_thresh;
1221
1222struct pipe_wait {
1223 struct trace_iterator *iter;
1224 int wait_index;
1225};
1226
1227static bool wait_pipe_cond(void *data)
1228{
1229 struct pipe_wait *pwait = data;
1230 struct trace_iterator *iter = pwait->iter;
1231
1232 if (atomic_read_acquire(&iter->wait_index) != pwait->wait_index)
1233 return true;
1234
1235 return iter->closed;
1236}
1237
1238static int wait_on_pipe(struct trace_iterator *iter, int full)
1239{
1240 struct pipe_wait pwait;
1241 int ret;
1242
1243 /* Iterators are static, they should be filled or empty */
1244 if (trace_buffer_iter(iter, iter->cpu_file))
1245 return 0;
1246
1247 pwait.wait_index = atomic_read_acquire(&iter->wait_index);
1248 pwait.iter = iter;
1249
1250 ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full,
1251 wait_pipe_cond, &pwait);
1252
1253#ifdef CONFIG_TRACER_SNAPSHOT
1254 /*
1255 * Make sure this is still the snapshot buffer, as if a snapshot were
1256 * to happen, this would now be the main buffer.
1257 */
1258 if (iter->snapshot)
1259 iter->array_buffer = &iter->tr->snapshot_buffer;
1260#endif
1261 return ret;
1262}
1263
1264#ifdef CONFIG_FTRACE_STARTUP_TEST
1265static bool selftests_can_run;
1266
1267struct trace_selftests {
1268 struct list_head list;
1269 struct tracer *type;
1270};
1271
1272static LIST_HEAD(postponed_selftests);
1273
1274static int save_selftest(struct tracer *type)
1275{
1276 struct trace_selftests *selftest;
1277
1278 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1279 if (!selftest)
1280 return -ENOMEM;
1281
1282 selftest->type = type;
1283 list_add(&selftest->list, &postponed_selftests);
1284 return 0;
1285}
1286
1287static int run_tracer_selftest(struct tracer *type)
1288{
1289 struct trace_array *tr = &global_trace;
1290 struct tracer_flags *saved_flags = tr->current_trace_flags;
1291 struct tracer *saved_tracer = tr->current_trace;
1292 int ret;
1293
1294 if (!type->selftest || tracing_selftest_disabled)
1295 return 0;
1296
1297 /*
1298 * If a tracer registers early in boot up (before scheduling is
1299 * initialized and such), then do not run its selftests yet.
1300 * Instead, run it a little later in the boot process.
1301 */
1302 if (!selftests_can_run)
1303 return save_selftest(type);
1304
1305 if (!tracing_is_on()) {
1306 pr_warn("Selftest for tracer %s skipped due to tracing disabled\n",
1307 type->name);
1308 return 0;
1309 }
1310
1311 /*
1312 * Run a selftest on this tracer.
1313 * Here we reset the trace buffer, and set the current
1314 * tracer to be this tracer. The tracer can then run some
1315 * internal tracing to verify that everything is in order.
1316 * If we fail, we do not register this tracer.
1317 */
1318 tracing_reset_online_cpus(&tr->array_buffer);
1319
1320 tr->current_trace = type;
1321 tr->current_trace_flags = type->flags ? : type->default_flags;
1322
1323#ifdef CONFIG_TRACER_MAX_TRACE
1324 if (tracer_uses_snapshot(type)) {
1325 /* If we expanded the buffers, make sure the max is expanded too */
1326 if (tr->ring_buffer_expanded)
1327 ring_buffer_resize(tr->snapshot_buffer.buffer, trace_buf_size,
1328 RING_BUFFER_ALL_CPUS);
1329 tr->allocated_snapshot = true;
1330 }
1331#endif
1332
1333 /* the test is responsible for initializing and enabling */
1334 pr_info("Testing tracer %s: ", type->name);
1335 ret = type->selftest(type, tr);
1336 /* the test is responsible for resetting too */
1337 tr->current_trace = saved_tracer;
1338 tr->current_trace_flags = saved_flags;
1339 if (ret) {
1340 printk(KERN_CONT "FAILED!\n");
1341 /* Add the warning after printing 'FAILED' */
1342 WARN_ON(1);
1343 return -1;
1344 }
1345 /* Only reset on passing, to avoid touching corrupted buffers */
1346 tracing_reset_online_cpus(&tr->array_buffer);
1347
1348#ifdef CONFIG_TRACER_MAX_TRACE
1349 if (tracer_uses_snapshot(type)) {
1350 tr->allocated_snapshot = false;
1351
1352 /* Shrink the max buffer again */
1353 if (tr->ring_buffer_expanded)
1354 ring_buffer_resize(tr->snapshot_buffer.buffer, 1,
1355 RING_BUFFER_ALL_CPUS);
1356 }
1357#endif
1358
1359 printk(KERN_CONT "PASSED\n");
1360 return 0;
1361}
1362
1363static int do_run_tracer_selftest(struct tracer *type)
1364{
1365 int ret;
1366
1367 /*
1368 * Tests can take a long time, especially if they are run one after the
1369 * other, as does happen during bootup when all the tracers are
1370 * registered. This could cause the soft lockup watchdog to trigger.
1371 */
1372 cond_resched();
1373
1374 tracing_selftest_running = true;
1375 ret = run_tracer_selftest(type);
1376 tracing_selftest_running = false;
1377
1378 return ret;
1379}
1380
1381static __init int init_trace_selftests(void)
1382{
1383 struct trace_selftests *p, *n;
1384 struct tracer *t, **last;
1385 int ret;
1386
1387 selftests_can_run = true;
1388
1389 guard(mutex)(&trace_types_lock);
1390
1391 if (list_empty(&postponed_selftests))
1392 return 0;
1393
1394 pr_info("Running postponed tracer tests:\n");
1395
1396 tracing_selftest_running = true;
1397 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1398 /* This loop can take minutes when sanitizers are enabled, so
1399 * lets make sure we allow RCU processing.
1400 */
1401 cond_resched();
1402 ret = run_tracer_selftest(p->type);
1403 /* If the test fails, then warn and remove from available_tracers */
1404 if (ret < 0) {
1405 WARN(1, "tracer: %s failed selftest, disabling\n",
1406 p->type->name);
1407 last = &trace_types;
1408 for (t = trace_types; t; t = t->next) {
1409 if (t == p->type) {
1410 *last = t->next;
1411 break;
1412 }
1413 last = &t->next;
1414 }
1415 }
1416 list_del(&p->list);
1417 kfree(p);
1418 }
1419 tracing_selftest_running = false;
1420
1421 return 0;
1422}
1423core_initcall(init_trace_selftests);
1424#else
1425static inline int do_run_tracer_selftest(struct tracer *type)
1426{
1427 return 0;
1428}
1429#endif /* CONFIG_FTRACE_STARTUP_TEST */
1430
1431static int add_tracer(struct trace_array *tr, struct tracer *t);
1432
1433static void __init apply_trace_boot_options(void);
1434
1435static void free_tracers(struct trace_array *tr)
1436{
1437 struct tracers *t, *n;
1438
1439 lockdep_assert_held(&trace_types_lock);
1440
1441 list_for_each_entry_safe(t, n, &tr->tracers, list) {
1442 list_del(&t->list);
1443 kfree(t->flags);
1444 kfree(t);
1445 }
1446}
1447
1448/**
1449 * register_tracer - register a tracer with the ftrace system.
1450 * @type: the plugin for the tracer
1451 *
1452 * Register a new plugin tracer.
1453 */
1454int __init register_tracer(struct tracer *type)
1455{
1456 struct trace_array *tr;
1457 struct tracer *t;
1458 int ret = 0;
1459
1460 if (!type->name) {
1461 pr_info("Tracer must have a name\n");
1462 return -1;
1463 }
1464
1465 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1466 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1467 return -1;
1468 }
1469
1470 if (security_locked_down(LOCKDOWN_TRACEFS)) {
1471 pr_warn("Can not register tracer %s due to lockdown\n",
1472 type->name);
1473 return -EPERM;
1474 }
1475
1476 mutex_lock(&trace_types_lock);
1477
1478 for (t = trace_types; t; t = t->next) {
1479 if (strcmp(type->name, t->name) == 0) {
1480 /* already found */
1481 pr_info("Tracer %s already registered\n",
1482 type->name);
1483 ret = -1;
1484 goto out;
1485 }
1486 }
1487
1488 /* store the tracer for __set_tracer_option */
1489 if (type->flags)
1490 type->flags->trace = type;
1491
1492 ret = do_run_tracer_selftest(type);
1493 if (ret < 0)
1494 goto out;
1495
1496 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1497 ret = add_tracer(tr, type);
1498 if (ret < 0) {
1499 /* The tracer will still exist but without options */
1500 pr_warn("Failed to create tracer options for %s\n", type->name);
1501 break;
1502 }
1503 }
1504
1505 type->next = trace_types;
1506 trace_types = type;
1507
1508 out:
1509 mutex_unlock(&trace_types_lock);
1510
1511 if (ret || !default_bootup_tracer)
1512 return ret;
1513
1514 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1515 return 0;
1516
1517 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1518 /* Do we want this tracer to start on bootup? */
1519 WARN_ON(tracing_set_tracer(&global_trace, type->name) < 0);
1520 default_bootup_tracer = NULL;
1521
1522 apply_trace_boot_options();
1523
1524 /* disable other selftests, since this will break it. */
1525 disable_tracing_selftest("running a tracer");
1526
1527 return 0;
1528}
1529
1530void tracing_reset_cpu(struct array_buffer *buf, int cpu)
1531{
1532 struct trace_buffer *buffer = buf->buffer;
1533
1534 if (!buffer)
1535 return;
1536
1537 ring_buffer_record_disable(buffer);
1538
1539 /* Make sure all commits have finished */
1540 synchronize_rcu();
1541 ring_buffer_reset_cpu(buffer, cpu);
1542
1543 ring_buffer_record_enable(buffer);
1544}
1545
1546void tracing_reset_online_cpus(struct array_buffer *buf)
1547{
1548 struct trace_buffer *buffer = buf->buffer;
1549
1550 if (!buffer)
1551 return;
1552
1553 ring_buffer_record_disable(buffer);
1554
1555 /* Make sure all commits have finished */
1556 synchronize_rcu();
1557
1558 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1559
1560 ring_buffer_reset_online_cpus(buffer);
1561
1562 ring_buffer_record_enable(buffer);
1563}
1564
1565static void tracing_reset_all_cpus(struct array_buffer *buf)
1566{
1567 struct trace_buffer *buffer = buf->buffer;
1568
1569 if (!buffer)
1570 return;
1571
1572 ring_buffer_record_disable(buffer);
1573
1574 /* Make sure all commits have finished */
1575 synchronize_rcu();
1576
1577 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1578
1579 ring_buffer_reset(buffer);
1580
1581 ring_buffer_record_enable(buffer);
1582}
1583
1584/* Must have trace_types_lock held */
1585void tracing_reset_all_online_cpus_unlocked(void)
1586{
1587 struct trace_array *tr;
1588
1589 lockdep_assert_held(&trace_types_lock);
1590
1591 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1592 if (!tr->clear_trace)
1593 continue;
1594 tr->clear_trace = false;
1595 tracing_reset_online_cpus(&tr->array_buffer);
1596#ifdef CONFIG_TRACER_SNAPSHOT
1597 tracing_reset_online_cpus(&tr->snapshot_buffer);
1598#endif
1599 }
1600}
1601
1602void tracing_reset_all_online_cpus(void)
1603{
1604 guard(mutex)(&trace_types_lock);
1605 tracing_reset_all_online_cpus_unlocked();
1606}
1607
1608int is_tracing_stopped(void)
1609{
1610 return global_trace.stop_count;
1611}
1612
1613static void tracing_start_tr(struct trace_array *tr)
1614{
1615 struct trace_buffer *buffer;
1616
1617 if (tracing_disabled)
1618 return;
1619
1620 guard(raw_spinlock_irqsave)(&tr->start_lock);
1621 if (--tr->stop_count) {
1622 if (WARN_ON_ONCE(tr->stop_count < 0)) {
1623 /* Someone screwed up their debugging */
1624 tr->stop_count = 0;
1625 }
1626 return;
1627 }
1628
1629 /* Prevent the buffers from switching */
1630 arch_spin_lock(&tr->max_lock);
1631
1632 buffer = tr->array_buffer.buffer;
1633 if (buffer)
1634 ring_buffer_record_enable(buffer);
1635
1636#ifdef CONFIG_TRACER_SNAPSHOT
1637 buffer = tr->snapshot_buffer.buffer;
1638 if (buffer)
1639 ring_buffer_record_enable(buffer);
1640#endif
1641
1642 arch_spin_unlock(&tr->max_lock);
1643}
1644
1645/**
1646 * tracing_start - quick start of the tracer
1647 *
1648 * If tracing is enabled but was stopped by tracing_stop,
1649 * this will start the tracer back up.
1650 */
1651void tracing_start(void)
1652
1653{
1654 return tracing_start_tr(&global_trace);
1655}
1656
1657static void tracing_stop_tr(struct trace_array *tr)
1658{
1659 struct trace_buffer *buffer;
1660
1661 guard(raw_spinlock_irqsave)(&tr->start_lock);
1662 if (tr->stop_count++)
1663 return;
1664
1665 /* Prevent the buffers from switching */
1666 arch_spin_lock(&tr->max_lock);
1667
1668 buffer = tr->array_buffer.buffer;
1669 if (buffer)
1670 ring_buffer_record_disable(buffer);
1671
1672#ifdef CONFIG_TRACER_SNAPSHOT
1673 buffer = tr->snapshot_buffer.buffer;
1674 if (buffer)
1675 ring_buffer_record_disable(buffer);
1676#endif
1677
1678 arch_spin_unlock(&tr->max_lock);
1679}
1680
1681/**
1682 * tracing_stop - quick stop of the tracer
1683 *
1684 * Light weight way to stop tracing. Use in conjunction with
1685 * tracing_start.
1686 */
1687void tracing_stop(void)
1688{
1689 return tracing_stop_tr(&global_trace);
1690}
1691
1692/*
1693 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
1694 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
1695 * simplifies those functions and keeps them in sync.
1696 */
1697enum print_line_t trace_handle_return(struct trace_seq *s)
1698{
1699 return trace_seq_has_overflowed(s) ?
1700 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
1701}
1702EXPORT_SYMBOL_GPL(trace_handle_return);
1703
1704static unsigned short migration_disable_value(void)
1705{
1706#if defined(CONFIG_SMP)
1707 return current->migration_disabled;
1708#else
1709 return 0;
1710#endif
1711}
1712
1713unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
1714{
1715 unsigned int trace_flags = irqs_status;
1716 unsigned int pc;
1717
1718 pc = preempt_count();
1719
1720 if (pc & NMI_MASK)
1721 trace_flags |= TRACE_FLAG_NMI;
1722 if (pc & HARDIRQ_MASK)
1723 trace_flags |= TRACE_FLAG_HARDIRQ;
1724 if (in_serving_softirq())
1725 trace_flags |= TRACE_FLAG_SOFTIRQ;
1726 if (softirq_count() >> (SOFTIRQ_SHIFT + 1))
1727 trace_flags |= TRACE_FLAG_BH_OFF;
1728
1729 if (tif_need_resched())
1730 trace_flags |= TRACE_FLAG_NEED_RESCHED;
1731 if (test_preempt_need_resched())
1732 trace_flags |= TRACE_FLAG_PREEMPT_RESCHED;
1733 if (IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY) && tif_test_bit(TIF_NEED_RESCHED_LAZY))
1734 trace_flags |= TRACE_FLAG_NEED_RESCHED_LAZY;
1735 return (trace_flags << 16) | (min_t(unsigned int, pc & 0xff, 0xf)) |
1736 (min_t(unsigned int, migration_disable_value(), 0xf)) << 4;
1737}
1738
1739struct ring_buffer_event *
1740trace_buffer_lock_reserve(struct trace_buffer *buffer,
1741 int type,
1742 unsigned long len,
1743 unsigned int trace_ctx)
1744{
1745 return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
1746}
1747
1748DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1749DEFINE_PER_CPU(int, trace_buffered_event_cnt);
1750static int trace_buffered_event_ref;
1751
1752/**
1753 * trace_buffered_event_enable - enable buffering events
1754 *
1755 * When events are being filtered, it is quicker to use a temporary
1756 * buffer to write the event data into if there's a likely chance
1757 * that it will not be committed. The discard of the ring buffer
1758 * is not as fast as committing, and is much slower than copying
1759 * a commit.
1760 *
1761 * When an event is to be filtered, allocate per cpu buffers to
1762 * write the event data into, and if the event is filtered and discarded
1763 * it is simply dropped, otherwise, the entire data is to be committed
1764 * in one shot.
1765 */
1766void trace_buffered_event_enable(void)
1767{
1768 struct ring_buffer_event *event;
1769 struct page *page;
1770 int cpu;
1771
1772 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1773
1774 if (trace_buffered_event_ref++)
1775 return;
1776
1777 for_each_tracing_cpu(cpu) {
1778 page = alloc_pages_node(cpu_to_node(cpu),
1779 GFP_KERNEL | __GFP_NORETRY, 0);
1780 /* This is just an optimization and can handle failures */
1781 if (!page) {
1782 pr_err("Failed to allocate event buffer\n");
1783 break;
1784 }
1785
1786 event = page_address(page);
1787 memset(event, 0, sizeof(*event));
1788
1789 per_cpu(trace_buffered_event, cpu) = event;
1790
1791 scoped_guard(preempt) {
1792 if (cpu == smp_processor_id() &&
1793 __this_cpu_read(trace_buffered_event) !=
1794 per_cpu(trace_buffered_event, cpu))
1795 WARN_ON_ONCE(1);
1796 }
1797 }
1798}
1799
1800static void enable_trace_buffered_event(void *data)
1801{
1802 this_cpu_dec(trace_buffered_event_cnt);
1803}
1804
1805static void disable_trace_buffered_event(void *data)
1806{
1807 this_cpu_inc(trace_buffered_event_cnt);
1808}
1809
1810/**
1811 * trace_buffered_event_disable - disable buffering events
1812 *
1813 * When a filter is removed, it is faster to not use the buffered
1814 * events, and to commit directly into the ring buffer. Free up
1815 * the temp buffers when there are no more users. This requires
1816 * special synchronization with current events.
1817 */
1818void trace_buffered_event_disable(void)
1819{
1820 int cpu;
1821
1822 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1823
1824 if (WARN_ON_ONCE(!trace_buffered_event_ref))
1825 return;
1826
1827 if (--trace_buffered_event_ref)
1828 return;
1829
1830 /* For each CPU, set the buffer as used. */
1831 on_each_cpu_mask(tracing_buffer_mask, disable_trace_buffered_event,
1832 NULL, true);
1833
1834 /* Wait for all current users to finish */
1835 synchronize_rcu();
1836
1837 for_each_tracing_cpu(cpu) {
1838 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
1839 per_cpu(trace_buffered_event, cpu) = NULL;
1840 }
1841
1842 /*
1843 * Wait for all CPUs that potentially started checking if they can use
1844 * their event buffer only after the previous synchronize_rcu() call and
1845 * they still read a valid pointer from trace_buffered_event. It must be
1846 * ensured they don't see cleared trace_buffered_event_cnt else they
1847 * could wrongly decide to use the pointed-to buffer which is now freed.
1848 */
1849 synchronize_rcu();
1850
1851 /* For each CPU, relinquish the buffer */
1852 on_each_cpu_mask(tracing_buffer_mask, enable_trace_buffered_event, NULL,
1853 true);
1854}
1855
1856static struct trace_buffer *temp_buffer;
1857
1858struct ring_buffer_event *
1859trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
1860 struct trace_event_file *trace_file,
1861 int type, unsigned long len,
1862 unsigned int trace_ctx)
1863{
1864 struct ring_buffer_event *entry;
1865 struct trace_array *tr = trace_file->tr;
1866 int val;
1867
1868 *current_rb = tr->array_buffer.buffer;
1869
1870 if (!tr->no_filter_buffering_ref &&
1871 (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) {
1872 preempt_disable_notrace();
1873 /*
1874 * Filtering is on, so try to use the per cpu buffer first.
1875 * This buffer will simulate a ring_buffer_event,
1876 * where the type_len is zero and the array[0] will
1877 * hold the full length.
1878 * (see include/linux/ring-buffer.h for details on
1879 * how the ring_buffer_event is structured).
1880 *
1881 * Using a temp buffer during filtering and copying it
1882 * on a matched filter is quicker than writing directly
1883 * into the ring buffer and then discarding it when
1884 * it doesn't match. That is because the discard
1885 * requires several atomic operations to get right.
1886 * Copying on match and doing nothing on a failed match
1887 * is still quicker than no copy on match, but having
1888 * to discard out of the ring buffer on a failed match.
1889 */
1890 if ((entry = __this_cpu_read(trace_buffered_event))) {
1891 int max_len = PAGE_SIZE - struct_size(entry, array, 1);
1892
1893 val = this_cpu_inc_return(trace_buffered_event_cnt);
1894
1895 /*
1896 * Preemption is disabled, but interrupts and NMIs
1897 * can still come in now. If that happens after
1898 * the above increment, then it will have to go
1899 * back to the old method of allocating the event
1900 * on the ring buffer, and if the filter fails, it
1901 * will have to call ring_buffer_discard_commit()
1902 * to remove it.
1903 *
1904 * Need to also check the unlikely case that the
1905 * length is bigger than the temp buffer size.
1906 * If that happens, then the reserve is pretty much
1907 * guaranteed to fail, as the ring buffer currently
1908 * only allows events less than a page. But that may
1909 * change in the future, so let the ring buffer reserve
1910 * handle the failure in that case.
1911 */
1912 if (val == 1 && likely(len <= max_len)) {
1913 trace_event_setup(entry, type, trace_ctx);
1914 entry->array[0] = len;
1915 /* Return with preemption disabled */
1916 return entry;
1917 }
1918 this_cpu_dec(trace_buffered_event_cnt);
1919 }
1920 /* __trace_buffer_lock_reserve() disables preemption */
1921 preempt_enable_notrace();
1922 }
1923
1924 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
1925 trace_ctx);
1926 /*
1927 * If tracing is off, but we have triggers enabled
1928 * we still need to look at the event data. Use the temp_buffer
1929 * to store the trace event for the trigger to use. It's recursive
1930 * safe and will not be recorded anywhere.
1931 */
1932 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
1933 *current_rb = temp_buffer;
1934 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
1935 trace_ctx);
1936 }
1937 return entry;
1938}
1939EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1940
1941static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock);
1942static DEFINE_MUTEX(tracepoint_printk_mutex);
1943
1944static void output_printk(struct trace_event_buffer *fbuffer)
1945{
1946 struct trace_event_call *event_call;
1947 struct trace_event_file *file;
1948 struct trace_event *event;
1949 unsigned long flags;
1950 struct trace_iterator *iter = tracepoint_print_iter;
1951
1952 /* We should never get here if iter is NULL */
1953 if (WARN_ON_ONCE(!iter))
1954 return;
1955
1956 event_call = fbuffer->trace_file->event_call;
1957 if (!event_call || !event_call->event.funcs ||
1958 !event_call->event.funcs->trace)
1959 return;
1960
1961 file = fbuffer->trace_file;
1962 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
1963 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
1964 !filter_match_preds(file->filter, fbuffer->entry)))
1965 return;
1966
1967 event = &fbuffer->trace_file->event_call->event;
1968
1969 raw_spin_lock_irqsave(&tracepoint_iter_lock, flags);
1970 trace_seq_init(&iter->seq);
1971 iter->ent = fbuffer->entry;
1972 event_call->event.funcs->trace(iter, 0, event);
1973 trace_seq_putc(&iter->seq, 0);
1974 printk("%s", iter->seq.buffer);
1975
1976 raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
1977}
1978
1979int tracepoint_printk_sysctl(const struct ctl_table *table, int write,
1980 void *buffer, size_t *lenp,
1981 loff_t *ppos)
1982{
1983 int save_tracepoint_printk;
1984 int ret;
1985
1986 guard(mutex)(&tracepoint_printk_mutex);
1987 save_tracepoint_printk = tracepoint_printk;
1988
1989 ret = proc_dointvec(table, write, buffer, lenp, ppos);
1990
1991 /*
1992 * This will force exiting early, as tracepoint_printk
1993 * is always zero when tracepoint_printk_iter is not allocated
1994 */
1995 if (!tracepoint_print_iter)
1996 tracepoint_printk = 0;
1997
1998 if (save_tracepoint_printk == tracepoint_printk)
1999 return ret;
2000
2001 if (tracepoint_printk)
2002 static_key_enable(&tracepoint_printk_key.key);
2003 else
2004 static_key_disable(&tracepoint_printk_key.key);
2005
2006 return ret;
2007}
2008
2009void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2010{
2011 enum event_trigger_type tt = ETT_NONE;
2012 struct trace_event_file *file = fbuffer->trace_file;
2013
2014 if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event,
2015 fbuffer->entry, &tt))
2016 goto discard;
2017
2018 if (static_key_false(&tracepoint_printk_key.key))
2019 output_printk(fbuffer);
2020
2021 if (static_branch_unlikely(&trace_event_exports_enabled))
2022 ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
2023
2024 trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer,
2025 fbuffer->event, fbuffer->trace_ctx, fbuffer->regs);
2026
2027discard:
2028 if (tt)
2029 event_triggers_post_call(file, tt);
2030
2031}
2032EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2033
2034/*
2035 * Skip 3:
2036 *
2037 * trace_buffer_unlock_commit_regs()
2038 * trace_event_buffer_commit()
2039 * trace_event_raw_event_xxx()
2040 */
2041# define STACK_SKIP 3
2042
2043void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2044 struct trace_buffer *buffer,
2045 struct ring_buffer_event *event,
2046 unsigned int trace_ctx,
2047 struct pt_regs *regs)
2048{
2049 __buffer_unlock_commit(buffer, event);
2050
2051 /*
2052 * If regs is not set, then skip the necessary functions.
2053 * Note, we can still get here via blktrace, wakeup tracer
2054 * and mmiotrace, but that's ok if they lose a function or
2055 * two. They are not that meaningful.
2056 */
2057 ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs);
2058 ftrace_trace_userstack(tr, buffer, trace_ctx);
2059}
2060
2061/*
2062 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2063 */
2064void
2065trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2066 struct ring_buffer_event *event)
2067{
2068 __buffer_unlock_commit(buffer, event);
2069}
2070
2071void
2072trace_function(struct trace_array *tr, unsigned long ip, unsigned long
2073 parent_ip, unsigned int trace_ctx, struct ftrace_regs *fregs)
2074{
2075 struct trace_buffer *buffer = tr->array_buffer.buffer;
2076 struct ring_buffer_event *event;
2077 struct ftrace_entry *entry;
2078 int size = sizeof(*entry);
2079
2080 size += FTRACE_REGS_MAX_ARGS * !!fregs * sizeof(long);
2081
2082 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, size,
2083 trace_ctx);
2084 if (!event)
2085 return;
2086 entry = ring_buffer_event_data(event);
2087 entry->ip = ip;
2088 entry->parent_ip = parent_ip;
2089
2090#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
2091 if (fregs) {
2092 for (int i = 0; i < FTRACE_REGS_MAX_ARGS; i++)
2093 entry->args[i] = ftrace_regs_get_argument(fregs, i);
2094 }
2095#endif
2096
2097 if (static_branch_unlikely(&trace_function_exports_enabled))
2098 ftrace_exports(event, TRACE_EXPORT_FUNCTION);
2099 __buffer_unlock_commit(buffer, event);
2100}
2101
2102#ifdef CONFIG_STACKTRACE
2103
2104/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2105#define FTRACE_KSTACK_NESTING 4
2106
2107#define FTRACE_KSTACK_ENTRIES (SZ_4K / FTRACE_KSTACK_NESTING)
2108
2109struct ftrace_stack {
2110 unsigned long calls[FTRACE_KSTACK_ENTRIES];
2111};
2112
2113
2114struct ftrace_stacks {
2115 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2116};
2117
2118static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2119static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2120
2121void __ftrace_trace_stack(struct trace_array *tr,
2122 struct trace_buffer *buffer,
2123 unsigned int trace_ctx,
2124 int skip, struct pt_regs *regs)
2125{
2126 struct ring_buffer_event *event;
2127 unsigned int size, nr_entries;
2128 struct ftrace_stack *fstack;
2129 struct stack_entry *entry;
2130 int stackidx;
2131 int bit;
2132
2133 bit = trace_test_and_set_recursion(_THIS_IP_, _RET_IP_, TRACE_EVENT_START);
2134 if (bit < 0)
2135 return;
2136
2137 /*
2138 * Add one, for this function and the call to save_stack_trace()
2139 * If regs is set, then these functions will not be in the way.
2140 */
2141#ifndef CONFIG_UNWINDER_ORC
2142 if (!regs)
2143 skip++;
2144#endif
2145
2146 guard(preempt_notrace)();
2147
2148 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2149
2150 /* This should never happen. If it does, yell once and skip */
2151 if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
2152 goto out;
2153
2154 /*
2155 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2156 * interrupt will either see the value pre increment or post
2157 * increment. If the interrupt happens pre increment it will have
2158 * restored the counter when it returns. We just need a barrier to
2159 * keep gcc from moving things around.
2160 */
2161 barrier();
2162
2163 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2164 size = ARRAY_SIZE(fstack->calls);
2165
2166 if (regs) {
2167 nr_entries = stack_trace_save_regs(regs, fstack->calls,
2168 size, skip);
2169 } else {
2170 nr_entries = stack_trace_save(fstack->calls, size, skip);
2171 }
2172
2173#ifdef CONFIG_DYNAMIC_FTRACE
2174 /* Mark entry of stack trace as trampoline code */
2175 if (tr->ops && tr->ops->trampoline) {
2176 unsigned long tramp_start = tr->ops->trampoline;
2177 unsigned long tramp_end = tramp_start + tr->ops->trampoline_size;
2178 unsigned long *calls = fstack->calls;
2179
2180 for (int i = 0; i < nr_entries; i++) {
2181 if (calls[i] >= tramp_start && calls[i] < tramp_end)
2182 calls[i] = FTRACE_TRAMPOLINE_MARKER;
2183 }
2184 }
2185#endif
2186
2187 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2188 struct_size(entry, caller, nr_entries),
2189 trace_ctx);
2190 if (!event)
2191 goto out;
2192 entry = ring_buffer_event_data(event);
2193
2194 entry->size = nr_entries;
2195 memcpy(&entry->caller, fstack->calls,
2196 flex_array_size(entry, caller, nr_entries));
2197
2198 __buffer_unlock_commit(buffer, event);
2199
2200 out:
2201 /* Again, don't let gcc optimize things here */
2202 barrier();
2203 __this_cpu_dec(ftrace_stack_reserve);
2204 trace_clear_recursion(bit);
2205}
2206
2207void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
2208 int skip)
2209{
2210 struct trace_buffer *buffer = tr->array_buffer.buffer;
2211
2212 if (rcu_is_watching()) {
2213 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL);
2214 return;
2215 }
2216
2217 if (WARN_ON_ONCE(IS_ENABLED(CONFIG_GENERIC_ENTRY)))
2218 return;
2219
2220 /*
2221 * When an NMI triggers, RCU is enabled via ct_nmi_enter(),
2222 * but if the above rcu_is_watching() failed, then the NMI
2223 * triggered someplace critical, and ct_irq_enter() should
2224 * not be called from NMI.
2225 */
2226 if (unlikely(in_nmi()))
2227 return;
2228
2229 ct_irq_enter_irqson();
2230 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL);
2231 ct_irq_exit_irqson();
2232}
2233
2234/**
2235 * trace_dump_stack - record a stack back trace in the trace buffer
2236 * @skip: Number of functions to skip (helper handlers)
2237 */
2238void trace_dump_stack(int skip)
2239{
2240 if (tracing_disabled || tracing_selftest_running)
2241 return;
2242
2243#ifndef CONFIG_UNWINDER_ORC
2244 /* Skip 1 to skip this function. */
2245 skip++;
2246#endif
2247 __ftrace_trace_stack(printk_trace, printk_trace->array_buffer.buffer,
2248 tracing_gen_ctx(), skip, NULL);
2249}
2250EXPORT_SYMBOL_GPL(trace_dump_stack);
2251
2252#ifdef CONFIG_USER_STACKTRACE_SUPPORT
2253static DEFINE_PER_CPU(int, user_stack_count);
2254
2255static void
2256ftrace_trace_userstack(struct trace_array *tr,
2257 struct trace_buffer *buffer, unsigned int trace_ctx)
2258{
2259 struct ring_buffer_event *event;
2260 struct userstack_entry *entry;
2261
2262 if (!(tr->trace_flags & TRACE_ITER(USERSTACKTRACE)))
2263 return;
2264
2265 /*
2266 * NMIs can not handle page faults, even with fix ups.
2267 * The save user stack can (and often does) fault.
2268 */
2269 if (unlikely(in_nmi()))
2270 return;
2271
2272 /*
2273 * prevent recursion, since the user stack tracing may
2274 * trigger other kernel events.
2275 */
2276 guard(preempt)();
2277 if (__this_cpu_read(user_stack_count))
2278 return;
2279
2280 __this_cpu_inc(user_stack_count);
2281
2282 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2283 sizeof(*entry), trace_ctx);
2284 if (!event)
2285 goto out_drop_count;
2286 entry = ring_buffer_event_data(event);
2287
2288 entry->tgid = current->tgid;
2289 memset(&entry->caller, 0, sizeof(entry->caller));
2290
2291 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
2292 __buffer_unlock_commit(buffer, event);
2293
2294 out_drop_count:
2295 __this_cpu_dec(user_stack_count);
2296}
2297#else /* CONFIG_USER_STACKTRACE_SUPPORT */
2298static void ftrace_trace_userstack(struct trace_array *tr,
2299 struct trace_buffer *buffer,
2300 unsigned int trace_ctx)
2301{
2302}
2303#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
2304
2305#endif /* CONFIG_STACKTRACE */
2306
2307static inline void
2308func_repeats_set_delta_ts(struct func_repeats_entry *entry,
2309 unsigned long long delta)
2310{
2311 entry->bottom_delta_ts = delta & U32_MAX;
2312 entry->top_delta_ts = (delta >> 32);
2313}
2314
2315void trace_last_func_repeats(struct trace_array *tr,
2316 struct trace_func_repeats *last_info,
2317 unsigned int trace_ctx)
2318{
2319 struct trace_buffer *buffer = tr->array_buffer.buffer;
2320 struct func_repeats_entry *entry;
2321 struct ring_buffer_event *event;
2322 u64 delta;
2323
2324 event = __trace_buffer_lock_reserve(buffer, TRACE_FUNC_REPEATS,
2325 sizeof(*entry), trace_ctx);
2326 if (!event)
2327 return;
2328
2329 delta = ring_buffer_event_time_stamp(buffer, event) -
2330 last_info->ts_last_call;
2331
2332 entry = ring_buffer_event_data(event);
2333 entry->ip = last_info->ip;
2334 entry->parent_ip = last_info->parent_ip;
2335 entry->count = last_info->count;
2336 func_repeats_set_delta_ts(entry, delta);
2337
2338 __buffer_unlock_commit(buffer, event);
2339}
2340
2341static struct trace_entry *
2342peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2343 unsigned long *lost_events)
2344{
2345 struct ring_buffer_event *event;
2346 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2347
2348 if (buf_iter) {
2349 event = ring_buffer_iter_peek(buf_iter, ts);
2350 if (lost_events)
2351 *lost_events = ring_buffer_iter_dropped(buf_iter) ?
2352 (unsigned long)-1 : 0;
2353 } else {
2354 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
2355 lost_events);
2356 }
2357
2358 if (event) {
2359 iter->ent_size = ring_buffer_event_length(event);
2360 return ring_buffer_event_data(event);
2361 }
2362 iter->ent_size = 0;
2363 return NULL;
2364}
2365
2366static struct trace_entry *
2367__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2368 unsigned long *missing_events, u64 *ent_ts)
2369{
2370 struct trace_buffer *buffer = iter->array_buffer->buffer;
2371 struct trace_entry *ent, *next = NULL;
2372 unsigned long lost_events = 0, next_lost = 0;
2373 int cpu_file = iter->cpu_file;
2374 u64 next_ts = 0, ts;
2375 int next_cpu = -1;
2376 int next_size = 0;
2377 int cpu;
2378
2379 /*
2380 * If we are in a per_cpu trace file, don't bother by iterating over
2381 * all cpu and peek directly.
2382 */
2383 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2384 if (ring_buffer_empty_cpu(buffer, cpu_file))
2385 return NULL;
2386 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2387 if (ent_cpu)
2388 *ent_cpu = cpu_file;
2389
2390 return ent;
2391 }
2392
2393 for_each_tracing_cpu(cpu) {
2394
2395 if (ring_buffer_empty_cpu(buffer, cpu))
2396 continue;
2397
2398 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2399
2400 /*
2401 * Pick the entry with the smallest timestamp:
2402 */
2403 if (ent && (!next || ts < next_ts)) {
2404 next = ent;
2405 next_cpu = cpu;
2406 next_ts = ts;
2407 next_lost = lost_events;
2408 next_size = iter->ent_size;
2409 }
2410 }
2411
2412 iter->ent_size = next_size;
2413
2414 if (ent_cpu)
2415 *ent_cpu = next_cpu;
2416
2417 if (ent_ts)
2418 *ent_ts = next_ts;
2419
2420 if (missing_events)
2421 *missing_events = next_lost;
2422
2423 return next;
2424}
2425
2426#define STATIC_FMT_BUF_SIZE 128
2427static char static_fmt_buf[STATIC_FMT_BUF_SIZE];
2428
2429char *trace_iter_expand_format(struct trace_iterator *iter)
2430{
2431 char *tmp;
2432
2433 /*
2434 * iter->tr is NULL when used with tp_printk, which makes
2435 * this get called where it is not safe to call krealloc().
2436 */
2437 if (!iter->tr || iter->fmt == static_fmt_buf)
2438 return NULL;
2439
2440 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE,
2441 GFP_KERNEL);
2442 if (tmp) {
2443 iter->fmt_size += STATIC_FMT_BUF_SIZE;
2444 iter->fmt = tmp;
2445 }
2446
2447 return tmp;
2448}
2449
2450/* Returns true if the string is safe to dereference from an event */
2451static bool trace_safe_str(struct trace_iterator *iter, const char *str)
2452{
2453 unsigned long addr = (unsigned long)str;
2454 struct trace_event *trace_event;
2455 struct trace_event_call *event;
2456
2457 /* OK if part of the event data */
2458 if ((addr >= (unsigned long)iter->ent) &&
2459 (addr < (unsigned long)iter->ent + iter->ent_size))
2460 return true;
2461
2462 /* OK if part of the temp seq buffer */
2463 if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&
2464 (addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE))
2465 return true;
2466
2467 /* Core rodata can not be freed */
2468 if (is_kernel_rodata(addr))
2469 return true;
2470
2471 if (trace_is_tracepoint_string(str))
2472 return true;
2473
2474 /*
2475 * Now this could be a module event, referencing core module
2476 * data, which is OK.
2477 */
2478 if (!iter->ent)
2479 return false;
2480
2481 trace_event = ftrace_find_event(iter->ent->type);
2482 if (!trace_event)
2483 return false;
2484
2485 event = container_of(trace_event, struct trace_event_call, event);
2486 if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module)
2487 return false;
2488
2489 /* Would rather have rodata, but this will suffice */
2490 if (within_module_core(addr, event->module))
2491 return true;
2492
2493 return false;
2494}
2495
2496/**
2497 * ignore_event - Check dereferenced fields while writing to the seq buffer
2498 * @iter: The iterator that holds the seq buffer and the event being printed
2499 *
2500 * At boot up, test_event_printk() will flag any event that dereferences
2501 * a string with "%s" that does exist in the ring buffer. It may still
2502 * be valid, as the string may point to a static string in the kernel
2503 * rodata that never gets freed. But if the string pointer is pointing
2504 * to something that was allocated, there's a chance that it can be freed
2505 * by the time the user reads the trace. This would cause a bad memory
2506 * access by the kernel and possibly crash the system.
2507 *
2508 * This function will check if the event has any fields flagged as needing
2509 * to be checked at runtime and perform those checks.
2510 *
2511 * If it is found that a field is unsafe, it will write into the @iter->seq
2512 * a message stating what was found to be unsafe.
2513 *
2514 * @return: true if the event is unsafe and should be ignored,
2515 * false otherwise.
2516 */
2517bool ignore_event(struct trace_iterator *iter)
2518{
2519 struct ftrace_event_field *field;
2520 struct trace_event *trace_event;
2521 struct trace_event_call *event;
2522 struct list_head *head;
2523 struct trace_seq *seq;
2524 const void *ptr;
2525
2526 trace_event = ftrace_find_event(iter->ent->type);
2527
2528 seq = &iter->seq;
2529
2530 if (!trace_event) {
2531 trace_seq_printf(seq, "EVENT ID %d NOT FOUND?\n", iter->ent->type);
2532 return true;
2533 }
2534
2535 event = container_of(trace_event, struct trace_event_call, event);
2536 if (!(event->flags & TRACE_EVENT_FL_TEST_STR))
2537 return false;
2538
2539 head = trace_get_fields(event);
2540 if (!head) {
2541 trace_seq_printf(seq, "FIELDS FOR EVENT '%s' NOT FOUND?\n",
2542 trace_event_name(event));
2543 return true;
2544 }
2545
2546 /* Offsets are from the iter->ent that points to the raw event */
2547 ptr = iter->ent;
2548
2549 list_for_each_entry(field, head, link) {
2550 const char *str;
2551 bool good;
2552
2553 if (!field->needs_test)
2554 continue;
2555
2556 str = *(const char **)(ptr + field->offset);
2557
2558 good = trace_safe_str(iter, str);
2559
2560 /*
2561 * If you hit this warning, it is likely that the
2562 * trace event in question used %s on a string that
2563 * was saved at the time of the event, but may not be
2564 * around when the trace is read. Use __string(),
2565 * __assign_str() and __get_str() helpers in the TRACE_EVENT()
2566 * instead. See samples/trace_events/trace-events-sample.h
2567 * for reference.
2568 */
2569 if (WARN_ONCE(!good, "event '%s' has unsafe pointer field '%s'",
2570 trace_event_name(event), field->name)) {
2571 trace_seq_printf(seq, "EVENT %s: HAS UNSAFE POINTER FIELD '%s'\n",
2572 trace_event_name(event), field->name);
2573 return true;
2574 }
2575 }
2576 return false;
2577}
2578
2579const char *trace_event_format(struct trace_iterator *iter, const char *fmt)
2580{
2581 const char *p, *new_fmt;
2582 char *q;
2583
2584 if (WARN_ON_ONCE(!fmt))
2585 return fmt;
2586
2587 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER(HASH_PTR))
2588 return fmt;
2589
2590 p = fmt;
2591 new_fmt = q = iter->fmt;
2592 while (*p) {
2593 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {
2594 if (!trace_iter_expand_format(iter))
2595 return fmt;
2596
2597 q += iter->fmt - new_fmt;
2598 new_fmt = iter->fmt;
2599 }
2600
2601 *q++ = *p++;
2602
2603 /* Replace %p with %px */
2604 if (p[-1] == '%') {
2605 if (p[0] == '%') {
2606 *q++ = *p++;
2607 } else if (p[0] == 'p' && !isalnum(p[1])) {
2608 *q++ = *p++;
2609 *q++ = 'x';
2610 }
2611 }
2612 }
2613 *q = '\0';
2614
2615 return new_fmt;
2616}
2617
2618#define STATIC_TEMP_BUF_SIZE 128
2619static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
2620
2621/* Find the next real entry, without updating the iterator itself */
2622struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2623 int *ent_cpu, u64 *ent_ts)
2624{
2625 /* __find_next_entry will reset ent_size */
2626 int ent_size = iter->ent_size;
2627 struct trace_entry *entry;
2628
2629 /*
2630 * If called from ftrace_dump(), then the iter->temp buffer
2631 * will be the static_temp_buf and not created from kmalloc.
2632 * If the entry size is greater than the buffer, we can
2633 * not save it. Just return NULL in that case. This is only
2634 * used to add markers when two consecutive events' time
2635 * stamps have a large delta. See trace_print_lat_context()
2636 */
2637 if (iter->temp == static_temp_buf &&
2638 STATIC_TEMP_BUF_SIZE < ent_size)
2639 return NULL;
2640
2641 /*
2642 * The __find_next_entry() may call peek_next_entry(), which may
2643 * call ring_buffer_peek() that may make the contents of iter->ent
2644 * undefined. Need to copy iter->ent now.
2645 */
2646 if (iter->ent && iter->ent != iter->temp) {
2647 if ((!iter->temp || iter->temp_size < iter->ent_size) &&
2648 !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
2649 void *temp;
2650 temp = kmalloc(iter->ent_size, GFP_KERNEL);
2651 if (!temp)
2652 return NULL;
2653 kfree(iter->temp);
2654 iter->temp = temp;
2655 iter->temp_size = iter->ent_size;
2656 }
2657 memcpy(iter->temp, iter->ent, iter->ent_size);
2658 iter->ent = iter->temp;
2659 }
2660 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2661 /* Put back the original ent_size */
2662 iter->ent_size = ent_size;
2663
2664 return entry;
2665}
2666
2667/* Find the next real entry, and increment the iterator to the next entry */
2668void *trace_find_next_entry_inc(struct trace_iterator *iter)
2669{
2670 struct ring_buffer_iter *buf_iter;
2671
2672 iter->ent = __find_next_entry(iter, &iter->cpu,
2673 &iter->lost_events, &iter->ts);
2674
2675 if (iter->ent) {
2676 iter->idx++;
2677 buf_iter = trace_buffer_iter(iter, iter->cpu);
2678 if (buf_iter)
2679 ring_buffer_iter_advance(buf_iter);
2680 }
2681
2682 return iter->ent ? iter : NULL;
2683}
2684
2685static void trace_consume(struct trace_iterator *iter)
2686{
2687 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
2688 &iter->lost_events);
2689}
2690
2691static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2692{
2693 struct trace_iterator *iter = m->private;
2694 int i = (int)*pos;
2695 void *ent;
2696
2697 WARN_ON_ONCE(iter->leftover);
2698
2699 (*pos)++;
2700
2701 /* can't go backwards */
2702 if (iter->idx > i)
2703 return NULL;
2704
2705 if (iter->idx < 0)
2706 ent = trace_find_next_entry_inc(iter);
2707 else
2708 ent = iter;
2709
2710 while (ent && iter->idx < i)
2711 ent = trace_find_next_entry_inc(iter);
2712
2713 iter->pos = *pos;
2714
2715 return ent;
2716}
2717
2718void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2719{
2720 struct ring_buffer_iter *buf_iter;
2721 unsigned long entries = 0;
2722 u64 ts;
2723
2724 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
2725
2726 buf_iter = trace_buffer_iter(iter, cpu);
2727 if (!buf_iter)
2728 return;
2729
2730 ring_buffer_iter_reset(buf_iter);
2731
2732 /*
2733 * We could have the case with the max latency tracers
2734 * that a reset never took place on a cpu. This is evident
2735 * by the timestamp being before the start of the buffer.
2736 */
2737 while (ring_buffer_iter_peek(buf_iter, &ts)) {
2738 if (ts >= iter->array_buffer->time_start)
2739 break;
2740 entries++;
2741 ring_buffer_iter_advance(buf_iter);
2742 /* This could be a big loop */
2743 cond_resched();
2744 }
2745
2746 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
2747}
2748
2749/*
2750 * The current tracer is copied to avoid a global locking
2751 * all around.
2752 */
2753static void *s_start(struct seq_file *m, loff_t *pos)
2754{
2755 struct trace_iterator *iter = m->private;
2756 struct trace_array *tr = iter->tr;
2757 int cpu_file = iter->cpu_file;
2758 void *p = NULL;
2759 loff_t l = 0;
2760 int cpu;
2761
2762 mutex_lock(&trace_types_lock);
2763 if (unlikely(tr->current_trace != iter->trace)) {
2764 /* Close iter->trace before switching to the new current tracer */
2765 if (iter->trace->close)
2766 iter->trace->close(iter);
2767 iter->trace = tr->current_trace;
2768 /* Reopen the new current tracer */
2769 if (iter->trace->open)
2770 iter->trace->open(iter);
2771 }
2772 mutex_unlock(&trace_types_lock);
2773
2774 if (iter->snapshot && tracer_uses_snapshot(iter->trace))
2775 return ERR_PTR(-EBUSY);
2776
2777 if (*pos != iter->pos) {
2778 iter->ent = NULL;
2779 iter->cpu = 0;
2780 iter->idx = -1;
2781
2782 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2783 for_each_tracing_cpu(cpu)
2784 tracing_iter_reset(iter, cpu);
2785 } else
2786 tracing_iter_reset(iter, cpu_file);
2787
2788 iter->leftover = 0;
2789 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2790 ;
2791
2792 } else {
2793 /*
2794 * If we overflowed the seq_file before, then we want
2795 * to just reuse the trace_seq buffer again.
2796 */
2797 if (iter->leftover)
2798 p = iter;
2799 else {
2800 l = *pos - 1;
2801 p = s_next(m, p, &l);
2802 }
2803 }
2804
2805 trace_event_read_lock();
2806 trace_access_lock(cpu_file);
2807 return p;
2808}
2809
2810static void s_stop(struct seq_file *m, void *p)
2811{
2812 struct trace_iterator *iter = m->private;
2813
2814 if (iter->snapshot && tracer_uses_snapshot(iter->trace))
2815 return;
2816
2817 trace_access_unlock(iter->cpu_file);
2818 trace_event_read_unlock();
2819}
2820
2821static void
2822get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
2823 unsigned long *entries, int cpu)
2824{
2825 unsigned long count;
2826
2827 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2828 /*
2829 * If this buffer has skipped entries, then we hold all
2830 * entries for the trace and we need to ignore the
2831 * ones before the time stamp.
2832 */
2833 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2834 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2835 /* total is the same as the entries */
2836 *total = count;
2837 } else
2838 *total = count +
2839 ring_buffer_overrun_cpu(buf->buffer, cpu);
2840 *entries = count;
2841}
2842
2843static void
2844get_total_entries(struct array_buffer *buf,
2845 unsigned long *total, unsigned long *entries)
2846{
2847 unsigned long t, e;
2848 int cpu;
2849
2850 *total = 0;
2851 *entries = 0;
2852
2853 for_each_tracing_cpu(cpu) {
2854 get_total_entries_cpu(buf, &t, &e, cpu);
2855 *total += t;
2856 *entries += e;
2857 }
2858}
2859
2860unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
2861{
2862 unsigned long total, entries;
2863
2864 if (!tr)
2865 tr = &global_trace;
2866
2867 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
2868
2869 return entries;
2870}
2871
2872unsigned long trace_total_entries(struct trace_array *tr)
2873{
2874 unsigned long total, entries;
2875
2876 if (!tr)
2877 tr = &global_trace;
2878
2879 get_total_entries(&tr->array_buffer, &total, &entries);
2880
2881 return entries;
2882}
2883
2884static void print_lat_help_header(struct seq_file *m)
2885{
2886 seq_puts(m, "# _------=> CPU# \n"
2887 "# / _-----=> irqs-off/BH-disabled\n"
2888 "# | / _----=> need-resched \n"
2889 "# || / _---=> hardirq/softirq \n"
2890 "# ||| / _--=> preempt-depth \n"
2891 "# |||| / _-=> migrate-disable \n"
2892 "# ||||| / delay \n"
2893 "# cmd pid |||||| time | caller \n"
2894 "# \\ / |||||| \\ | / \n");
2895}
2896
2897static void print_event_info(struct array_buffer *buf, struct seq_file *m)
2898{
2899 unsigned long total;
2900 unsigned long entries;
2901
2902 get_total_entries(buf, &total, &entries);
2903 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2904 entries, total, num_online_cpus());
2905 seq_puts(m, "#\n");
2906}
2907
2908static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
2909 unsigned int flags)
2910{
2911 bool tgid = flags & TRACE_ITER(RECORD_TGID);
2912
2913 print_event_info(buf, m);
2914
2915 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
2916 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
2917}
2918
2919static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
2920 unsigned int flags)
2921{
2922 bool tgid = flags & TRACE_ITER(RECORD_TGID);
2923 static const char space[] = " ";
2924 int prec = tgid ? 12 : 2;
2925
2926 print_event_info(buf, m);
2927
2928 seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space);
2929 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
2930 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
2931 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
2932 seq_printf(m, "# %.*s||| / _-=> migrate-disable\n", prec, space);
2933 seq_printf(m, "# %.*s|||| / delay\n", prec, space);
2934 seq_printf(m, "# TASK-PID %.*s CPU# ||||| TIMESTAMP FUNCTION\n", prec, " TGID ");
2935 seq_printf(m, "# | | %.*s | ||||| | |\n", prec, " | ");
2936}
2937
2938void
2939print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2940{
2941 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2942 struct array_buffer *buf = iter->array_buffer;
2943 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2944 struct tracer *type = iter->trace;
2945 unsigned long entries;
2946 unsigned long total;
2947 const char *name = type->name;
2948
2949 get_total_entries(buf, &total, &entries);
2950
2951 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2952 name, init_utsname()->release);
2953 seq_puts(m, "# -----------------------------------"
2954 "---------------------------------\n");
2955 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2956 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2957 nsecs_to_usecs(data->saved_latency),
2958 entries,
2959 total,
2960 buf->cpu,
2961 preempt_model_str(),
2962 /* These are reserved for later use */
2963 0, 0, 0, 0);
2964#ifdef CONFIG_SMP
2965 seq_printf(m, " #P:%d)\n", num_online_cpus());
2966#else
2967 seq_puts(m, ")\n");
2968#endif
2969 seq_puts(m, "# -----------------\n");
2970 seq_printf(m, "# | task: %.16s-%d "
2971 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2972 data->comm, data->pid,
2973 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2974 data->policy, data->rt_priority);
2975 seq_puts(m, "# -----------------\n");
2976
2977 if (data->critical_start) {
2978 seq_puts(m, "# => started at: ");
2979 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2980 trace_print_seq(m, &iter->seq);
2981 seq_puts(m, "\n# => ended at: ");
2982 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2983 trace_print_seq(m, &iter->seq);
2984 seq_puts(m, "\n#\n");
2985 }
2986
2987 seq_puts(m, "#\n");
2988}
2989
2990static void test_cpu_buff_start(struct trace_iterator *iter)
2991{
2992 struct trace_seq *s = &iter->seq;
2993 struct trace_array *tr = iter->tr;
2994
2995 if (!(tr->trace_flags & TRACE_ITER(ANNOTATE)))
2996 return;
2997
2998 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2999 return;
3000
3001 if (cpumask_available(iter->started) &&
3002 cpumask_test_cpu(iter->cpu, iter->started))
3003 return;
3004
3005 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
3006 return;
3007
3008 if (cpumask_available(iter->started))
3009 cpumask_set_cpu(iter->cpu, iter->started);
3010
3011 /* Don't print started cpu buffer for the first entry of the trace */
3012 if (iter->idx > 1)
3013 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3014 iter->cpu);
3015}
3016
3017#ifdef CONFIG_FTRACE_SYSCALLS
3018static bool is_syscall_event(struct trace_event *event)
3019{
3020 return (event->funcs == &enter_syscall_print_funcs) ||
3021 (event->funcs == &exit_syscall_print_funcs);
3022
3023}
3024#define syscall_buf_size CONFIG_TRACE_SYSCALL_BUF_SIZE_DEFAULT
3025#else
3026static inline bool is_syscall_event(struct trace_event *event)
3027{
3028 return false;
3029}
3030#define syscall_buf_size 0
3031#endif /* CONFIG_FTRACE_SYSCALLS */
3032
3033static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3034{
3035 struct trace_array *tr = iter->tr;
3036 struct trace_seq *s = &iter->seq;
3037 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3038 struct trace_entry *entry;
3039 struct trace_event *event;
3040
3041 entry = iter->ent;
3042
3043 test_cpu_buff_start(iter);
3044
3045 event = ftrace_find_event(entry->type);
3046
3047 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) {
3048 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3049 trace_print_lat_context(iter);
3050 else
3051 trace_print_context(iter);
3052 }
3053
3054 if (trace_seq_has_overflowed(s))
3055 return TRACE_TYPE_PARTIAL_LINE;
3056
3057 if (event) {
3058 if (tr->trace_flags & TRACE_ITER(FIELDS))
3059 return print_event_fields(iter, event);
3060 /*
3061 * For TRACE_EVENT() events, the print_fmt is not
3062 * safe to use if the array has delta offsets
3063 * Force printing via the fields.
3064 */
3065 if ((tr->text_delta)) {
3066 /* ftrace and system call events are still OK */
3067 if ((event->type > __TRACE_LAST_TYPE) &&
3068 !is_syscall_event(event))
3069 return print_event_fields(iter, event);
3070 }
3071 return event->funcs->trace(iter, sym_flags, event);
3072 }
3073
3074 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3075
3076 return trace_handle_return(s);
3077}
3078
3079static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3080{
3081 struct trace_array *tr = iter->tr;
3082 struct trace_seq *s = &iter->seq;
3083 struct trace_entry *entry;
3084 struct trace_event *event;
3085
3086 entry = iter->ent;
3087
3088 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO))
3089 trace_seq_printf(s, "%d %d %llu ",
3090 entry->pid, iter->cpu, iter->ts);
3091
3092 if (trace_seq_has_overflowed(s))
3093 return TRACE_TYPE_PARTIAL_LINE;
3094
3095 event = ftrace_find_event(entry->type);
3096 if (event)
3097 return event->funcs->raw(iter, 0, event);
3098
3099 trace_seq_printf(s, "%d ?\n", entry->type);
3100
3101 return trace_handle_return(s);
3102}
3103
3104static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3105{
3106 struct trace_array *tr = iter->tr;
3107 struct trace_seq *s = &iter->seq;
3108 unsigned char newline = '\n';
3109 struct trace_entry *entry;
3110 struct trace_event *event;
3111
3112 entry = iter->ent;
3113
3114 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) {
3115 SEQ_PUT_HEX_FIELD(s, entry->pid);
3116 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3117 SEQ_PUT_HEX_FIELD(s, iter->ts);
3118 if (trace_seq_has_overflowed(s))
3119 return TRACE_TYPE_PARTIAL_LINE;
3120 }
3121
3122 event = ftrace_find_event(entry->type);
3123 if (event) {
3124 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3125 if (ret != TRACE_TYPE_HANDLED)
3126 return ret;
3127 }
3128
3129 SEQ_PUT_FIELD(s, newline);
3130
3131 return trace_handle_return(s);
3132}
3133
3134static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3135{
3136 struct trace_array *tr = iter->tr;
3137 struct trace_seq *s = &iter->seq;
3138 struct trace_entry *entry;
3139 struct trace_event *event;
3140
3141 entry = iter->ent;
3142
3143 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) {
3144 SEQ_PUT_FIELD(s, entry->pid);
3145 SEQ_PUT_FIELD(s, iter->cpu);
3146 SEQ_PUT_FIELD(s, iter->ts);
3147 if (trace_seq_has_overflowed(s))
3148 return TRACE_TYPE_PARTIAL_LINE;
3149 }
3150
3151 event = ftrace_find_event(entry->type);
3152 return event ? event->funcs->binary(iter, 0, event) :
3153 TRACE_TYPE_HANDLED;
3154}
3155
3156int trace_empty(struct trace_iterator *iter)
3157{
3158 struct ring_buffer_iter *buf_iter;
3159 int cpu;
3160
3161 /* If we are looking at one CPU buffer, only check that one */
3162 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3163 cpu = iter->cpu_file;
3164 buf_iter = trace_buffer_iter(iter, cpu);
3165 if (buf_iter) {
3166 if (!ring_buffer_iter_empty(buf_iter))
3167 return 0;
3168 } else {
3169 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
3170 return 0;
3171 }
3172 return 1;
3173 }
3174
3175 for_each_tracing_cpu(cpu) {
3176 buf_iter = trace_buffer_iter(iter, cpu);
3177 if (buf_iter) {
3178 if (!ring_buffer_iter_empty(buf_iter))
3179 return 0;
3180 } else {
3181 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
3182 return 0;
3183 }
3184 }
3185
3186 return 1;
3187}
3188
3189/* Called with trace_event_read_lock() held. */
3190enum print_line_t print_trace_line(struct trace_iterator *iter)
3191{
3192 struct trace_array *tr = iter->tr;
3193 unsigned long trace_flags = tr->trace_flags;
3194 enum print_line_t ret;
3195
3196 if (iter->lost_events) {
3197 if (iter->lost_events == (unsigned long)-1)
3198 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
3199 iter->cpu);
3200 else
3201 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3202 iter->cpu, iter->lost_events);
3203 if (trace_seq_has_overflowed(&iter->seq))
3204 return TRACE_TYPE_PARTIAL_LINE;
3205 }
3206
3207 if (iter->trace && iter->trace->print_line) {
3208 ret = iter->trace->print_line(iter);
3209 if (ret != TRACE_TYPE_UNHANDLED)
3210 return ret;
3211 }
3212
3213 if (iter->ent->type == TRACE_BPUTS &&
3214 trace_flags & TRACE_ITER(PRINTK) &&
3215 trace_flags & TRACE_ITER(PRINTK_MSGONLY))
3216 return trace_print_bputs_msg_only(iter);
3217
3218 if (iter->ent->type == TRACE_BPRINT &&
3219 trace_flags & TRACE_ITER(PRINTK) &&
3220 trace_flags & TRACE_ITER(PRINTK_MSGONLY))
3221 return trace_print_bprintk_msg_only(iter);
3222
3223 if (iter->ent->type == TRACE_PRINT &&
3224 trace_flags & TRACE_ITER(PRINTK) &&
3225 trace_flags & TRACE_ITER(PRINTK_MSGONLY))
3226 return trace_print_printk_msg_only(iter);
3227
3228 if (trace_flags & TRACE_ITER(BIN))
3229 return print_bin_fmt(iter);
3230
3231 if (trace_flags & TRACE_ITER(HEX))
3232 return print_hex_fmt(iter);
3233
3234 if (trace_flags & TRACE_ITER(RAW))
3235 return print_raw_fmt(iter);
3236
3237 return print_trace_fmt(iter);
3238}
3239
3240void trace_latency_header(struct seq_file *m)
3241{
3242 struct trace_iterator *iter = m->private;
3243 struct trace_array *tr = iter->tr;
3244
3245 /* print nothing if the buffers are empty */
3246 if (trace_empty(iter))
3247 return;
3248
3249 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3250 print_trace_header(m, iter);
3251
3252 if (!(tr->trace_flags & TRACE_ITER(VERBOSE)))
3253 print_lat_help_header(m);
3254}
3255
3256void trace_default_header(struct seq_file *m)
3257{
3258 struct trace_iterator *iter = m->private;
3259 struct trace_array *tr = iter->tr;
3260 unsigned long trace_flags = tr->trace_flags;
3261
3262 if (!(trace_flags & TRACE_ITER(CONTEXT_INFO)))
3263 return;
3264
3265 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3266 /* print nothing if the buffers are empty */
3267 if (trace_empty(iter))
3268 return;
3269 print_trace_header(m, iter);
3270 if (!(trace_flags & TRACE_ITER(VERBOSE)))
3271 print_lat_help_header(m);
3272 } else {
3273 if (!(trace_flags & TRACE_ITER(VERBOSE))) {
3274 if (trace_flags & TRACE_ITER(IRQ_INFO))
3275 print_func_help_header_irq(iter->array_buffer,
3276 m, trace_flags);
3277 else
3278 print_func_help_header(iter->array_buffer, m,
3279 trace_flags);
3280 }
3281 }
3282}
3283
3284static void test_ftrace_alive(struct seq_file *m)
3285{
3286 if (!ftrace_is_dead())
3287 return;
3288 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3289 "# MAY BE MISSING FUNCTION EVENTS\n");
3290}
3291
3292static int s_show(struct seq_file *m, void *v)
3293{
3294 struct trace_iterator *iter = v;
3295 int ret;
3296
3297 if (iter->ent == NULL) {
3298 if (iter->tr) {
3299 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3300 seq_puts(m, "#\n");
3301 test_ftrace_alive(m);
3302 }
3303 if (iter->snapshot && trace_empty(iter))
3304 print_snapshot_help(m, iter);
3305 else if (iter->trace && iter->trace->print_header)
3306 iter->trace->print_header(m);
3307 else
3308 trace_default_header(m);
3309
3310 } else if (iter->leftover) {
3311 /*
3312 * If we filled the seq_file buffer earlier, we
3313 * want to just show it now.
3314 */
3315 ret = trace_print_seq(m, &iter->seq);
3316
3317 /* ret should this time be zero, but you never know */
3318 iter->leftover = ret;
3319
3320 } else {
3321 ret = print_trace_line(iter);
3322 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3323 iter->seq.full = 0;
3324 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
3325 }
3326 ret = trace_print_seq(m, &iter->seq);
3327 /*
3328 * If we overflow the seq_file buffer, then it will
3329 * ask us for this data again at start up.
3330 * Use that instead.
3331 * ret is 0 if seq_file write succeeded.
3332 * -1 otherwise.
3333 */
3334 iter->leftover = ret;
3335 }
3336
3337 return 0;
3338}
3339
3340static const struct seq_operations tracer_seq_ops = {
3341 .start = s_start,
3342 .next = s_next,
3343 .stop = s_stop,
3344 .show = s_show,
3345};
3346
3347/*
3348 * Note, as iter itself can be allocated and freed in different
3349 * ways, this function is only used to free its content, and not
3350 * the iterator itself. The only requirement to all the allocations
3351 * is that it must zero all fields (kzalloc), as freeing works with
3352 * ethier allocated content or NULL.
3353 */
3354static void free_trace_iter_content(struct trace_iterator *iter)
3355{
3356 /* The fmt is either NULL, allocated or points to static_fmt_buf */
3357 if (iter->fmt != static_fmt_buf)
3358 kfree(iter->fmt);
3359
3360 kfree(iter->temp);
3361 kfree(iter->buffer_iter);
3362 mutex_destroy(&iter->mutex);
3363 free_cpumask_var(iter->started);
3364}
3365
3366struct trace_iterator *
3367__tracing_open(struct inode *inode, struct file *file, bool snapshot)
3368{
3369 struct trace_array *tr = inode->i_private;
3370 struct trace_iterator *iter;
3371 int cpu;
3372
3373 if (tracing_disabled)
3374 return ERR_PTR(-ENODEV);
3375
3376 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3377 if (!iter)
3378 return ERR_PTR(-ENOMEM);
3379
3380 iter->buffer_iter = kzalloc_objs(*iter->buffer_iter, nr_cpu_ids);
3381 if (!iter->buffer_iter)
3382 goto release;
3383
3384 /*
3385 * trace_find_next_entry() may need to save off iter->ent.
3386 * It will place it into the iter->temp buffer. As most
3387 * events are less than 128, allocate a buffer of that size.
3388 * If one is greater, then trace_find_next_entry() will
3389 * allocate a new buffer to adjust for the bigger iter->ent.
3390 * It's not critical if it fails to get allocated here.
3391 */
3392 iter->temp = kmalloc(128, GFP_KERNEL);
3393 if (iter->temp)
3394 iter->temp_size = 128;
3395
3396 /*
3397 * trace_event_printf() may need to modify given format
3398 * string to replace %p with %px so that it shows real address
3399 * instead of hash value. However, that is only for the event
3400 * tracing, other tracer may not need. Defer the allocation
3401 * until it is needed.
3402 */
3403 iter->fmt = NULL;
3404 iter->fmt_size = 0;
3405
3406 mutex_lock(&trace_types_lock);
3407 iter->trace = tr->current_trace;
3408
3409 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3410 goto fail;
3411
3412 iter->tr = tr;
3413
3414#ifdef CONFIG_TRACER_SNAPSHOT
3415 /* Currently only the top directory has a snapshot */
3416 if (tr->current_trace->print_max || snapshot)
3417 iter->array_buffer = &tr->snapshot_buffer;
3418 else
3419#endif
3420 iter->array_buffer = &tr->array_buffer;
3421 iter->snapshot = snapshot;
3422 iter->pos = -1;
3423 iter->cpu_file = tracing_get_cpu(inode);
3424 mutex_init(&iter->mutex);
3425
3426 /* Notify the tracer early; before we stop tracing. */
3427 if (iter->trace->open)
3428 iter->trace->open(iter);
3429
3430 /* Annotate start of buffers if we had overruns */
3431 if (ring_buffer_overruns(iter->array_buffer->buffer))
3432 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3433
3434 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3435 if (trace_clocks[tr->clock_id].in_ns)
3436 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3437
3438 /*
3439 * If pause-on-trace is enabled, then stop the trace while
3440 * dumping, unless this is the "snapshot" file
3441 */
3442 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER(PAUSE_ON_TRACE))) {
3443 iter->iter_flags |= TRACE_FILE_PAUSE;
3444 tracing_stop_tr(tr);
3445 }
3446
3447 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3448 for_each_tracing_cpu(cpu) {
3449 iter->buffer_iter[cpu] =
3450 ring_buffer_read_start(iter->array_buffer->buffer,
3451 cpu, GFP_KERNEL);
3452 tracing_iter_reset(iter, cpu);
3453 }
3454 } else {
3455 cpu = iter->cpu_file;
3456 iter->buffer_iter[cpu] =
3457 ring_buffer_read_start(iter->array_buffer->buffer,
3458 cpu, GFP_KERNEL);
3459 tracing_iter_reset(iter, cpu);
3460 }
3461
3462 mutex_unlock(&trace_types_lock);
3463
3464 return iter;
3465
3466 fail:
3467 mutex_unlock(&trace_types_lock);
3468 free_trace_iter_content(iter);
3469release:
3470 seq_release_private(inode, file);
3471 return ERR_PTR(-ENOMEM);
3472}
3473
3474int tracing_open_generic(struct inode *inode, struct file *filp)
3475{
3476 int ret;
3477
3478 ret = tracing_check_open_get_tr(NULL);
3479 if (ret)
3480 return ret;
3481
3482 filp->private_data = inode->i_private;
3483 return 0;
3484}
3485
3486/*
3487 * Open and update trace_array ref count.
3488 * Must have the current trace_array passed to it.
3489 */
3490int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3491{
3492 struct trace_array *tr = inode->i_private;
3493 int ret;
3494
3495 ret = tracing_check_open_get_tr(tr);
3496 if (ret)
3497 return ret;
3498
3499 if ((filp->f_mode & FMODE_WRITE) && trace_array_is_readonly(tr)) {
3500 trace_array_put(tr);
3501 return -EACCES;
3502 }
3503
3504 filp->private_data = inode->i_private;
3505
3506 return 0;
3507}
3508
3509/*
3510 * The private pointer of the inode is the trace_event_file.
3511 * Update the tr ref count associated to it.
3512 */
3513int tracing_open_file_tr(struct inode *inode, struct file *filp)
3514{
3515 struct trace_event_file *file = inode->i_private;
3516 int ret;
3517
3518 ret = tracing_check_open_get_tr(file->tr);
3519 if (ret)
3520 return ret;
3521
3522 guard(mutex)(&event_mutex);
3523
3524 /* Fail if the file is marked for removal */
3525 if (file->flags & EVENT_FILE_FL_FREED) {
3526 trace_array_put(file->tr);
3527 return -ENODEV;
3528 } else {
3529 event_file_get(file);
3530 }
3531
3532 return 0;
3533}
3534
3535int tracing_release_file_tr(struct inode *inode, struct file *filp)
3536{
3537 struct trace_event_file *file = inode->i_private;
3538
3539 trace_array_put(file->tr);
3540 event_file_put(file);
3541
3542 return 0;
3543}
3544
3545int tracing_single_release_file_tr(struct inode *inode, struct file *filp)
3546{
3547 tracing_release_file_tr(inode, filp);
3548 return single_release(inode, filp);
3549}
3550
3551int tracing_release(struct inode *inode, struct file *file)
3552{
3553 struct trace_array *tr = inode->i_private;
3554 struct seq_file *m = file->private_data;
3555 struct trace_iterator *iter;
3556 int cpu;
3557
3558 if (!(file->f_mode & FMODE_READ)) {
3559 trace_array_put(tr);
3560 return 0;
3561 }
3562
3563 /* Writes do not use seq_file */
3564 iter = m->private;
3565 mutex_lock(&trace_types_lock);
3566
3567 for_each_tracing_cpu(cpu) {
3568 if (iter->buffer_iter[cpu])
3569 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3570 }
3571
3572 if (iter->trace && iter->trace->close)
3573 iter->trace->close(iter);
3574
3575 if (iter->iter_flags & TRACE_FILE_PAUSE)
3576 /* reenable tracing if it was previously enabled */
3577 tracing_start_tr(tr);
3578
3579 __trace_array_put(tr);
3580
3581 mutex_unlock(&trace_types_lock);
3582
3583 free_trace_iter_content(iter);
3584 seq_release_private(inode, file);
3585
3586 return 0;
3587}
3588
3589int tracing_release_generic_tr(struct inode *inode, struct file *file)
3590{
3591 struct trace_array *tr = inode->i_private;
3592
3593 trace_array_put(tr);
3594 return 0;
3595}
3596
3597static int tracing_single_release_tr(struct inode *inode, struct file *file)
3598{
3599 struct trace_array *tr = inode->i_private;
3600
3601 trace_array_put(tr);
3602
3603 return single_release(inode, file);
3604}
3605
3606static bool update_last_data_if_empty(struct trace_array *tr);
3607
3608static int tracing_open(struct inode *inode, struct file *file)
3609{
3610 struct trace_array *tr = inode->i_private;
3611 struct trace_iterator *iter;
3612 int ret;
3613
3614 ret = tracing_check_open_get_tr(tr);
3615 if (ret)
3616 return ret;
3617
3618 /* If this file was open for write, then erase contents */
3619 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3620 int cpu = tracing_get_cpu(inode);
3621 struct array_buffer *trace_buf = &tr->array_buffer;
3622
3623#ifdef CONFIG_TRACER_MAX_TRACE
3624 if (tr->current_trace->print_max)
3625 trace_buf = &tr->snapshot_buffer;
3626#endif
3627
3628 if (cpu == RING_BUFFER_ALL_CPUS)
3629 tracing_reset_online_cpus(trace_buf);
3630 else
3631 tracing_reset_cpu(trace_buf, cpu);
3632
3633 update_last_data_if_empty(tr);
3634 }
3635
3636 if (file->f_mode & FMODE_READ) {
3637 iter = __tracing_open(inode, file, false);
3638 if (IS_ERR(iter))
3639 ret = PTR_ERR(iter);
3640 else if (tr->trace_flags & TRACE_ITER(LATENCY_FMT))
3641 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3642 }
3643
3644 if (ret < 0)
3645 trace_array_put(tr);
3646
3647 return ret;
3648}
3649
3650/*
3651 * Some tracers are not suitable for instance buffers.
3652 * A tracer is always available for the global array (toplevel)
3653 * or if it explicitly states that it is.
3654 */
3655static bool
3656trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3657{
3658 /* arrays with mapped buffer range do not have snapshots */
3659 if (tr->range_addr_start && tracer_uses_snapshot(t))
3660 return false;
3661 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3662}
3663
3664/* Find the next tracer that this trace array may use */
3665static struct tracer *
3666get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3667{
3668 while (t && !trace_ok_for_array(t, tr))
3669 t = t->next;
3670
3671 return t;
3672}
3673
3674static void *
3675t_next(struct seq_file *m, void *v, loff_t *pos)
3676{
3677 struct trace_array *tr = m->private;
3678 struct tracer *t = v;
3679
3680 (*pos)++;
3681
3682 if (t)
3683 t = get_tracer_for_array(tr, t->next);
3684
3685 return t;
3686}
3687
3688static void *t_start(struct seq_file *m, loff_t *pos)
3689{
3690 struct trace_array *tr = m->private;
3691 struct tracer *t;
3692 loff_t l = 0;
3693
3694 mutex_lock(&trace_types_lock);
3695
3696 t = get_tracer_for_array(tr, trace_types);
3697 for (; t && l < *pos; t = t_next(m, t, &l))
3698 ;
3699
3700 return t;
3701}
3702
3703static void t_stop(struct seq_file *m, void *p)
3704{
3705 mutex_unlock(&trace_types_lock);
3706}
3707
3708static int t_show(struct seq_file *m, void *v)
3709{
3710 struct tracer *t = v;
3711
3712 if (!t)
3713 return 0;
3714
3715 seq_puts(m, t->name);
3716 if (t->next)
3717 seq_putc(m, ' ');
3718 else
3719 seq_putc(m, '\n');
3720
3721 return 0;
3722}
3723
3724static const struct seq_operations show_traces_seq_ops = {
3725 .start = t_start,
3726 .next = t_next,
3727 .stop = t_stop,
3728 .show = t_show,
3729};
3730
3731static int show_traces_open(struct inode *inode, struct file *file)
3732{
3733 struct trace_array *tr = inode->i_private;
3734 struct seq_file *m;
3735 int ret;
3736
3737 ret = tracing_check_open_get_tr(tr);
3738 if (ret)
3739 return ret;
3740
3741 ret = seq_open(file, &show_traces_seq_ops);
3742 if (ret) {
3743 trace_array_put(tr);
3744 return ret;
3745 }
3746
3747 m = file->private_data;
3748 m->private = tr;
3749
3750 return 0;
3751}
3752
3753static int tracing_seq_release(struct inode *inode, struct file *file)
3754{
3755 struct trace_array *tr = inode->i_private;
3756
3757 trace_array_put(tr);
3758 return seq_release(inode, file);
3759}
3760
3761static ssize_t
3762tracing_write_stub(struct file *filp, const char __user *ubuf,
3763 size_t count, loff_t *ppos)
3764{
3765 return count;
3766}
3767
3768loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3769{
3770 int ret;
3771
3772 if (file->f_mode & FMODE_READ)
3773 ret = seq_lseek(file, offset, whence);
3774 else
3775 file->f_pos = ret = 0;
3776
3777 return ret;
3778}
3779
3780static const struct file_operations tracing_fops = {
3781 .open = tracing_open,
3782 .read = seq_read,
3783 .read_iter = seq_read_iter,
3784 .splice_read = copy_splice_read,
3785 .write = tracing_write_stub,
3786 .llseek = tracing_lseek,
3787 .release = tracing_release,
3788};
3789
3790static const struct file_operations show_traces_fops = {
3791 .open = show_traces_open,
3792 .read = seq_read,
3793 .llseek = seq_lseek,
3794 .release = tracing_seq_release,
3795};
3796
3797static ssize_t
3798tracing_cpumask_read(struct file *filp, char __user *ubuf,
3799 size_t count, loff_t *ppos)
3800{
3801 struct trace_array *tr = file_inode(filp)->i_private;
3802 char *mask_str __free(kfree) = NULL;
3803 int len;
3804
3805 len = snprintf(NULL, 0, "%*pb\n",
3806 cpumask_pr_args(tr->tracing_cpumask)) + 1;
3807 mask_str = kmalloc(len, GFP_KERNEL);
3808 if (!mask_str)
3809 return -ENOMEM;
3810
3811 len = snprintf(mask_str, len, "%*pb\n",
3812 cpumask_pr_args(tr->tracing_cpumask));
3813 if (len >= count)
3814 return -EINVAL;
3815
3816 return simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
3817}
3818
3819int tracing_set_cpumask(struct trace_array *tr,
3820 cpumask_var_t tracing_cpumask_new)
3821{
3822 int cpu;
3823
3824 if (!tr)
3825 return -EINVAL;
3826
3827 local_irq_disable();
3828 arch_spin_lock(&tr->max_lock);
3829 for_each_tracing_cpu(cpu) {
3830 /*
3831 * Increase/decrease the disabled counter if we are
3832 * about to flip a bit in the cpumask:
3833 */
3834 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3835 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3836 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
3837#ifdef CONFIG_TRACER_SNAPSHOT
3838 ring_buffer_record_disable_cpu(tr->snapshot_buffer.buffer, cpu);
3839#endif
3840 }
3841 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3842 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3843 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
3844#ifdef CONFIG_TRACER_SNAPSHOT
3845 ring_buffer_record_enable_cpu(tr->snapshot_buffer.buffer, cpu);
3846#endif
3847 }
3848 }
3849 arch_spin_unlock(&tr->max_lock);
3850 local_irq_enable();
3851
3852 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3853
3854 return 0;
3855}
3856
3857static ssize_t
3858tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3859 size_t count, loff_t *ppos)
3860{
3861 struct trace_array *tr = file_inode(filp)->i_private;
3862 cpumask_var_t tracing_cpumask_new;
3863 int err;
3864
3865 if (count == 0 || count > KMALLOC_MAX_SIZE)
3866 return -EINVAL;
3867
3868 if (!zalloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3869 return -ENOMEM;
3870
3871 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3872 if (err)
3873 goto err_free;
3874
3875 err = tracing_set_cpumask(tr, tracing_cpumask_new);
3876 if (err)
3877 goto err_free;
3878
3879 free_cpumask_var(tracing_cpumask_new);
3880
3881 return count;
3882
3883err_free:
3884 free_cpumask_var(tracing_cpumask_new);
3885
3886 return err;
3887}
3888
3889static const struct file_operations tracing_cpumask_fops = {
3890 .open = tracing_open_generic_tr,
3891 .read = tracing_cpumask_read,
3892 .write = tracing_cpumask_write,
3893 .release = tracing_release_generic_tr,
3894 .llseek = generic_file_llseek,
3895};
3896
3897static int tracing_trace_options_show(struct seq_file *m, void *v)
3898{
3899 struct tracer_opt *trace_opts;
3900 struct trace_array *tr = m->private;
3901 struct tracer_flags *flags;
3902 u32 tracer_flags;
3903 int i;
3904
3905 guard(mutex)(&trace_types_lock);
3906
3907 for (i = 0; trace_options[i]; i++) {
3908 if (tr->trace_flags & (1ULL << i))
3909 seq_printf(m, "%s\n", trace_options[i]);
3910 else
3911 seq_printf(m, "no%s\n", trace_options[i]);
3912 }
3913
3914 flags = tr->current_trace_flags;
3915 if (!flags || !flags->opts)
3916 return 0;
3917
3918 tracer_flags = flags->val;
3919 trace_opts = flags->opts;
3920
3921 for (i = 0; trace_opts[i].name; i++) {
3922 if (tracer_flags & trace_opts[i].bit)
3923 seq_printf(m, "%s\n", trace_opts[i].name);
3924 else
3925 seq_printf(m, "no%s\n", trace_opts[i].name);
3926 }
3927
3928 return 0;
3929}
3930
3931static int __set_tracer_option(struct trace_array *tr,
3932 struct tracer_flags *tracer_flags,
3933 struct tracer_opt *opts, int neg)
3934{
3935 struct tracer *trace = tracer_flags->trace;
3936 int ret = 0;
3937
3938 if (trace->set_flag)
3939 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3940 if (ret)
3941 return ret;
3942
3943 if (neg)
3944 tracer_flags->val &= ~opts->bit;
3945 else
3946 tracer_flags->val |= opts->bit;
3947 return 0;
3948}
3949
3950/* Try to assign a tracer specific option */
3951static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3952{
3953 struct tracer_flags *tracer_flags = tr->current_trace_flags;
3954 struct tracer_opt *opts = NULL;
3955 int i;
3956
3957 if (!tracer_flags || !tracer_flags->opts)
3958 return 0;
3959
3960 for (i = 0; tracer_flags->opts[i].name; i++) {
3961 opts = &tracer_flags->opts[i];
3962
3963 if (strcmp(cmp, opts->name) == 0)
3964 return __set_tracer_option(tr, tracer_flags, opts, neg);
3965 }
3966
3967 return -EINVAL;
3968}
3969
3970/* Some tracers require overwrite to stay enabled */
3971int trace_keep_overwrite(struct tracer *tracer, u64 mask, int set)
3972{
3973 if (tracer->enabled && (mask & TRACE_ITER(OVERWRITE)) && !set)
3974 return -1;
3975
3976 return 0;
3977}
3978
3979int set_tracer_flag(struct trace_array *tr, u64 mask, int enabled)
3980{
3981 switch (mask) {
3982 case TRACE_ITER(RECORD_TGID):
3983 case TRACE_ITER(RECORD_CMD):
3984 case TRACE_ITER(TRACE_PRINTK):
3985 case TRACE_ITER(COPY_MARKER):
3986 lockdep_assert_held(&event_mutex);
3987 }
3988
3989 /* do nothing if flag is already set */
3990 if (!!(tr->trace_flags & mask) == !!enabled)
3991 return 0;
3992
3993 /* Give the tracer a chance to approve the change */
3994 if (tr->current_trace->flag_changed)
3995 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3996 return -EINVAL;
3997
3998 switch (mask) {
3999 case TRACE_ITER(TRACE_PRINTK):
4000 if (enabled) {
4001 update_printk_trace(tr);
4002 } else {
4003 /*
4004 * The global_trace cannot clear this.
4005 * It's flag only gets cleared if another instance sets it.
4006 */
4007 if (printk_trace == &global_trace)
4008 return -EINVAL;
4009 /*
4010 * An instance must always have it set.
4011 * by default, that's the global_trace instance.
4012 */
4013 if (printk_trace == tr)
4014 update_printk_trace(&global_trace);
4015 }
4016 break;
4017
4018 case TRACE_ITER(COPY_MARKER):
4019 update_marker_trace(tr, enabled);
4020 /* update_marker_trace updates the tr->trace_flags */
4021 return 0;
4022 }
4023
4024 if (enabled)
4025 tr->trace_flags |= mask;
4026 else
4027 tr->trace_flags &= ~mask;
4028
4029 switch (mask) {
4030 case TRACE_ITER(RECORD_CMD):
4031 trace_event_enable_cmd_record(enabled);
4032 break;
4033
4034 case TRACE_ITER(RECORD_TGID):
4035
4036 if (trace_alloc_tgid_map() < 0) {
4037 tr->trace_flags &= ~TRACE_ITER(RECORD_TGID);
4038 return -ENOMEM;
4039 }
4040
4041 trace_event_enable_tgid_record(enabled);
4042 break;
4043
4044 case TRACE_ITER(EVENT_FORK):
4045 trace_event_follow_fork(tr, enabled);
4046 break;
4047
4048 case TRACE_ITER(FUNC_FORK):
4049 ftrace_pid_follow_fork(tr, enabled);
4050 break;
4051
4052 case TRACE_ITER(OVERWRITE):
4053 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
4054#ifdef CONFIG_TRACER_SNAPSHOT
4055 ring_buffer_change_overwrite(tr->snapshot_buffer.buffer, enabled);
4056#endif
4057 break;
4058
4059 case TRACE_ITER(PRINTK):
4060 trace_printk_start_stop_comm(enabled);
4061 trace_printk_control(enabled);
4062 break;
4063
4064#if defined(CONFIG_FUNCTION_PROFILER) && defined(CONFIG_FUNCTION_GRAPH_TRACER)
4065 case TRACE_GRAPH_GRAPH_TIME:
4066 ftrace_graph_graph_time_control(enabled);
4067 break;
4068#endif
4069 }
4070
4071 return 0;
4072}
4073
4074int trace_set_options(struct trace_array *tr, char *option)
4075{
4076 char *cmp;
4077 int neg = 0;
4078 int ret;
4079 size_t orig_len = strlen(option);
4080 int len;
4081
4082 cmp = strstrip(option);
4083
4084 len = str_has_prefix(cmp, "no");
4085 if (len)
4086 neg = 1;
4087
4088 cmp += len;
4089
4090 mutex_lock(&event_mutex);
4091 mutex_lock(&trace_types_lock);
4092
4093 ret = match_string(trace_options, -1, cmp);
4094 /* If no option could be set, test the specific tracer options */
4095 if (ret < 0)
4096 ret = set_tracer_option(tr, cmp, neg);
4097 else
4098 ret = set_tracer_flag(tr, 1ULL << ret, !neg);
4099
4100 mutex_unlock(&trace_types_lock);
4101 mutex_unlock(&event_mutex);
4102
4103 /*
4104 * If the first trailing whitespace is replaced with '\0' by strstrip,
4105 * turn it back into a space.
4106 */
4107 if (orig_len > strlen(option))
4108 option[strlen(option)] = ' ';
4109
4110 return ret;
4111}
4112
4113static void __init apply_trace_boot_options(void)
4114{
4115 char *buf = trace_boot_options_buf;
4116 char *option;
4117
4118 while (true) {
4119 option = strsep(&buf, ",");
4120
4121 if (!option)
4122 break;
4123
4124 if (*option)
4125 trace_set_options(&global_trace, option);
4126
4127 /* Put back the comma to allow this to be called again */
4128 if (buf)
4129 *(buf - 1) = ',';
4130 }
4131}
4132
4133static ssize_t
4134tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4135 size_t cnt, loff_t *ppos)
4136{
4137 struct seq_file *m = filp->private_data;
4138 struct trace_array *tr = m->private;
4139 char buf[64];
4140 int ret;
4141
4142 if (cnt >= sizeof(buf))
4143 return -EINVAL;
4144
4145 if (copy_from_user(buf, ubuf, cnt))
4146 return -EFAULT;
4147
4148 buf[cnt] = 0;
4149
4150 ret = trace_set_options(tr, buf);
4151 if (ret < 0)
4152 return ret;
4153
4154 *ppos += cnt;
4155
4156 return cnt;
4157}
4158
4159static int tracing_trace_options_open(struct inode *inode, struct file *file)
4160{
4161 struct trace_array *tr = inode->i_private;
4162 int ret;
4163
4164 ret = tracing_check_open_get_tr(tr);
4165 if (ret)
4166 return ret;
4167
4168 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4169 if (ret < 0)
4170 trace_array_put(tr);
4171
4172 return ret;
4173}
4174
4175static const struct file_operations tracing_iter_fops = {
4176 .open = tracing_trace_options_open,
4177 .read = seq_read,
4178 .llseek = seq_lseek,
4179 .release = tracing_single_release_tr,
4180 .write = tracing_trace_options_write,
4181};
4182
4183static const char readme_msg[] =
4184 "tracing mini-HOWTO:\n\n"
4185 "By default tracefs removes all OTH file permission bits.\n"
4186 "When mounting tracefs an optional group id can be specified\n"
4187 "which adds the group to every directory and file in tracefs:\n\n"
4188 "\t e.g. mount -t tracefs [-o [gid=<gid>]] nodev /sys/kernel/tracing\n\n"
4189 "# echo 0 > tracing_on : quick way to disable tracing\n"
4190 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4191 " Important files:\n"
4192 " trace\t\t\t- The static contents of the buffer\n"
4193 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4194 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4195 " current_tracer\t- function and latency tracers\n"
4196 " available_tracers\t- list of configured tracers for current_tracer\n"
4197 " error_log\t- error log for failed commands (that support it)\n"
4198 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4199 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4200 " trace_clock\t\t- change the clock used to order events\n"
4201 " local: Per cpu clock but may not be synced across CPUs\n"
4202 " global: Synced across CPUs but slows tracing down.\n"
4203 " counter: Not a clock, but just an increment\n"
4204 " uptime: Jiffy counter from time of boot\n"
4205 " perf: Same clock that perf events use\n"
4206#ifdef CONFIG_X86_64
4207 " x86-tsc: TSC cycle counter\n"
4208#endif
4209 "\n timestamp_mode\t- view the mode used to timestamp events\n"
4210 " delta: Delta difference against a buffer-wide timestamp\n"
4211 " absolute: Absolute (standalone) timestamp\n"
4212 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4213 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4214 " tracing_cpumask\t- Limit which CPUs to trace\n"
4215 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4216 "\t\t\t Remove sub-buffer with rmdir\n"
4217 " trace_options\t\t- Set format or modify how tracing happens\n"
4218 "\t\t\t Disable an option by prefixing 'no' to the\n"
4219 "\t\t\t option name\n"
4220 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4221#ifdef CONFIG_DYNAMIC_FTRACE
4222 "\n available_filter_functions - list of functions that can be filtered on\n"
4223 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4224 "\t\t\t functions\n"
4225 "\t accepts: func_full_name or glob-matching-pattern\n"
4226 "\t modules: Can select a group via module\n"
4227 "\t Format: :mod:<module-name>\n"
4228 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4229 "\t triggers: a command to perform when function is hit\n"
4230 "\t Format: <function>:<trigger>[:count]\n"
4231 "\t trigger: traceon, traceoff\n"
4232 "\t\t enable_event:<system>:<event>\n"
4233 "\t\t disable_event:<system>:<event>\n"
4234#ifdef CONFIG_STACKTRACE
4235 "\t\t stacktrace\n"
4236#endif
4237#ifdef CONFIG_TRACER_SNAPSHOT
4238 "\t\t snapshot\n"
4239#endif
4240 "\t\t dump\n"
4241 "\t\t cpudump\n"
4242 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4243 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4244 "\t The first one will disable tracing every time do_fault is hit\n"
4245 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4246 "\t The first time do trap is hit and it disables tracing, the\n"
4247 "\t counter will decrement to 2. If tracing is already disabled,\n"
4248 "\t the counter will not decrement. It only decrements when the\n"
4249 "\t trigger did work\n"
4250 "\t To remove trigger without count:\n"
4251 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4252 "\t To remove trigger with a count:\n"
4253 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4254 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4255 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4256 "\t modules: Can select a group via module command :mod:\n"
4257 "\t Does not accept triggers\n"
4258#endif /* CONFIG_DYNAMIC_FTRACE */
4259#ifdef CONFIG_FUNCTION_TRACER
4260 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4261 "\t\t (function)\n"
4262 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
4263 "\t\t (function)\n"
4264#endif
4265#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4266 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4267 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4268 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4269#endif
4270#ifdef CONFIG_TRACER_SNAPSHOT
4271 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4272 "\t\t\t snapshot buffer. Read the contents for more\n"
4273 "\t\t\t information\n"
4274#endif
4275#ifdef CONFIG_STACK_TRACER
4276 " stack_trace\t\t- Shows the max stack trace when active\n"
4277 " stack_max_size\t- Shows current max stack size that was traced\n"
4278 "\t\t\t Write into this file to reset the max size (trigger a\n"
4279 "\t\t\t new trace)\n"
4280#ifdef CONFIG_DYNAMIC_FTRACE
4281 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4282 "\t\t\t traces\n"
4283#endif
4284#endif /* CONFIG_STACK_TRACER */
4285#ifdef CONFIG_DYNAMIC_EVENTS
4286 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
4287 "\t\t\t Write into this file to define/undefine new trace events.\n"
4288#endif
4289#ifdef CONFIG_KPROBE_EVENTS
4290 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
4291 "\t\t\t Write into this file to define/undefine new trace events.\n"
4292#endif
4293#ifdef CONFIG_UPROBE_EVENTS
4294 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
4295 "\t\t\t Write into this file to define/undefine new trace events.\n"
4296#endif
4297#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) || \
4298 defined(CONFIG_FPROBE_EVENTS)
4299 "\t accepts: event-definitions (one definition per line)\n"
4300#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
4301 "\t Format: p[:[<group>/][<event>]] <place> [<args>]\n"
4302 "\t r[maxactive][:[<group>/][<event>]] <place> [<args>]\n"
4303#endif
4304#ifdef CONFIG_FPROBE_EVENTS
4305 "\t f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n"
4306 "\t t[:[<group>/][<event>]] <tracepoint> [<args>]\n"
4307#endif
4308#ifdef CONFIG_HIST_TRIGGERS
4309 "\t s:[synthetic/]<event> <field> [<field>]\n"
4310#endif
4311 "\t e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n"
4312 "\t -:[<group>/][<event>]\n"
4313#ifdef CONFIG_KPROBE_EVENTS
4314 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4315 "place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n"
4316#endif
4317#ifdef CONFIG_UPROBE_EVENTS
4318 " place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
4319#endif
4320 "\t args: <name>=fetcharg[:type]\n"
4321 "\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
4322#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
4323 "\t $stack<index>, $stack, $retval, $comm, $arg<N>, $current\n"
4324#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
4325 "\t [(structname[,field])]<argname>[->field[->field|.field...]],\n"
4326 "\t [(structname[,field])](fetcharg)->field[->field|.field...],\n"
4327#endif
4328#else
4329 "\t $stack<index>, $stack, $retval, $comm, $current\n"
4330#endif
4331 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
4332 "\t this_cpu_read(<fetcharg>), this_cpu_ptr(<fetcharg>)\n"
4333 "\t kernel return probes support: $retval, $arg<N>, $comm\n"
4334 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n"
4335 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
4336 "\t symstr, %pd/%pD, <type>\\[<array-size>\\]\n"
4337#ifdef CONFIG_HIST_TRIGGERS
4338 "\t field: <stype> <name>;\n"
4339 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
4340 "\t [unsigned] char/int/long\n"
4341#endif
4342 "\t efield: For event probes ('e' types), the field is on of the fields\n"
4343 "\t of the <attached-group>/<attached-event>.\n"
4344#endif
4345 " set_event\t\t- Enables events by name written into it\n"
4346 "\t\t\t Can enable module events via: :mod:<module>\n"
4347 " events/\t\t- Directory containing all trace event subsystems:\n"
4348 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4349 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4350 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4351 "\t\t\t events\n"
4352 " filter\t\t- If set, only events passing filter are traced\n"
4353 " events/<system>/<event>/\t- Directory containing control files for\n"
4354 "\t\t\t <event>:\n"
4355 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4356 " filter\t\t- If set, only events passing filter are traced\n"
4357 " trigger\t\t- If set, a command to perform when event is hit\n"
4358 "\t Format: <trigger>[:count][if <filter>]\n"
4359 "\t trigger: traceon, traceoff\n"
4360 "\t enable_event:<system>:<event>\n"
4361 "\t disable_event:<system>:<event>\n"
4362#ifdef CONFIG_HIST_TRIGGERS
4363 "\t enable_hist:<system>:<event>\n"
4364 "\t disable_hist:<system>:<event>\n"
4365#endif
4366#ifdef CONFIG_STACKTRACE
4367 "\t\t stacktrace\n"
4368#endif
4369#ifdef CONFIG_TRACER_SNAPSHOT
4370 "\t\t snapshot\n"
4371#endif
4372#ifdef CONFIG_HIST_TRIGGERS
4373 "\t\t hist (see below)\n"
4374#endif
4375 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4376 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4377 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4378 "\t events/block/block_unplug/trigger\n"
4379 "\t The first disables tracing every time block_unplug is hit.\n"
4380 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4381 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4382 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4383 "\t Like function triggers, the counter is only decremented if it\n"
4384 "\t enabled or disabled tracing.\n"
4385 "\t To remove a trigger without a count:\n"
4386 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4387 "\t To remove a trigger with a count:\n"
4388 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4389 "\t Filters can be ignored when removing a trigger.\n"
4390#ifdef CONFIG_HIST_TRIGGERS
4391 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4392 "\t Format: hist:keys=<field1[,field2,...]>\n"
4393 "\t [:<var1>=<field|var_ref|numeric_literal>[,<var2>=...]]\n"
4394 "\t [:values=<field1[,field2,...]>]\n"
4395 "\t [:sort=<field1[,field2,...]>]\n"
4396 "\t [:size=#entries]\n"
4397 "\t [:pause][:continue][:clear]\n"
4398 "\t [:name=histname1]\n"
4399 "\t [:nohitcount]\n"
4400 "\t [:<handler>.<action>]\n"
4401 "\t [if <filter>]\n\n"
4402 "\t Note, special fields can be used as well:\n"
4403 "\t common_timestamp - to record current timestamp\n"
4404 "\t common_cpu - to record the CPU the event happened on\n"
4405 "\n"
4406 "\t A hist trigger variable can be:\n"
4407 "\t - a reference to a field e.g. x=current_timestamp,\n"
4408 "\t - a reference to another variable e.g. y=$x,\n"
4409 "\t - a numeric literal: e.g. ms_per_sec=1000,\n"
4410 "\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
4411 "\n"
4412 "\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
4413 "\t multiplication(*) and division(/) operators. An operand can be either a\n"
4414 "\t variable reference, field or numeric literal.\n"
4415 "\n"
4416 "\t When a matching event is hit, an entry is added to a hash\n"
4417 "\t table using the key(s) and value(s) named, and the value of a\n"
4418 "\t sum called 'hitcount' is incremented. Keys and values\n"
4419 "\t correspond to fields in the event's format description. Keys\n"
4420 "\t can be any field, or the special string 'common_stacktrace'.\n"
4421 "\t Compound keys consisting of up to two fields can be specified\n"
4422 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4423 "\t fields. Sort keys consisting of up to two fields can be\n"
4424 "\t specified using the 'sort' keyword. The sort direction can\n"
4425 "\t be modified by appending '.descending' or '.ascending' to a\n"
4426 "\t sort field. The 'size' parameter can be used to specify more\n"
4427 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4428 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4429 "\t its histogram data will be shared with other triggers of the\n"
4430 "\t same name, and trigger hits will update this common data.\n\n"
4431 "\t Reading the 'hist' file for the event will dump the hash\n"
4432 "\t table in its entirety to stdout. If there are multiple hist\n"
4433 "\t triggers attached to an event, there will be a table for each\n"
4434 "\t trigger in the output. The table displayed for a named\n"
4435 "\t trigger will be the same as any other instance having the\n"
4436 "\t same name. The default format used to display a given field\n"
4437 "\t can be modified by appending any of the following modifiers\n"
4438 "\t to the field name, as applicable:\n\n"
4439 "\t .hex display a number as a hex value\n"
4440 "\t .sym display an address as a symbol\n"
4441 "\t .sym-offset display an address as a symbol and offset\n"
4442 "\t .execname display a common_pid as a program name\n"
4443 "\t .syscall display a syscall id as a syscall name\n"
4444 "\t .log2 display log2 value rather than raw number\n"
4445 "\t .buckets=size display values in groups of size rather than raw number\n"
4446 "\t .usecs display a common_timestamp in microseconds\n"
4447 "\t .percent display a number of percentage value\n"
4448 "\t .graph display a bar-graph of a value\n\n"
4449 "\t The 'pause' parameter can be used to pause an existing hist\n"
4450 "\t trigger or to start a hist trigger but not log any events\n"
4451 "\t until told to do so. 'continue' can be used to start or\n"
4452 "\t restart a paused hist trigger.\n\n"
4453 "\t The 'clear' parameter will clear the contents of a running\n"
4454 "\t hist trigger and leave its current paused/active state\n"
4455 "\t unchanged.\n\n"
4456 "\t The 'nohitcount' (or NOHC) parameter will suppress display of\n"
4457 "\t raw hitcount in the histogram.\n\n"
4458 "\t The enable_hist and disable_hist triggers can be used to\n"
4459 "\t have one event conditionally start and stop another event's\n"
4460 "\t already-attached hist trigger. The syntax is analogous to\n"
4461 "\t the enable_event and disable_event triggers.\n\n"
4462 "\t Hist trigger handlers and actions are executed whenever a\n"
4463 "\t a histogram entry is added or updated. They take the form:\n\n"
4464 "\t <handler>.<action>\n\n"
4465 "\t The available handlers are:\n\n"
4466 "\t onmatch(matching.event) - invoke on addition or update\n"
4467 "\t onmax(var) - invoke if var exceeds current max\n"
4468 "\t onchange(var) - invoke action if var changes\n\n"
4469 "\t The available actions are:\n\n"
4470 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
4471 "\t save(field,...) - save current event fields\n"
4472#ifdef CONFIG_TRACER_SNAPSHOT
4473 "\t snapshot() - snapshot the trace buffer\n\n"
4474#endif
4475#ifdef CONFIG_SYNTH_EVENTS
4476 " synthetic_events\t- Create/append/remove/show synthetic events\n"
4477 "\t Write into this file to define/undefine new synthetic events.\n"
4478 "\t example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n"
4479#endif
4480#endif
4481;
4482
4483static ssize_t
4484tracing_readme_read(struct file *filp, char __user *ubuf,
4485 size_t cnt, loff_t *ppos)
4486{
4487 return simple_read_from_buffer(ubuf, cnt, ppos,
4488 readme_msg, strlen(readme_msg));
4489}
4490
4491static const struct file_operations tracing_readme_fops = {
4492 .open = tracing_open_generic,
4493 .read = tracing_readme_read,
4494 .llseek = generic_file_llseek,
4495};
4496
4497#ifdef CONFIG_TRACE_EVAL_MAP_FILE
4498static union trace_eval_map_item *
4499update_eval_map(union trace_eval_map_item *ptr)
4500{
4501 if (!ptr->map.eval_string) {
4502 if (ptr->tail.next) {
4503 ptr = ptr->tail.next;
4504 /* Set ptr to the next real item (skip head) */
4505 ptr++;
4506 } else
4507 return NULL;
4508 }
4509 return ptr;
4510}
4511
4512static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
4513{
4514 union trace_eval_map_item *ptr = v;
4515
4516 /*
4517 * Paranoid! If ptr points to end, we don't want to increment past it.
4518 * This really should never happen.
4519 */
4520 (*pos)++;
4521 ptr = update_eval_map(ptr);
4522 if (WARN_ON_ONCE(!ptr))
4523 return NULL;
4524
4525 ptr++;
4526 ptr = update_eval_map(ptr);
4527
4528 return ptr;
4529}
4530
4531static void *eval_map_start(struct seq_file *m, loff_t *pos)
4532{
4533 union trace_eval_map_item *v;
4534 loff_t l = 0;
4535
4536 mutex_lock(&trace_eval_mutex);
4537
4538 v = trace_eval_maps;
4539 if (v)
4540 v++;
4541
4542 while (v && l < *pos) {
4543 v = eval_map_next(m, v, &l);
4544 }
4545
4546 return v;
4547}
4548
4549static void eval_map_stop(struct seq_file *m, void *v)
4550{
4551 mutex_unlock(&trace_eval_mutex);
4552}
4553
4554static int eval_map_show(struct seq_file *m, void *v)
4555{
4556 union trace_eval_map_item *ptr = v;
4557
4558 seq_printf(m, "%s %ld (%s)\n",
4559 ptr->map.eval_string, ptr->map.eval_value,
4560 ptr->map.system);
4561
4562 return 0;
4563}
4564
4565static const struct seq_operations tracing_eval_map_seq_ops = {
4566 .start = eval_map_start,
4567 .next = eval_map_next,
4568 .stop = eval_map_stop,
4569 .show = eval_map_show,
4570};
4571
4572static int tracing_eval_map_open(struct inode *inode, struct file *filp)
4573{
4574 int ret;
4575
4576 ret = tracing_check_open_get_tr(NULL);
4577 if (ret)
4578 return ret;
4579
4580 return seq_open(filp, &tracing_eval_map_seq_ops);
4581}
4582
4583static const struct file_operations tracing_eval_map_fops = {
4584 .open = tracing_eval_map_open,
4585 .read = seq_read,
4586 .llseek = seq_lseek,
4587 .release = seq_release,
4588};
4589
4590static inline union trace_eval_map_item *
4591trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
4592{
4593 /* Return tail of array given the head */
4594 return ptr + ptr->head.length + 1;
4595}
4596
4597static void
4598trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
4599 int len)
4600{
4601 struct trace_eval_map **stop;
4602 struct trace_eval_map **map;
4603 union trace_eval_map_item *map_array;
4604 union trace_eval_map_item *ptr;
4605
4606 stop = start + len;
4607
4608 /*
4609 * The trace_eval_maps contains the map plus a head and tail item,
4610 * where the head holds the module and length of array, and the
4611 * tail holds a pointer to the next list.
4612 */
4613 map_array = kmalloc_objs(*map_array, len + 2);
4614 if (!map_array) {
4615 pr_warn("Unable to allocate trace eval mapping\n");
4616 return;
4617 }
4618
4619 guard(mutex)(&trace_eval_mutex);
4620
4621 if (!trace_eval_maps)
4622 trace_eval_maps = map_array;
4623 else {
4624 ptr = trace_eval_maps;
4625 for (;;) {
4626 ptr = trace_eval_jmp_to_tail(ptr);
4627 if (!ptr->tail.next)
4628 break;
4629 ptr = ptr->tail.next;
4630
4631 }
4632 ptr->tail.next = map_array;
4633 }
4634 map_array->head.mod = mod;
4635 map_array->head.length = len;
4636 map_array++;
4637
4638 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4639 map_array->map = **map;
4640 map_array++;
4641 }
4642 memset(map_array, 0, sizeof(*map_array));
4643}
4644
4645static void trace_create_eval_file(struct dentry *d_tracer)
4646{
4647 trace_create_file("eval_map", TRACE_MODE_READ, d_tracer,
4648 NULL, &tracing_eval_map_fops);
4649}
4650
4651#else /* CONFIG_TRACE_EVAL_MAP_FILE */
4652static inline void trace_create_eval_file(struct dentry *d_tracer) { }
4653static inline void trace_insert_eval_map_file(struct module *mod,
4654 struct trace_eval_map **start, int len) { }
4655#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
4656
4657static void
4658trace_event_update_with_eval_map(struct module *mod,
4659 struct trace_eval_map **start,
4660 int len)
4661{
4662 struct trace_eval_map **map;
4663
4664 /* Always run sanitizer only if btf_type_tag attr exists. */
4665 if (len <= 0) {
4666 if (!(IS_ENABLED(CONFIG_DEBUG_INFO_BTF) &&
4667 IS_ENABLED(CONFIG_PAHOLE_HAS_BTF_TAG) &&
4668 __has_attribute(btf_type_tag)))
4669 return;
4670 }
4671
4672 map = start;
4673
4674 trace_event_update_all(map, len, mod);
4675
4676 if (len <= 0)
4677 return;
4678
4679 trace_insert_eval_map_file(mod, start, len);
4680}
4681
4682static ssize_t
4683tracing_set_trace_read(struct file *filp, char __user *ubuf,
4684 size_t cnt, loff_t *ppos)
4685{
4686 struct trace_array *tr = filp->private_data;
4687 char buf[MAX_TRACER_SIZE+2];
4688 int r;
4689
4690 scoped_guard(mutex, &trace_types_lock) {
4691 r = sprintf(buf, "%s\n", tr->current_trace->name);
4692 }
4693
4694 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4695}
4696
4697int tracer_init(struct tracer *t, struct trace_array *tr)
4698{
4699 tracing_reset_online_cpus(&tr->array_buffer);
4700 update_last_data_if_empty(tr);
4701 return t->init(tr);
4702}
4703
4704void trace_set_buffer_entries(struct array_buffer *buf, unsigned long val)
4705{
4706 int cpu;
4707
4708 for_each_tracing_cpu(cpu)
4709 per_cpu_ptr(buf->data, cpu)->entries = val;
4710}
4711
4712static void update_buffer_entries(struct array_buffer *buf, int cpu)
4713{
4714 if (cpu == RING_BUFFER_ALL_CPUS) {
4715 trace_set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0));
4716 } else {
4717 per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu);
4718 }
4719}
4720
4721static int __tracing_resize_ring_buffer(struct trace_array *tr,
4722 unsigned long size, int cpu)
4723{
4724 int ret;
4725
4726 /*
4727 * If kernel or user changes the size of the ring buffer
4728 * we use the size that was given, and we can forget about
4729 * expanding it later.
4730 */
4731 trace_set_ring_buffer_expanded(tr);
4732
4733 /* May be called before buffers are initialized */
4734 if (!tr->array_buffer.buffer)
4735 return 0;
4736
4737 /* Do not allow tracing while resizing ring buffer */
4738 tracing_stop_tr(tr);
4739
4740 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
4741 if (ret < 0)
4742 goto out_start;
4743
4744#ifdef CONFIG_TRACER_SNAPSHOT
4745 if (!tr->allocated_snapshot)
4746 goto out;
4747
4748 ret = ring_buffer_resize(tr->snapshot_buffer.buffer, size, cpu);
4749 if (ret < 0) {
4750 int r = resize_buffer_duplicate_size(&tr->array_buffer,
4751 &tr->array_buffer, cpu);
4752 if (r < 0) {
4753 /*
4754 * AARGH! We are left with different
4755 * size max buffer!!!!
4756 * The max buffer is our "snapshot" buffer.
4757 * When a tracer needs a snapshot (one of the
4758 * latency tracers), it swaps the max buffer
4759 * with the saved snap shot. We succeeded to
4760 * update the size of the main buffer, but failed to
4761 * update the size of the max buffer. But when we tried
4762 * to reset the main buffer to the original size, we
4763 * failed there too. This is very unlikely to
4764 * happen, but if it does, warn and kill all
4765 * tracing.
4766 */
4767 WARN_ON(1);
4768 tracing_disabled = 1;
4769 }
4770 goto out_start;
4771 }
4772
4773 update_buffer_entries(&tr->snapshot_buffer, cpu);
4774
4775 out:
4776#endif /* CONFIG_TRACER_SNAPSHOT */
4777
4778 update_buffer_entries(&tr->array_buffer, cpu);
4779 out_start:
4780 tracing_start_tr(tr);
4781 return ret;
4782}
4783
4784ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4785 unsigned long size, int cpu_id)
4786{
4787 guard(mutex)(&trace_types_lock);
4788
4789 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4790 /* make sure, this cpu is enabled in the mask */
4791 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask))
4792 return -EINVAL;
4793 }
4794
4795 return __tracing_resize_ring_buffer(tr, size, cpu_id);
4796}
4797
4798struct trace_mod_entry {
4799 unsigned long mod_addr;
4800 char mod_name[MODULE_NAME_LEN];
4801};
4802
4803struct trace_scratch {
4804 unsigned int clock_id;
4805 unsigned long text_addr;
4806 unsigned long nr_entries;
4807 struct trace_mod_entry entries[];
4808};
4809
4810static DEFINE_MUTEX(scratch_mutex);
4811
4812static int cmp_mod_entry(const void *key, const void *pivot)
4813{
4814 unsigned long addr = (unsigned long)key;
4815 const struct trace_mod_entry *ent = pivot;
4816
4817 if (addr < ent[0].mod_addr)
4818 return -1;
4819
4820 return addr >= ent[1].mod_addr;
4821}
4822
4823/**
4824 * trace_adjust_address() - Adjust prev boot address to current address.
4825 * @tr: Persistent ring buffer's trace_array.
4826 * @addr: Address in @tr which is adjusted.
4827 */
4828unsigned long trace_adjust_address(struct trace_array *tr, unsigned long addr)
4829{
4830 struct trace_module_delta *module_delta;
4831 struct trace_scratch *tscratch;
4832 struct trace_mod_entry *entry;
4833 unsigned long raddr;
4834 int idx = 0, nr_entries;
4835
4836 /* If we don't have last boot delta, return the address */
4837 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
4838 return addr;
4839
4840 /* tr->module_delta must be protected by rcu. */
4841 guard(rcu)();
4842 tscratch = tr->scratch;
4843 /* if there is no tscrach, module_delta must be NULL. */
4844 module_delta = READ_ONCE(tr->module_delta);
4845 if (!module_delta || !tscratch->nr_entries ||
4846 tscratch->entries[0].mod_addr > addr) {
4847 raddr = addr + tr->text_delta;
4848 return __is_kernel(raddr) || is_kernel_core_data(raddr) ||
4849 is_kernel_rodata(raddr) ? raddr : addr;
4850 }
4851
4852 /* Note that entries must be sorted. */
4853 nr_entries = tscratch->nr_entries;
4854 if (nr_entries == 1 ||
4855 tscratch->entries[nr_entries - 1].mod_addr < addr)
4856 idx = nr_entries - 1;
4857 else {
4858 entry = __inline_bsearch((void *)addr,
4859 tscratch->entries,
4860 nr_entries - 1,
4861 sizeof(tscratch->entries[0]),
4862 cmp_mod_entry);
4863 if (entry)
4864 idx = entry - tscratch->entries;
4865 }
4866
4867 return addr + module_delta->delta[idx];
4868}
4869
4870#ifdef CONFIG_MODULES
4871static int save_mod(struct module *mod, void *data)
4872{
4873 struct trace_array *tr = data;
4874 struct trace_scratch *tscratch;
4875 struct trace_mod_entry *entry;
4876 unsigned int size;
4877
4878 tscratch = tr->scratch;
4879 if (!tscratch)
4880 return -1;
4881 size = tr->scratch_size;
4882
4883 if (struct_size(tscratch, entries, tscratch->nr_entries + 1) > size)
4884 return -1;
4885
4886 entry = &tscratch->entries[tscratch->nr_entries];
4887
4888 tscratch->nr_entries++;
4889
4890 entry->mod_addr = (unsigned long)mod->mem[MOD_TEXT].base;
4891 strscpy(entry->mod_name, mod->name);
4892
4893 return 0;
4894}
4895#else
4896static int save_mod(struct module *mod, void *data)
4897{
4898 return 0;
4899}
4900#endif
4901
4902static void update_last_data(struct trace_array *tr)
4903{
4904 struct trace_module_delta *module_delta;
4905 struct trace_scratch *tscratch;
4906
4907 if (!(tr->flags & TRACE_ARRAY_FL_BOOT))
4908 return;
4909
4910 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
4911 return;
4912
4913 /* Only if the buffer has previous boot data clear and update it. */
4914 tr->flags &= ~TRACE_ARRAY_FL_LAST_BOOT;
4915
4916 /* If this is a backup instance, mark it for autoremove. */
4917 if (tr->flags & TRACE_ARRAY_FL_VMALLOC)
4918 tr->free_on_close = true;
4919
4920 /* Reset the module list and reload them */
4921 if (tr->scratch) {
4922 struct trace_scratch *tscratch = tr->scratch;
4923
4924 tscratch->clock_id = tr->clock_id;
4925 memset(tscratch->entries, 0,
4926 flex_array_size(tscratch, entries, tscratch->nr_entries));
4927 tscratch->nr_entries = 0;
4928
4929 guard(mutex)(&scratch_mutex);
4930 module_for_each_mod(save_mod, tr);
4931 }
4932
4933 /*
4934 * Need to clear all CPU buffers as there cannot be events
4935 * from the previous boot mixed with events with this boot
4936 * as that will cause a confusing trace. Need to clear all
4937 * CPU buffers, even for those that may currently be offline.
4938 */
4939 tracing_reset_all_cpus(&tr->array_buffer);
4940
4941 /* Using current data now */
4942 tr->text_delta = 0;
4943
4944 if (!tr->scratch)
4945 return;
4946
4947 tscratch = tr->scratch;
4948 module_delta = READ_ONCE(tr->module_delta);
4949 WRITE_ONCE(tr->module_delta, NULL);
4950 kfree_rcu(module_delta, rcu);
4951
4952 /* Set the persistent ring buffer meta data to this address */
4953 tscratch->text_addr = (unsigned long)_text;
4954}
4955
4956/**
4957 * tracing_update_buffers - used by tracing facility to expand ring buffers
4958 * @tr: The tracing instance
4959 *
4960 * To save on memory when the tracing is never used on a system with it
4961 * configured in. The ring buffers are set to a minimum size. But once
4962 * a user starts to use the tracing facility, then they need to grow
4963 * to their default size.
4964 *
4965 * This function is to be called when a tracer is about to be used.
4966 */
4967int tracing_update_buffers(struct trace_array *tr)
4968{
4969 int ret = 0;
4970
4971 if (!tr)
4972 tr = &global_trace;
4973
4974 guard(mutex)(&trace_types_lock);
4975
4976 update_last_data(tr);
4977
4978 if (!tr->ring_buffer_expanded)
4979 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4980 RING_BUFFER_ALL_CPUS);
4981 return ret;
4982}
4983
4984/*
4985 * Used to clear out the tracer before deletion of an instance.
4986 * Must have trace_types_lock held.
4987 */
4988static void tracing_set_nop(struct trace_array *tr)
4989{
4990 if (tr->current_trace == &nop_trace)
4991 return;
4992
4993 tr->current_trace->enabled--;
4994
4995 if (tr->current_trace->reset)
4996 tr->current_trace->reset(tr);
4997
4998 tr->current_trace = &nop_trace;
4999 tr->current_trace_flags = nop_trace.flags;
5000}
5001
5002static bool tracer_options_updated;
5003
5004int tracing_set_tracer(struct trace_array *tr, const char *buf)
5005{
5006 struct tracer *trace = NULL;
5007 struct tracers *t;
5008 bool had_max_tr;
5009 int ret;
5010
5011 guard(mutex)(&trace_types_lock);
5012
5013 update_last_data(tr);
5014
5015 if (!tr->ring_buffer_expanded) {
5016 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5017 RING_BUFFER_ALL_CPUS);
5018 if (ret < 0)
5019 return ret;
5020 }
5021
5022 list_for_each_entry(t, &tr->tracers, list) {
5023 if (strcmp(t->tracer->name, buf) == 0) {
5024 trace = t->tracer;
5025 break;
5026 }
5027 }
5028 if (!trace)
5029 return -EINVAL;
5030
5031 if (trace == tr->current_trace)
5032 return 0;
5033
5034#ifdef CONFIG_TRACER_SNAPSHOT
5035 if (tracer_uses_snapshot(trace)) {
5036 local_irq_disable();
5037 arch_spin_lock(&tr->max_lock);
5038 ret = tr->cond_snapshot ? -EBUSY : 0;
5039 arch_spin_unlock(&tr->max_lock);
5040 local_irq_enable();
5041 if (ret)
5042 return ret;
5043 }
5044#endif
5045 /* Some tracers won't work on kernel command line */
5046 if (system_state < SYSTEM_RUNNING && trace->noboot) {
5047 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5048 trace->name);
5049 return -EINVAL;
5050 }
5051
5052 /* Some tracers are only allowed for the top level buffer */
5053 if (!trace_ok_for_array(trace, tr))
5054 return -EINVAL;
5055
5056 /* If trace pipe files are being read, we can't change the tracer */
5057 if (tr->trace_ref)
5058 return -EBUSY;
5059
5060 trace_branch_disable();
5061
5062 tr->current_trace->enabled--;
5063
5064 if (tr->current_trace->reset)
5065 tr->current_trace->reset(tr);
5066
5067 had_max_tr = tracer_uses_snapshot(tr->current_trace);
5068
5069 /* Current trace needs to be nop_trace before synchronize_rcu */
5070 tr->current_trace = &nop_trace;
5071 tr->current_trace_flags = nop_trace.flags;
5072
5073 if (had_max_tr && !tracer_uses_snapshot(trace)) {
5074 /*
5075 * We need to make sure that the update_max_tr sees that
5076 * current_trace changed to nop_trace to keep it from
5077 * swapping the buffers after we resize it.
5078 * The update_max_tr is called from interrupts disabled
5079 * so a synchronized_sched() is sufficient.
5080 */
5081 synchronize_rcu();
5082 free_snapshot(tr);
5083 tracing_disarm_snapshot(tr);
5084 }
5085
5086 if (!had_max_tr && tracer_uses_snapshot(trace)) {
5087 ret = tracing_arm_snapshot_locked(tr);
5088 if (ret)
5089 return ret;
5090 }
5091
5092 tr->current_trace_flags = t->flags ? : t->tracer->flags;
5093
5094 if (trace->init) {
5095 ret = tracer_init(trace, tr);
5096 if (ret) {
5097 if (tracer_uses_snapshot(trace))
5098 tracing_disarm_snapshot(tr);
5099 tr->current_trace_flags = nop_trace.flags;
5100 return ret;
5101 }
5102 }
5103
5104 tr->current_trace = trace;
5105 tr->current_trace->enabled++;
5106 trace_branch_enable(tr);
5107
5108 return 0;
5109}
5110
5111static ssize_t
5112tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5113 size_t cnt, loff_t *ppos)
5114{
5115 struct trace_array *tr = filp->private_data;
5116 char buf[MAX_TRACER_SIZE+1];
5117 char *name;
5118 size_t ret;
5119 int err;
5120
5121 ret = cnt;
5122
5123 if (cnt > MAX_TRACER_SIZE)
5124 cnt = MAX_TRACER_SIZE;
5125
5126 if (copy_from_user(buf, ubuf, cnt))
5127 return -EFAULT;
5128
5129 buf[cnt] = 0;
5130
5131 name = strim(buf);
5132
5133 err = tracing_set_tracer(tr, name);
5134 if (err)
5135 return err;
5136
5137 *ppos += ret;
5138
5139 return ret;
5140}
5141
5142ssize_t tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5143 size_t cnt, loff_t *ppos)
5144{
5145 char buf[64];
5146 int r;
5147
5148 r = snprintf(buf, sizeof(buf), "%ld\n",
5149 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5150 if (r > sizeof(buf))
5151 r = sizeof(buf);
5152 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5153}
5154
5155ssize_t tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5156 size_t cnt, loff_t *ppos)
5157{
5158 unsigned long val;
5159 int ret;
5160
5161 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5162 if (ret)
5163 return ret;
5164
5165 *ptr = val * 1000;
5166
5167 return cnt;
5168}
5169
5170static ssize_t
5171tracing_thresh_read(struct file *filp, char __user *ubuf,
5172 size_t cnt, loff_t *ppos)
5173{
5174 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5175}
5176
5177static ssize_t
5178tracing_thresh_write(struct file *filp, const char __user *ubuf,
5179 size_t cnt, loff_t *ppos)
5180{
5181 struct trace_array *tr = filp->private_data;
5182 int ret;
5183
5184 guard(mutex)(&trace_types_lock);
5185 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5186 if (ret < 0)
5187 return ret;
5188
5189 if (tr->current_trace->update_thresh) {
5190 ret = tr->current_trace->update_thresh(tr);
5191 if (ret < 0)
5192 return ret;
5193 }
5194
5195 return cnt;
5196}
5197
5198static int open_pipe_on_cpu(struct trace_array *tr, int cpu)
5199{
5200 if (cpu == RING_BUFFER_ALL_CPUS) {
5201 if (cpumask_empty(tr->pipe_cpumask)) {
5202 cpumask_setall(tr->pipe_cpumask);
5203 return 0;
5204 }
5205 } else if (!cpumask_test_cpu(cpu, tr->pipe_cpumask)) {
5206 cpumask_set_cpu(cpu, tr->pipe_cpumask);
5207 return 0;
5208 }
5209 return -EBUSY;
5210}
5211
5212static void close_pipe_on_cpu(struct trace_array *tr, int cpu)
5213{
5214 if (cpu == RING_BUFFER_ALL_CPUS) {
5215 WARN_ON(!cpumask_full(tr->pipe_cpumask));
5216 cpumask_clear(tr->pipe_cpumask);
5217 } else {
5218 WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask));
5219 cpumask_clear_cpu(cpu, tr->pipe_cpumask);
5220 }
5221}
5222
5223static int tracing_open_pipe(struct inode *inode, struct file *filp)
5224{
5225 struct trace_array *tr = inode->i_private;
5226 struct trace_iterator *iter;
5227 int cpu;
5228 int ret;
5229
5230 ret = tracing_check_open_get_tr(tr);
5231 if (ret)
5232 return ret;
5233
5234 guard(mutex)(&trace_types_lock);
5235 cpu = tracing_get_cpu(inode);
5236 ret = open_pipe_on_cpu(tr, cpu);
5237 if (ret)
5238 goto fail_pipe_on_cpu;
5239
5240 /* create a buffer to store the information to pass to userspace */
5241 iter = kzalloc_obj(*iter);
5242 if (!iter) {
5243 ret = -ENOMEM;
5244 goto fail_alloc_iter;
5245 }
5246
5247 trace_seq_init(&iter->seq);
5248 iter->trace = tr->current_trace;
5249
5250 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5251 ret = -ENOMEM;
5252 goto fail;
5253 }
5254
5255 /* trace pipe does not show start of buffer */
5256 cpumask_setall(iter->started);
5257
5258 if (tr->trace_flags & TRACE_ITER(LATENCY_FMT))
5259 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5260
5261 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5262 if (trace_clocks[tr->clock_id].in_ns)
5263 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5264
5265 iter->tr = tr;
5266 iter->array_buffer = &tr->array_buffer;
5267 iter->cpu_file = cpu;
5268 mutex_init(&iter->mutex);
5269 filp->private_data = iter;
5270
5271 if (iter->trace->pipe_open)
5272 iter->trace->pipe_open(iter);
5273
5274 nonseekable_open(inode, filp);
5275
5276 tr->trace_ref++;
5277
5278 return ret;
5279
5280fail:
5281 kfree(iter);
5282fail_alloc_iter:
5283 close_pipe_on_cpu(tr, cpu);
5284fail_pipe_on_cpu:
5285 __trace_array_put(tr);
5286 return ret;
5287}
5288
5289static int tracing_release_pipe(struct inode *inode, struct file *file)
5290{
5291 struct trace_iterator *iter = file->private_data;
5292 struct trace_array *tr = inode->i_private;
5293
5294 scoped_guard(mutex, &trace_types_lock) {
5295 tr->trace_ref--;
5296
5297 if (iter->trace->pipe_close)
5298 iter->trace->pipe_close(iter);
5299 close_pipe_on_cpu(tr, iter->cpu_file);
5300 }
5301
5302 free_trace_iter_content(iter);
5303 kfree(iter);
5304
5305 trace_array_put(tr);
5306
5307 return 0;
5308}
5309
5310static __poll_t
5311trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
5312{
5313 struct trace_array *tr = iter->tr;
5314
5315 /* Iterators are static, they should be filled or empty */
5316 if (trace_buffer_iter(iter, iter->cpu_file))
5317 return EPOLLIN | EPOLLRDNORM;
5318
5319 if (tr->trace_flags & TRACE_ITER(BLOCK))
5320 /*
5321 * Always select as readable when in blocking mode
5322 */
5323 return EPOLLIN | EPOLLRDNORM;
5324 else
5325 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
5326 filp, poll_table, iter->tr->buffer_percent);
5327}
5328
5329static __poll_t
5330tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5331{
5332 struct trace_iterator *iter = filp->private_data;
5333
5334 return trace_poll(iter, filp, poll_table);
5335}
5336
5337/* Must be called with iter->mutex held. */
5338static int tracing_wait_pipe(struct file *filp)
5339{
5340 struct trace_iterator *iter = filp->private_data;
5341 int ret;
5342
5343 while (trace_empty(iter)) {
5344
5345 if ((filp->f_flags & O_NONBLOCK)) {
5346 return -EAGAIN;
5347 }
5348
5349 /*
5350 * We block until we read something and tracing is disabled.
5351 * We still block if tracing is disabled, but we have never
5352 * read anything. This allows a user to cat this file, and
5353 * then enable tracing. But after we have read something,
5354 * we give an EOF when tracing is again disabled.
5355 *
5356 * iter->pos will be 0 if we haven't read anything.
5357 */
5358 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
5359 break;
5360
5361 mutex_unlock(&iter->mutex);
5362
5363 ret = wait_on_pipe(iter, 0);
5364
5365 mutex_lock(&iter->mutex);
5366
5367 if (ret)
5368 return ret;
5369 }
5370
5371 return 1;
5372}
5373
5374static bool update_last_data_if_empty(struct trace_array *tr)
5375{
5376 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
5377 return false;
5378
5379 if (!ring_buffer_empty(tr->array_buffer.buffer))
5380 return false;
5381
5382 /*
5383 * If the buffer contains the last boot data and all per-cpu
5384 * buffers are empty, reset it from the kernel side.
5385 */
5386 update_last_data(tr);
5387 return true;
5388}
5389
5390/*
5391 * Consumer reader.
5392 */
5393static ssize_t
5394tracing_read_pipe(struct file *filp, char __user *ubuf,
5395 size_t cnt, loff_t *ppos)
5396{
5397 struct trace_iterator *iter = filp->private_data;
5398 ssize_t sret;
5399
5400 /*
5401 * Avoid more than one consumer on a single file descriptor
5402 * This is just a matter of traces coherency, the ring buffer itself
5403 * is protected.
5404 */
5405 guard(mutex)(&iter->mutex);
5406
5407 /* return any leftover data */
5408 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5409 if (sret != -EBUSY)
5410 return sret;
5411
5412 trace_seq_init(&iter->seq);
5413
5414 if (iter->trace->read) {
5415 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5416 if (sret)
5417 return sret;
5418 }
5419
5420waitagain:
5421 if (update_last_data_if_empty(iter->tr))
5422 return 0;
5423
5424 sret = tracing_wait_pipe(filp);
5425 if (sret <= 0)
5426 return sret;
5427
5428 /* stop when tracing is finished */
5429 if (trace_empty(iter))
5430 return 0;
5431
5432 if (cnt >= TRACE_SEQ_BUFFER_SIZE)
5433 cnt = TRACE_SEQ_BUFFER_SIZE - 1;
5434
5435 /* reset all but tr, trace, and overruns */
5436 trace_iterator_reset(iter);
5437 cpumask_clear(iter->started);
5438 trace_seq_init(&iter->seq);
5439
5440 trace_event_read_lock();
5441 trace_access_lock(iter->cpu_file);
5442 while (trace_find_next_entry_inc(iter) != NULL) {
5443 enum print_line_t ret;
5444 int save_len = iter->seq.seq.len;
5445
5446 ret = print_trace_line(iter);
5447 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5448 /*
5449 * If one print_trace_line() fills entire trace_seq in one shot,
5450 * trace_seq_to_user() will returns -EBUSY because save_len == 0,
5451 * In this case, we need to consume it, otherwise, loop will peek
5452 * this event next time, resulting in an infinite loop.
5453 */
5454 if (save_len == 0) {
5455 iter->seq.full = 0;
5456 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
5457 trace_consume(iter);
5458 break;
5459 }
5460
5461 /* In other cases, don't print partial lines */
5462 iter->seq.seq.len = save_len;
5463 break;
5464 }
5465 if (ret != TRACE_TYPE_NO_CONSUME)
5466 trace_consume(iter);
5467
5468 if (trace_seq_used(&iter->seq) >= cnt)
5469 break;
5470
5471 /*
5472 * Setting the full flag means we reached the trace_seq buffer
5473 * size and we should leave by partial output condition above.
5474 * One of the trace_seq_* functions is not used properly.
5475 */
5476 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
5477 iter->ent->type);
5478 }
5479 trace_access_unlock(iter->cpu_file);
5480 trace_event_read_unlock();
5481
5482 /* Now copy what we have to the user */
5483 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5484 if (iter->seq.readpos >= trace_seq_used(&iter->seq))
5485 trace_seq_init(&iter->seq);
5486
5487 /*
5488 * If there was nothing to send to user, in spite of consuming trace
5489 * entries, go back to wait for more entries.
5490 */
5491 if (sret == -EBUSY)
5492 goto waitagain;
5493
5494 return sret;
5495}
5496
5497static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
5498 unsigned int idx)
5499{
5500 __free_page(spd->pages[idx]);
5501}
5502
5503static size_t
5504tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5505{
5506 size_t count;
5507 int save_len;
5508 int ret;
5509
5510 /* Seq buffer is page-sized, exactly what we need. */
5511 for (;;) {
5512 save_len = iter->seq.seq.len;
5513 ret = print_trace_line(iter);
5514
5515 if (trace_seq_has_overflowed(&iter->seq)) {
5516 iter->seq.seq.len = save_len;
5517 break;
5518 }
5519
5520 /*
5521 * This should not be hit, because it should only
5522 * be set if the iter->seq overflowed. But check it
5523 * anyway to be safe.
5524 */
5525 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5526 iter->seq.seq.len = save_len;
5527 break;
5528 }
5529
5530 count = trace_seq_used(&iter->seq) - save_len;
5531 if (rem < count) {
5532 rem = 0;
5533 iter->seq.seq.len = save_len;
5534 break;
5535 }
5536
5537 if (ret != TRACE_TYPE_NO_CONSUME)
5538 trace_consume(iter);
5539 rem -= count;
5540 if (!trace_find_next_entry_inc(iter)) {
5541 rem = 0;
5542 iter->ent = NULL;
5543 break;
5544 }
5545 }
5546
5547 return rem;
5548}
5549
5550static ssize_t tracing_splice_read_pipe(struct file *filp,
5551 loff_t *ppos,
5552 struct pipe_inode_info *pipe,
5553 size_t len,
5554 unsigned int flags)
5555{
5556 struct page *pages_def[PIPE_DEF_BUFFERS];
5557 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5558 struct trace_iterator *iter = filp->private_data;
5559 struct splice_pipe_desc spd = {
5560 .pages = pages_def,
5561 .partial = partial_def,
5562 .nr_pages = 0, /* This gets updated below. */
5563 .nr_pages_max = PIPE_DEF_BUFFERS,
5564 .ops = &default_pipe_buf_ops,
5565 .spd_release = tracing_spd_release_pipe,
5566 };
5567 ssize_t ret;
5568 size_t rem;
5569 unsigned int i;
5570
5571 if (splice_grow_spd(pipe, &spd))
5572 return -ENOMEM;
5573
5574 mutex_lock(&iter->mutex);
5575
5576 if (iter->trace->splice_read) {
5577 ret = iter->trace->splice_read(iter, filp,
5578 ppos, pipe, len, flags);
5579 if (ret)
5580 goto out_err;
5581 }
5582
5583 ret = tracing_wait_pipe(filp);
5584 if (ret <= 0)
5585 goto out_err;
5586
5587 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
5588 ret = -EFAULT;
5589 goto out_err;
5590 }
5591
5592 trace_event_read_lock();
5593 trace_access_lock(iter->cpu_file);
5594
5595 /* Fill as many pages as possible. */
5596 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
5597 spd.pages[i] = alloc_page(GFP_KERNEL);
5598 if (!spd.pages[i])
5599 break;
5600
5601 rem = tracing_fill_pipe_page(rem, iter);
5602
5603 /* Copy the data into the page, so we can start over. */
5604 ret = trace_seq_to_buffer(&iter->seq,
5605 page_address(spd.pages[i]),
5606 min((size_t)trace_seq_used(&iter->seq),
5607 (size_t)PAGE_SIZE));
5608 if (ret < 0) {
5609 __free_page(spd.pages[i]);
5610 break;
5611 }
5612 spd.partial[i].offset = 0;
5613 spd.partial[i].len = ret;
5614
5615 trace_seq_init(&iter->seq);
5616 }
5617
5618 trace_access_unlock(iter->cpu_file);
5619 trace_event_read_unlock();
5620 mutex_unlock(&iter->mutex);
5621
5622 spd.nr_pages = i;
5623
5624 if (i)
5625 ret = splice_to_pipe(pipe, &spd);
5626 else
5627 ret = 0;
5628out:
5629 splice_shrink_spd(&spd);
5630 return ret;
5631
5632out_err:
5633 mutex_unlock(&iter->mutex);
5634 goto out;
5635}
5636
5637static ssize_t
5638tracing_syscall_buf_read(struct file *filp, char __user *ubuf,
5639 size_t cnt, loff_t *ppos)
5640{
5641 struct inode *inode = file_inode(filp);
5642 struct trace_array *tr = inode->i_private;
5643 char buf[64];
5644 int r;
5645
5646 r = snprintf(buf, 64, "%d\n", tr->syscall_buf_sz);
5647
5648 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5649}
5650
5651static ssize_t
5652tracing_syscall_buf_write(struct file *filp, const char __user *ubuf,
5653 size_t cnt, loff_t *ppos)
5654{
5655 struct inode *inode = file_inode(filp);
5656 struct trace_array *tr = inode->i_private;
5657 unsigned long val;
5658 int ret;
5659
5660 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5661 if (ret)
5662 return ret;
5663
5664 if (val > SYSCALL_FAULT_USER_MAX)
5665 val = SYSCALL_FAULT_USER_MAX;
5666
5667 tr->syscall_buf_sz = val;
5668
5669 *ppos += cnt;
5670
5671 return cnt;
5672}
5673
5674static ssize_t
5675tracing_entries_read(struct file *filp, char __user *ubuf,
5676 size_t cnt, loff_t *ppos)
5677{
5678 struct inode *inode = file_inode(filp);
5679 struct trace_array *tr = inode->i_private;
5680 int cpu = tracing_get_cpu(inode);
5681 char buf[64];
5682 int r = 0;
5683 ssize_t ret;
5684
5685 mutex_lock(&trace_types_lock);
5686
5687 if (cpu == RING_BUFFER_ALL_CPUS) {
5688 int cpu, buf_size_same;
5689 unsigned long size;
5690
5691 size = 0;
5692 buf_size_same = 1;
5693 /* check if all cpu sizes are same */
5694 for_each_tracing_cpu(cpu) {
5695 /* fill in the size from first enabled cpu */
5696 if (size == 0)
5697 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
5698 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
5699 buf_size_same = 0;
5700 break;
5701 }
5702 }
5703
5704 if (buf_size_same) {
5705 if (!tr->ring_buffer_expanded)
5706 r = sprintf(buf, "%lu (expanded: %lu)\n",
5707 size >> 10,
5708 trace_buf_size >> 10);
5709 else
5710 r = sprintf(buf, "%lu\n", size >> 10);
5711 } else
5712 r = sprintf(buf, "X\n");
5713 } else
5714 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
5715
5716 mutex_unlock(&trace_types_lock);
5717
5718 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5719 return ret;
5720}
5721
5722static ssize_t
5723tracing_entries_write(struct file *filp, const char __user *ubuf,
5724 size_t cnt, loff_t *ppos)
5725{
5726 struct inode *inode = file_inode(filp);
5727 struct trace_array *tr = inode->i_private;
5728 unsigned long val;
5729 int ret;
5730
5731 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5732 if (ret)
5733 return ret;
5734
5735 /* must have at least 1 entry */
5736 if (!val)
5737 return -EINVAL;
5738
5739 /* value is in KB */
5740 val <<= 10;
5741 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5742 if (ret < 0)
5743 return ret;
5744
5745 *ppos += cnt;
5746
5747 return cnt;
5748}
5749
5750static ssize_t
5751tracing_total_entries_read(struct file *filp, char __user *ubuf,
5752 size_t cnt, loff_t *ppos)
5753{
5754 struct trace_array *tr = filp->private_data;
5755 char buf[64];
5756 int r, cpu;
5757 unsigned long size = 0, expanded_size = 0;
5758
5759 mutex_lock(&trace_types_lock);
5760 for_each_tracing_cpu(cpu) {
5761 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
5762 if (!tr->ring_buffer_expanded)
5763 expanded_size += trace_buf_size >> 10;
5764 }
5765 if (tr->ring_buffer_expanded)
5766 r = sprintf(buf, "%lu\n", size);
5767 else
5768 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5769 mutex_unlock(&trace_types_lock);
5770
5771 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5772}
5773
5774#define LAST_BOOT_HEADER ((void *)1)
5775
5776static void *l_next(struct seq_file *m, void *v, loff_t *pos)
5777{
5778 struct trace_array *tr = m->private;
5779 struct trace_scratch *tscratch = tr->scratch;
5780 unsigned int index = *pos;
5781
5782 (*pos)++;
5783
5784 if (*pos == 1)
5785 return LAST_BOOT_HEADER;
5786
5787 /* Only show offsets of the last boot data */
5788 if (!tscratch || !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
5789 return NULL;
5790
5791 /* *pos 0 is for the header, 1 is for the first module */
5792 index--;
5793
5794 if (index >= tscratch->nr_entries)
5795 return NULL;
5796
5797 return &tscratch->entries[index];
5798}
5799
5800static void *l_start(struct seq_file *m, loff_t *pos)
5801{
5802 mutex_lock(&scratch_mutex);
5803
5804 return l_next(m, NULL, pos);
5805}
5806
5807static void l_stop(struct seq_file *m, void *p)
5808{
5809 mutex_unlock(&scratch_mutex);
5810}
5811
5812static void show_last_boot_header(struct seq_file *m, struct trace_array *tr)
5813{
5814 struct trace_scratch *tscratch = tr->scratch;
5815
5816 /*
5817 * Do not leak KASLR address. This only shows the KASLR address of
5818 * the last boot. When the ring buffer is started, the LAST_BOOT
5819 * flag gets cleared, and this should only report "current".
5820 * Otherwise it shows the KASLR address from the previous boot which
5821 * should not be the same as the current boot.
5822 */
5823 if (tscratch && (tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
5824 seq_printf(m, "%lx\t[kernel]\n", tscratch->text_addr);
5825 else
5826 seq_puts(m, "# Current\n");
5827}
5828
5829static int l_show(struct seq_file *m, void *v)
5830{
5831 struct trace_array *tr = m->private;
5832 struct trace_mod_entry *entry = v;
5833
5834 if (v == LAST_BOOT_HEADER) {
5835 show_last_boot_header(m, tr);
5836 return 0;
5837 }
5838
5839 seq_printf(m, "%lx\t%s\n", entry->mod_addr, entry->mod_name);
5840 return 0;
5841}
5842
5843static const struct seq_operations last_boot_seq_ops = {
5844 .start = l_start,
5845 .next = l_next,
5846 .stop = l_stop,
5847 .show = l_show,
5848};
5849
5850static int tracing_last_boot_open(struct inode *inode, struct file *file)
5851{
5852 struct trace_array *tr = inode->i_private;
5853 struct seq_file *m;
5854 int ret;
5855
5856 ret = tracing_check_open_get_tr(tr);
5857 if (ret)
5858 return ret;
5859
5860 ret = seq_open(file, &last_boot_seq_ops);
5861 if (ret) {
5862 trace_array_put(tr);
5863 return ret;
5864 }
5865
5866 m = file->private_data;
5867 m->private = tr;
5868
5869 return 0;
5870}
5871
5872static int tracing_buffer_meta_open(struct inode *inode, struct file *filp)
5873{
5874 struct trace_array *tr = inode->i_private;
5875 int cpu = tracing_get_cpu(inode);
5876 int ret;
5877
5878 ret = tracing_check_open_get_tr(tr);
5879 if (ret)
5880 return ret;
5881
5882 ret = ring_buffer_meta_seq_init(filp, tr->array_buffer.buffer, cpu);
5883 if (ret < 0)
5884 __trace_array_put(tr);
5885 return ret;
5886}
5887
5888static ssize_t
5889tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5890 size_t cnt, loff_t *ppos)
5891{
5892 /*
5893 * There is no need to read what the user has written, this function
5894 * is just to make sure that there is no error when "echo" is used
5895 */
5896
5897 *ppos += cnt;
5898
5899 return cnt;
5900}
5901
5902static int
5903tracing_free_buffer_release(struct inode *inode, struct file *filp)
5904{
5905 struct trace_array *tr = inode->i_private;
5906
5907 /* disable tracing ? */
5908 if (tr->trace_flags & TRACE_ITER(STOP_ON_FREE))
5909 tracer_tracing_off(tr);
5910 /* resize the ring buffer to 0 */
5911 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5912
5913 trace_array_put(tr);
5914
5915 return 0;
5916}
5917
5918#define TRACE_MARKER_MAX_SIZE 4096
5919
5920static ssize_t write_marker_to_buffer(struct trace_array *tr, const char *buf,
5921 size_t cnt, unsigned long ip)
5922{
5923 struct ring_buffer_event *event;
5924 enum event_trigger_type tt = ETT_NONE;
5925 struct trace_buffer *buffer;
5926 struct print_entry *entry;
5927 int meta_size;
5928 ssize_t written;
5929 size_t size;
5930
5931 meta_size = sizeof(*entry) + 2; /* add '\0' and possible '\n' */
5932 again:
5933 size = cnt + meta_size;
5934
5935 buffer = tr->array_buffer.buffer;
5936 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5937 tracing_gen_ctx());
5938 if (unlikely(!event)) {
5939 /*
5940 * If the size was greater than what was allowed, then
5941 * make it smaller and try again.
5942 */
5943 if (size > ring_buffer_max_event_size(buffer)) {
5944 cnt = ring_buffer_max_event_size(buffer) - meta_size;
5945 /* The above should only happen once */
5946 if (WARN_ON_ONCE(cnt + meta_size == size))
5947 return -EBADF;
5948 goto again;
5949 }
5950
5951 /* Ring buffer disabled, return as if not open for write */
5952 return -EBADF;
5953 }
5954
5955 entry = ring_buffer_event_data(event);
5956 entry->ip = ip;
5957 memcpy(&entry->buf, buf, cnt);
5958 written = cnt;
5959
5960 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
5961 /* do not add \n before testing triggers, but add \0 */
5962 entry->buf[cnt] = '\0';
5963 tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event);
5964 }
5965
5966 if (entry->buf[cnt - 1] != '\n') {
5967 entry->buf[cnt] = '\n';
5968 entry->buf[cnt + 1] = '\0';
5969 } else
5970 entry->buf[cnt] = '\0';
5971
5972 if (static_branch_unlikely(&trace_marker_exports_enabled))
5973 ftrace_exports(event, TRACE_EXPORT_MARKER);
5974 __buffer_unlock_commit(buffer, event);
5975
5976 if (tt)
5977 event_triggers_post_call(tr->trace_marker_file, tt);
5978
5979 return written;
5980}
5981
5982struct trace_user_buf {
5983 char *buf;
5984};
5985
5986static DEFINE_MUTEX(trace_user_buffer_mutex);
5987static struct trace_user_buf_info *trace_user_buffer;
5988
5989/**
5990 * trace_user_fault_destroy - free up allocated memory of a trace user buffer
5991 * @tinfo: The descriptor to free up
5992 *
5993 * Frees any data allocated in the trace info dsecriptor.
5994 */
5995void trace_user_fault_destroy(struct trace_user_buf_info *tinfo)
5996{
5997 char *buf;
5998 int cpu;
5999
6000 if (!tinfo || !tinfo->tbuf)
6001 return;
6002
6003 for_each_possible_cpu(cpu) {
6004 buf = per_cpu_ptr(tinfo->tbuf, cpu)->buf;
6005 kfree(buf);
6006 }
6007 free_percpu(tinfo->tbuf);
6008}
6009
6010static int user_fault_buffer_enable(struct trace_user_buf_info *tinfo, size_t size)
6011{
6012 char *buf;
6013 int cpu;
6014
6015 lockdep_assert_held(&trace_user_buffer_mutex);
6016
6017 tinfo->tbuf = alloc_percpu(struct trace_user_buf);
6018 if (!tinfo->tbuf)
6019 return -ENOMEM;
6020
6021 tinfo->ref = 1;
6022 tinfo->size = size;
6023
6024 /* Clear each buffer in case of error */
6025 for_each_possible_cpu(cpu) {
6026 per_cpu_ptr(tinfo->tbuf, cpu)->buf = NULL;
6027 }
6028
6029 for_each_possible_cpu(cpu) {
6030 buf = kmalloc_node(size, GFP_KERNEL,
6031 cpu_to_node(cpu));
6032 if (!buf)
6033 return -ENOMEM;
6034 per_cpu_ptr(tinfo->tbuf, cpu)->buf = buf;
6035 }
6036
6037 return 0;
6038}
6039
6040/* For internal use. Free and reinitialize */
6041static void user_buffer_free(struct trace_user_buf_info **tinfo)
6042{
6043 lockdep_assert_held(&trace_user_buffer_mutex);
6044
6045 trace_user_fault_destroy(*tinfo);
6046 kfree(*tinfo);
6047 *tinfo = NULL;
6048}
6049
6050/* For internal use. Initialize and allocate */
6051static int user_buffer_init(struct trace_user_buf_info **tinfo, size_t size)
6052{
6053 bool alloc = false;
6054 int ret;
6055
6056 lockdep_assert_held(&trace_user_buffer_mutex);
6057
6058 if (!*tinfo) {
6059 alloc = true;
6060 *tinfo = kzalloc_obj(**tinfo);
6061 if (!*tinfo)
6062 return -ENOMEM;
6063 }
6064
6065 ret = user_fault_buffer_enable(*tinfo, size);
6066 if (ret < 0 && alloc)
6067 user_buffer_free(tinfo);
6068
6069 return ret;
6070}
6071
6072/* For internal use, derefrence and free if necessary */
6073static void user_buffer_put(struct trace_user_buf_info **tinfo)
6074{
6075 guard(mutex)(&trace_user_buffer_mutex);
6076
6077 if (WARN_ON_ONCE(!*tinfo || !(*tinfo)->ref))
6078 return;
6079
6080 if (--(*tinfo)->ref)
6081 return;
6082
6083 user_buffer_free(tinfo);
6084}
6085
6086/**
6087 * trace_user_fault_init - Allocated or reference a per CPU buffer
6088 * @tinfo: A pointer to the trace buffer descriptor
6089 * @size: The size to allocate each per CPU buffer
6090 *
6091 * Create a per CPU buffer that can be used to copy from user space
6092 * in a task context. When calling trace_user_fault_read(), preemption
6093 * must be disabled, and it will enable preemption and copy user
6094 * space data to the buffer. If any schedule switches occur, it will
6095 * retry until it succeeds without a schedule switch knowing the buffer
6096 * is still valid.
6097 *
6098 * Returns 0 on success, negative on failure.
6099 */
6100int trace_user_fault_init(struct trace_user_buf_info *tinfo, size_t size)
6101{
6102 int ret;
6103
6104 if (!tinfo)
6105 return -EINVAL;
6106
6107 guard(mutex)(&trace_user_buffer_mutex);
6108
6109 ret = user_buffer_init(&tinfo, size);
6110 if (ret < 0)
6111 trace_user_fault_destroy(tinfo);
6112
6113 return ret;
6114}
6115
6116/**
6117 * trace_user_fault_get - up the ref count for the user buffer
6118 * @tinfo: A pointer to a pointer to the trace buffer descriptor
6119 *
6120 * Ups the ref count of the trace buffer.
6121 *
6122 * Returns the new ref count.
6123 */
6124int trace_user_fault_get(struct trace_user_buf_info *tinfo)
6125{
6126 if (!tinfo)
6127 return -1;
6128
6129 guard(mutex)(&trace_user_buffer_mutex);
6130
6131 tinfo->ref++;
6132 return tinfo->ref;
6133}
6134
6135/**
6136 * trace_user_fault_put - dereference a per cpu trace buffer
6137 * @tinfo: The @tinfo that was passed to trace_user_fault_get()
6138 *
6139 * Decrement the ref count of @tinfo.
6140 *
6141 * Returns the new refcount (negative on error).
6142 */
6143int trace_user_fault_put(struct trace_user_buf_info *tinfo)
6144{
6145 guard(mutex)(&trace_user_buffer_mutex);
6146
6147 if (WARN_ON_ONCE(!tinfo || !tinfo->ref))
6148 return -1;
6149
6150 --tinfo->ref;
6151 return tinfo->ref;
6152}
6153
6154/**
6155 * trace_user_fault_read - Read user space into a per CPU buffer
6156 * @tinfo: The @tinfo allocated by trace_user_fault_get()
6157 * @ptr: The user space pointer to read
6158 * @size: The size of user space to read.
6159 * @copy_func: Optional function to use to copy from user space
6160 * @data: Data to pass to copy_func if it was supplied
6161 *
6162 * Preemption must be disabled when this is called, and must not
6163 * be enabled while using the returned buffer.
6164 * This does the copying from user space into a per CPU buffer.
6165 *
6166 * The @size must not be greater than the size passed in to
6167 * trace_user_fault_init().
6168 *
6169 * If @copy_func is NULL, trace_user_fault_read() will use copy_from_user(),
6170 * otherwise it will call @copy_func. It will call @copy_func with:
6171 *
6172 * buffer: the per CPU buffer of the @tinfo.
6173 * ptr: The pointer @ptr to user space to read
6174 * size: The @size of the ptr to read
6175 * data: The @data parameter
6176 *
6177 * It is expected that @copy_func will return 0 on success and non zero
6178 * if there was a fault.
6179 *
6180 * Returns a pointer to the buffer with the content read from @ptr.
6181 * Preemption must remain disabled while the caller accesses the
6182 * buffer returned by this function.
6183 * Returns NULL if there was a fault, or the size passed in is
6184 * greater than the size passed to trace_user_fault_init().
6185 */
6186char *trace_user_fault_read(struct trace_user_buf_info *tinfo,
6187 const char __user *ptr, size_t size,
6188 trace_user_buf_copy copy_func, void *data)
6189{
6190 int cpu = smp_processor_id();
6191 char *buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf;
6192 unsigned long long cnt;
6193 int trys = 0;
6194 int ret;
6195
6196 lockdep_assert_preemption_disabled();
6197
6198 /*
6199 * It's up to the caller to not try to copy more than it said
6200 * it would.
6201 */
6202 if (size > tinfo->size)
6203 return NULL;
6204
6205 /*
6206 * This acts similar to a seqcount. The per CPU context switches are
6207 * recorded, migration is disabled and preemption is enabled. The
6208 * read of the user space memory is copied into the per CPU buffer.
6209 * Preemption is disabled again, and if the per CPU context switches count
6210 * is still the same, it means the buffer has not been corrupted.
6211 * If the count is different, it is assumed the buffer is corrupted
6212 * and reading must be tried again.
6213 */
6214
6215 do {
6216 /*
6217 * It is possible that something is trying to migrate this
6218 * task. What happens then, is when preemption is enabled,
6219 * the migration thread will preempt this task, try to
6220 * migrate it, fail, then let it run again. That will
6221 * cause this to loop again and never succeed.
6222 * On failures, enabled and disable preemption with
6223 * migration enabled, to allow the migration thread to
6224 * migrate this task.
6225 */
6226 if (trys) {
6227 preempt_enable_notrace();
6228 preempt_disable_notrace();
6229 cpu = smp_processor_id();
6230 buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf;
6231 }
6232
6233 /*
6234 * If for some reason, copy_from_user() always causes a context
6235 * switch, this would then cause an infinite loop.
6236 * If this task is preempted by another user space task, it
6237 * will cause this task to try again. But just in case something
6238 * changes where the copying from user space causes another task
6239 * to run, prevent this from going into an infinite loop.
6240 * 100 tries should be plenty.
6241 */
6242 if (WARN_ONCE(trys++ > 100, "Error: Too many tries to read user space"))
6243 return NULL;
6244
6245 /* Read the current CPU context switch counter */
6246 cnt = nr_context_switches_cpu(cpu);
6247
6248 /*
6249 * Preemption is going to be enabled, but this task must
6250 * remain on this CPU.
6251 */
6252 migrate_disable();
6253
6254 /*
6255 * Now preemption is being enabled and another task can come in
6256 * and use the same buffer and corrupt our data.
6257 */
6258 preempt_enable_notrace();
6259
6260 /* Make sure preemption is enabled here */
6261 lockdep_assert_preemption_enabled();
6262
6263 if (copy_func) {
6264 ret = copy_func(buffer, ptr, size, data);
6265 } else {
6266 ret = __copy_from_user(buffer, ptr, size);
6267 }
6268
6269 preempt_disable_notrace();
6270 migrate_enable();
6271
6272 /* if it faulted, no need to test if the buffer was corrupted */
6273 if (ret)
6274 return NULL;
6275
6276 /*
6277 * Preemption is disabled again, now check the per CPU context
6278 * switch counter. If it doesn't match, then another user space
6279 * process may have schedule in and corrupted our buffer. In that
6280 * case the copying must be retried.
6281 */
6282 } while (nr_context_switches_cpu(cpu) != cnt);
6283
6284 return buffer;
6285}
6286
6287static ssize_t
6288tracing_mark_write(struct file *filp, const char __user *ubuf,
6289 size_t cnt, loff_t *fpos)
6290{
6291 struct trace_array *tr = filp->private_data;
6292 ssize_t written = -ENODEV;
6293 unsigned long ip;
6294 char *buf;
6295
6296 if (unlikely(tracing_disabled))
6297 return -EINVAL;
6298
6299 if (!(tr->trace_flags & TRACE_ITER(MARKERS)))
6300 return -EINVAL;
6301
6302 if ((ssize_t)cnt < 0)
6303 return -EINVAL;
6304
6305 if (cnt > TRACE_MARKER_MAX_SIZE)
6306 cnt = TRACE_MARKER_MAX_SIZE;
6307
6308 /* Must have preemption disabled while having access to the buffer */
6309 guard(preempt_notrace)();
6310
6311 buf = trace_user_fault_read(trace_user_buffer, ubuf, cnt, NULL, NULL);
6312 if (!buf)
6313 return -EFAULT;
6314
6315 /* The selftests expect this function to be the IP address */
6316 ip = _THIS_IP_;
6317
6318 /* The global trace_marker can go to multiple instances */
6319 if (tr == &global_trace) {
6320 guard(rcu)();
6321 list_for_each_entry_rcu(tr, &marker_copies, marker_list) {
6322 written = write_marker_to_buffer(tr, buf, cnt, ip);
6323 if (written < 0)
6324 break;
6325 }
6326 } else {
6327 written = write_marker_to_buffer(tr, buf, cnt, ip);
6328 }
6329
6330 return written;
6331}
6332
6333static ssize_t write_raw_marker_to_buffer(struct trace_array *tr,
6334 const char *buf, size_t cnt)
6335{
6336 struct ring_buffer_event *event;
6337 struct trace_buffer *buffer;
6338 struct raw_data_entry *entry;
6339 ssize_t written;
6340 size_t size;
6341
6342 /* cnt includes both the entry->id and the data behind it. */
6343 size = struct_offset(entry, id) + cnt;
6344
6345 buffer = tr->array_buffer.buffer;
6346
6347 if (size > ring_buffer_max_event_size(buffer))
6348 return -EINVAL;
6349
6350 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6351 tracing_gen_ctx());
6352 if (!event)
6353 /* Ring buffer disabled, return as if not open for write */
6354 return -EBADF;
6355
6356 entry = ring_buffer_event_data(event);
6357 unsafe_memcpy(&entry->id, buf, cnt,
6358 "id and content already reserved on ring buffer"
6359 "'buf' includes the 'id' and the data."
6360 "'entry' was allocated with cnt from 'id'.");
6361 written = cnt;
6362
6363 __buffer_unlock_commit(buffer, event);
6364
6365 return written;
6366}
6367
6368static ssize_t
6369tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6370 size_t cnt, loff_t *fpos)
6371{
6372 struct trace_array *tr = filp->private_data;
6373 ssize_t written = -ENODEV;
6374 char *buf;
6375
6376 if (unlikely(tracing_disabled))
6377 return -EINVAL;
6378
6379 if (!(tr->trace_flags & TRACE_ITER(MARKERS)))
6380 return -EINVAL;
6381
6382 /* The marker must at least have a tag id */
6383 if (cnt < sizeof(unsigned int))
6384 return -EINVAL;
6385
6386 /* raw write is all or nothing */
6387 if (cnt > TRACE_MARKER_MAX_SIZE)
6388 return -EINVAL;
6389
6390 /* Must have preemption disabled while having access to the buffer */
6391 guard(preempt_notrace)();
6392
6393 buf = trace_user_fault_read(trace_user_buffer, ubuf, cnt, NULL, NULL);
6394 if (!buf)
6395 return -EFAULT;
6396
6397 /* The global trace_marker_raw can go to multiple instances */
6398 if (tr == &global_trace) {
6399 guard(rcu)();
6400 list_for_each_entry_rcu(tr, &marker_copies, marker_list) {
6401 written = write_raw_marker_to_buffer(tr, buf, cnt);
6402 if (written < 0)
6403 break;
6404 }
6405 } else {
6406 written = write_raw_marker_to_buffer(tr, buf, cnt);
6407 }
6408
6409 return written;
6410}
6411
6412static int tracing_mark_open(struct inode *inode, struct file *filp)
6413{
6414 int ret;
6415
6416 scoped_guard(mutex, &trace_user_buffer_mutex) {
6417 if (!trace_user_buffer) {
6418 ret = user_buffer_init(&trace_user_buffer, TRACE_MARKER_MAX_SIZE);
6419 if (ret < 0)
6420 return ret;
6421 } else {
6422 trace_user_buffer->ref++;
6423 }
6424 }
6425
6426 stream_open(inode, filp);
6427 ret = tracing_open_generic_tr(inode, filp);
6428 if (ret < 0)
6429 user_buffer_put(&trace_user_buffer);
6430 return ret;
6431}
6432
6433static int tracing_mark_release(struct inode *inode, struct file *file)
6434{
6435 user_buffer_put(&trace_user_buffer);
6436 return tracing_release_generic_tr(inode, file);
6437}
6438
6439static int tracing_clock_show(struct seq_file *m, void *v)
6440{
6441 struct trace_array *tr = m->private;
6442 int i;
6443
6444 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6445 seq_printf(m,
6446 "%s%s%s%s", i ? " " : "",
6447 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6448 i == tr->clock_id ? "]" : "");
6449 seq_putc(m, '\n');
6450
6451 return 0;
6452}
6453
6454int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6455{
6456 int i;
6457
6458 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6459 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6460 break;
6461 }
6462 if (i == ARRAY_SIZE(trace_clocks))
6463 return -EINVAL;
6464
6465 guard(mutex)(&trace_types_lock);
6466
6467 tr->clock_id = i;
6468
6469 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
6470
6471 /*
6472 * New clock may not be consistent with the previous clock.
6473 * Reset the buffer so that it doesn't have incomparable timestamps.
6474 */
6475 tracing_reset_online_cpus(&tr->array_buffer);
6476
6477#ifdef CONFIG_TRACER_SNAPSHOT
6478 if (tr->snapshot_buffer.buffer)
6479 ring_buffer_set_clock(tr->snapshot_buffer.buffer, trace_clocks[i].func);
6480 tracing_reset_online_cpus(&tr->snapshot_buffer);
6481#endif
6482 update_last_data_if_empty(tr);
6483
6484 if (tr->scratch && !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) {
6485 struct trace_scratch *tscratch = tr->scratch;
6486
6487 tscratch->clock_id = i;
6488 }
6489
6490 return 0;
6491}
6492
6493static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6494 size_t cnt, loff_t *fpos)
6495{
6496 struct seq_file *m = filp->private_data;
6497 struct trace_array *tr = m->private;
6498 char buf[64];
6499 const char *clockstr;
6500 int ret;
6501
6502 if (cnt >= sizeof(buf))
6503 return -EINVAL;
6504
6505 if (copy_from_user(buf, ubuf, cnt))
6506 return -EFAULT;
6507
6508 buf[cnt] = 0;
6509
6510 clockstr = strstrip(buf);
6511
6512 ret = tracing_set_clock(tr, clockstr);
6513 if (ret)
6514 return ret;
6515
6516 *fpos += cnt;
6517
6518 return cnt;
6519}
6520
6521static int tracing_clock_open(struct inode *inode, struct file *file)
6522{
6523 struct trace_array *tr = inode->i_private;
6524 int ret;
6525
6526 ret = tracing_check_open_get_tr(tr);
6527 if (ret)
6528 return ret;
6529
6530 if ((file->f_mode & FMODE_WRITE) && trace_array_is_readonly(tr)) {
6531 trace_array_put(tr);
6532 return -EACCES;
6533 }
6534
6535 ret = single_open(file, tracing_clock_show, inode->i_private);
6536 if (ret < 0)
6537 trace_array_put(tr);
6538
6539 return ret;
6540}
6541
6542static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6543{
6544 struct trace_array *tr = m->private;
6545
6546 guard(mutex)(&trace_types_lock);
6547
6548 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
6549 seq_puts(m, "delta [absolute]\n");
6550 else
6551 seq_puts(m, "[delta] absolute\n");
6552
6553 return 0;
6554}
6555
6556static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6557{
6558 struct trace_array *tr = inode->i_private;
6559 int ret;
6560
6561 ret = tracing_check_open_get_tr(tr);
6562 if (ret)
6563 return ret;
6564
6565 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6566 if (ret < 0)
6567 trace_array_put(tr);
6568
6569 return ret;
6570}
6571
6572u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_event *rbe)
6573{
6574 if (rbe == this_cpu_read(trace_buffered_event))
6575 return ring_buffer_time_stamp(buffer);
6576
6577 return ring_buffer_event_time_stamp(buffer, rbe);
6578}
6579
6580static const struct file_operations tracing_thresh_fops = {
6581 .open = tracing_open_generic,
6582 .read = tracing_thresh_read,
6583 .write = tracing_thresh_write,
6584 .llseek = generic_file_llseek,
6585};
6586
6587static const struct file_operations set_tracer_fops = {
6588 .open = tracing_open_generic_tr,
6589 .read = tracing_set_trace_read,
6590 .write = tracing_set_trace_write,
6591 .llseek = generic_file_llseek,
6592 .release = tracing_release_generic_tr,
6593};
6594
6595static const struct file_operations tracing_pipe_fops = {
6596 .open = tracing_open_pipe,
6597 .poll = tracing_poll_pipe,
6598 .read = tracing_read_pipe,
6599 .splice_read = tracing_splice_read_pipe,
6600 .release = tracing_release_pipe,
6601};
6602
6603static const struct file_operations tracing_entries_fops = {
6604 .open = tracing_open_generic_tr,
6605 .read = tracing_entries_read,
6606 .write = tracing_entries_write,
6607 .llseek = generic_file_llseek,
6608 .release = tracing_release_generic_tr,
6609};
6610
6611static const struct file_operations tracing_syscall_buf_fops = {
6612 .open = tracing_open_generic_tr,
6613 .read = tracing_syscall_buf_read,
6614 .write = tracing_syscall_buf_write,
6615 .llseek = generic_file_llseek,
6616 .release = tracing_release_generic_tr,
6617};
6618
6619static const struct file_operations tracing_buffer_meta_fops = {
6620 .open = tracing_buffer_meta_open,
6621 .read = seq_read,
6622 .llseek = seq_lseek,
6623 .release = tracing_seq_release,
6624};
6625
6626static const struct file_operations tracing_total_entries_fops = {
6627 .open = tracing_open_generic_tr,
6628 .read = tracing_total_entries_read,
6629 .llseek = generic_file_llseek,
6630 .release = tracing_release_generic_tr,
6631};
6632
6633static const struct file_operations tracing_free_buffer_fops = {
6634 .open = tracing_open_generic_tr,
6635 .write = tracing_free_buffer_write,
6636 .release = tracing_free_buffer_release,
6637};
6638
6639static const struct file_operations tracing_mark_fops = {
6640 .open = tracing_mark_open,
6641 .write = tracing_mark_write,
6642 .release = tracing_mark_release,
6643};
6644
6645static const struct file_operations tracing_mark_raw_fops = {
6646 .open = tracing_mark_open,
6647 .write = tracing_mark_raw_write,
6648 .release = tracing_mark_release,
6649};
6650
6651static const struct file_operations trace_clock_fops = {
6652 .open = tracing_clock_open,
6653 .read = seq_read,
6654 .llseek = seq_lseek,
6655 .release = tracing_single_release_tr,
6656 .write = tracing_clock_write,
6657};
6658
6659static const struct file_operations trace_time_stamp_mode_fops = {
6660 .open = tracing_time_stamp_mode_open,
6661 .read = seq_read,
6662 .llseek = seq_lseek,
6663 .release = tracing_single_release_tr,
6664};
6665
6666static const struct file_operations last_boot_fops = {
6667 .open = tracing_last_boot_open,
6668 .read = seq_read,
6669 .llseek = seq_lseek,
6670 .release = tracing_seq_release,
6671};
6672
6673/*
6674 * trace_min_max_write - Write a u64 value to a trace_min_max_param struct
6675 * @filp: The active open file structure
6676 * @ubuf: The userspace provided buffer to read value into
6677 * @cnt: The maximum number of bytes to read
6678 * @ppos: The current "file" position
6679 *
6680 * This function implements the write interface for a struct trace_min_max_param.
6681 * The filp->private_data must point to a trace_min_max_param structure that
6682 * defines where to write the value, the min and the max acceptable values,
6683 * and a lock to protect the write.
6684 */
6685static ssize_t
6686trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
6687{
6688 struct trace_min_max_param *param = filp->private_data;
6689 u64 val;
6690 int err;
6691
6692 if (!param)
6693 return -EFAULT;
6694
6695 err = kstrtoull_from_user(ubuf, cnt, 10, &val);
6696 if (err)
6697 return err;
6698
6699 if (param->lock)
6700 mutex_lock(param->lock);
6701
6702 if (param->min && val < *param->min)
6703 err = -EINVAL;
6704
6705 if (param->max && val > *param->max)
6706 err = -EINVAL;
6707
6708 if (!err)
6709 *param->val = val;
6710
6711 if (param->lock)
6712 mutex_unlock(param->lock);
6713
6714 if (err)
6715 return err;
6716
6717 return cnt;
6718}
6719
6720/*
6721 * trace_min_max_read - Read a u64 value from a trace_min_max_param struct
6722 * @filp: The active open file structure
6723 * @ubuf: The userspace provided buffer to read value into
6724 * @cnt: The maximum number of bytes to read
6725 * @ppos: The current "file" position
6726 *
6727 * This function implements the read interface for a struct trace_min_max_param.
6728 * The filp->private_data must point to a trace_min_max_param struct with valid
6729 * data.
6730 */
6731static ssize_t
6732trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
6733{
6734 struct trace_min_max_param *param = filp->private_data;
6735 char buf[U64_STR_SIZE];
6736 int len;
6737 u64 val;
6738
6739 if (!param)
6740 return -EFAULT;
6741
6742 val = *param->val;
6743
6744 if (cnt > sizeof(buf))
6745 cnt = sizeof(buf);
6746
6747 len = snprintf(buf, sizeof(buf), "%llu\n", val);
6748
6749 return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
6750}
6751
6752const struct file_operations trace_min_max_fops = {
6753 .open = tracing_open_generic,
6754 .read = trace_min_max_read,
6755 .write = trace_min_max_write,
6756};
6757
6758#define TRACING_LOG_ERRS_MAX 8
6759#define TRACING_LOG_LOC_MAX 128
6760
6761#define CMD_PREFIX " Command: "
6762
6763struct err_info {
6764 const char **errs; /* ptr to loc-specific array of err strings */
6765 u8 type; /* index into errs -> specific err string */
6766 u16 pos; /* caret position */
6767 u64 ts;
6768};
6769
6770struct tracing_log_err {
6771 struct list_head list;
6772 struct err_info info;
6773 char loc[TRACING_LOG_LOC_MAX]; /* err location */
6774 char *cmd; /* what caused err */
6775};
6776
6777static DEFINE_MUTEX(tracing_err_log_lock);
6778
6779static struct tracing_log_err *alloc_tracing_log_err(int len)
6780{
6781 struct tracing_log_err *err;
6782
6783 err = kzalloc_obj(*err);
6784 if (!err)
6785 return ERR_PTR(-ENOMEM);
6786
6787 err->cmd = kzalloc(len, GFP_KERNEL);
6788 if (!err->cmd) {
6789 kfree(err);
6790 return ERR_PTR(-ENOMEM);
6791 }
6792
6793 return err;
6794}
6795
6796static void free_tracing_log_err(struct tracing_log_err *err)
6797{
6798 kfree(err->cmd);
6799 kfree(err);
6800}
6801
6802static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr,
6803 int len)
6804{
6805 struct tracing_log_err *err;
6806 char *cmd;
6807
6808 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
6809 err = alloc_tracing_log_err(len);
6810 if (PTR_ERR(err) != -ENOMEM)
6811 tr->n_err_log_entries++;
6812
6813 return err;
6814 }
6815 cmd = kzalloc(len, GFP_KERNEL);
6816 if (!cmd)
6817 return ERR_PTR(-ENOMEM);
6818 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
6819 kfree(err->cmd);
6820 err->cmd = cmd;
6821 list_del(&err->list);
6822
6823 return err;
6824}
6825
6826/**
6827 * err_pos - find the position of a string within a command for error careting
6828 * @cmd: The tracing command that caused the error
6829 * @str: The string to position the caret at within @cmd
6830 *
6831 * Finds the position of the first occurrence of @str within @cmd. The
6832 * return value can be passed to tracing_log_err() for caret placement
6833 * within @cmd.
6834 *
6835 * Returns the index within @cmd of the first occurrence of @str or 0
6836 * if @str was not found.
6837 */
6838unsigned int err_pos(char *cmd, const char *str)
6839{
6840 char *found;
6841
6842 if (WARN_ON(!strlen(cmd)))
6843 return 0;
6844
6845 found = strstr(cmd, str);
6846 if (found)
6847 return found - cmd;
6848
6849 return 0;
6850}
6851
6852/**
6853 * tracing_log_err - write an error to the tracing error log
6854 * @tr: The associated trace array for the error (NULL for top level array)
6855 * @loc: A string describing where the error occurred
6856 * @cmd: The tracing command that caused the error
6857 * @errs: The array of loc-specific static error strings
6858 * @type: The index into errs[], which produces the specific static err string
6859 * @pos: The position the caret should be placed in the cmd
6860 *
6861 * Writes an error into tracing/error_log of the form:
6862 *
6863 * <loc>: error: <text>
6864 * Command: <cmd>
6865 * ^
6866 *
6867 * tracing/error_log is a small log file containing the last
6868 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
6869 * unless there has been a tracing error, and the error log can be
6870 * cleared and have its memory freed by writing the empty string in
6871 * truncation mode to it i.e. echo > tracing/error_log.
6872 *
6873 * NOTE: the @errs array along with the @type param are used to
6874 * produce a static error string - this string is not copied and saved
6875 * when the error is logged - only a pointer to it is saved. See
6876 * existing callers for examples of how static strings are typically
6877 * defined for use with tracing_log_err().
6878 */
6879void tracing_log_err(struct trace_array *tr,
6880 const char *loc, const char *cmd,
6881 const char **errs, u8 type, u16 pos)
6882{
6883 struct tracing_log_err *err;
6884 int len = 0;
6885
6886 if (!tr)
6887 tr = &global_trace;
6888
6889 len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1;
6890
6891 guard(mutex)(&tracing_err_log_lock);
6892
6893 err = get_tracing_log_err(tr, len);
6894 if (PTR_ERR(err) == -ENOMEM)
6895 return;
6896
6897 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
6898 snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd);
6899
6900 err->info.errs = errs;
6901 err->info.type = type;
6902 err->info.pos = pos;
6903 err->info.ts = local_clock();
6904
6905 list_add_tail(&err->list, &tr->err_log);
6906}
6907
6908static void clear_tracing_err_log(struct trace_array *tr)
6909{
6910 struct tracing_log_err *err, *next;
6911
6912 guard(mutex)(&tracing_err_log_lock);
6913
6914 list_for_each_entry_safe(err, next, &tr->err_log, list) {
6915 list_del(&err->list);
6916 free_tracing_log_err(err);
6917 }
6918
6919 tr->n_err_log_entries = 0;
6920}
6921
6922static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
6923{
6924 struct trace_array *tr = m->private;
6925
6926 mutex_lock(&tracing_err_log_lock);
6927
6928 return seq_list_start(&tr->err_log, *pos);
6929}
6930
6931static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
6932{
6933 struct trace_array *tr = m->private;
6934
6935 return seq_list_next(v, &tr->err_log, pos);
6936}
6937
6938static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
6939{
6940 mutex_unlock(&tracing_err_log_lock);
6941}
6942
6943static void tracing_err_log_show_pos(struct seq_file *m, u16 pos)
6944{
6945 u16 i;
6946
6947 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
6948 seq_putc(m, ' ');
6949 for (i = 0; i < pos; i++)
6950 seq_putc(m, ' ');
6951 seq_puts(m, "^\n");
6952}
6953
6954static int tracing_err_log_seq_show(struct seq_file *m, void *v)
6955{
6956 struct tracing_log_err *err = v;
6957
6958 if (err) {
6959 const char *err_text = err->info.errs[err->info.type];
6960 u64 sec = err->info.ts;
6961 u32 nsec;
6962
6963 nsec = do_div(sec, NSEC_PER_SEC);
6964 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
6965 err->loc, err_text);
6966 seq_printf(m, "%s", err->cmd);
6967 tracing_err_log_show_pos(m, err->info.pos);
6968 }
6969
6970 return 0;
6971}
6972
6973static const struct seq_operations tracing_err_log_seq_ops = {
6974 .start = tracing_err_log_seq_start,
6975 .next = tracing_err_log_seq_next,
6976 .stop = tracing_err_log_seq_stop,
6977 .show = tracing_err_log_seq_show
6978};
6979
6980static int tracing_err_log_open(struct inode *inode, struct file *file)
6981{
6982 struct trace_array *tr = inode->i_private;
6983 int ret = 0;
6984
6985 ret = tracing_check_open_get_tr(tr);
6986 if (ret)
6987 return ret;
6988
6989 /* If this file was opened for write, then erase contents */
6990 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
6991 clear_tracing_err_log(tr);
6992
6993 if (file->f_mode & FMODE_READ) {
6994 ret = seq_open(file, &tracing_err_log_seq_ops);
6995 if (!ret) {
6996 struct seq_file *m = file->private_data;
6997 m->private = tr;
6998 } else {
6999 trace_array_put(tr);
7000 }
7001 }
7002 return ret;
7003}
7004
7005static ssize_t tracing_err_log_write(struct file *file,
7006 const char __user *buffer,
7007 size_t count, loff_t *ppos)
7008{
7009 return count;
7010}
7011
7012static int tracing_err_log_release(struct inode *inode, struct file *file)
7013{
7014 struct trace_array *tr = inode->i_private;
7015
7016 trace_array_put(tr);
7017
7018 if (file->f_mode & FMODE_READ)
7019 seq_release(inode, file);
7020
7021 return 0;
7022}
7023
7024static const struct file_operations tracing_err_log_fops = {
7025 .open = tracing_err_log_open,
7026 .write = tracing_err_log_write,
7027 .read = seq_read,
7028 .llseek = tracing_lseek,
7029 .release = tracing_err_log_release,
7030};
7031
7032int tracing_buffers_open(struct inode *inode, struct file *filp)
7033{
7034 struct trace_array *tr = inode->i_private;
7035 struct ftrace_buffer_info *info;
7036 int ret;
7037
7038 ret = tracing_check_open_get_tr(tr);
7039 if (ret)
7040 return ret;
7041
7042 info = kvzalloc_obj(*info);
7043 if (!info) {
7044 trace_array_put(tr);
7045 return -ENOMEM;
7046 }
7047
7048 mutex_lock(&trace_types_lock);
7049
7050 info->iter.tr = tr;
7051 info->iter.cpu_file = tracing_get_cpu(inode);
7052 info->iter.trace = tr->current_trace;
7053 info->iter.array_buffer = &tr->array_buffer;
7054 info->spare = NULL;
7055 /* Force reading ring buffer for first read */
7056 info->read = (unsigned int)-1;
7057
7058 filp->private_data = info;
7059
7060 tr->trace_ref++;
7061
7062 mutex_unlock(&trace_types_lock);
7063
7064 ret = nonseekable_open(inode, filp);
7065 if (ret < 0)
7066 trace_array_put(tr);
7067
7068 return ret;
7069}
7070
7071static __poll_t
7072tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7073{
7074 struct ftrace_buffer_info *info = filp->private_data;
7075 struct trace_iterator *iter = &info->iter;
7076
7077 return trace_poll(iter, filp, poll_table);
7078}
7079
7080ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7081 size_t count, loff_t *ppos)
7082{
7083 struct ftrace_buffer_info *info = filp->private_data;
7084 struct trace_iterator *iter = &info->iter;
7085 unsigned int spare_size;
7086 void *trace_data;
7087 ssize_t ret = 0;
7088 ssize_t size;
7089
7090 if (!count)
7091 return 0;
7092
7093 if (iter->snapshot && tracer_uses_snapshot(iter->tr->current_trace))
7094 return -EBUSY;
7095
7096 spare_size = ring_buffer_read_page_size(info->spare);
7097
7098again:
7099 /* Do we have previous read data to read? */
7100 if (info->read < spare_size)
7101 goto read;
7102
7103 ret = ring_buffer_alloc_read_page(iter->array_buffer->buffer, iter->cpu_file,
7104 &info->spare);
7105 if (ret)
7106 return ret;
7107
7108 spare_size = ring_buffer_read_page_size(info->spare);
7109 info->read = spare_size;
7110 info->spare_cpu = iter->cpu_file;
7111
7112 trace_access_lock(iter->cpu_file);
7113 ret = ring_buffer_read_page(iter->array_buffer->buffer,
7114 info->spare,
7115 count,
7116 iter->cpu_file, 0);
7117 trace_access_unlock(iter->cpu_file);
7118
7119 if (ret < 0) {
7120 if (trace_empty(iter) && !iter->closed) {
7121 if (update_last_data_if_empty(iter->tr))
7122 return 0;
7123
7124 if ((filp->f_flags & O_NONBLOCK))
7125 return -EAGAIN;
7126
7127 ret = wait_on_pipe(iter, 0);
7128 if (ret)
7129 return ret;
7130
7131 goto again;
7132 }
7133 return 0;
7134 }
7135
7136 info->read = 0;
7137
7138 read:
7139 size = spare_size - info->read;
7140 if (size > count)
7141 size = count;
7142 trace_data = ring_buffer_read_page_data(info->spare);
7143 ret = copy_to_user(ubuf, trace_data + info->read, size);
7144 if (ret == size)
7145 return -EFAULT;
7146
7147 size -= ret;
7148
7149 *ppos += size;
7150 info->read += size;
7151
7152 return size;
7153}
7154
7155static int tracing_buffers_flush(struct file *file, fl_owner_t id)
7156{
7157 struct ftrace_buffer_info *info = file->private_data;
7158 struct trace_iterator *iter = &info->iter;
7159
7160 iter->closed = true;
7161 /* Make sure the waiters see the new wait_index */
7162 (void)atomic_fetch_inc_release(&iter->wait_index);
7163
7164 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
7165
7166 return 0;
7167}
7168
7169int tracing_buffers_release(struct inode *inode, struct file *file)
7170{
7171 struct ftrace_buffer_info *info = file->private_data;
7172 struct trace_iterator *iter = &info->iter;
7173
7174 guard(mutex)(&trace_types_lock);
7175
7176 iter->tr->trace_ref--;
7177
7178 __trace_array_put(iter->tr);
7179
7180 ring_buffer_free_read_page(iter->array_buffer->buffer, info->spare_cpu, info->spare);
7181 kvfree(info);
7182
7183 return 0;
7184}
7185
7186struct buffer_ref {
7187 struct trace_buffer *buffer;
7188 struct buffer_data_read_page *rpage;
7189 int cpu;
7190 refcount_t refcount;
7191};
7192
7193static void buffer_ref_release(struct buffer_ref *ref)
7194{
7195 if (!refcount_dec_and_test(&ref->refcount))
7196 return;
7197 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->rpage);
7198 kfree(ref);
7199}
7200
7201static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7202 struct pipe_buffer *buf)
7203{
7204 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7205
7206 buffer_ref_release(ref);
7207 buf->private = 0;
7208}
7209
7210static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
7211 struct pipe_buffer *buf)
7212{
7213 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7214
7215 if (refcount_read(&ref->refcount) > INT_MAX/2)
7216 return false;
7217
7218 refcount_inc(&ref->refcount);
7219 return true;
7220}
7221
7222/* Pipe buffer operations for a buffer. */
7223static const struct pipe_buf_operations buffer_pipe_buf_ops = {
7224 .release = buffer_pipe_buf_release,
7225 .get = buffer_pipe_buf_get,
7226};
7227
7228/*
7229 * Callback from splice_to_pipe(), if we need to release some pages
7230 * at the end of the spd in case we error'ed out in filling the pipe.
7231 */
7232static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7233{
7234 struct buffer_ref *ref =
7235 (struct buffer_ref *)spd->partial[i].private;
7236
7237 buffer_ref_release(ref);
7238 spd->partial[i].private = 0;
7239}
7240
7241ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7242 struct pipe_inode_info *pipe, size_t len,
7243 unsigned int flags)
7244{
7245 struct ftrace_buffer_info *info = file->private_data;
7246 struct trace_iterator *iter = &info->iter;
7247 struct partial_page partial_def[PIPE_DEF_BUFFERS];
7248 struct page *pages_def[PIPE_DEF_BUFFERS];
7249 struct splice_pipe_desc spd = {
7250 .pages = pages_def,
7251 .partial = partial_def,
7252 .nr_pages_max = PIPE_DEF_BUFFERS,
7253 .ops = &buffer_pipe_buf_ops,
7254 .spd_release = buffer_spd_release,
7255 };
7256 unsigned int page_size = 0;
7257 struct buffer_ref *ref;
7258 bool woken = false;
7259 int entries, i;
7260 ssize_t ret = 0;
7261
7262 if (iter->snapshot && tracer_uses_snapshot(iter->tr->current_trace))
7263 return -EBUSY;
7264
7265 if (splice_grow_spd(pipe, &spd))
7266 return -ENOMEM;
7267
7268 again:
7269 trace_access_lock(iter->cpu_file);
7270 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7271
7272 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) {
7273 struct page *page;
7274 int r;
7275
7276 ref = kzalloc_obj(*ref);
7277 if (!ref) {
7278 ret = -ENOMEM;
7279 break;
7280 }
7281
7282 refcount_set(&ref->refcount, 1);
7283 ref->buffer = iter->array_buffer->buffer;
7284
7285 ret = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file, &ref->rpage);
7286 if (ret) {
7287 kfree(ref);
7288 break;
7289 }
7290 ref->cpu = iter->cpu_file;
7291
7292 page_size = ring_buffer_read_page_size(ref->rpage);
7293
7294 r = -EINVAL;
7295 if (IS_ALIGNED(*ppos, page_size) && len >= page_size) {
7296 r = ring_buffer_read_page(ref->buffer, ref->rpage, len, iter->cpu_file, 1);
7297 } else if (!i) {
7298 /*
7299 * If this fails to read on the first iteration, it
7300 * means the length was too small and an error should
7301 * be returned to user space. Otherwise, at least
7302 * one sub-buffer was successfully read but this failed
7303 * due to either the length was unaligned or the
7304 * subbuf order changed. Either case, do not report
7305 * an error.
7306 */
7307 ret = -EINVAL;
7308 }
7309
7310 if (r < 0) {
7311 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->rpage);
7312 kfree(ref);
7313 break;
7314 }
7315
7316 page = virt_to_page(ring_buffer_read_page_data(ref->rpage));
7317
7318 spd.pages[i] = page;
7319 spd.partial[i].len = page_size;
7320 spd.partial[i].offset = 0;
7321 spd.partial[i].private = (unsigned long)ref;
7322 spd.nr_pages++;
7323 *ppos += page_size;
7324
7325 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7326 }
7327
7328 trace_access_unlock(iter->cpu_file);
7329 spd.nr_pages = i;
7330
7331 /* did we read anything? */
7332 if (!spd.nr_pages) {
7333
7334 if (ret)
7335 goto out;
7336
7337 if (woken)
7338 goto out;
7339
7340 ret = -EAGAIN;
7341 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
7342 goto out;
7343
7344 ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent);
7345 if (ret)
7346 goto out;
7347
7348 /* No need to wait after waking up when tracing is off */
7349 if (!tracer_tracing_is_on(iter->tr))
7350 goto out;
7351
7352 /* Iterate one more time to collect any new data then exit */
7353 woken = true;
7354
7355 goto again;
7356 }
7357
7358 ret = splice_to_pipe(pipe, &spd);
7359out:
7360 splice_shrink_spd(&spd);
7361
7362 return ret;
7363}
7364
7365static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
7366{
7367 struct ftrace_buffer_info *info = file->private_data;
7368 struct trace_iterator *iter = &info->iter;
7369 int err;
7370
7371 if (cmd == TRACE_MMAP_IOCTL_GET_READER) {
7372 if (!(file->f_flags & O_NONBLOCK)) {
7373 err = ring_buffer_wait(iter->array_buffer->buffer,
7374 iter->cpu_file,
7375 iter->tr->buffer_percent,
7376 NULL, NULL);
7377 if (err)
7378 return err;
7379 }
7380
7381 return ring_buffer_map_get_reader(iter->array_buffer->buffer,
7382 iter->cpu_file);
7383 } else if (cmd) {
7384 return -ENOTTY;
7385 }
7386
7387 /*
7388 * An ioctl call with cmd 0 to the ring buffer file will wake up all
7389 * waiters
7390 */
7391 guard(mutex)(&trace_types_lock);
7392
7393 /* Make sure the waiters see the new wait_index */
7394 (void)atomic_fetch_inc_release(&iter->wait_index);
7395
7396 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
7397
7398 return 0;
7399}
7400
7401/*
7402 * This is called when a VMA is duplicated (e.g., on fork()) to increment
7403 * the user_mapped counter without remapping pages.
7404 */
7405static void tracing_buffers_mmap_open(struct vm_area_struct *vma)
7406{
7407 struct ftrace_buffer_info *info = vma->vm_file->private_data;
7408 struct trace_iterator *iter = &info->iter;
7409
7410 ring_buffer_map_dup(iter->array_buffer->buffer, iter->cpu_file);
7411}
7412
7413static void tracing_buffers_mmap_close(struct vm_area_struct *vma)
7414{
7415 struct ftrace_buffer_info *info = vma->vm_file->private_data;
7416 struct trace_iterator *iter = &info->iter;
7417
7418 WARN_ON(ring_buffer_unmap(iter->array_buffer->buffer, iter->cpu_file));
7419 put_snapshot_map(iter->tr);
7420}
7421
7422static int tracing_buffers_may_split(struct vm_area_struct *vma, unsigned long addr)
7423{
7424 /*
7425 * Trace buffer mappings require the complete buffer including
7426 * the meta page. Partial mappings are not supported.
7427 */
7428 return -EINVAL;
7429}
7430
7431static const struct vm_operations_struct tracing_buffers_vmops = {
7432 .open = tracing_buffers_mmap_open,
7433 .close = tracing_buffers_mmap_close,
7434 .may_split = tracing_buffers_may_split,
7435};
7436
7437static int tracing_buffers_mmap(struct file *filp, struct vm_area_struct *vma)
7438{
7439 struct ftrace_buffer_info *info = filp->private_data;
7440 struct trace_iterator *iter = &info->iter;
7441 int ret = 0;
7442
7443 /* A memmap'ed and backup buffers are not supported for user space mmap */
7444 if (iter->tr->flags & (TRACE_ARRAY_FL_MEMMAP | TRACE_ARRAY_FL_VMALLOC))
7445 return -ENODEV;
7446
7447 ret = get_snapshot_map(iter->tr);
7448 if (ret)
7449 return ret;
7450
7451 ret = ring_buffer_map(iter->array_buffer->buffer, iter->cpu_file, vma);
7452 if (ret)
7453 put_snapshot_map(iter->tr);
7454
7455 vma->vm_ops = &tracing_buffers_vmops;
7456
7457 return ret;
7458}
7459
7460static const struct file_operations tracing_buffers_fops = {
7461 .open = tracing_buffers_open,
7462 .read = tracing_buffers_read,
7463 .poll = tracing_buffers_poll,
7464 .release = tracing_buffers_release,
7465 .flush = tracing_buffers_flush,
7466 .splice_read = tracing_buffers_splice_read,
7467 .unlocked_ioctl = tracing_buffers_ioctl,
7468 .mmap = tracing_buffers_mmap,
7469};
7470
7471static ssize_t
7472tracing_stats_read(struct file *filp, char __user *ubuf,
7473 size_t count, loff_t *ppos)
7474{
7475 struct inode *inode = file_inode(filp);
7476 struct trace_array *tr = inode->i_private;
7477 struct array_buffer *trace_buf = &tr->array_buffer;
7478 int cpu = tracing_get_cpu(inode);
7479 struct trace_seq *s;
7480 unsigned long cnt;
7481 unsigned long long t;
7482 unsigned long usec_rem;
7483
7484 s = kmalloc_obj(*s);
7485 if (!s)
7486 return -ENOMEM;
7487
7488 trace_seq_init(s);
7489
7490 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
7491 trace_seq_printf(s, "entries: %ld\n", cnt);
7492
7493 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
7494 trace_seq_printf(s, "overrun: %ld\n", cnt);
7495
7496 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
7497 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7498
7499 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
7500 trace_seq_printf(s, "bytes: %ld\n", cnt);
7501
7502 if (trace_clocks[tr->clock_id].in_ns) {
7503 /* local or global for trace_clock */
7504 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7505 usec_rem = do_div(t, USEC_PER_SEC);
7506 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7507 t, usec_rem);
7508
7509 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer));
7510 usec_rem = do_div(t, USEC_PER_SEC);
7511 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7512 } else {
7513 /* counter or tsc mode for trace_clock */
7514 trace_seq_printf(s, "oldest event ts: %llu\n",
7515 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7516
7517 trace_seq_printf(s, "now ts: %llu\n",
7518 ring_buffer_time_stamp(trace_buf->buffer));
7519 }
7520
7521 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7522 trace_seq_printf(s, "dropped events: %ld\n", cnt);
7523
7524 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7525 trace_seq_printf(s, "read events: %ld\n", cnt);
7526
7527 count = simple_read_from_buffer(ubuf, count, ppos,
7528 s->buffer, trace_seq_used(s));
7529
7530 kfree(s);
7531
7532 return count;
7533}
7534
7535static const struct file_operations tracing_stats_fops = {
7536 .open = tracing_open_generic_tr,
7537 .read = tracing_stats_read,
7538 .llseek = generic_file_llseek,
7539 .release = tracing_release_generic_tr,
7540};
7541
7542#ifdef CONFIG_DYNAMIC_FTRACE
7543
7544static ssize_t
7545tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7546 size_t cnt, loff_t *ppos)
7547{
7548 ssize_t ret;
7549 char *buf;
7550 int r;
7551
7552 /* 512 should be plenty to hold the amount needed */
7553#define DYN_INFO_BUF_SIZE 512
7554
7555 buf = kmalloc(DYN_INFO_BUF_SIZE, GFP_KERNEL);
7556 if (!buf)
7557 return -ENOMEM;
7558
7559 r = scnprintf(buf, DYN_INFO_BUF_SIZE,
7560 "%ld pages:%ld groups: %ld\n"
7561 "ftrace boot update time = %llu (ns)\n"
7562 "ftrace module total update time = %llu (ns)\n",
7563 ftrace_update_tot_cnt,
7564 ftrace_number_of_pages,
7565 ftrace_number_of_groups,
7566 ftrace_update_time,
7567 ftrace_total_mod_time);
7568
7569 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7570 kfree(buf);
7571 return ret;
7572}
7573
7574static const struct file_operations tracing_dyn_info_fops = {
7575 .open = tracing_open_generic,
7576 .read = tracing_read_dyn_info,
7577 .llseek = generic_file_llseek,
7578};
7579#endif /* CONFIG_DYNAMIC_FTRACE */
7580
7581static struct dentry *tracing_get_dentry(struct trace_array *tr)
7582{
7583 /* Top directory uses NULL as the parent */
7584 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7585 return NULL;
7586
7587 if (WARN_ON(!tr->dir))
7588 return ERR_PTR(-ENODEV);
7589
7590 /* All sub buffers have a descriptor */
7591 return tr->dir;
7592}
7593
7594static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7595{
7596 struct dentry *d_tracer;
7597
7598 if (tr->percpu_dir)
7599 return tr->percpu_dir;
7600
7601 d_tracer = tracing_get_dentry(tr);
7602 if (IS_ERR(d_tracer))
7603 return NULL;
7604
7605 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
7606
7607 MEM_FAIL(!tr->percpu_dir,
7608 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
7609
7610 return tr->percpu_dir;
7611}
7612
7613struct dentry *
7614trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
7615 void *data, long cpu, const struct file_operations *fops)
7616{
7617 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
7618
7619 if (ret) /* See tracing_get_cpu() */
7620 d_inode(ret)->i_cdev = (void *)(cpu + 1);
7621 return ret;
7622}
7623
7624static void
7625tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
7626{
7627 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
7628 struct dentry *d_cpu;
7629 char cpu_dir[30]; /* 30 characters should be more than enough */
7630
7631 if (!d_percpu)
7632 return;
7633
7634 snprintf(cpu_dir, 30, "cpu%ld", cpu);
7635 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
7636 if (!d_cpu) {
7637 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
7638 return;
7639 }
7640
7641 /* per cpu trace_pipe */
7642 trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu,
7643 tr, cpu, &tracing_pipe_fops);
7644
7645 /* per cpu trace */
7646 trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu,
7647 tr, cpu, &tracing_fops);
7648
7649 trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu,
7650 tr, cpu, &tracing_buffers_fops);
7651
7652 trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu,
7653 tr, cpu, &tracing_stats_fops);
7654
7655 trace_create_cpu_file("buffer_size_kb", TRACE_MODE_WRITE, d_cpu,
7656 tr, cpu, &tracing_entries_fops);
7657
7658 if (tr->range_addr_start)
7659 trace_create_cpu_file("buffer_meta", TRACE_MODE_READ, d_cpu,
7660 tr, cpu, &tracing_buffer_meta_fops);
7661#ifdef CONFIG_TRACER_SNAPSHOT
7662 if (!tr->range_addr_start) {
7663 trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu,
7664 tr, cpu, &snapshot_fops);
7665
7666 trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu,
7667 tr, cpu, &snapshot_raw_fops);
7668 }
7669#endif
7670}
7671
7672#ifdef CONFIG_FTRACE_SELFTEST
7673/* Let selftest have access to static functions in this file */
7674#include "trace_selftest.c"
7675#endif
7676
7677static ssize_t
7678trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
7679 loff_t *ppos)
7680{
7681 struct trace_option_dentry *topt = filp->private_data;
7682 char *buf;
7683
7684 if (topt->flags->val & topt->opt->bit)
7685 buf = "1\n";
7686 else
7687 buf = "0\n";
7688
7689 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7690}
7691
7692static ssize_t
7693trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
7694 loff_t *ppos)
7695{
7696 struct trace_option_dentry *topt = filp->private_data;
7697 unsigned long val;
7698 int ret;
7699
7700 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7701 if (ret)
7702 return ret;
7703
7704 if (val != 0 && val != 1)
7705 return -EINVAL;
7706
7707 if (!!(topt->flags->val & topt->opt->bit) != val) {
7708 guard(mutex)(&trace_types_lock);
7709 ret = __set_tracer_option(topt->tr, topt->flags,
7710 topt->opt, !val);
7711 if (ret)
7712 return ret;
7713 }
7714
7715 *ppos += cnt;
7716
7717 return cnt;
7718}
7719
7720static bool tr_option_match(struct trace_array *tr, void *topt)
7721{
7722 for (int i = 0; i < tr->nr_topts; i++) {
7723 struct trace_options *tr_topts = &tr->topts[i];
7724
7725 if (topt >= (void *)&tr_topts->topts[0] &&
7726 topt < (void *)&tr_topts->topts[tr_topts->nr_topts])
7727 return true;
7728 }
7729 return false;
7730}
7731
7732/*
7733 * The topt is the address of a trace_array->topts[] element that holds the
7734 * the tracer options descriptor. But since the trace_array reference has not
7735 * been taken yet, it cannot be dereferenced as it could have been freed by
7736 * a rmdir of the instance the trace_array represents.
7737 *
7738 * Search the list of trace_arrays and compare the topt to the address of
7739 * the entire trace_array topts array for each trace_array in the list.
7740 * If one is matched, then take the reference and return it. If not, the
7741 * trace_array no longer exits.
7742 */
7743static int trace_array_tracer_options_get(void *topt)
7744{
7745 struct trace_array *tr;
7746 int ret;
7747
7748 ret = security_locked_down(LOCKDOWN_TRACEFS);
7749 if (ret)
7750 return ret;
7751
7752 if (tracing_disabled)
7753 return -ENODEV;
7754
7755 guard(mutex)(&trace_types_lock);
7756 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7757 if (tr_option_match(tr, topt))
7758 return __trace_array_get(tr);
7759 }
7760 return -ENODEV;
7761}
7762
7763static int tracing_open_options(struct inode *inode, struct file *filp)
7764{
7765 struct trace_option_dentry *topt = inode->i_private;
7766 int ret;
7767
7768 ret = trace_array_tracer_options_get(topt);
7769 if (ret)
7770 return ret;
7771
7772 filp->private_data = inode->i_private;
7773 return 0;
7774}
7775
7776static int tracing_release_options(struct inode *inode, struct file *file)
7777{
7778 struct trace_option_dentry *topt = file->private_data;
7779
7780 trace_array_put(topt->tr);
7781 return 0;
7782}
7783
7784static const struct file_operations trace_options_fops = {
7785 .open = tracing_open_options,
7786 .read = trace_options_read,
7787 .write = trace_options_write,
7788 .llseek = generic_file_llseek,
7789 .release = tracing_release_options,
7790};
7791
7792/*
7793 * In order to pass in both the trace_array descriptor as well as the index
7794 * to the flag that the trace option file represents, the trace_array
7795 * has a character array of trace_flags_index[], which holds the index
7796 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
7797 * The address of this character array is passed to the flag option file
7798 * read/write callbacks.
7799 *
7800 * In order to extract both the index and the trace_array descriptor,
7801 * get_tr_index() uses the following algorithm.
7802 *
7803 * idx = *ptr;
7804 *
7805 * As the pointer itself contains the address of the index (remember
7806 * index[1] == 1).
7807 *
7808 * Then to get the trace_array descriptor, by subtracting that index
7809 * from the ptr, we get to the start of the index itself.
7810 *
7811 * ptr - idx == &index[0]
7812 *
7813 * Then a simple container_of() from that pointer gets us to the
7814 * trace_array descriptor.
7815 */
7816static void get_tr_index(void *data, struct trace_array **ptr,
7817 unsigned int *pindex)
7818{
7819 *pindex = *(unsigned char *)data;
7820
7821 *ptr = container_of(data - *pindex, struct trace_array,
7822 trace_flags_index);
7823}
7824
7825static ssize_t
7826trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
7827 loff_t *ppos)
7828{
7829 void *tr_index = filp->private_data;
7830 struct trace_array *tr;
7831 unsigned int index;
7832 char *buf;
7833
7834 get_tr_index(tr_index, &tr, &index);
7835
7836 if (tr->trace_flags & (1ULL << index))
7837 buf = "1\n";
7838 else
7839 buf = "0\n";
7840
7841 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7842}
7843
7844static ssize_t
7845trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
7846 loff_t *ppos)
7847{
7848 void *tr_index = filp->private_data;
7849 struct trace_array *tr;
7850 unsigned int index;
7851 unsigned long val;
7852 int ret;
7853
7854 get_tr_index(tr_index, &tr, &index);
7855
7856 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7857 if (ret)
7858 return ret;
7859
7860 if (val != 0 && val != 1)
7861 return -EINVAL;
7862
7863 mutex_lock(&event_mutex);
7864 mutex_lock(&trace_types_lock);
7865 ret = set_tracer_flag(tr, 1ULL << index, val);
7866 mutex_unlock(&trace_types_lock);
7867 mutex_unlock(&event_mutex);
7868
7869 if (ret < 0)
7870 return ret;
7871
7872 *ppos += cnt;
7873
7874 return cnt;
7875}
7876
7877/*
7878 * The tr_index is the address of a trace_array->trace_flags_index[]
7879 * element that holds the index of the trace flag. But since the
7880 * trace_array reference has not been taken yet, it cannot be referenced
7881 * as it could have been freed by a rmdir of the instance the trace_array
7882 * represents.
7883 *
7884 * Search the list of trace_arrays and compare the tr_index to the
7885 * address of the entire trace_array trace_flags_index array for each
7886 * trace_array in the list. If one is matched, then take the reference
7887 * and return it. If not, the trace_array no longer exits.
7888 */
7889static int trace_array_options_get(void *tr_index)
7890{
7891 struct trace_array *tr;
7892 int ret;
7893
7894 ret = security_locked_down(LOCKDOWN_TRACEFS);
7895 if (ret)
7896 return ret;
7897
7898 if (tracing_disabled)
7899 return -ENODEV;
7900
7901 guard(mutex)(&trace_types_lock);
7902 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7903 if (tr_index >= (void *)&tr->trace_flags_index[0] &&
7904 tr_index < (void *)&tr->trace_flags_index[TRACE_FLAGS_MAX_SIZE])
7905 return __trace_array_get(tr);
7906 }
7907 return -ENODEV;
7908}
7909
7910static int trace_options_open(struct inode *inode, struct file *filp)
7911{
7912 void *tr_index = inode->i_private;
7913
7914 if (trace_array_options_get(tr_index) < 0)
7915 return -ENODEV;
7916
7917 filp->private_data = tr_index;
7918
7919 return 0;
7920}
7921
7922static int trace_options_release(struct inode *inode, struct file *filp)
7923{
7924 void *tr_index = filp->private_data;
7925 struct trace_array *tr;
7926 unsigned int index;
7927
7928 get_tr_index(tr_index, &tr, &index);
7929
7930 trace_array_put(tr);
7931
7932 return 0;
7933}
7934
7935static const struct file_operations trace_options_core_fops = {
7936 .open = trace_options_open,
7937 .read = trace_options_core_read,
7938 .write = trace_options_core_write,
7939 .llseek = generic_file_llseek,
7940 .release = trace_options_release,
7941};
7942
7943struct dentry *trace_create_file(const char *name,
7944 umode_t mode,
7945 struct dentry *parent,
7946 void *data,
7947 const struct file_operations *fops)
7948{
7949 struct dentry *ret;
7950
7951 ret = tracefs_create_file(name, mode, parent, data, fops);
7952 if (!ret)
7953 pr_warn("Could not create tracefs '%s' entry\n", name);
7954
7955 return ret;
7956}
7957
7958
7959static struct dentry *trace_options_init_dentry(struct trace_array *tr)
7960{
7961 struct dentry *d_tracer;
7962
7963 if (tr->options)
7964 return tr->options;
7965
7966 d_tracer = tracing_get_dentry(tr);
7967 if (IS_ERR(d_tracer))
7968 return NULL;
7969
7970 tr->options = tracefs_create_dir("options", d_tracer);
7971 if (!tr->options) {
7972 pr_warn("Could not create tracefs directory 'options'\n");
7973 return NULL;
7974 }
7975
7976 return tr->options;
7977}
7978
7979static void
7980create_trace_option_file(struct trace_array *tr,
7981 struct trace_option_dentry *topt,
7982 struct tracer_flags *flags,
7983 struct tracer_opt *opt)
7984{
7985 struct dentry *t_options;
7986
7987 t_options = trace_options_init_dentry(tr);
7988 if (!t_options)
7989 return;
7990
7991 topt->flags = flags;
7992 topt->opt = opt;
7993 topt->tr = tr;
7994
7995 topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,
7996 t_options, topt, &trace_options_fops);
7997}
7998
7999static int
8000create_trace_option_files(struct trace_array *tr, struct tracer *tracer,
8001 struct tracer_flags *flags)
8002{
8003 struct trace_option_dentry *topts;
8004 struct trace_options *tr_topts;
8005 struct tracer_opt *opts;
8006 int cnt;
8007
8008 if (!flags || !flags->opts)
8009 return 0;
8010
8011 opts = flags->opts;
8012
8013 for (cnt = 0; opts[cnt].name; cnt++)
8014 ;
8015
8016 topts = kzalloc_objs(*topts, cnt + 1);
8017 if (!topts)
8018 return 0;
8019
8020 tr_topts = krealloc_array(tr->topts, tr->nr_topts + 1, sizeof(*tr->topts),
8021 GFP_KERNEL);
8022 if (!tr_topts) {
8023 kfree(topts);
8024 return -ENOMEM;
8025 }
8026
8027 tr->topts = tr_topts;
8028 tr->topts[tr->nr_topts].tracer = tracer;
8029 tr->topts[tr->nr_topts].topts = topts;
8030 tr->topts[tr->nr_topts].nr_topts = cnt;
8031 tr->nr_topts++;
8032
8033 for (cnt = 0; opts[cnt].name; cnt++) {
8034 create_trace_option_file(tr, &topts[cnt], flags,
8035 &opts[cnt]);
8036 MEM_FAIL(topts[cnt].entry == NULL,
8037 "Failed to create trace option: %s",
8038 opts[cnt].name);
8039 }
8040 return 0;
8041}
8042
8043static int get_global_flags_val(struct tracer *tracer)
8044{
8045 struct tracers *t;
8046
8047 list_for_each_entry(t, &global_trace.tracers, list) {
8048 if (t->tracer != tracer)
8049 continue;
8050 if (!t->flags)
8051 return -1;
8052 return t->flags->val;
8053 }
8054 return -1;
8055}
8056
8057static int add_tracer_options(struct trace_array *tr, struct tracers *t)
8058{
8059 struct tracer *tracer = t->tracer;
8060 struct tracer_flags *flags = t->flags ?: tracer->flags;
8061
8062 if (!flags)
8063 return 0;
8064
8065 /* Only add tracer options after update_tracer_options finish */
8066 if (!tracer_options_updated)
8067 return 0;
8068
8069 return create_trace_option_files(tr, tracer, flags);
8070}
8071
8072static int add_tracer(struct trace_array *tr, struct tracer *tracer)
8073{
8074 struct tracer_flags *flags;
8075 struct tracers *t;
8076 int ret;
8077
8078 /* Only enable if the directory has been created already. */
8079 if (!tr->dir && !(tr->flags & TRACE_ARRAY_FL_GLOBAL))
8080 return 0;
8081
8082 /*
8083 * If this is an instance, only create flags for tracers
8084 * the instance may have.
8085 */
8086 if (!trace_ok_for_array(tracer, tr))
8087 return 0;
8088
8089 t = kmalloc_obj(*t);
8090 if (!t)
8091 return -ENOMEM;
8092
8093 t->tracer = tracer;
8094 t->flags = NULL;
8095 list_add(&t->list, &tr->tracers);
8096
8097 flags = tracer->flags;
8098 if (!flags) {
8099 if (!tracer->default_flags)
8100 return 0;
8101
8102 /*
8103 * If the tracer defines default flags, it means the flags are
8104 * per trace instance.
8105 */
8106 flags = kmalloc_obj(*flags);
8107 if (!flags)
8108 return -ENOMEM;
8109
8110 *flags = *tracer->default_flags;
8111 flags->trace = tracer;
8112
8113 t->flags = flags;
8114
8115 /* If this is an instance, inherit the global_trace flags */
8116 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL)) {
8117 int val = get_global_flags_val(tracer);
8118 if (!WARN_ON_ONCE(val < 0))
8119 flags->val = val;
8120 }
8121 }
8122
8123 ret = add_tracer_options(tr, t);
8124 if (ret < 0) {
8125 list_del(&t->list);
8126 kfree(t->flags);
8127 kfree(t);
8128 }
8129
8130 return ret;
8131}
8132
8133static struct dentry *
8134create_trace_option_core_file(struct trace_array *tr,
8135 const char *option, long index)
8136{
8137 struct dentry *t_options;
8138
8139 t_options = trace_options_init_dentry(tr);
8140 if (!t_options)
8141 return NULL;
8142
8143 return trace_create_file(option, TRACE_MODE_WRITE, t_options,
8144 (void *)&tr->trace_flags_index[index],
8145 &trace_options_core_fops);
8146}
8147
8148static void create_trace_options_dir(struct trace_array *tr)
8149{
8150 struct dentry *t_options;
8151 bool top_level = tr == &global_trace;
8152 int i;
8153
8154 t_options = trace_options_init_dentry(tr);
8155 if (!t_options)
8156 return;
8157
8158 for (i = 0; trace_options[i]; i++) {
8159 if (top_level ||
8160 !((1ULL << i) & TOP_LEVEL_TRACE_FLAGS)) {
8161 create_trace_option_core_file(tr, trace_options[i], i);
8162 }
8163 }
8164}
8165
8166static ssize_t
8167rb_simple_read(struct file *filp, char __user *ubuf,
8168 size_t cnt, loff_t *ppos)
8169{
8170 struct trace_array *tr = filp->private_data;
8171 char buf[64];
8172 int r;
8173
8174 r = tracer_tracing_is_on(tr);
8175 r = sprintf(buf, "%d\n", r);
8176
8177 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8178}
8179
8180static ssize_t
8181rb_simple_write(struct file *filp, const char __user *ubuf,
8182 size_t cnt, loff_t *ppos)
8183{
8184 struct trace_array *tr = filp->private_data;
8185 struct trace_buffer *buffer = tr->array_buffer.buffer;
8186 unsigned long val;
8187 int ret;
8188
8189 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8190 if (ret)
8191 return ret;
8192
8193 if (buffer) {
8194 guard(mutex)(&trace_types_lock);
8195 if (!!val == tracer_tracing_is_on(tr)) {
8196 val = 0; /* do nothing */
8197 } else if (val) {
8198 tracer_tracing_on(tr);
8199 if (tr->current_trace->start)
8200 tr->current_trace->start(tr);
8201 } else {
8202 tracer_tracing_off(tr);
8203 if (tr->current_trace->stop)
8204 tr->current_trace->stop(tr);
8205 /* Wake up any waiters */
8206 ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS);
8207 }
8208 }
8209
8210 (*ppos)++;
8211
8212 return cnt;
8213}
8214
8215static const struct file_operations rb_simple_fops = {
8216 .open = tracing_open_generic_tr,
8217 .read = rb_simple_read,
8218 .write = rb_simple_write,
8219 .release = tracing_release_generic_tr,
8220 .llseek = default_llseek,
8221};
8222
8223static ssize_t
8224buffer_percent_read(struct file *filp, char __user *ubuf,
8225 size_t cnt, loff_t *ppos)
8226{
8227 struct trace_array *tr = filp->private_data;
8228 char buf[64];
8229 int r;
8230
8231 r = tr->buffer_percent;
8232 r = sprintf(buf, "%d\n", r);
8233
8234 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8235}
8236
8237static ssize_t
8238buffer_percent_write(struct file *filp, const char __user *ubuf,
8239 size_t cnt, loff_t *ppos)
8240{
8241 struct trace_array *tr = filp->private_data;
8242 unsigned long val;
8243 int ret;
8244
8245 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8246 if (ret)
8247 return ret;
8248
8249 if (val > 100)
8250 return -EINVAL;
8251
8252 tr->buffer_percent = val;
8253
8254 (*ppos)++;
8255
8256 return cnt;
8257}
8258
8259static const struct file_operations buffer_percent_fops = {
8260 .open = tracing_open_generic_tr,
8261 .read = buffer_percent_read,
8262 .write = buffer_percent_write,
8263 .release = tracing_release_generic_tr,
8264 .llseek = default_llseek,
8265};
8266
8267static ssize_t
8268buffer_subbuf_size_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
8269{
8270 struct trace_array *tr = filp->private_data;
8271 size_t size;
8272 char buf[64];
8273 int order;
8274 int r;
8275
8276 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
8277 size = (PAGE_SIZE << order) / 1024;
8278
8279 r = sprintf(buf, "%zd\n", size);
8280
8281 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8282}
8283
8284static ssize_t
8285buffer_subbuf_size_write(struct file *filp, const char __user *ubuf,
8286 size_t cnt, loff_t *ppos)
8287{
8288 struct trace_array *tr = filp->private_data;
8289 unsigned long val;
8290 int old_order;
8291 int order;
8292 int pages;
8293 int ret;
8294
8295 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8296 if (ret)
8297 return ret;
8298
8299 val *= 1024; /* value passed in is in KB */
8300
8301 pages = DIV_ROUND_UP(val, PAGE_SIZE);
8302 order = fls(pages - 1);
8303
8304 /* limit between 1 and 128 system pages */
8305 if (order < 0 || order > 7)
8306 return -EINVAL;
8307
8308 /* Do not allow tracing while changing the order of the ring buffer */
8309 tracing_stop_tr(tr);
8310
8311 trace_access_lock(RING_BUFFER_ALL_CPUS);
8312
8313 old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
8314 if (old_order == order)
8315 goto out;
8316
8317 ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order);
8318 if (ret)
8319 goto out;
8320
8321#ifdef CONFIG_TRACER_SNAPSHOT
8322
8323 if (!tr->allocated_snapshot)
8324 goto out_max;
8325
8326 ret = ring_buffer_subbuf_order_set(tr->snapshot_buffer.buffer, order);
8327 if (ret) {
8328 /* Put back the old order */
8329 cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order);
8330 if (WARN_ON_ONCE(cnt)) {
8331 /*
8332 * AARGH! We are left with different orders!
8333 * The max buffer is our "snapshot" buffer.
8334 * When a tracer needs a snapshot (one of the
8335 * latency tracers), it swaps the max buffer
8336 * with the saved snap shot. We succeeded to
8337 * update the order of the main buffer, but failed to
8338 * update the order of the max buffer. But when we tried
8339 * to reset the main buffer to the original size, we
8340 * failed there too. This is very unlikely to
8341 * happen, but if it does, warn and kill all
8342 * tracing.
8343 */
8344 tracing_disabled = 1;
8345 }
8346 goto out;
8347 }
8348 out_max:
8349#endif
8350 (*ppos)++;
8351 out:
8352 trace_access_unlock(RING_BUFFER_ALL_CPUS);
8353 if (ret)
8354 cnt = ret;
8355 tracing_start_tr(tr);
8356 return cnt;
8357}
8358
8359static const struct file_operations buffer_subbuf_size_fops = {
8360 .open = tracing_open_generic_tr,
8361 .read = buffer_subbuf_size_read,
8362 .write = buffer_subbuf_size_write,
8363 .release = tracing_release_generic_tr,
8364 .llseek = default_llseek,
8365};
8366
8367static struct dentry *trace_instance_dir;
8368
8369static void
8370init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
8371
8372#ifdef CONFIG_MODULES
8373static int make_mod_delta(struct module *mod, void *data)
8374{
8375 struct trace_module_delta *module_delta;
8376 struct trace_scratch *tscratch;
8377 struct trace_mod_entry *entry;
8378 struct trace_array *tr = data;
8379 int i;
8380
8381 tscratch = tr->scratch;
8382 module_delta = READ_ONCE(tr->module_delta);
8383 for (i = 0; i < tscratch->nr_entries; i++) {
8384 entry = &tscratch->entries[i];
8385 if (strcmp(mod->name, entry->mod_name))
8386 continue;
8387 if (mod->state == MODULE_STATE_GOING)
8388 module_delta->delta[i] = 0;
8389 else
8390 module_delta->delta[i] = (unsigned long)mod->mem[MOD_TEXT].base
8391 - entry->mod_addr;
8392 break;
8393 }
8394 return 0;
8395}
8396#else
8397static int make_mod_delta(struct module *mod, void *data)
8398{
8399 return 0;
8400}
8401#endif
8402
8403static int mod_addr_comp(const void *a, const void *b, const void *data)
8404{
8405 const struct trace_mod_entry *e1 = a;
8406 const struct trace_mod_entry *e2 = b;
8407
8408 return e1->mod_addr > e2->mod_addr ? 1 : -1;
8409}
8410
8411static void setup_trace_scratch(struct trace_array *tr,
8412 struct trace_scratch *tscratch, unsigned int size)
8413{
8414 struct trace_module_delta *module_delta;
8415 struct trace_mod_entry *entry;
8416 int i, nr_entries;
8417
8418 if (!tscratch)
8419 return;
8420
8421 tr->scratch = tscratch;
8422 tr->scratch_size = size;
8423
8424 if (tscratch->text_addr)
8425 tr->text_delta = (unsigned long)_text - tscratch->text_addr;
8426
8427 if (struct_size(tscratch, entries, tscratch->nr_entries) > size)
8428 goto reset;
8429
8430 /* Check if each module name is a valid string */
8431 for (i = 0; i < tscratch->nr_entries; i++) {
8432 int n;
8433
8434 entry = &tscratch->entries[i];
8435
8436 for (n = 0; n < MODULE_NAME_LEN; n++) {
8437 if (entry->mod_name[n] == '\0')
8438 break;
8439 if (!isprint(entry->mod_name[n]))
8440 goto reset;
8441 }
8442 if (n == MODULE_NAME_LEN)
8443 goto reset;
8444 }
8445
8446 /* Sort the entries so that we can find appropriate module from address. */
8447 nr_entries = tscratch->nr_entries;
8448 sort_r(tscratch->entries, nr_entries, sizeof(struct trace_mod_entry),
8449 mod_addr_comp, NULL, NULL);
8450
8451 if (IS_ENABLED(CONFIG_MODULES)) {
8452 module_delta = kzalloc_flex(*module_delta, delta, nr_entries);
8453 if (!module_delta) {
8454 pr_info("module_delta allocation failed. Not able to decode module address.");
8455 goto reset;
8456 }
8457 init_rcu_head(&module_delta->rcu);
8458 } else
8459 module_delta = NULL;
8460 WRITE_ONCE(tr->module_delta, module_delta);
8461
8462 /* Scan modules to make text delta for modules. */
8463 module_for_each_mod(make_mod_delta, tr);
8464
8465 /* Set trace_clock as the same of the previous boot. */
8466 if (tscratch->clock_id != tr->clock_id) {
8467 if (tscratch->clock_id >= ARRAY_SIZE(trace_clocks) ||
8468 tracing_set_clock(tr, trace_clocks[tscratch->clock_id].name) < 0) {
8469 pr_info("the previous trace_clock info is not valid.");
8470 goto reset;
8471 }
8472 }
8473 return;
8474 reset:
8475 /* Invalid trace modules */
8476 memset(tscratch, 0, size);
8477}
8478
8479#define TRACE_TEST_PTRACING_NAME "ptracingtest"
8480
8481int allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
8482{
8483 enum ring_buffer_flags rb_flags;
8484 struct trace_scratch *tscratch;
8485 unsigned int scratch_size = 0;
8486
8487 rb_flags = tr->trace_flags & TRACE_ITER(OVERWRITE) ? RB_FL_OVERWRITE : 0;
8488
8489 buf->tr = tr;
8490
8491 if (tr->range_addr_start && tr->range_addr_size) {
8492 if (tr->name && !strcmp(tr->name, TRACE_TEST_PTRACING_NAME))
8493 rb_flags |= RB_FL_TESTING;
8494 /* Add scratch buffer to handle 128 modules */
8495 buf->buffer = ring_buffer_alloc_range(size, rb_flags, 0,
8496 tr->range_addr_start,
8497 tr->range_addr_size,
8498 struct_size(tscratch, entries, 128));
8499
8500 tscratch = ring_buffer_meta_scratch(buf->buffer, &scratch_size);
8501 setup_trace_scratch(tr, tscratch, scratch_size);
8502
8503 /*
8504 * This is basically the same as a mapped buffer,
8505 * with the same restrictions.
8506 */
8507 tr->mapped++;
8508 } else {
8509 buf->buffer = ring_buffer_alloc(size, rb_flags);
8510 }
8511 if (!buf->buffer)
8512 return -ENOMEM;
8513
8514 buf->data = alloc_percpu(struct trace_array_cpu);
8515 if (!buf->data) {
8516 ring_buffer_free(buf->buffer);
8517 buf->buffer = NULL;
8518 return -ENOMEM;
8519 }
8520
8521 /* Allocate the first page for all buffers */
8522 trace_set_buffer_entries(&tr->array_buffer,
8523 ring_buffer_size(tr->array_buffer.buffer, 0));
8524
8525 return 0;
8526}
8527
8528static void free_trace_buffer(struct array_buffer *buf)
8529{
8530 if (buf->buffer) {
8531 ring_buffer_free(buf->buffer);
8532 buf->buffer = NULL;
8533 free_percpu(buf->data);
8534 buf->data = NULL;
8535 }
8536}
8537
8538static int allocate_trace_buffers(struct trace_array *tr, unsigned long size)
8539{
8540 int ret;
8541
8542 ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
8543 if (ret)
8544 return ret;
8545
8546 ret = trace_allocate_snapshot(tr, size);
8547 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n"))
8548 free_trace_buffer(&tr->array_buffer);
8549
8550 return ret;
8551}
8552
8553static void free_trace_buffers(struct trace_array *tr)
8554{
8555 if (!tr)
8556 return;
8557
8558 free_trace_buffer(&tr->array_buffer);
8559 kfree(tr->module_delta);
8560
8561#ifdef CONFIG_TRACER_SNAPSHOT
8562 free_trace_buffer(&tr->snapshot_buffer);
8563#endif
8564}
8565
8566static void init_trace_flags_index(struct trace_array *tr)
8567{
8568 int i;
8569
8570 /* Used by the trace options files */
8571 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8572 tr->trace_flags_index[i] = i;
8573}
8574
8575static int __update_tracer(struct trace_array *tr)
8576{
8577 struct tracer *t;
8578 int ret = 0;
8579
8580 for (t = trace_types; t && !ret; t = t->next)
8581 ret = add_tracer(tr, t);
8582
8583 return ret;
8584}
8585
8586static __init int __update_tracer_options(struct trace_array *tr)
8587{
8588 struct tracers *t;
8589 int ret = 0;
8590
8591 list_for_each_entry(t, &tr->tracers, list) {
8592 ret = add_tracer_options(tr, t);
8593 if (ret < 0)
8594 break;
8595 }
8596
8597 return ret;
8598}
8599
8600static __init void update_tracer_options(void)
8601{
8602 struct trace_array *tr;
8603
8604 guard(mutex)(&trace_types_lock);
8605 tracer_options_updated = true;
8606 list_for_each_entry(tr, &ftrace_trace_arrays, list)
8607 __update_tracer_options(tr);
8608}
8609
8610/* Must have trace_types_lock held */
8611struct trace_array *trace_array_find(const char *instance)
8612{
8613 struct trace_array *tr, *found = NULL;
8614
8615 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8616 if (tr->name && strcmp(tr->name, instance) == 0) {
8617 found = tr;
8618 break;
8619 }
8620 }
8621
8622 return found;
8623}
8624
8625struct trace_array *trace_array_find_get(const char *instance)
8626{
8627 struct trace_array *tr;
8628
8629 guard(mutex)(&trace_types_lock);
8630 tr = trace_array_find(instance);
8631 if (tr && __trace_array_get(tr) < 0)
8632 tr = NULL;
8633
8634 return tr;
8635}
8636
8637static int trace_array_create_dir(struct trace_array *tr)
8638{
8639 int ret;
8640
8641 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
8642 if (!tr->dir)
8643 return -EINVAL;
8644
8645 ret = event_trace_add_tracer(tr->dir, tr);
8646 if (ret) {
8647 tracefs_remove(tr->dir);
8648 return ret;
8649 }
8650
8651 init_tracer_tracefs(tr, tr->dir);
8652 ret = __update_tracer(tr);
8653 if (ret) {
8654 event_trace_del_tracer(tr);
8655 tracefs_remove(tr->dir);
8656 return ret;
8657 }
8658 return 0;
8659}
8660
8661static struct trace_array *
8662trace_array_create_systems(const char *name, const char *systems,
8663 unsigned long range_addr_start,
8664 unsigned long range_addr_size)
8665{
8666 struct trace_array *tr;
8667 int ret;
8668
8669 ret = -ENOMEM;
8670 tr = kzalloc_obj(*tr);
8671 if (!tr)
8672 return ERR_PTR(ret);
8673
8674 tr->name = kstrdup(name, GFP_KERNEL);
8675 if (!tr->name)
8676 goto out_free_tr;
8677
8678 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8679 goto out_free_tr;
8680
8681 if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL))
8682 goto out_free_tr;
8683
8684 if (systems) {
8685 tr->system_names = kstrdup_const(systems, GFP_KERNEL);
8686 if (!tr->system_names)
8687 goto out_free_tr;
8688 }
8689
8690 /* Only for boot up memory mapped ring buffers */
8691 tr->range_addr_start = range_addr_start;
8692 tr->range_addr_size = range_addr_size;
8693
8694 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
8695
8696 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8697
8698 raw_spin_lock_init(&tr->start_lock);
8699
8700 tr->syscall_buf_sz = global_trace.syscall_buf_sz;
8701
8702 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8703#ifdef CONFIG_TRACER_SNAPSHOT
8704 spin_lock_init(&tr->snapshot_trigger_lock);
8705#endif
8706 tr->current_trace = &nop_trace;
8707 tr->current_trace_flags = nop_trace.flags;
8708
8709 INIT_LIST_HEAD(&tr->systems);
8710 INIT_LIST_HEAD(&tr->events);
8711 INIT_LIST_HEAD(&tr->hist_vars);
8712 INIT_LIST_HEAD(&tr->err_log);
8713 INIT_LIST_HEAD(&tr->tracers);
8714 INIT_LIST_HEAD(&tr->marker_list);
8715
8716#ifdef CONFIG_MODULES
8717 INIT_LIST_HEAD(&tr->mod_events);
8718#endif
8719
8720 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
8721 goto out_free_tr;
8722
8723 /* The ring buffer is defaultly expanded */
8724 trace_set_ring_buffer_expanded(tr);
8725
8726 if (ftrace_allocate_ftrace_ops(tr) < 0)
8727 goto out_free_tr;
8728
8729 trace_array_init_autoremove(tr);
8730
8731 ftrace_init_trace_array(tr);
8732
8733 init_trace_flags_index(tr);
8734
8735 if (trace_instance_dir) {
8736 ret = trace_array_create_dir(tr);
8737 if (ret)
8738 goto out_free_tr;
8739 } else
8740 __trace_early_add_events(tr);
8741
8742 list_add(&tr->list, &ftrace_trace_arrays);
8743
8744 tr->ref++;
8745
8746 return tr;
8747
8748 out_free_tr:
8749 ftrace_free_ftrace_ops(tr);
8750 free_trace_buffers(tr);
8751 free_cpumask_var(tr->pipe_cpumask);
8752 free_cpumask_var(tr->tracing_cpumask);
8753 kfree_const(tr->system_names);
8754 kfree(tr->range_name);
8755 kfree(tr->name);
8756 kfree(tr);
8757
8758 return ERR_PTR(ret);
8759}
8760
8761static struct trace_array *trace_array_create(const char *name)
8762{
8763 return trace_array_create_systems(name, NULL, 0, 0);
8764}
8765
8766static int instance_mkdir(const char *name)
8767{
8768 struct trace_array *tr;
8769 int ret;
8770
8771 guard(mutex)(&event_mutex);
8772 guard(mutex)(&trace_types_lock);
8773
8774 ret = -EEXIST;
8775 if (trace_array_find(name))
8776 return -EEXIST;
8777
8778 tr = trace_array_create(name);
8779
8780 ret = PTR_ERR_OR_ZERO(tr);
8781
8782 return ret;
8783}
8784
8785#ifdef CONFIG_MMU
8786static u64 map_pages(unsigned long start, unsigned long size)
8787{
8788 unsigned long vmap_start, vmap_end;
8789 struct vm_struct *area;
8790 int ret;
8791
8792 area = get_vm_area(size, VM_IOREMAP);
8793 if (!area)
8794 return 0;
8795
8796 vmap_start = (unsigned long) area->addr;
8797 vmap_end = vmap_start + size;
8798
8799 ret = vmap_page_range(vmap_start, vmap_end,
8800 start, pgprot_nx(PAGE_KERNEL));
8801 if (ret < 0) {
8802 free_vm_area(area);
8803 return 0;
8804 }
8805
8806 return (u64)vmap_start;
8807}
8808#else
8809static inline u64 map_pages(unsigned long start, unsigned long size)
8810{
8811 return 0;
8812}
8813#endif
8814
8815/**
8816 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
8817 * @name: The name of the trace array to be looked up/created.
8818 * @systems: A list of systems to create event directories for (NULL for all)
8819 *
8820 * Returns pointer to trace array with given name.
8821 * NULL, if it cannot be created.
8822 *
8823 * NOTE: This function increments the reference counter associated with the
8824 * trace array returned. This makes sure it cannot be freed while in use.
8825 * Use trace_array_put() once the trace array is no longer needed.
8826 * If the trace_array is to be freed, trace_array_destroy() needs to
8827 * be called after the trace_array_put(), or simply let user space delete
8828 * it from the tracefs instances directory. But until the
8829 * trace_array_put() is called, user space can not delete it.
8830 *
8831 */
8832struct trace_array *trace_array_get_by_name(const char *name, const char *systems)
8833{
8834 struct trace_array *tr;
8835
8836 guard(mutex)(&event_mutex);
8837 guard(mutex)(&trace_types_lock);
8838
8839 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8840 if (tr->name && strcmp(tr->name, name) == 0) {
8841 /* if this fails, @tr is going to be removed. */
8842 if (__trace_array_get(tr) < 0)
8843 tr = NULL;
8844 return tr;
8845 }
8846 }
8847
8848 tr = trace_array_create_systems(name, systems, 0, 0);
8849
8850 if (IS_ERR(tr))
8851 tr = NULL;
8852 else
8853 tr->ref++;
8854
8855 return tr;
8856}
8857EXPORT_SYMBOL_GPL(trace_array_get_by_name);
8858
8859static int __remove_instance(struct trace_array *tr)
8860{
8861 int i;
8862
8863 /* Reference counter for a newly created trace array = 1. */
8864 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))
8865 return -EBUSY;
8866
8867 list_del(&tr->list);
8868
8869 if (printk_trace == tr)
8870 update_printk_trace(&global_trace);
8871
8872 /* Must be done before disabling all the flags */
8873 if (update_marker_trace(tr, 0))
8874 synchronize_rcu();
8875
8876 /* Disable all the flags that were enabled coming in */
8877 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8878 if ((1ULL << i) & ZEROED_TRACE_FLAGS)
8879 set_tracer_flag(tr, 1ULL << i, 0);
8880 }
8881
8882 trace_array_cancel_autoremove(tr);
8883 tracing_set_nop(tr);
8884 clear_ftrace_function_probes(tr);
8885 event_trace_del_tracer(tr);
8886 ftrace_clear_pids(tr);
8887 ftrace_destroy_function_files(tr);
8888 tracefs_remove(tr->dir);
8889 free_percpu(tr->last_func_repeats);
8890 free_trace_buffers(tr);
8891 clear_tracing_err_log(tr);
8892 free_tracers(tr);
8893
8894 if (tr->range_name) {
8895 reserve_mem_release_by_name(tr->range_name);
8896 kfree(tr->range_name);
8897 }
8898 if (tr->flags & TRACE_ARRAY_FL_VMALLOC)
8899 vfree((void *)tr->range_addr_start);
8900
8901 for (i = 0; i < tr->nr_topts; i++) {
8902 kfree(tr->topts[i].topts);
8903 }
8904 kfree(tr->topts);
8905
8906 free_cpumask_var(tr->pipe_cpumask);
8907 free_cpumask_var(tr->tracing_cpumask);
8908 kfree_const(tr->system_names);
8909 kfree(tr->name);
8910 kfree(tr);
8911
8912 return 0;
8913}
8914
8915int trace_array_destroy(struct trace_array *this_tr)
8916{
8917 struct trace_array *tr;
8918
8919 if (!this_tr)
8920 return -EINVAL;
8921
8922 guard(mutex)(&event_mutex);
8923 guard(mutex)(&trace_types_lock);
8924
8925
8926 /* Making sure trace array exists before destroying it. */
8927 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8928 if (tr == this_tr)
8929 return __remove_instance(tr);
8930 }
8931
8932 return -ENODEV;
8933}
8934EXPORT_SYMBOL_GPL(trace_array_destroy);
8935
8936static int instance_rmdir(const char *name)
8937{
8938 struct trace_array *tr;
8939
8940 guard(mutex)(&event_mutex);
8941 guard(mutex)(&trace_types_lock);
8942
8943 tr = trace_array_find(name);
8944 if (!tr)
8945 return -ENODEV;
8946
8947 return __remove_instance(tr);
8948}
8949
8950static __init void create_trace_instances(struct dentry *d_tracer)
8951{
8952 struct trace_array *tr;
8953
8954 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8955 instance_mkdir,
8956 instance_rmdir);
8957 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
8958 return;
8959
8960 guard(mutex)(&event_mutex);
8961 guard(mutex)(&trace_types_lock);
8962
8963 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8964 if (!tr->name)
8965 continue;
8966 if (MEM_FAIL(trace_array_create_dir(tr) < 0,
8967 "Failed to create instance directory\n"))
8968 return;
8969 }
8970}
8971
8972static void
8973init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8974{
8975 umode_t writable_mode = TRACE_MODE_WRITE;
8976 int cpu;
8977
8978 if (trace_array_is_readonly(tr))
8979 writable_mode = TRACE_MODE_READ;
8980
8981 trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer,
8982 tr, &show_traces_fops);
8983
8984 trace_create_file("current_tracer", writable_mode, d_tracer,
8985 tr, &set_tracer_fops);
8986
8987 trace_create_file("tracing_cpumask", writable_mode, d_tracer,
8988 tr, &tracing_cpumask_fops);
8989
8990 /* Options are used for changing print-format even for readonly instance. */
8991 trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer,
8992 tr, &tracing_iter_fops);
8993
8994 trace_create_file("trace", TRACE_MODE_WRITE, d_tracer,
8995 tr, &tracing_fops);
8996
8997 trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer,
8998 tr, &tracing_pipe_fops);
8999
9000 trace_create_file("buffer_size_kb", writable_mode, d_tracer,
9001 tr, &tracing_entries_fops);
9002
9003 trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer,
9004 tr, &tracing_total_entries_fops);
9005
9006 trace_create_file("trace_clock", writable_mode, d_tracer, tr,
9007 &trace_clock_fops);
9008
9009 trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr,
9010 &trace_time_stamp_mode_fops);
9011
9012 tr->buffer_percent = 50;
9013
9014 trace_create_file("buffer_subbuf_size_kb", writable_mode, d_tracer,
9015 tr, &buffer_subbuf_size_fops);
9016
9017 create_trace_options_dir(tr);
9018
9019 if (tr->range_addr_start)
9020 trace_create_file("last_boot_info", TRACE_MODE_READ, d_tracer,
9021 tr, &last_boot_fops);
9022
9023 for_each_tracing_cpu(cpu)
9024 tracing_init_tracefs_percpu(tr, cpu);
9025
9026 /* Read-only instance has above files only. */
9027 if (trace_array_is_readonly(tr))
9028 return;
9029
9030 trace_create_file("free_buffer", 0200, d_tracer,
9031 tr, &tracing_free_buffer_fops);
9032
9033 trace_create_file("trace_marker", 0220, d_tracer,
9034 tr, &tracing_mark_fops);
9035
9036 tr->trace_marker_file = __find_event_file(tr, "ftrace", "print");
9037
9038 trace_create_file("trace_marker_raw", 0220, d_tracer,
9039 tr, &tracing_mark_raw_fops);
9040
9041 trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer,
9042 tr, &buffer_percent_fops);
9043
9044 trace_create_file("syscall_user_buf_size", TRACE_MODE_WRITE, d_tracer,
9045 tr, &tracing_syscall_buf_fops);
9046
9047 trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer,
9048 tr, &rb_simple_fops);
9049
9050 trace_create_maxlat_file(tr, d_tracer);
9051
9052 if (ftrace_create_function_files(tr, d_tracer))
9053 MEM_FAIL(1, "Could not allocate function filter files");
9054
9055#ifdef CONFIG_TRACER_SNAPSHOT
9056 if (!tr->range_addr_start)
9057 trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer,
9058 tr, &snapshot_fops);
9059#endif
9060
9061 trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer,
9062 tr, &tracing_err_log_fops);
9063
9064 ftrace_init_tracefs(tr, d_tracer);
9065}
9066
9067#ifdef CONFIG_TRACEFS_AUTOMOUNT_DEPRECATED
9068static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
9069{
9070 struct vfsmount *mnt;
9071 struct file_system_type *type;
9072 struct fs_context *fc;
9073 int ret;
9074
9075 /*
9076 * To maintain backward compatibility for tools that mount
9077 * debugfs to get to the tracing facility, tracefs is automatically
9078 * mounted to the debugfs/tracing directory.
9079 */
9080 type = get_fs_type("tracefs");
9081 if (!type)
9082 return NULL;
9083
9084 fc = fs_context_for_submount(type, mntpt);
9085 put_filesystem(type);
9086 if (IS_ERR(fc))
9087 return ERR_CAST(fc);
9088
9089 pr_warn("NOTICE: Automounting of tracing to debugfs is deprecated and will be removed in 2030\n");
9090
9091 ret = vfs_parse_fs_string(fc, "source", "tracefs");
9092 if (!ret)
9093 mnt = fc_mount(fc);
9094 else
9095 mnt = ERR_PTR(ret);
9096
9097 put_fs_context(fc);
9098 return mnt;
9099}
9100#endif
9101
9102/**
9103 * tracing_init_dentry - initialize top level trace array
9104 *
9105 * This is called when creating files or directories in the tracing
9106 * directory. It is called via fs_initcall() by any of the boot up code
9107 * and expects to return the dentry of the top level tracing directory.
9108 */
9109int tracing_init_dentry(void)
9110{
9111 struct trace_array *tr = &global_trace;
9112
9113 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9114 pr_warn("Tracing disabled due to lockdown\n");
9115 return -EPERM;
9116 }
9117
9118 /* The top level trace array uses NULL as parent */
9119 if (tr->dir)
9120 return 0;
9121
9122 if (WARN_ON(!tracefs_initialized()))
9123 return -ENODEV;
9124
9125#ifdef CONFIG_TRACEFS_AUTOMOUNT_DEPRECATED
9126 /*
9127 * As there may still be users that expect the tracing
9128 * files to exist in debugfs/tracing, we must automount
9129 * the tracefs file system there, so older tools still
9130 * work with the newer kernel.
9131 */
9132 tr->dir = debugfs_create_automount("tracing", NULL,
9133 trace_automount, NULL);
9134#endif
9135
9136 return 0;
9137}
9138
9139extern struct trace_eval_map *__start_ftrace_eval_maps[];
9140extern struct trace_eval_map *__stop_ftrace_eval_maps[];
9141
9142struct workqueue_struct *trace_init_wq __initdata;
9143static struct work_struct eval_map_work __initdata;
9144static struct work_struct tracerfs_init_work __initdata;
9145
9146static void __init eval_map_work_func(struct work_struct *work)
9147{
9148 int len;
9149
9150 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
9151 trace_event_update_with_eval_map(NULL, __start_ftrace_eval_maps, len);
9152}
9153
9154static int __init trace_eval_init(void)
9155{
9156 INIT_WORK(&eval_map_work, eval_map_work_func);
9157
9158 trace_init_wq = alloc_workqueue("trace_init_wq", WQ_UNBOUND, 0);
9159 if (!trace_init_wq) {
9160 pr_err("Unable to allocate trace_init_wq\n");
9161 /* Do work here */
9162 eval_map_work_func(&eval_map_work);
9163 return -ENOMEM;
9164 }
9165
9166 queue_work(trace_init_wq, &eval_map_work);
9167 return 0;
9168}
9169
9170subsys_initcall(trace_eval_init);
9171
9172static int __init trace_eval_sync(void)
9173{
9174 /* Make sure the eval map updates are finished */
9175 if (trace_init_wq)
9176 destroy_workqueue(trace_init_wq);
9177 return 0;
9178}
9179
9180late_initcall_sync(trace_eval_sync);
9181
9182
9183#ifdef CONFIG_MODULES
9184
9185bool module_exists(const char *module)
9186{
9187 /* All modules have the symbol __this_module */
9188 static const char this_mod[] = "__this_module";
9189 char modname[MODULE_NAME_LEN + sizeof(this_mod) + 2];
9190 unsigned long val;
9191 int n;
9192
9193 n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
9194
9195 if (n > sizeof(modname) - 1)
9196 return false;
9197
9198 val = module_kallsyms_lookup_name(modname);
9199 return val != 0;
9200}
9201
9202static void trace_module_add_evals(struct module *mod)
9203{
9204 /*
9205 * Modules with bad taint do not have events created, do
9206 * not bother with enums either.
9207 */
9208 if (trace_module_has_bad_taint(mod))
9209 return;
9210
9211 /* Even if no trace_evals, this need to sanitize field types. */
9212 trace_event_update_with_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
9213}
9214
9215#ifdef CONFIG_TRACE_EVAL_MAP_FILE
9216static void trace_module_remove_evals(struct module *mod)
9217{
9218 union trace_eval_map_item *map;
9219 union trace_eval_map_item **last = &trace_eval_maps;
9220
9221 if (!mod->num_trace_evals)
9222 return;
9223
9224 guard(mutex)(&trace_eval_mutex);
9225
9226 map = trace_eval_maps;
9227
9228 while (map) {
9229 if (map->head.mod == mod)
9230 break;
9231 map = trace_eval_jmp_to_tail(map);
9232 last = &map->tail.next;
9233 map = map->tail.next;
9234 }
9235 if (!map)
9236 return;
9237
9238 *last = trace_eval_jmp_to_tail(map)->tail.next;
9239 kfree(map);
9240}
9241#else
9242static inline void trace_module_remove_evals(struct module *mod) { }
9243#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9244
9245static void trace_module_record(struct module *mod, bool add)
9246{
9247 struct trace_array *tr;
9248 unsigned long flags;
9249
9250 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9251 flags = tr->flags & (TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT);
9252 /* Update any persistent trace array that has already been started */
9253 if (flags == TRACE_ARRAY_FL_BOOT && add) {
9254 guard(mutex)(&scratch_mutex);
9255 save_mod(mod, tr);
9256 } else if (flags & TRACE_ARRAY_FL_LAST_BOOT) {
9257 /* Update delta if the module loaded in previous boot */
9258 make_mod_delta(mod, tr);
9259 }
9260 }
9261}
9262
9263static int trace_module_notify(struct notifier_block *self,
9264 unsigned long val, void *data)
9265{
9266 struct module *mod = data;
9267
9268 switch (val) {
9269 case MODULE_STATE_COMING:
9270 trace_module_add_evals(mod);
9271 trace_module_record(mod, true);
9272 break;
9273 case MODULE_STATE_GOING:
9274 trace_module_remove_evals(mod);
9275 trace_module_record(mod, false);
9276 break;
9277 }
9278
9279 return NOTIFY_OK;
9280}
9281
9282static struct notifier_block trace_module_nb = {
9283 .notifier_call = trace_module_notify,
9284 .priority = 0,
9285};
9286#endif /* CONFIG_MODULES */
9287
9288static __init void tracer_init_tracefs_work_func(struct work_struct *work)
9289{
9290
9291 event_trace_init();
9292
9293 init_tracer_tracefs(&global_trace, NULL);
9294 ftrace_init_tracefs_toplevel(&global_trace, NULL);
9295
9296 trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL,
9297 &global_trace, &tracing_thresh_fops);
9298
9299 trace_create_file("README", TRACE_MODE_READ, NULL,
9300 NULL, &tracing_readme_fops);
9301
9302 trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL,
9303 NULL, &tracing_saved_cmdlines_fops);
9304
9305 trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
9306 NULL, &tracing_saved_cmdlines_size_fops);
9307
9308 trace_create_file("saved_tgids", TRACE_MODE_READ, NULL,
9309 NULL, &tracing_saved_tgids_fops);
9310
9311 trace_create_eval_file(NULL);
9312
9313#ifdef CONFIG_MODULES
9314 register_module_notifier(&trace_module_nb);
9315#endif
9316
9317#ifdef CONFIG_DYNAMIC_FTRACE
9318 trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
9319 NULL, &tracing_dyn_info_fops);
9320#endif
9321
9322 create_trace_instances(NULL);
9323
9324 update_tracer_options();
9325}
9326
9327static __init int tracer_init_tracefs(void)
9328{
9329 int ret;
9330
9331 trace_access_lock_init();
9332
9333 ret = tracing_init_dentry();
9334 if (ret)
9335 return 0;
9336
9337 if (trace_init_wq) {
9338 INIT_WORK(&tracerfs_init_work, tracer_init_tracefs_work_func);
9339 queue_work(trace_init_wq, &tracerfs_init_work);
9340 } else {
9341 tracer_init_tracefs_work_func(NULL);
9342 }
9343
9344 if (rv_init_interface())
9345 pr_err("RV: Error while creating the RV interface\n");
9346
9347 return 0;
9348}
9349
9350fs_initcall(tracer_init_tracefs);
9351
9352static int trace_die_panic_handler(struct notifier_block *self,
9353 unsigned long ev, void *unused);
9354
9355static struct notifier_block trace_panic_notifier = {
9356 .notifier_call = trace_die_panic_handler,
9357 .priority = INT_MAX - 1,
9358};
9359
9360static struct notifier_block trace_die_notifier = {
9361 .notifier_call = trace_die_panic_handler,
9362 .priority = INT_MAX - 1,
9363};
9364
9365/*
9366 * The idea is to execute the following die/panic callback early, in order
9367 * to avoid showing irrelevant information in the trace (like other panic
9368 * notifier functions); we are the 2nd to run, after hung_task/rcu_stall
9369 * warnings get disabled (to prevent potential log flooding).
9370 */
9371static int trace_die_panic_handler(struct notifier_block *self,
9372 unsigned long ev, void *unused)
9373{
9374 if (!ftrace_dump_on_oops_enabled())
9375 return NOTIFY_DONE;
9376
9377 /* The die notifier requires DIE_OOPS to trigger */
9378 if (self == &trace_die_notifier && ev != DIE_OOPS)
9379 return NOTIFY_DONE;
9380
9381 ftrace_dump(DUMP_PARAM);
9382
9383 return NOTIFY_DONE;
9384}
9385
9386/*
9387 * printk is set to max of 1024, we really don't need it that big.
9388 * Nothing should be printing 1000 characters anyway.
9389 */
9390#define TRACE_MAX_PRINT 1000
9391
9392/*
9393 * Define here KERN_TRACE so that we have one place to modify
9394 * it if we decide to change what log level the ftrace dump
9395 * should be at.
9396 */
9397#define KERN_TRACE KERN_EMERG
9398
9399void
9400trace_printk_seq(struct trace_seq *s)
9401{
9402 /* Probably should print a warning here. */
9403 if (s->seq.len >= TRACE_MAX_PRINT)
9404 s->seq.len = TRACE_MAX_PRINT;
9405
9406 /*
9407 * More paranoid code. Although the buffer size is set to
9408 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9409 * an extra layer of protection.
9410 */
9411 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9412 s->seq.len = s->seq.size - 1;
9413
9414 /* should be zero ended, but we are paranoid. */
9415 s->buffer[s->seq.len] = 0;
9416
9417 printk(KERN_TRACE "%s", s->buffer);
9418
9419 trace_seq_init(s);
9420}
9421
9422static void trace_init_iter(struct trace_iterator *iter, struct trace_array *tr)
9423{
9424 iter->tr = tr;
9425 iter->trace = iter->tr->current_trace;
9426 iter->cpu_file = RING_BUFFER_ALL_CPUS;
9427 iter->array_buffer = &tr->array_buffer;
9428
9429 if (iter->trace && iter->trace->open)
9430 iter->trace->open(iter);
9431
9432 /* Annotate start of buffers if we had overruns */
9433 if (ring_buffer_overruns(iter->array_buffer->buffer))
9434 iter->iter_flags |= TRACE_FILE_ANNOTATE;
9435
9436 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
9437 if (trace_clocks[iter->tr->clock_id].in_ns)
9438 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9439
9440 /* Can not use kmalloc for iter.temp and iter.fmt */
9441 iter->temp = static_temp_buf;
9442 iter->temp_size = STATIC_TEMP_BUF_SIZE;
9443 iter->fmt = static_fmt_buf;
9444 iter->fmt_size = STATIC_FMT_BUF_SIZE;
9445}
9446
9447void trace_init_global_iter(struct trace_iterator *iter)
9448{
9449 trace_init_iter(iter, &global_trace);
9450}
9451
9452static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_mode)
9453{
9454 /* use static because iter can be a bit big for the stack */
9455 static struct trace_iterator iter;
9456 unsigned int old_userobj;
9457 unsigned long flags;
9458 int cnt = 0;
9459
9460 /*
9461 * Always turn off tracing when we dump.
9462 * We don't need to show trace output of what happens
9463 * between multiple crashes.
9464 *
9465 * If the user does a sysrq-z, then they can re-enable
9466 * tracing with echo 1 > tracing_on.
9467 */
9468 tracer_tracing_off(tr);
9469
9470 local_irq_save(flags);
9471
9472 /* Simulate the iterator */
9473 trace_init_iter(&iter, tr);
9474
9475 /* While dumping, do not allow the buffer to be enable */
9476 tracer_tracing_disable(tr);
9477
9478 old_userobj = tr->trace_flags & TRACE_ITER(SYM_USEROBJ);
9479
9480 /* don't look at user memory in panic mode */
9481 tr->trace_flags &= ~TRACE_ITER(SYM_USEROBJ);
9482
9483 if (dump_mode == DUMP_ORIG)
9484 iter.cpu_file = raw_smp_processor_id();
9485 else
9486 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9487
9488 if (tr == &global_trace)
9489 printk(KERN_TRACE "Dumping ftrace buffer:\n");
9490 else
9491 printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name);
9492
9493 /* Did function tracer already get disabled? */
9494 if (ftrace_is_dead()) {
9495 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9496 printk("# MAY BE MISSING FUNCTION EVENTS\n");
9497 }
9498
9499 /*
9500 * We need to stop all tracing on all CPUS to read
9501 * the next buffer. This is a bit expensive, but is
9502 * not done often. We fill all what we can read,
9503 * and then release the locks again.
9504 */
9505
9506 while (!trace_empty(&iter)) {
9507
9508 if (!cnt)
9509 printk(KERN_TRACE "---------------------------------\n");
9510
9511 cnt++;
9512
9513 trace_iterator_reset(&iter);
9514 iter.iter_flags |= TRACE_FILE_LAT_FMT;
9515
9516 if (trace_find_next_entry_inc(&iter) != NULL) {
9517 int ret;
9518
9519 ret = print_trace_line(&iter);
9520 if (ret != TRACE_TYPE_NO_CONSUME)
9521 trace_consume(&iter);
9522
9523 trace_printk_seq(&iter.seq);
9524 }
9525 touch_nmi_watchdog();
9526 }
9527
9528 if (!cnt)
9529 printk(KERN_TRACE " (ftrace buffer empty)\n");
9530 else
9531 printk(KERN_TRACE "---------------------------------\n");
9532
9533 tr->trace_flags |= old_userobj;
9534
9535 tracer_tracing_enable(tr);
9536 local_irq_restore(flags);
9537}
9538
9539static void ftrace_dump_by_param(void)
9540{
9541 bool first_param = true;
9542 char dump_param[MAX_TRACER_SIZE];
9543 char *buf, *token, *inst_name;
9544 struct trace_array *tr;
9545
9546 strscpy(dump_param, ftrace_dump_on_oops, MAX_TRACER_SIZE);
9547 buf = dump_param;
9548
9549 while ((token = strsep(&buf, ",")) != NULL) {
9550 if (first_param) {
9551 first_param = false;
9552 if (!strcmp("0", token))
9553 continue;
9554 else if (!strcmp("1", token)) {
9555 ftrace_dump_one(&global_trace, DUMP_ALL);
9556 continue;
9557 }
9558 else if (!strcmp("2", token) ||
9559 !strcmp("orig_cpu", token)) {
9560 ftrace_dump_one(&global_trace, DUMP_ORIG);
9561 continue;
9562 }
9563 }
9564
9565 inst_name = strsep(&token, "=");
9566 tr = trace_array_find(inst_name);
9567 if (!tr) {
9568 printk(KERN_TRACE "Instance %s not found\n", inst_name);
9569 continue;
9570 }
9571
9572 if (token && (!strcmp("2", token) ||
9573 !strcmp("orig_cpu", token)))
9574 ftrace_dump_one(tr, DUMP_ORIG);
9575 else
9576 ftrace_dump_one(tr, DUMP_ALL);
9577 }
9578}
9579
9580void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
9581{
9582 static atomic_t dump_running;
9583
9584 /* Only allow one dump user at a time. */
9585 if (atomic_inc_return(&dump_running) != 1) {
9586 atomic_dec(&dump_running);
9587 return;
9588 }
9589
9590 switch (oops_dump_mode) {
9591 case DUMP_ALL:
9592 ftrace_dump_one(&global_trace, DUMP_ALL);
9593 break;
9594 case DUMP_ORIG:
9595 ftrace_dump_one(&global_trace, DUMP_ORIG);
9596 break;
9597 case DUMP_PARAM:
9598 ftrace_dump_by_param();
9599 break;
9600 case DUMP_NONE:
9601 break;
9602 default:
9603 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
9604 ftrace_dump_one(&global_trace, DUMP_ALL);
9605 }
9606
9607 atomic_dec(&dump_running);
9608}
9609EXPORT_SYMBOL_GPL(ftrace_dump);
9610
9611#define WRITE_BUFSIZE 4096
9612
9613ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9614 size_t count, loff_t *ppos,
9615 int (*createfn)(const char *))
9616{
9617 char *kbuf __free(kfree) = NULL;
9618 char *buf, *tmp;
9619 int ret = 0;
9620 size_t done = 0;
9621 size_t size;
9622
9623 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9624 if (!kbuf)
9625 return -ENOMEM;
9626
9627 while (done < count) {
9628 size = count - done;
9629
9630 if (size >= WRITE_BUFSIZE)
9631 size = WRITE_BUFSIZE - 1;
9632
9633 if (copy_from_user(kbuf, buffer + done, size))
9634 return -EFAULT;
9635
9636 kbuf[size] = '\0';
9637 buf = kbuf;
9638 do {
9639 tmp = strchr(buf, '\n');
9640 if (tmp) {
9641 *tmp = '\0';
9642 size = tmp - buf + 1;
9643 } else {
9644 size = strlen(buf);
9645 if (done + size < count) {
9646 if (buf != kbuf)
9647 break;
9648 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9649 pr_warn("Line length is too long: Should be less than %d\n",
9650 WRITE_BUFSIZE - 2);
9651 return -EINVAL;
9652 }
9653 }
9654 done += size;
9655
9656 /* Remove comments */
9657 tmp = strchr(buf, '#');
9658
9659 if (tmp)
9660 *tmp = '\0';
9661
9662 ret = createfn(buf);
9663 if (ret)
9664 return ret;
9665 buf += size;
9666
9667 } while (done < count);
9668 }
9669 return done;
9670}
9671
9672__init static int backup_instance_area(const char *backup,
9673 unsigned long *addr, phys_addr_t *size)
9674{
9675 struct trace_array *backup_tr;
9676 void *allocated_vaddr = NULL;
9677
9678 backup_tr = trace_array_get_by_name(backup, NULL);
9679 if (!backup_tr) {
9680 pr_warn("Tracing: Instance %s is not found.\n", backup);
9681 return -ENOENT;
9682 }
9683
9684 if (!(backup_tr->flags & TRACE_ARRAY_FL_BOOT)) {
9685 pr_warn("Tracing: Instance %s is not boot mapped.\n", backup);
9686 trace_array_put(backup_tr);
9687 return -EINVAL;
9688 }
9689
9690 *size = backup_tr->range_addr_size;
9691
9692 allocated_vaddr = vzalloc(*size);
9693 if (!allocated_vaddr) {
9694 pr_warn("Tracing: Failed to allocate memory for copying instance %s (size 0x%lx)\n",
9695 backup, (unsigned long)*size);
9696 trace_array_put(backup_tr);
9697 return -ENOMEM;
9698 }
9699
9700 memcpy(allocated_vaddr,
9701 (void *)backup_tr->range_addr_start, (size_t)*size);
9702 *addr = (unsigned long)allocated_vaddr;
9703
9704 trace_array_put(backup_tr);
9705 return 0;
9706}
9707
9708__init static void enable_instances(void)
9709{
9710 struct trace_array *tr;
9711 bool memmap_area = false;
9712 char *curr_str;
9713 char *name;
9714 char *str;
9715 char *tok;
9716
9717 /* A tab is always appended */
9718 boot_instance_info[boot_instance_index - 1] = '\0';
9719 str = boot_instance_info;
9720
9721 while ((curr_str = strsep(&str, "\t"))) {
9722 phys_addr_t start = 0;
9723 phys_addr_t size = 0;
9724 unsigned long addr = 0;
9725 bool traceprintk = false;
9726 bool traceoff = false;
9727 char *flag_delim;
9728 char *addr_delim;
9729 char *rname __free(kfree) = NULL;
9730 char *backup;
9731
9732 tok = strsep(&curr_str, ",");
9733
9734 name = strsep(&tok, "=");
9735 backup = tok;
9736
9737 flag_delim = strchr(name, '^');
9738 addr_delim = strchr(name, '@');
9739
9740 if (addr_delim)
9741 *addr_delim++ = '\0';
9742
9743 if (flag_delim)
9744 *flag_delim++ = '\0';
9745
9746 if (trace_array_find(name)) {
9747 pr_warn("Tracing: Instance %s already exists\n", name);
9748 continue;
9749 }
9750
9751 if (backup) {
9752 if (backup_instance_area(backup, &addr, &size) < 0)
9753 continue;
9754 }
9755
9756 if (flag_delim) {
9757 char *flag;
9758
9759 while ((flag = strsep(&flag_delim, "^"))) {
9760 if (strcmp(flag, "traceoff") == 0) {
9761 traceoff = true;
9762 } else if ((strcmp(flag, "printk") == 0) ||
9763 (strcmp(flag, "traceprintk") == 0) ||
9764 (strcmp(flag, "trace_printk") == 0)) {
9765 traceprintk = true;
9766 } else {
9767 pr_info("Tracing: Invalid instance flag '%s' for %s\n",
9768 flag, name);
9769 }
9770 }
9771 }
9772
9773 tok = addr_delim;
9774 if (tok && isdigit(*tok)) {
9775 start = memparse(tok, &tok);
9776 if (!start) {
9777 pr_warn("Tracing: Invalid boot instance address for %s\n",
9778 name);
9779 continue;
9780 }
9781 if (*tok != ':') {
9782 pr_warn("Tracing: No size specified for instance %s\n", name);
9783 continue;
9784 }
9785 tok++;
9786 size = memparse(tok, &tok);
9787 if (!size) {
9788 pr_warn("Tracing: Invalid boot instance size for %s\n",
9789 name);
9790 continue;
9791 }
9792 memmap_area = true;
9793 } else if (tok) {
9794 if (!reserve_mem_find_by_name(tok, &start, &size)) {
9795 start = 0;
9796 pr_warn("Failed to map boot instance %s to %s\n", name, tok);
9797 continue;
9798 }
9799 rname = kstrdup(tok, GFP_KERNEL);
9800 }
9801
9802 if (start) {
9803 /* Start and size must be page aligned */
9804 if (start & ~PAGE_MASK) {
9805 pr_warn("Tracing: mapping start addr %pa is not page aligned\n", &start);
9806 continue;
9807 }
9808 if (size & ~PAGE_MASK) {
9809 pr_warn("Tracing: mapping size %pa is not page aligned\n", &size);
9810 continue;
9811 }
9812
9813 if (memmap_area)
9814 addr = map_pages(start, size);
9815 else
9816 addr = (unsigned long)phys_to_virt(start);
9817 if (addr) {
9818 pr_info("Tracing: mapped boot instance %s at physical memory %pa of size 0x%lx\n",
9819 name, &start, (unsigned long)size);
9820 } else {
9821 pr_warn("Tracing: Failed to map boot instance %s\n", name);
9822 continue;
9823 }
9824 } else {
9825 /* Only non mapped buffers have snapshot buffers */
9826 do_allocate_snapshot(name);
9827 }
9828
9829 tr = trace_array_create_systems(name, NULL, addr, size);
9830 if (IS_ERR(tr)) {
9831 pr_warn("Tracing: Failed to create instance buffer '%s' (%ld)\n", name,
9832 PTR_ERR(tr));
9833 continue;
9834 }
9835
9836 if (traceoff)
9837 tracer_tracing_off(tr);
9838
9839 if (traceprintk)
9840 update_printk_trace(tr);
9841
9842 /*
9843 * memmap'd buffers can not be freed.
9844 */
9845 if (memmap_area) {
9846 tr->flags |= TRACE_ARRAY_FL_MEMMAP;
9847 tr->ref++;
9848 }
9849
9850 /*
9851 * Backup buffers can be freed but need vfree().
9852 */
9853 if (backup) {
9854 tr->flags |= TRACE_ARRAY_FL_VMALLOC | TRACE_ARRAY_FL_RDONLY;
9855 trace_array_start_autoremove();
9856 }
9857
9858 if (start || backup) {
9859 tr->flags |= TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT;
9860 tr->range_name = no_free_ptr(rname);
9861 }
9862
9863 /*
9864 * Save the events to start and enabled them after all boot instances
9865 * have been created.
9866 */
9867 tr->boot_events = curr_str;
9868 }
9869
9870 /* Enable the events after all boot instances have been created */
9871 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9872
9873 if (!tr->boot_events || !(*tr->boot_events)) {
9874 tr->boot_events = NULL;
9875 continue;
9876 }
9877
9878 curr_str = tr->boot_events;
9879
9880 /* Clear the instance if this is a persistent buffer */
9881 if (tr->flags & TRACE_ARRAY_FL_LAST_BOOT)
9882 update_last_data(tr);
9883
9884 while ((tok = strsep(&curr_str, ","))) {
9885 early_enable_events(tr, tok, true);
9886 }
9887 tr->boot_events = NULL;
9888 }
9889}
9890
9891__init static int tracer_alloc_buffers(void)
9892{
9893 unsigned long ring_buf_size;
9894 int ret = -ENOMEM;
9895
9896
9897 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9898 pr_warn("Tracing disabled due to lockdown\n");
9899 return -EPERM;
9900 }
9901
9902 /*
9903 * Make sure we don't accidentally add more trace options
9904 * than we have bits for.
9905 */
9906 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
9907
9908 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
9909 return -ENOMEM;
9910
9911 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9912 goto out_free_buffer_mask;
9913
9914 /* Only allocate trace_printk buffers if a trace_printk exists */
9915 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
9916 /* Must be called before global_trace.buffer is allocated */
9917 trace_printk_init_buffers();
9918
9919 /* To save memory, keep the ring buffer size to its minimum */
9920 if (global_trace.ring_buffer_expanded)
9921 ring_buf_size = trace_buf_size;
9922 else
9923 ring_buf_size = 1;
9924
9925 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
9926 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9927
9928 raw_spin_lock_init(&global_trace.start_lock);
9929
9930 /*
9931 * The prepare callbacks allocates some memory for the ring buffer. We
9932 * don't free the buffer if the CPU goes down. If we were to free
9933 * the buffer, then the user would lose any trace that was in the
9934 * buffer. The memory will be removed once the "instance" is removed.
9935 */
9936 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
9937 "trace/RB:prepare", trace_rb_cpu_prepare,
9938 NULL);
9939 if (ret < 0)
9940 goto out_free_cpumask;
9941 /* Used for event triggers */
9942 ret = -ENOMEM;
9943 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
9944 if (!temp_buffer)
9945 goto out_rm_hp_state;
9946
9947 if (trace_create_savedcmd() < 0)
9948 goto out_free_temp_buffer;
9949
9950 if (!zalloc_cpumask_var(&global_trace.pipe_cpumask, GFP_KERNEL))
9951 goto out_free_savedcmd;
9952
9953 /* TODO: make the number of buffers hot pluggable with CPUS */
9954 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
9955 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
9956 goto out_free_pipe_cpumask;
9957 }
9958 if (global_trace.buffer_disabled)
9959 tracing_off();
9960
9961 if (trace_boot_clock) {
9962 ret = tracing_set_clock(&global_trace, trace_boot_clock);
9963 if (ret < 0)
9964 pr_warn("Trace clock %s not defined, going back to default\n",
9965 trace_boot_clock);
9966 }
9967
9968 /*
9969 * register_tracer() might reference current_trace, so it
9970 * needs to be set before we register anything. This is
9971 * just a bootstrap of current_trace anyway.
9972 */
9973 global_trace.current_trace = &nop_trace;
9974 global_trace.current_trace_flags = nop_trace.flags;
9975
9976 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9977#ifdef CONFIG_TRACER_SNAPSHOT
9978 spin_lock_init(&global_trace.snapshot_trigger_lock);
9979#endif
9980 ftrace_init_global_array_ops(&global_trace);
9981
9982#ifdef CONFIG_MODULES
9983 INIT_LIST_HEAD(&global_trace.mod_events);
9984#endif
9985
9986 init_trace_flags_index(&global_trace);
9987
9988 INIT_LIST_HEAD(&global_trace.tracers);
9989
9990 /* All seems OK, enable tracing */
9991 tracing_disabled = 0;
9992
9993 atomic_notifier_chain_register(&panic_notifier_list,
9994 &trace_panic_notifier);
9995
9996 register_die_notifier(&trace_die_notifier);
9997
9998 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
9999
10000 global_trace.syscall_buf_sz = syscall_buf_size;
10001
10002 INIT_LIST_HEAD(&global_trace.systems);
10003 INIT_LIST_HEAD(&global_trace.events);
10004 INIT_LIST_HEAD(&global_trace.hist_vars);
10005 INIT_LIST_HEAD(&global_trace.err_log);
10006 list_add(&global_trace.marker_list, &marker_copies);
10007 list_add(&global_trace.list, &ftrace_trace_arrays);
10008
10009 register_tracer(&nop_trace);
10010
10011 /* Function tracing may start here (via kernel command line) */
10012 init_function_trace();
10013
10014 apply_trace_boot_options();
10015
10016 register_snapshot_cmd();
10017
10018 return 0;
10019
10020out_free_pipe_cpumask:
10021 free_cpumask_var(global_trace.pipe_cpumask);
10022out_free_savedcmd:
10023 trace_free_saved_cmdlines_buffer();
10024out_free_temp_buffer:
10025 ring_buffer_free(temp_buffer);
10026out_rm_hp_state:
10027 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
10028out_free_cpumask:
10029 free_cpumask_var(global_trace.tracing_cpumask);
10030out_free_buffer_mask:
10031 free_cpumask_var(tracing_buffer_mask);
10032 return ret;
10033}
10034
10035#ifdef CONFIG_FUNCTION_TRACER
10036/* Used to set module cached ftrace filtering at boot up */
10037struct trace_array *trace_get_global_array(void)
10038{
10039 return &global_trace;
10040}
10041#endif
10042
10043void __init early_trace_init(void)
10044{
10045 if (tracepoint_printk) {
10046 tracepoint_print_iter = kzalloc_obj(*tracepoint_print_iter);
10047 if (MEM_FAIL(!tracepoint_print_iter,
10048 "Failed to allocate trace iterator\n"))
10049 tracepoint_printk = 0;
10050 else
10051 static_key_enable(&tracepoint_printk_key.key);
10052 }
10053 tracer_alloc_buffers();
10054
10055 init_events();
10056}
10057
10058void __init trace_init(void)
10059{
10060 trace_event_init();
10061
10062 if (boot_instance_index)
10063 enable_instances();
10064}
10065
10066__init static void clear_boot_tracer(void)
10067{
10068 /*
10069 * The default tracer at boot buffer is an init section.
10070 * This function is called in lateinit. If we did not
10071 * find the boot tracer, then clear it out, to prevent
10072 * later registration from accessing the buffer that is
10073 * about to be freed.
10074 */
10075 if (!default_bootup_tracer)
10076 return;
10077
10078 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
10079 default_bootup_tracer);
10080 default_bootup_tracer = NULL;
10081}
10082
10083#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
10084__init static void tracing_set_default_clock(void)
10085{
10086 /* sched_clock_stable() is determined in late_initcall */
10087 if (!trace_boot_clock && !sched_clock_stable()) {
10088 if (security_locked_down(LOCKDOWN_TRACEFS)) {
10089 pr_warn("Can not set tracing clock due to lockdown\n");
10090 return;
10091 }
10092
10093 printk(KERN_WARNING
10094 "Unstable clock detected, switching default tracing clock to \"global\"\n"
10095 "If you want to keep using the local clock, then add:\n"
10096 " \"trace_clock=local\"\n"
10097 "on the kernel command line\n");
10098 tracing_set_clock(&global_trace, "global");
10099 }
10100}
10101#else
10102static inline void tracing_set_default_clock(void) { }
10103#endif
10104
10105__init static int late_trace_init(void)
10106{
10107 if (tracepoint_printk && tracepoint_printk_stop_on_boot) {
10108 static_key_disable(&tracepoint_printk_key.key);
10109 tracepoint_printk = 0;
10110 }
10111
10112 if (traceoff_after_boot)
10113 tracing_off();
10114
10115 tracing_set_default_clock();
10116 clear_boot_tracer();
10117 return 0;
10118}
10119
10120late_initcall_sync(late_trace_init);