projects/linux·blob master · refs

10142 lines · 250.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * ring buffer based function tracer
4 *
5 * Copyright (C) 2007-2012 Steven Rostedt <[email protected]>
6 * Copyright (C) 2008 Ingo Molnar <[email protected]>
7 *
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <[email protected]>
10 *
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
14 */
15#include <linux/ring_buffer.h>
16#include <linux/utsname.h>
17#include <linux/stacktrace.h>
18#include <linux/writeback.h>
19#include <linux/kallsyms.h>
20#include <linux/security.h>
21#include <linux/seq_file.h>
22#include <linux/irqflags.h>
23#include <linux/syscalls.h>
24#include <linux/debugfs.h>
25#include <linux/tracefs.h>
26#include <linux/pagemap.h>
27#include <linux/hardirq.h>
28#include <linux/linkage.h>
29#include <linux/uaccess.h>
30#include <linux/cleanup.h>
31#include <linux/vmalloc.h>
32#include <linux/ftrace.h>
33#include <linux/module.h>
34#include <linux/percpu.h>
35#include <linux/splice.h>
36#include <linux/kdebug.h>
37#include <linux/string.h>
38#include <linux/mount.h>
39#include <linux/rwsem.h>
40#include <linux/slab.h>
41#include <linux/ctype.h>
42#include <linux/init.h>
43#include <linux/panic_notifier.h>
44#include <linux/poll.h>
45#include <linux/nmi.h>
46#include <linux/fs.h>
47#include <linux/trace.h>
48#include <linux/sched/clock.h>
49#include <linux/sched/rt.h>
50#include <linux/irq_work.h>
51#include <linux/workqueue.h>
52#include <linux/sort.h>
53#include <linux/io.h> /* vmap_page_range() */
54#include <linux/fs_context.h>
55
56#include <asm/setup.h> /* COMMAND_LINE_SIZE */
57
58#include "trace.h"
59#include "trace_output.h"
60
61#ifdef CONFIG_FTRACE_STARTUP_TEST
62/*
63 * We need to change this state when a selftest is running.
64 * A selftest will lurk into the ring-buffer to count the
65 * entries inserted during the selftest although some concurrent
66 * insertions into the ring-buffer such as trace_printk could occurred
67 * at the same time, giving false positive or negative results.
68 */
69bool __read_mostly tracing_selftest_running;
70
71/*
72 * If boot-time tracing including tracers/events via kernel cmdline
73 * is running, we do not want to run SELFTEST.
74 */
75bool __read_mostly tracing_selftest_disabled;
76
77void __init disable_tracing_selftest(const char *reason)
78{
79 if (!tracing_selftest_disabled) {
80 tracing_selftest_disabled = true;
81 pr_info("Ftrace startup test is disabled due to %s\n", reason);
82 }
83}
84#else
85#define tracing_selftest_disabled 0
86#endif
87
88/* Pipe tracepoints to printk */
89static struct trace_iterator *tracepoint_print_iter;
90static int tracepoint_printk;
91static bool tracepoint_printk_stop_on_boot __initdata;
92static bool traceoff_after_boot __initdata;
93static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
94
95/* Store tracers and their flags per instance */
96struct tracers {
97 struct list_head list;
98 struct tracer *tracer;
99 struct tracer_flags *flags;
100};
101
102/*
103 * Tracers that do not define their own flags use these empty flags,
104 * as tr->current_trace_flags must never be NULL.
105 */
106static struct tracer_flags no_tracer_flags;
107
108/*
109 * To prevent the comm cache from being overwritten when no
110 * tracing is active, only save the comm when a trace event
111 * occurred.
112 */
113DEFINE_PER_CPU(bool, trace_taskinfo_save);
114
115/*
116 * Kill all tracing for good (never come back).
117 * It is initialized to 1 but will turn to zero if the initialization
118 * of the tracer is successful. But that is the only place that sets
119 * this back to zero.
120 */
121int tracing_disabled = 1;
122
123cpumask_var_t __read_mostly tracing_buffer_mask;
124
125#define MAX_TRACER_SIZE 100
126/*
127 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
128 *
129 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
130 * is set, then ftrace_dump is called. This will output the contents
131 * of the ftrace buffers to the console. This is very useful for
132 * capturing traces that lead to crashes and outputting it to a
133 * serial console.
134 *
135 * It is default off, but you can enable it with either specifying
136 * "ftrace_dump_on_oops" in the kernel command line, or setting
137 * /proc/sys/kernel/ftrace_dump_on_oops
138 * Set 1 if you want to dump buffers of all CPUs
139 * Set 2 if you want to dump the buffer of the CPU that triggered oops
140 * Set instance name if you want to dump the specific trace instance
141 * Multiple instance dump is also supported, and instances are separated
142 * by commas.
143 */
144/* Set to string format zero to disable by default */
145static char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0";
146
147/* When set, tracing will stop when a WARN*() is hit */
148static int __disable_trace_on_warning;
149
150int tracepoint_printk_sysctl(const struct ctl_table *table, int write,
151 void *buffer, size_t *lenp, loff_t *ppos);
152static const struct ctl_table trace_sysctl_table[] = {
153 {
154 .procname = "ftrace_dump_on_oops",
155 .data = &ftrace_dump_on_oops,
156 .maxlen = MAX_TRACER_SIZE,
157 .mode = 0644,
158 .proc_handler = proc_dostring,
159 },
160 {
161 .procname = "traceoff_on_warning",
162 .data = &__disable_trace_on_warning,
163 .maxlen = sizeof(__disable_trace_on_warning),
164 .mode = 0644,
165 .proc_handler = proc_dointvec,
166 },
167 {
168 .procname = "tracepoint_printk",
169 .data = &tracepoint_printk,
170 .maxlen = sizeof(tracepoint_printk),
171 .mode = 0644,
172 .proc_handler = tracepoint_printk_sysctl,
173 },
174};
175
176static int __init init_trace_sysctls(void)
177{
178 register_sysctl_init("kernel", trace_sysctl_table);
179 return 0;
180}
181subsys_initcall(init_trace_sysctls);
182
183#ifdef CONFIG_TRACE_EVAL_MAP_FILE
184/* Map of enums to their values, for "eval_map" file */
185struct trace_eval_map_head {
186 struct module *mod;
187 unsigned long length;
188};
189
190union trace_eval_map_item;
191
192struct trace_eval_map_tail {
193 /*
194 * "end" is first and points to NULL as it must be different
195 * than "mod" or "eval_string"
196 */
197 union trace_eval_map_item *next;
198 const char *end; /* points to NULL */
199};
200
201static DEFINE_MUTEX(trace_eval_mutex);
202
203/*
204 * The trace_eval_maps are saved in an array with two extra elements,
205 * one at the beginning, and one at the end. The beginning item contains
206 * the count of the saved maps (head.length), and the module they
207 * belong to if not built in (head.mod). The ending item contains a
208 * pointer to the next array of saved eval_map items.
209 */
210union trace_eval_map_item {
211 struct trace_eval_map map;
212 struct trace_eval_map_head head;
213 struct trace_eval_map_tail tail;
214};
215
216static union trace_eval_map_item *trace_eval_maps;
217#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
218
219int tracing_set_tracer(struct trace_array *tr, const char *buf);
220static void ftrace_trace_userstack(struct trace_array *tr,
221 struct trace_buffer *buffer,
222 unsigned int trace_ctx);
223
224static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
225static char *default_bootup_tracer;
226
227static char boot_instance_info[COMMAND_LINE_SIZE] __initdata;
228static int boot_instance_index;
229
230/*
231 * Repeated boot parameters, including Bootconfig array expansions, need
232 * to stay in the delimiter form that the existing parser consumes.
233 */
234void __init trace_append_boot_param(char *buf, const char *str, char sep,
235 int size)
236{
237 int len, needed, str_len;
238
239 if (!*str)
240 return;
241
242 len = strlen(buf);
243 str_len = strlen(str);
244 needed = len + str_len + 1;
245
246 /* For continuation, account for the separator. */
247 if (len)
248 needed++;
249 if (needed > size)
250 return;
251
252 if (len)
253 buf[len++] = sep;
254
255 strscpy(buf + len, str, size - len);
256}
257
258static int __init set_cmdline_ftrace(char *str)
259{
260 strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
261 default_bootup_tracer = bootup_tracer_buf;
262 /* We are using ftrace early, expand it */
263 trace_set_ring_buffer_expanded(NULL);
264 return 1;
265}
266__setup("ftrace=", set_cmdline_ftrace);
267
268int ftrace_dump_on_oops_enabled(void)
269{
270 if (!strcmp("0", ftrace_dump_on_oops))
271 return 0;
272 else
273 return 1;
274}
275
276static int __init set_ftrace_dump_on_oops(char *str)
277{
278 if (!*str) {
279 strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
280 return 1;
281 }
282
283 if (*str == ',') {
284 strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
285 strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1);
286 return 1;
287 }
288
289 if (*str++ == '=') {
290 strscpy(ftrace_dump_on_oops, str, MAX_TRACER_SIZE);
291 return 1;
292 }
293
294 return 0;
295}
296__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
297
298static int __init stop_trace_on_warning(char *str)
299{
300 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
301 __disable_trace_on_warning = 1;
302 return 1;
303}
304__setup("traceoff_on_warning", stop_trace_on_warning);
305
306static int __init boot_instance(char *str)
307{
308 char *slot = boot_instance_info + boot_instance_index;
309 int left = sizeof(boot_instance_info) - boot_instance_index;
310 int ret;
311
312 if (strlen(str) >= left)
313 return -1;
314
315 ret = snprintf(slot, left, "%s\t", str);
316 boot_instance_index += ret;
317
318 return 1;
319}
320__setup("trace_instance=", boot_instance);
321
322
323static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
324
325static int __init set_trace_boot_options(char *str)
326{
327 trace_append_boot_param(trace_boot_options_buf, str, ',',
328 MAX_TRACER_SIZE);
329 return 1;
330}
331__setup("trace_options=", set_trace_boot_options);
332
333static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
334static char *trace_boot_clock __initdata;
335
336static int __init set_trace_boot_clock(char *str)
337{
338 strscpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
339 trace_boot_clock = trace_boot_clock_buf;
340 return 1;
341}
342__setup("trace_clock=", set_trace_boot_clock);
343
344static int __init set_tracepoint_printk(char *str)
345{
346 /* Ignore the "tp_printk_stop_on_boot" param */
347 if (*str == '_')
348 return 0;
349
350 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
351 tracepoint_printk = 1;
352 return 1;
353}
354__setup("tp_printk", set_tracepoint_printk);
355
356static int __init set_tracepoint_printk_stop(char *str)
357{
358 tracepoint_printk_stop_on_boot = true;
359 return 1;
360}
361__setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);
362
363static int __init set_traceoff_after_boot(char *str)
364{
365 traceoff_after_boot = true;
366 return 1;
367}
368__setup("traceoff_after_boot", set_traceoff_after_boot);
369
370unsigned long long ns2usecs(u64 nsec)
371{
372 nsec += 500;
373 do_div(nsec, 1000);
374 return nsec;
375}
376
377static void
378trace_process_export(struct trace_export *export,
379 struct ring_buffer_event *event, int flag)
380{
381 struct trace_entry *entry;
382 unsigned int size = 0;
383
384 if (export->flags & flag) {
385 entry = ring_buffer_event_data(event);
386 size = ring_buffer_event_length(event);
387 export->write(export, entry, size);
388 }
389}
390
391static DEFINE_MUTEX(ftrace_export_lock);
392
393static struct trace_export __rcu *ftrace_exports_list __read_mostly;
394
395static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
396static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
397static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
398
399static inline void ftrace_exports_enable(struct trace_export *export)
400{
401 if (export->flags & TRACE_EXPORT_FUNCTION)
402 static_branch_inc(&trace_function_exports_enabled);
403
404 if (export->flags & TRACE_EXPORT_EVENT)
405 static_branch_inc(&trace_event_exports_enabled);
406
407 if (export->flags & TRACE_EXPORT_MARKER)
408 static_branch_inc(&trace_marker_exports_enabled);
409}
410
411static inline void ftrace_exports_disable(struct trace_export *export)
412{
413 if (export->flags & TRACE_EXPORT_FUNCTION)
414 static_branch_dec(&trace_function_exports_enabled);
415
416 if (export->flags & TRACE_EXPORT_EVENT)
417 static_branch_dec(&trace_event_exports_enabled);
418
419 if (export->flags & TRACE_EXPORT_MARKER)
420 static_branch_dec(&trace_marker_exports_enabled);
421}
422
423static void ftrace_exports(struct ring_buffer_event *event, int flag)
424{
425 struct trace_export *export;
426
427 guard(preempt_notrace)();
428
429 export = rcu_dereference_raw_check(ftrace_exports_list);
430 while (export) {
431 trace_process_export(export, event, flag);
432 export = rcu_dereference_raw_check(export->next);
433 }
434}
435
436static inline void
437add_trace_export(struct trace_export **list, struct trace_export *export)
438{
439 rcu_assign_pointer(export->next, *list);
440 /*
441 * We are entering export into the list but another
442 * CPU might be walking that list. We need to make sure
443 * the export->next pointer is valid before another CPU sees
444 * the export pointer included into the list.
445 */
446 rcu_assign_pointer(*list, export);
447}
448
449static inline int
450rm_trace_export(struct trace_export **list, struct trace_export *export)
451{
452 struct trace_export **p;
453
454 for (p = list; *p != NULL; p = &(*p)->next)
455 if (*p == export)
456 break;
457
458 if (*p != export)
459 return -1;
460
461 rcu_assign_pointer(*p, (*p)->next);
462
463 return 0;
464}
465
466static inline void
467add_ftrace_export(struct trace_export **list, struct trace_export *export)
468{
469 ftrace_exports_enable(export);
470
471 add_trace_export(list, export);
472}
473
474static inline int
475rm_ftrace_export(struct trace_export **list, struct trace_export *export)
476{
477 int ret;
478
479 ret = rm_trace_export(list, export);
480 ftrace_exports_disable(export);
481
482 return ret;
483}
484
485int register_ftrace_export(struct trace_export *export)
486{
487 if (WARN_ON_ONCE(!export->write))
488 return -1;
489
490 guard(mutex)(&ftrace_export_lock);
491
492 add_ftrace_export(&ftrace_exports_list, export);
493
494 return 0;
495}
496EXPORT_SYMBOL_GPL(register_ftrace_export);
497
498int unregister_ftrace_export(struct trace_export *export)
499{
500 guard(mutex)(&ftrace_export_lock);
501 return rm_ftrace_export(&ftrace_exports_list, export);
502}
503EXPORT_SYMBOL_GPL(unregister_ftrace_export);
504
505/* trace_flags holds trace_options default values */
506#define TRACE_DEFAULT_FLAGS \
507 (FUNCTION_DEFAULT_FLAGS | FPROFILE_DEFAULT_FLAGS | \
508 TRACE_ITER(PRINT_PARENT) | TRACE_ITER(PRINTK) | \
509 TRACE_ITER(ANNOTATE) | TRACE_ITER(CONTEXT_INFO) | \
510 TRACE_ITER(RECORD_CMD) | TRACE_ITER(OVERWRITE) | \
511 TRACE_ITER(IRQ_INFO) | TRACE_ITER(MARKERS) | \
512 TRACE_ITER(HASH_PTR) | TRACE_ITER(TRACE_PRINTK) | \
513 TRACE_ITER(COPY_MARKER))
514
515/* trace_options that are only supported by global_trace */
516#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER(PRINTK) | \
517 TRACE_ITER(PRINTK_MSGONLY) | TRACE_ITER(RECORD_CMD) | \
518 TRACE_ITER(PROF_TEXT_OFFSET) | FPROFILE_DEFAULT_FLAGS)
519
520/* trace_flags that are default zero for instances */
521#define ZEROED_TRACE_FLAGS \
522 (TRACE_ITER(EVENT_FORK) | TRACE_ITER(FUNC_FORK) | TRACE_ITER(TRACE_PRINTK) | \
523 TRACE_ITER(COPY_MARKER))
524
525/*
526 * The global_trace is the descriptor that holds the top-level tracing
527 * buffers for the live tracing.
528 */
529static struct trace_array global_trace = {
530 .trace_flags = TRACE_DEFAULT_FLAGS,
531};
532
533struct trace_array *printk_trace = &global_trace;
534
535/* List of trace_arrays interested in the top level trace_marker */
536static LIST_HEAD(marker_copies);
537
538static void update_printk_trace(struct trace_array *tr)
539{
540 if (printk_trace == tr)
541 return;
542
543 printk_trace->trace_flags &= ~TRACE_ITER(TRACE_PRINTK);
544 printk_trace = tr;
545 tr->trace_flags |= TRACE_ITER(TRACE_PRINTK);
546}
547
548/* Returns true if the status of tr changed */
549static bool update_marker_trace(struct trace_array *tr, int enabled)
550{
551 lockdep_assert_held(&event_mutex);
552
553 if (enabled) {
554 if (tr->trace_flags & TRACE_ITER(COPY_MARKER))
555 return false;
556
557 list_add_rcu(&tr->marker_list, &marker_copies);
558 tr->trace_flags |= TRACE_ITER(COPY_MARKER);
559 return true;
560 }
561
562 if (!(tr->trace_flags & TRACE_ITER(COPY_MARKER)))
563 return false;
564
565 list_del_rcu(&tr->marker_list);
566 tr->trace_flags &= ~TRACE_ITER(COPY_MARKER);
567 return true;
568}
569
570void trace_set_ring_buffer_expanded(struct trace_array *tr)
571{
572 if (!tr)
573 tr = &global_trace;
574 tr->ring_buffer_expanded = true;
575}
576
577static void trace_array_autoremove(struct work_struct *work)
578{
579 struct trace_array *tr = container_of(work, struct trace_array, autoremove_work);
580
581 trace_array_destroy(tr);
582}
583
584static struct workqueue_struct *autoremove_wq;
585
586static void trace_array_kick_autoremove(struct trace_array *tr)
587{
588 if (autoremove_wq)
589 queue_work(autoremove_wq, &tr->autoremove_work);
590}
591
592static void trace_array_cancel_autoremove(struct trace_array *tr)
593{
594 /*
595 * Since this can be called inside trace_array_autoremove(),
596 * it has to avoid deadlock of the workqueue.
597 */
598 if (work_pending(&tr->autoremove_work))
599 cancel_work_sync(&tr->autoremove_work);
600}
601
602static void trace_array_init_autoremove(struct trace_array *tr)
603{
604 INIT_WORK(&tr->autoremove_work, trace_array_autoremove);
605}
606
607static void trace_array_start_autoremove(void)
608{
609 if (autoremove_wq)
610 return;
611
612 autoremove_wq = alloc_workqueue("tr_autoremove_wq",
613 WQ_UNBOUND | WQ_HIGHPRI, 0);
614 if (!autoremove_wq)
615 pr_warn("Unable to allocate tr_autoremove_wq. autoremove disabled.\n");
616}
617
618LIST_HEAD(ftrace_trace_arrays);
619
620static int __trace_array_get(struct trace_array *this_tr)
621{
622 /* When free_on_close is set, this is not available anymore. */
623 if (autoremove_wq && this_tr->free_on_close)
624 return -ENODEV;
625
626 this_tr->ref++;
627 return 0;
628}
629
630int trace_array_get(struct trace_array *this_tr)
631{
632 struct trace_array *tr;
633
634 guard(mutex)(&trace_types_lock);
635 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
636 if (tr == this_tr) {
637 return __trace_array_get(tr);
638 }
639 }
640
641 return -ENODEV;
642}
643
644static void __trace_array_put(struct trace_array *this_tr)
645{
646 WARN_ON(!this_tr->ref);
647 this_tr->ref--;
648 /*
649 * When free_on_close is set, prepare removing the array
650 * when the last reference is released.
651 */
652 if (this_tr->ref == 1 && this_tr->free_on_close)
653 trace_array_kick_autoremove(this_tr);
654}
655
656/**
657 * trace_array_put - Decrement the reference counter for this trace array.
658 * @this_tr : pointer to the trace array
659 *
660 * NOTE: Use this when we no longer need the trace array returned by
661 * trace_array_get_by_name(). This ensures the trace array can be later
662 * destroyed.
663 *
664 */
665void trace_array_put(struct trace_array *this_tr)
666{
667 if (!this_tr)
668 return;
669
670 guard(mutex)(&trace_types_lock);
671 __trace_array_put(this_tr);
672}
673EXPORT_SYMBOL_GPL(trace_array_put);
674
675int tracing_check_open_get_tr(struct trace_array *tr)
676{
677 int ret;
678
679 ret = security_locked_down(LOCKDOWN_TRACEFS);
680 if (ret)
681 return ret;
682
683 if (tracing_disabled)
684 return -ENODEV;
685
686 if (tr && trace_array_get(tr) < 0)
687 return -ENODEV;
688
689 return 0;
690}
691
692static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
693{
694 u64 ts;
695
696 /* Early boot up does not have a buffer yet */
697 if (!buf->buffer)
698 return trace_clock_local();
699
700 ts = ring_buffer_time_stamp(buf->buffer);
701 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
702
703 return ts;
704}
705
706u64 ftrace_now(int cpu)
707{
708 return buffer_ftrace_now(&global_trace.array_buffer, cpu);
709}
710
711/**
712 * tracing_is_enabled - Show if global_trace has been enabled
713 *
714 * Shows if the global trace has been enabled or not. It uses the
715 * mirror flag "buffer_disabled" to be used in fast paths such as for
716 * the irqsoff tracer. But it may be inaccurate due to races. If you
717 * need to know the accurate state, use tracing_is_on() which is a little
718 * slower, but accurate.
719 */
720int tracing_is_enabled(void)
721{
722 /*
723 * For quick access (irqsoff uses this in fast path), just
724 * return the mirror variable of the state of the ring buffer.
725 * It's a little racy, but we don't really care.
726 */
727 return !global_trace.buffer_disabled;
728}
729
730/*
731 * trace_buf_size is the size in bytes that is allocated
732 * for a buffer. Note, the number of bytes is always rounded
733 * to page size.
734 *
735 * This number is purposely set to a low number of 16384.
736 * If the dump on oops happens, it will be much appreciated
737 * to not have to wait for all that output. Anyway this can be
738 * boot time and run time configurable.
739 */
740#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
741
742static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
743
744/* trace_types holds a link list of available tracers. */
745static struct tracer *trace_types __read_mostly;
746
747/*
748 * trace_types_lock is used to protect the trace_types list.
749 */
750DEFINE_MUTEX(trace_types_lock);
751
752/*
753 * serialize the access of the ring buffer
754 *
755 * ring buffer serializes readers, but it is low level protection.
756 * The validity of the events (which returns by ring_buffer_peek() ..etc)
757 * are not protected by ring buffer.
758 *
759 * The content of events may become garbage if we allow other process consumes
760 * these events concurrently:
761 * A) the page of the consumed events may become a normal page
762 * (not reader page) in ring buffer, and this page will be rewritten
763 * by events producer.
764 * B) The page of the consumed events may become a page for splice_read,
765 * and this page will be returned to system.
766 *
767 * These primitives allow multi process access to different cpu ring buffer
768 * concurrently.
769 *
770 * These primitives don't distinguish read-only and read-consume access.
771 * Multi read-only access are also serialized.
772 */
773
774#ifdef CONFIG_SMP
775static DECLARE_RWSEM(all_cpu_access_lock);
776static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
777
778static inline void trace_access_lock(int cpu)
779{
780 if (cpu == RING_BUFFER_ALL_CPUS) {
781 /* gain it for accessing the whole ring buffer. */
782 down_write(&all_cpu_access_lock);
783 } else {
784 /* gain it for accessing a cpu ring buffer. */
785
786 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
787 down_read(&all_cpu_access_lock);
788
789 /* Secondly block other access to this @cpu ring buffer. */
790 mutex_lock(&per_cpu(cpu_access_lock, cpu));
791 }
792}
793
794static inline void trace_access_unlock(int cpu)
795{
796 if (cpu == RING_BUFFER_ALL_CPUS) {
797 up_write(&all_cpu_access_lock);
798 } else {
799 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
800 up_read(&all_cpu_access_lock);
801 }
802}
803
804static inline void trace_access_lock_init(void)
805{
806 int cpu;
807
808 for_each_possible_cpu(cpu)
809 mutex_init(&per_cpu(cpu_access_lock, cpu));
810}
811
812#else
813
814static DEFINE_MUTEX(access_lock);
815
816static inline void trace_access_lock(int cpu)
817{
818 (void)cpu;
819 mutex_lock(&access_lock);
820}
821
822static inline void trace_access_unlock(int cpu)
823{
824 (void)cpu;
825 mutex_unlock(&access_lock);
826}
827
828static inline void trace_access_lock_init(void)
829{
830}
831
832#endif
833
834void tracer_tracing_on(struct trace_array *tr)
835{
836 if (tr->array_buffer.buffer)
837 ring_buffer_record_on(tr->array_buffer.buffer);
838 /*
839 * This flag is looked at when buffers haven't been allocated
840 * yet, or by some tracers (like irqsoff), that just want to
841 * know if the ring buffer has been disabled, but it can handle
842 * races of where it gets disabled but we still do a record.
843 * As the check is in the fast path of the tracers, it is more
844 * important to be fast than accurate.
845 */
846 tr->buffer_disabled = 0;
847}
848
849/**
850 * tracing_on - enable tracing buffers
851 *
852 * This function enables tracing buffers that may have been
853 * disabled with tracing_off.
854 */
855void tracing_on(void)
856{
857 tracer_tracing_on(&global_trace);
858}
859EXPORT_SYMBOL_GPL(tracing_on);
860
861#ifdef CONFIG_TRACER_SNAPSHOT
862/**
863 * tracing_snapshot - take a snapshot of the current buffer.
864 *
865 * This causes a swap between the snapshot buffer and the current live
866 * tracing buffer. You can use this to take snapshots of the live
867 * trace when some condition is triggered, but continue to trace.
868 *
869 * Note, make sure to allocate the snapshot with either
870 * a tracing_snapshot_alloc(), or by doing it manually
871 * with: echo 1 > /sys/kernel/tracing/snapshot
872 *
873 * If the snapshot buffer is not allocated, it will stop tracing.
874 * Basically making a permanent snapshot.
875 */
876void tracing_snapshot(void)
877{
878 struct trace_array *tr = &global_trace;
879
880 tracing_snapshot_instance(tr);
881}
882EXPORT_SYMBOL_GPL(tracing_snapshot);
883
884/**
885 * tracing_alloc_snapshot - allocate snapshot buffer.
886 *
887 * This only allocates the snapshot buffer if it isn't already
888 * allocated - it doesn't also take a snapshot.
889 *
890 * This is meant to be used in cases where the snapshot buffer needs
891 * to be set up for events that can't sleep but need to be able to
892 * trigger a snapshot.
893 */
894int tracing_alloc_snapshot(void)
895{
896 struct trace_array *tr = &global_trace;
897 int ret;
898
899 ret = tracing_alloc_snapshot_instance(tr);
900 WARN_ON(ret < 0);
901
902 return ret;
903}
904#else
905void tracing_snapshot(void)
906{
907 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
908}
909EXPORT_SYMBOL_GPL(tracing_snapshot);
910void tracing_snapshot_alloc(void)
911{
912 /* Give warning */
913 tracing_snapshot();
914}
915EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
916#endif /* CONFIG_TRACER_SNAPSHOT */
917
918void tracer_tracing_off(struct trace_array *tr)
919{
920 if (tr->array_buffer.buffer)
921 ring_buffer_record_off(tr->array_buffer.buffer);
922 /*
923 * This flag is looked at when buffers haven't been allocated
924 * yet, or by some tracers (like irqsoff), that just want to
925 * know if the ring buffer has been disabled, but it can handle
926 * races of where it gets disabled but we still do a record.
927 * As the check is in the fast path of the tracers, it is more
928 * important to be fast than accurate.
929 */
930 tr->buffer_disabled = 1;
931}
932
933/**
934 * tracer_tracing_disable() - temporary disable the buffer from write
935 * @tr: The trace array to disable its buffer for
936 *
937 * Expects trace_tracing_enable() to re-enable tracing.
938 * The difference between this and tracer_tracing_off() is that this
939 * is a counter and can nest, whereas, tracer_tracing_off() can
940 * be called multiple times and a single trace_tracing_on() will
941 * enable it.
942 */
943void tracer_tracing_disable(struct trace_array *tr)
944{
945 if (WARN_ON_ONCE(!tr->array_buffer.buffer))
946 return;
947
948 ring_buffer_record_disable(tr->array_buffer.buffer);
949}
950
951/**
952 * tracer_tracing_enable() - counter part of tracer_tracing_disable()
953 * @tr: The trace array that had tracer_tracincg_disable() called on it
954 *
955 * This is called after tracer_tracing_disable() has been called on @tr,
956 * when it's safe to re-enable tracing.
957 */
958void tracer_tracing_enable(struct trace_array *tr)
959{
960 if (WARN_ON_ONCE(!tr->array_buffer.buffer))
961 return;
962
963 ring_buffer_record_enable(tr->array_buffer.buffer);
964}
965
966/**
967 * tracing_off - turn off tracing buffers
968 *
969 * This function stops the tracing buffers from recording data.
970 * It does not disable any overhead the tracers themselves may
971 * be causing. This function simply causes all recording to
972 * the ring buffers to fail.
973 */
974void tracing_off(void)
975{
976 tracer_tracing_off(&global_trace);
977}
978EXPORT_SYMBOL_GPL(tracing_off);
979
980void disable_trace_on_warning(void)
981{
982 if (__disable_trace_on_warning) {
983 struct trace_array *tr = READ_ONCE(printk_trace);
984
985 trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
986 "Disabling tracing due to warning\n");
987 tracing_off();
988
989 /* Disable trace_printk() buffer too */
990 if (tr != &global_trace) {
991 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
992 "Disabling tracing due to warning\n");
993 tracer_tracing_off(tr);
994 }
995 }
996}
997
998/**
999 * tracer_tracing_is_on - show real state of ring buffer enabled
1000 * @tr : the trace array to know if ring buffer is enabled
1001 *
1002 * Shows real state of the ring buffer if it is enabled or not.
1003 */
1004bool tracer_tracing_is_on(struct trace_array *tr)
1005{
1006 if (tr->array_buffer.buffer)
1007 return ring_buffer_record_is_set_on(tr->array_buffer.buffer);
1008 return !tr->buffer_disabled;
1009}
1010
1011/**
1012 * tracing_is_on - show state of ring buffers enabled
1013 */
1014int tracing_is_on(void)
1015{
1016 return tracer_tracing_is_on(&global_trace);
1017}
1018EXPORT_SYMBOL_GPL(tracing_is_on);
1019
1020static int __init set_buf_size(char *str)
1021{
1022 unsigned long buf_size;
1023
1024 if (!str)
1025 return 0;
1026 buf_size = memparse(str, &str);
1027 /*
1028 * nr_entries can not be zero and the startup
1029 * tests require some buffer space. Therefore
1030 * ensure we have at least 4096 bytes of buffer.
1031 */
1032 trace_buf_size = max(4096UL, buf_size);
1033 return 1;
1034}
1035__setup("trace_buf_size=", set_buf_size);
1036
1037static int __init set_tracing_thresh(char *str)
1038{
1039 unsigned long threshold;
1040 int ret;
1041
1042 if (!str)
1043 return 0;
1044 ret = kstrtoul(str, 0, &threshold);
1045 if (ret < 0)
1046 return 0;
1047 tracing_thresh = threshold * 1000;
1048 return 1;
1049}
1050__setup("tracing_thresh=", set_tracing_thresh);
1051
1052unsigned long nsecs_to_usecs(unsigned long nsecs)
1053{
1054 return nsecs / 1000;
1055}
1056
1057/*
1058 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1059 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1060 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1061 * of strings in the order that the evals (enum) were defined.
1062 */
1063#undef C
1064#define C(a, b) b
1065
1066/* These must match the bit positions in trace_iterator_flags */
1067static const char *trace_options[] = {
1068 TRACE_FLAGS
1069 NULL
1070};
1071
1072static struct {
1073 u64 (*func)(void);
1074 const char *name;
1075 int in_ns; /* is this clock in nanoseconds? */
1076} trace_clocks[] = {
1077 { trace_clock_local, "local", 1 },
1078 { trace_clock_global, "global", 1 },
1079 { trace_clock_counter, "counter", 0 },
1080 { trace_clock_jiffies, "uptime", 0 },
1081 { trace_clock, "perf", 1 },
1082 { ktime_get_mono_fast_ns, "mono", 1 },
1083 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1084 { ktime_get_boot_fast_ns, "boot", 1 },
1085 { ktime_get_tai_fast_ns, "tai", 1 },
1086 ARCH_TRACE_CLOCKS
1087};
1088
1089bool trace_clock_in_ns(struct trace_array *tr)
1090{
1091 if (trace_clocks[tr->clock_id].in_ns)
1092 return true;
1093
1094 return false;
1095}
1096
1097/*
1098 * trace_parser_get_init - gets the buffer for trace parser
1099 */
1100int trace_parser_get_init(struct trace_parser *parser, int size)
1101{
1102 memset(parser, 0, sizeof(*parser));
1103
1104 parser->buffer = kmalloc(size, GFP_KERNEL);
1105 if (!parser->buffer)
1106 return 1;
1107
1108 parser->size = size;
1109 return 0;
1110}
1111
1112/*
1113 * trace_parser_put - frees the buffer for trace parser
1114 */
1115void trace_parser_put(struct trace_parser *parser)
1116{
1117 kfree(parser->buffer);
1118 parser->buffer = NULL;
1119}
1120
1121/*
1122 * trace_get_user - reads the user input string separated by space
1123 * (matched by isspace(ch))
1124 *
1125 * For each string found the 'struct trace_parser' is updated,
1126 * and the function returns.
1127 *
1128 * Returns number of bytes read.
1129 *
1130 * See kernel/trace/trace.h for 'struct trace_parser' details.
1131 */
1132int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1133 size_t cnt, loff_t *ppos)
1134{
1135 char ch;
1136 size_t read = 0;
1137 ssize_t ret;
1138
1139 if (!*ppos)
1140 trace_parser_clear(parser);
1141
1142 ret = get_user(ch, ubuf++);
1143 if (ret)
1144 goto fail;
1145
1146 read++;
1147 cnt--;
1148
1149 /*
1150 * The parser is not finished with the last write,
1151 * continue reading the user input without skipping spaces.
1152 */
1153 if (!parser->cont) {
1154 /* skip white space */
1155 while (cnt && isspace(ch)) {
1156 ret = get_user(ch, ubuf++);
1157 if (ret)
1158 goto fail;
1159 read++;
1160 cnt--;
1161 }
1162
1163 parser->idx = 0;
1164
1165 /* only spaces were written */
1166 if (isspace(ch) || !ch) {
1167 *ppos += read;
1168 return read;
1169 }
1170 }
1171
1172 /* read the non-space input */
1173 while (cnt && !isspace(ch) && ch) {
1174 if (parser->idx < parser->size - 1)
1175 parser->buffer[parser->idx++] = ch;
1176 else {
1177 ret = -EINVAL;
1178 goto fail;
1179 }
1180
1181 ret = get_user(ch, ubuf++);
1182 if (ret)
1183 goto fail;
1184 read++;
1185 cnt--;
1186 }
1187
1188 /* We either got finished input or we have to wait for another call. */
1189 if (isspace(ch) || !ch) {
1190 parser->buffer[parser->idx] = 0;
1191 parser->cont = false;
1192 } else if (parser->idx < parser->size - 1) {
1193 parser->cont = true;
1194 parser->buffer[parser->idx++] = ch;
1195 /* Make sure the parsed string always terminates with '\0'. */
1196 parser->buffer[parser->idx] = 0;
1197 } else {
1198 ret = -EINVAL;
1199 goto fail;
1200 }
1201
1202 *ppos += read;
1203 return read;
1204fail:
1205 trace_parser_fail(parser);
1206 return ret;
1207}
1208
1209/* TODO add a seq_buf_to_buffer() */
1210static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1211{
1212 int len;
1213
1214 if (trace_seq_used(s) <= s->readpos)
1215 return -EBUSY;
1216
1217 len = trace_seq_used(s) - s->readpos;
1218 if (cnt > len)
1219 cnt = len;
1220 memcpy(buf, s->buffer + s->readpos, cnt);
1221
1222 s->readpos += cnt;
1223 return cnt;
1224}
1225
1226unsigned long __read_mostly tracing_thresh;
1227
1228struct pipe_wait {
1229 struct trace_iterator *iter;
1230 int wait_index;
1231};
1232
1233static bool wait_pipe_cond(void *data)
1234{
1235 struct pipe_wait *pwait = data;
1236 struct trace_iterator *iter = pwait->iter;
1237
1238 if (atomic_read_acquire(&iter->wait_index) != pwait->wait_index)
1239 return true;
1240
1241 return iter->closed;
1242}
1243
1244static int wait_on_pipe(struct trace_iterator *iter, int full)
1245{
1246 struct pipe_wait pwait;
1247 int ret;
1248
1249 /* Iterators are static, they should be filled or empty */
1250 if (trace_buffer_iter(iter, iter->cpu_file))
1251 return 0;
1252
1253 pwait.wait_index = atomic_read_acquire(&iter->wait_index);
1254 pwait.iter = iter;
1255
1256 ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full,
1257 wait_pipe_cond, &pwait);
1258
1259#ifdef CONFIG_TRACER_SNAPSHOT
1260 /*
1261 * Make sure this is still the snapshot buffer, as if a snapshot were
1262 * to happen, this would now be the main buffer.
1263 */
1264 if (iter->snapshot)
1265 iter->array_buffer = &iter->tr->snapshot_buffer;
1266#endif
1267 return ret;
1268}
1269
1270#ifdef CONFIG_FTRACE_STARTUP_TEST
1271static bool selftests_can_run;
1272
1273struct trace_selftests {
1274 struct list_head list;
1275 struct tracer *type;
1276};
1277
1278static LIST_HEAD(postponed_selftests);
1279
1280static int save_selftest(struct tracer *type)
1281{
1282 struct trace_selftests *selftest;
1283
1284 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1285 if (!selftest)
1286 return -ENOMEM;
1287
1288 selftest->type = type;
1289 list_add(&selftest->list, &postponed_selftests);
1290 return 0;
1291}
1292
1293static int run_tracer_selftest(struct tracer *type)
1294{
1295 struct trace_array *tr = &global_trace;
1296 struct tracer_flags *saved_flags = tr->current_trace_flags;
1297 struct tracer *saved_tracer = tr->current_trace;
1298 int ret;
1299
1300 if (!type->selftest || tracing_selftest_disabled)
1301 return 0;
1302
1303 /*
1304 * If a tracer registers early in boot up (before scheduling is
1305 * initialized and such), then do not run its selftests yet.
1306 * Instead, run it a little later in the boot process.
1307 */
1308 if (!selftests_can_run)
1309 return save_selftest(type);
1310
1311 if (!tracing_is_on()) {
1312 pr_warn("Selftest for tracer %s skipped due to tracing disabled\n",
1313 type->name);
1314 return 0;
1315 }
1316
1317 /*
1318 * Run a selftest on this tracer.
1319 * Here we reset the trace buffer, and set the current
1320 * tracer to be this tracer. The tracer can then run some
1321 * internal tracing to verify that everything is in order.
1322 * If we fail, we do not register this tracer.
1323 */
1324 tracing_reset_online_cpus(&tr->array_buffer);
1325
1326 tr->current_trace = type;
1327 if (type->flags)
1328 tr->current_trace_flags = type->flags;
1329 else if (type->default_flags)
1330 tr->current_trace_flags = type->default_flags;
1331 else
1332 tr->current_trace_flags = &no_tracer_flags;
1333
1334#ifdef CONFIG_TRACER_MAX_TRACE
1335 if (tracer_uses_snapshot(type)) {
1336 /* If we expanded the buffers, make sure the max is expanded too */
1337 if (tr->ring_buffer_expanded)
1338 ring_buffer_resize(tr->snapshot_buffer.buffer, trace_buf_size,
1339 RING_BUFFER_ALL_CPUS);
1340 tr->allocated_snapshot = true;
1341 }
1342#endif
1343
1344 /* the test is responsible for initializing and enabling */
1345 pr_info("Testing tracer %s: ", type->name);
1346 ret = type->selftest(type, tr);
1347 /* the test is responsible for resetting too */
1348 tr->current_trace = saved_tracer;
1349 tr->current_trace_flags = saved_flags;
1350 if (ret) {
1351 printk(KERN_CONT "FAILED!\n");
1352 /* Add the warning after printing 'FAILED' */
1353 WARN_ON(1);
1354 return -1;
1355 }
1356 /* Only reset on passing, to avoid touching corrupted buffers */
1357 tracing_reset_online_cpus(&tr->array_buffer);
1358
1359#ifdef CONFIG_TRACER_MAX_TRACE
1360 if (tracer_uses_snapshot(type)) {
1361 tr->allocated_snapshot = false;
1362
1363 /* Shrink the max buffer again */
1364 if (tr->ring_buffer_expanded)
1365 ring_buffer_resize(tr->snapshot_buffer.buffer, 1,
1366 RING_BUFFER_ALL_CPUS);
1367 }
1368#endif
1369
1370 printk(KERN_CONT "PASSED\n");
1371 return 0;
1372}
1373
1374static int do_run_tracer_selftest(struct tracer *type)
1375{
1376 int ret;
1377
1378 /*
1379 * Tests can take a long time, especially if they are run one after the
1380 * other, as does happen during bootup when all the tracers are
1381 * registered. This could cause the soft lockup watchdog to trigger.
1382 */
1383 cond_resched();
1384
1385 tracing_selftest_running = true;
1386 ret = run_tracer_selftest(type);
1387 tracing_selftest_running = false;
1388
1389 return ret;
1390}
1391
1392static __init int init_trace_selftests(void)
1393{
1394 struct trace_selftests *p, *n;
1395 struct tracer *t, **last;
1396 int ret;
1397
1398 selftests_can_run = true;
1399
1400 guard(mutex)(&trace_types_lock);
1401
1402 if (list_empty(&postponed_selftests))
1403 return 0;
1404
1405 pr_info("Running postponed tracer tests:\n");
1406
1407 tracing_selftest_running = true;
1408 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1409 /* This loop can take minutes when sanitizers are enabled, so
1410 * lets make sure we allow RCU processing.
1411 */
1412 cond_resched();
1413 ret = run_tracer_selftest(p->type);
1414 /* If the test fails, then warn and remove from available_tracers */
1415 if (ret < 0) {
1416 WARN(1, "tracer: %s failed selftest, disabling\n",
1417 p->type->name);
1418 last = &trace_types;
1419 for (t = trace_types; t; t = t->next) {
1420 if (t == p->type) {
1421 *last = t->next;
1422 break;
1423 }
1424 last = &t->next;
1425 }
1426 }
1427 list_del(&p->list);
1428 kfree(p);
1429 }
1430 tracing_selftest_running = false;
1431
1432 return 0;
1433}
1434core_initcall(init_trace_selftests);
1435#else
1436static inline int do_run_tracer_selftest(struct tracer *type)
1437{
1438 return 0;
1439}
1440#endif /* CONFIG_FTRACE_STARTUP_TEST */
1441
1442static int add_tracer(struct trace_array *tr, struct tracer *t);
1443
1444static void __init apply_trace_boot_options(void);
1445
1446static void free_tracers(struct trace_array *tr)
1447{
1448 struct tracers *t, *n;
1449
1450 lockdep_assert_held(&trace_types_lock);
1451
1452 list_for_each_entry_safe(t, n, &tr->tracers, list) {
1453 list_del(&t->list);
1454 kfree(t->flags);
1455 kfree(t);
1456 }
1457}
1458
1459/**
1460 * register_tracer - register a tracer with the ftrace system.
1461 * @type: the plugin for the tracer
1462 *
1463 * Register a new plugin tracer.
1464 */
1465int __init register_tracer(struct tracer *type)
1466{
1467 struct trace_array *tr;
1468 struct tracer *t;
1469 int ret = 0;
1470
1471 if (!type->name) {
1472 pr_info("Tracer must have a name\n");
1473 return -1;
1474 }
1475
1476 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1477 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1478 return -1;
1479 }
1480
1481 if (security_locked_down(LOCKDOWN_TRACEFS)) {
1482 pr_warn("Can not register tracer %s due to lockdown\n",
1483 type->name);
1484 return -EPERM;
1485 }
1486
1487 mutex_lock(&trace_types_lock);
1488
1489 for (t = trace_types; t; t = t->next) {
1490 if (strcmp(type->name, t->name) == 0) {
1491 /* already found */
1492 pr_info("Tracer %s already registered\n",
1493 type->name);
1494 ret = -1;
1495 goto out;
1496 }
1497 }
1498
1499 /* store the tracer for __set_tracer_option */
1500 if (type->flags)
1501 type->flags->trace = type;
1502
1503 ret = do_run_tracer_selftest(type);
1504 if (ret < 0)
1505 goto out;
1506
1507 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1508 ret = add_tracer(tr, type);
1509 if (ret < 0) {
1510 /* The tracer will still exist but without options */
1511 pr_warn("Failed to create tracer options for %s\n", type->name);
1512 break;
1513 }
1514 }
1515
1516 type->next = trace_types;
1517 trace_types = type;
1518
1519 out:
1520 mutex_unlock(&trace_types_lock);
1521
1522 if (ret || !default_bootup_tracer)
1523 return ret;
1524
1525 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1526 return 0;
1527
1528 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1529 /* Do we want this tracer to start on bootup? */
1530 WARN_ON(tracing_set_tracer(&global_trace, type->name) < 0);
1531 default_bootup_tracer = NULL;
1532
1533 apply_trace_boot_options();
1534
1535 /* disable other selftests, since this will break it. */
1536 disable_tracing_selftest("running a tracer");
1537
1538 return 0;
1539}
1540
1541void tracing_reset_cpu(struct array_buffer *buf, int cpu)
1542{
1543 struct trace_buffer *buffer = buf->buffer;
1544
1545 if (!buffer)
1546 return;
1547
1548 ring_buffer_record_disable(buffer);
1549
1550 /* Make sure all commits have finished */
1551 synchronize_rcu();
1552 ring_buffer_reset_cpu(buffer, cpu);
1553
1554 ring_buffer_record_enable(buffer);
1555}
1556
1557void tracing_reset_online_cpus(struct array_buffer *buf)
1558{
1559 struct trace_buffer *buffer = buf->buffer;
1560
1561 if (!buffer)
1562 return;
1563
1564 ring_buffer_record_disable(buffer);
1565
1566 /* Make sure all commits have finished */
1567 synchronize_rcu();
1568
1569 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1570
1571 ring_buffer_reset_online_cpus(buffer);
1572
1573 ring_buffer_record_enable(buffer);
1574}
1575
1576static void tracing_reset_all_cpus(struct array_buffer *buf)
1577{
1578 struct trace_buffer *buffer = buf->buffer;
1579
1580 if (!buffer)
1581 return;
1582
1583 ring_buffer_record_disable(buffer);
1584
1585 /* Make sure all commits have finished */
1586 synchronize_rcu();
1587
1588 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1589
1590 ring_buffer_reset(buffer);
1591
1592 ring_buffer_record_enable(buffer);
1593}
1594
1595/* Must have trace_types_lock held */
1596void tracing_reset_all_online_cpus_unlocked(void)
1597{
1598 struct trace_array *tr;
1599
1600 lockdep_assert_held(&trace_types_lock);
1601
1602 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1603 if (!tr->clear_trace)
1604 continue;
1605 tr->clear_trace = false;
1606 tracing_reset_online_cpus(&tr->array_buffer);
1607#ifdef CONFIG_TRACER_SNAPSHOT
1608 tracing_reset_online_cpus(&tr->snapshot_buffer);
1609#endif
1610 }
1611}
1612
1613void tracing_reset_all_online_cpus(void)
1614{
1615 guard(mutex)(&trace_types_lock);
1616 tracing_reset_all_online_cpus_unlocked();
1617}
1618
1619int is_tracing_stopped(void)
1620{
1621 return global_trace.stop_count;
1622}
1623
1624static void tracing_start_tr(struct trace_array *tr)
1625{
1626 struct trace_buffer *buffer;
1627
1628 if (tracing_disabled)
1629 return;
1630
1631 guard(raw_spinlock_irqsave)(&tr->start_lock);
1632 if (--tr->stop_count) {
1633 if (WARN_ON_ONCE(tr->stop_count < 0)) {
1634 /* Someone screwed up their debugging */
1635 tr->stop_count = 0;
1636 }
1637 return;
1638 }
1639
1640 /* Prevent the buffers from switching */
1641 arch_spin_lock(&tr->max_lock);
1642
1643 buffer = tr->array_buffer.buffer;
1644 if (buffer)
1645 ring_buffer_record_enable(buffer);
1646
1647#ifdef CONFIG_TRACER_SNAPSHOT
1648 buffer = tr->snapshot_buffer.buffer;
1649 if (buffer)
1650 ring_buffer_record_enable(buffer);
1651#endif
1652
1653 arch_spin_unlock(&tr->max_lock);
1654}
1655
1656/**
1657 * tracing_start - quick start of the tracer
1658 *
1659 * If tracing is enabled but was stopped by tracing_stop,
1660 * this will start the tracer back up.
1661 */
1662void tracing_start(void)
1663
1664{
1665 return tracing_start_tr(&global_trace);
1666}
1667
1668static void tracing_stop_tr(struct trace_array *tr)
1669{
1670 struct trace_buffer *buffer;
1671
1672 guard(raw_spinlock_irqsave)(&tr->start_lock);
1673 if (tr->stop_count++)
1674 return;
1675
1676 /* Prevent the buffers from switching */
1677 arch_spin_lock(&tr->max_lock);
1678
1679 buffer = tr->array_buffer.buffer;
1680 if (buffer)
1681 ring_buffer_record_disable(buffer);
1682
1683#ifdef CONFIG_TRACER_SNAPSHOT
1684 buffer = tr->snapshot_buffer.buffer;
1685 if (buffer)
1686 ring_buffer_record_disable(buffer);
1687#endif
1688
1689 arch_spin_unlock(&tr->max_lock);
1690}
1691
1692/**
1693 * tracing_stop - quick stop of the tracer
1694 *
1695 * Light weight way to stop tracing. Use in conjunction with
1696 * tracing_start.
1697 */
1698void tracing_stop(void)
1699{
1700 return tracing_stop_tr(&global_trace);
1701}
1702
1703/*
1704 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
1705 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
1706 * simplifies those functions and keeps them in sync.
1707 */
1708enum print_line_t trace_handle_return(struct trace_seq *s)
1709{
1710 return trace_seq_has_overflowed(s) ?
1711 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
1712}
1713EXPORT_SYMBOL_GPL(trace_handle_return);
1714
1715static unsigned short migration_disable_value(void)
1716{
1717#if defined(CONFIG_SMP)
1718 return current->migration_disabled;
1719#else
1720 return 0;
1721#endif
1722}
1723
1724unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
1725{
1726 unsigned int trace_flags = irqs_status;
1727 unsigned int pc;
1728
1729 pc = preempt_count();
1730
1731 if (pc & NMI_MASK)
1732 trace_flags |= TRACE_FLAG_NMI;
1733 if (pc & HARDIRQ_MASK)
1734 trace_flags |= TRACE_FLAG_HARDIRQ;
1735 if (in_serving_softirq())
1736 trace_flags |= TRACE_FLAG_SOFTIRQ;
1737 if (softirq_count() >> (SOFTIRQ_SHIFT + 1))
1738 trace_flags |= TRACE_FLAG_BH_OFF;
1739
1740 if (tif_need_resched())
1741 trace_flags |= TRACE_FLAG_NEED_RESCHED;
1742 if (test_preempt_need_resched())
1743 trace_flags |= TRACE_FLAG_PREEMPT_RESCHED;
1744 if (IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY) && tif_test_bit(TIF_NEED_RESCHED_LAZY))
1745 trace_flags |= TRACE_FLAG_NEED_RESCHED_LAZY;
1746 return (trace_flags << 16) | (min_t(unsigned int, pc & 0xff, 0xf)) |
1747 (min_t(unsigned int, migration_disable_value(), 0xf)) << 4;
1748}
1749
1750struct ring_buffer_event *
1751trace_buffer_lock_reserve(struct trace_buffer *buffer,
1752 int type,
1753 unsigned long len,
1754 unsigned int trace_ctx)
1755{
1756 return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
1757}
1758
1759DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1760DEFINE_PER_CPU(int, trace_buffered_event_cnt);
1761static int trace_buffered_event_ref;
1762
1763/**
1764 * trace_buffered_event_enable - enable buffering events
1765 *
1766 * When events are being filtered, it is quicker to use a temporary
1767 * buffer to write the event data into if there's a likely chance
1768 * that it will not be committed. The discard of the ring buffer
1769 * is not as fast as committing, and is much slower than copying
1770 * a commit.
1771 *
1772 * When an event is to be filtered, allocate per cpu buffers to
1773 * write the event data into, and if the event is filtered and discarded
1774 * it is simply dropped, otherwise, the entire data is to be committed
1775 * in one shot.
1776 */
1777void trace_buffered_event_enable(void)
1778{
1779 struct ring_buffer_event *event;
1780 struct page *page;
1781 int cpu;
1782
1783 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1784
1785 if (trace_buffered_event_ref++)
1786 return;
1787
1788 for_each_tracing_cpu(cpu) {
1789 page = alloc_pages_node(cpu_to_node(cpu),
1790 GFP_KERNEL | __GFP_NORETRY, 0);
1791 /* This is just an optimization and can handle failures */
1792 if (!page) {
1793 pr_err("Failed to allocate event buffer\n");
1794 break;
1795 }
1796
1797 event = page_address(page);
1798 memset(event, 0, sizeof(*event));
1799
1800 per_cpu(trace_buffered_event, cpu) = event;
1801
1802 scoped_guard(preempt) {
1803 if (cpu == smp_processor_id() &&
1804 __this_cpu_read(trace_buffered_event) !=
1805 per_cpu(trace_buffered_event, cpu))
1806 WARN_ON_ONCE(1);
1807 }
1808 }
1809}
1810
1811static void enable_trace_buffered_event(void *data)
1812{
1813 this_cpu_dec(trace_buffered_event_cnt);
1814}
1815
1816static void disable_trace_buffered_event(void *data)
1817{
1818 this_cpu_inc(trace_buffered_event_cnt);
1819}
1820
1821/**
1822 * trace_buffered_event_disable - disable buffering events
1823 *
1824 * When a filter is removed, it is faster to not use the buffered
1825 * events, and to commit directly into the ring buffer. Free up
1826 * the temp buffers when there are no more users. This requires
1827 * special synchronization with current events.
1828 */
1829void trace_buffered_event_disable(void)
1830{
1831 int cpu;
1832
1833 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1834
1835 if (WARN_ON_ONCE(!trace_buffered_event_ref))
1836 return;
1837
1838 if (--trace_buffered_event_ref)
1839 return;
1840
1841 /* For each CPU, set the buffer as used. */
1842 on_each_cpu_mask(tracing_buffer_mask, disable_trace_buffered_event,
1843 NULL, true);
1844
1845 /* Wait for all current users to finish */
1846 synchronize_rcu();
1847
1848 for_each_tracing_cpu(cpu) {
1849 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
1850 per_cpu(trace_buffered_event, cpu) = NULL;
1851 }
1852
1853 /*
1854 * Wait for all CPUs that potentially started checking if they can use
1855 * their event buffer only after the previous synchronize_rcu() call and
1856 * they still read a valid pointer from trace_buffered_event. It must be
1857 * ensured they don't see cleared trace_buffered_event_cnt else they
1858 * could wrongly decide to use the pointed-to buffer which is now freed.
1859 */
1860 synchronize_rcu();
1861
1862 /* For each CPU, relinquish the buffer */
1863 on_each_cpu_mask(tracing_buffer_mask, enable_trace_buffered_event, NULL,
1864 true);
1865}
1866
1867static struct trace_buffer *temp_buffer;
1868
1869struct ring_buffer_event *
1870trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
1871 struct trace_event_file *trace_file,
1872 int type, unsigned long len,
1873 unsigned int trace_ctx)
1874{
1875 struct ring_buffer_event *entry;
1876 struct trace_array *tr = trace_file->tr;
1877 int val;
1878
1879 *current_rb = tr->array_buffer.buffer;
1880
1881 if (!tr->no_filter_buffering_ref &&
1882 (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) {
1883 preempt_disable_notrace();
1884 /*
1885 * Filtering is on, so try to use the per cpu buffer first.
1886 * This buffer will simulate a ring_buffer_event,
1887 * where the type_len is zero and the array[0] will
1888 * hold the full length.
1889 * (see include/linux/ring-buffer.h for details on
1890 * how the ring_buffer_event is structured).
1891 *
1892 * Using a temp buffer during filtering and copying it
1893 * on a matched filter is quicker than writing directly
1894 * into the ring buffer and then discarding it when
1895 * it doesn't match. That is because the discard
1896 * requires several atomic operations to get right.
1897 * Copying on match and doing nothing on a failed match
1898 * is still quicker than no copy on match, but having
1899 * to discard out of the ring buffer on a failed match.
1900 */
1901 if ((entry = __this_cpu_read(trace_buffered_event))) {
1902 int max_len = PAGE_SIZE - struct_size(entry, array, 1);
1903
1904 val = this_cpu_inc_return(trace_buffered_event_cnt);
1905
1906 /*
1907 * Preemption is disabled, but interrupts and NMIs
1908 * can still come in now. If that happens after
1909 * the above increment, then it will have to go
1910 * back to the old method of allocating the event
1911 * on the ring buffer, and if the filter fails, it
1912 * will have to call ring_buffer_discard_commit()
1913 * to remove it.
1914 *
1915 * Need to also check the unlikely case that the
1916 * length is bigger than the temp buffer size.
1917 * If that happens, then the reserve is pretty much
1918 * guaranteed to fail, as the ring buffer currently
1919 * only allows events less than a page. But that may
1920 * change in the future, so let the ring buffer reserve
1921 * handle the failure in that case.
1922 */
1923 if (val == 1 && likely(len <= max_len)) {
1924 trace_event_setup(entry, type, trace_ctx);
1925 entry->array[0] = len;
1926 /* Return with preemption disabled */
1927 return entry;
1928 }
1929 this_cpu_dec(trace_buffered_event_cnt);
1930 }
1931 /* __trace_buffer_lock_reserve() disables preemption */
1932 preempt_enable_notrace();
1933 }
1934
1935 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
1936 trace_ctx);
1937 /*
1938 * If tracing is off, but we have triggers enabled
1939 * we still need to look at the event data. Use the temp_buffer
1940 * to store the trace event for the trigger to use. It's recursive
1941 * safe and will not be recorded anywhere.
1942 */
1943 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
1944 *current_rb = temp_buffer;
1945 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
1946 trace_ctx);
1947 }
1948 return entry;
1949}
1950EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1951
1952static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock);
1953static DEFINE_MUTEX(tracepoint_printk_mutex);
1954
1955static void output_printk(struct trace_event_buffer *fbuffer)
1956{
1957 struct trace_event_call *event_call;
1958 struct trace_event_file *file;
1959 struct trace_event *event;
1960 unsigned long flags;
1961 struct trace_iterator *iter = tracepoint_print_iter;
1962
1963 /* We should never get here if iter is NULL */
1964 if (WARN_ON_ONCE(!iter))
1965 return;
1966
1967 event_call = fbuffer->trace_file->event_call;
1968 if (!event_call || !event_call->event.funcs ||
1969 !event_call->event.funcs->trace)
1970 return;
1971
1972 file = fbuffer->trace_file;
1973 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
1974 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
1975 !filter_match_preds(file->filter, fbuffer->entry)))
1976 return;
1977
1978 event = &fbuffer->trace_file->event_call->event;
1979
1980 raw_spin_lock_irqsave(&tracepoint_iter_lock, flags);
1981 trace_seq_init(&iter->seq);
1982 iter->ent = fbuffer->entry;
1983 event_call->event.funcs->trace(iter, 0, event);
1984 trace_seq_putc(&iter->seq, 0);
1985 printk("%s", iter->seq.buffer);
1986
1987 raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
1988}
1989
1990int tracepoint_printk_sysctl(const struct ctl_table *table, int write,
1991 void *buffer, size_t *lenp,
1992 loff_t *ppos)
1993{
1994 int save_tracepoint_printk;
1995 int ret;
1996
1997 guard(mutex)(&tracepoint_printk_mutex);
1998 save_tracepoint_printk = tracepoint_printk;
1999
2000 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2001
2002 /*
2003 * This will force exiting early, as tracepoint_printk
2004 * is always zero when tracepoint_printk_iter is not allocated
2005 */
2006 if (!tracepoint_print_iter)
2007 tracepoint_printk = 0;
2008
2009 if (save_tracepoint_printk == tracepoint_printk)
2010 return ret;
2011
2012 if (tracepoint_printk)
2013 static_key_enable(&tracepoint_printk_key.key);
2014 else
2015 static_key_disable(&tracepoint_printk_key.key);
2016
2017 return ret;
2018}
2019
2020void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2021{
2022 enum event_trigger_type tt = ETT_NONE;
2023 struct trace_event_file *file = fbuffer->trace_file;
2024
2025 if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event,
2026 fbuffer->entry, &tt))
2027 goto discard;
2028
2029 if (static_key_false(&tracepoint_printk_key.key))
2030 output_printk(fbuffer);
2031
2032 if (static_branch_unlikely(&trace_event_exports_enabled))
2033 ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
2034
2035 trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer,
2036 fbuffer->event, fbuffer->trace_ctx, fbuffer->regs);
2037
2038discard:
2039 if (tt)
2040 event_triggers_post_call(file, tt);
2041
2042}
2043EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2044
2045/*
2046 * Skip 3:
2047 *
2048 * trace_buffer_unlock_commit_regs()
2049 * trace_event_buffer_commit()
2050 * trace_event_raw_event_xxx()
2051 */
2052# define STACK_SKIP 3
2053
2054void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2055 struct trace_buffer *buffer,
2056 struct ring_buffer_event *event,
2057 unsigned int trace_ctx,
2058 struct pt_regs *regs)
2059{
2060 __buffer_unlock_commit(buffer, event);
2061
2062 /*
2063 * If regs is not set, then skip the necessary functions.
2064 * Note, we can still get here via blktrace, wakeup tracer
2065 * and mmiotrace, but that's ok if they lose a function or
2066 * two. They are not that meaningful.
2067 */
2068 ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs);
2069 ftrace_trace_userstack(tr, buffer, trace_ctx);
2070}
2071
2072/*
2073 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2074 */
2075void
2076trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2077 struct ring_buffer_event *event)
2078{
2079 __buffer_unlock_commit(buffer, event);
2080}
2081
2082void
2083trace_function(struct trace_array *tr, unsigned long ip, unsigned long
2084 parent_ip, unsigned int trace_ctx, struct ftrace_regs *fregs)
2085{
2086 struct trace_buffer *buffer = tr->array_buffer.buffer;
2087 struct ring_buffer_event *event;
2088 struct ftrace_entry *entry;
2089 int size = sizeof(*entry);
2090
2091 size += FTRACE_REGS_MAX_ARGS * !!fregs * sizeof(long);
2092
2093 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, size,
2094 trace_ctx);
2095 if (!event)
2096 return;
2097 entry = ring_buffer_event_data(event);
2098 entry->ip = ip;
2099 entry->parent_ip = parent_ip;
2100
2101#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
2102 if (fregs) {
2103 for (int i = 0; i < FTRACE_REGS_MAX_ARGS; i++)
2104 entry->args[i] = ftrace_regs_get_argument(fregs, i);
2105 }
2106#endif
2107
2108 if (static_branch_unlikely(&trace_function_exports_enabled))
2109 ftrace_exports(event, TRACE_EXPORT_FUNCTION);
2110 __buffer_unlock_commit(buffer, event);
2111}
2112
2113#ifdef CONFIG_STACKTRACE
2114
2115/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2116#define FTRACE_KSTACK_NESTING 4
2117
2118#define FTRACE_KSTACK_ENTRIES (SZ_4K / FTRACE_KSTACK_NESTING)
2119
2120struct ftrace_stack {
2121 unsigned long calls[FTRACE_KSTACK_ENTRIES];
2122};
2123
2124
2125struct ftrace_stacks {
2126 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2127};
2128
2129static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2130static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2131
2132void __ftrace_trace_stack(struct trace_array *tr,
2133 struct trace_buffer *buffer,
2134 unsigned int trace_ctx,
2135 int skip, struct pt_regs *regs)
2136{
2137 struct ring_buffer_event *event;
2138 unsigned int size, nr_entries;
2139 struct ftrace_stack *fstack;
2140 struct stack_entry *entry;
2141 int stackidx;
2142 int bit;
2143
2144 bit = trace_test_and_set_recursion(_THIS_IP_, _RET_IP_, TRACE_EVENT_START);
2145 if (bit < 0)
2146 return;
2147
2148 /*
2149 * Add one, for this function and the call to save_stack_trace()
2150 * If regs is set, then these functions will not be in the way.
2151 */
2152#ifndef CONFIG_UNWINDER_ORC
2153 if (!regs)
2154 skip++;
2155#endif
2156
2157 guard(preempt_notrace)();
2158
2159 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2160
2161 /* This should never happen. If it does, yell once and skip */
2162 if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
2163 goto out;
2164
2165 /*
2166 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2167 * interrupt will either see the value pre increment or post
2168 * increment. If the interrupt happens pre increment it will have
2169 * restored the counter when it returns. We just need a barrier to
2170 * keep gcc from moving things around.
2171 */
2172 barrier();
2173
2174 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2175 size = ARRAY_SIZE(fstack->calls);
2176
2177 if (regs) {
2178 nr_entries = stack_trace_save_regs(regs, fstack->calls,
2179 size, skip);
2180 } else {
2181 nr_entries = stack_trace_save(fstack->calls, size, skip);
2182 }
2183
2184#ifdef CONFIG_DYNAMIC_FTRACE
2185 /* Mark entry of stack trace as trampoline code */
2186 if (tr->ops && tr->ops->trampoline) {
2187 unsigned long tramp_start = tr->ops->trampoline;
2188 unsigned long tramp_end = tramp_start + tr->ops->trampoline_size;
2189 unsigned long *calls = fstack->calls;
2190
2191 for (int i = 0; i < nr_entries; i++) {
2192 if (calls[i] >= tramp_start && calls[i] < tramp_end)
2193 calls[i] = FTRACE_TRAMPOLINE_MARKER;
2194 }
2195 }
2196#endif
2197
2198 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2199 struct_size(entry, caller, nr_entries),
2200 trace_ctx);
2201 if (!event)
2202 goto out;
2203 entry = ring_buffer_event_data(event);
2204
2205 entry->size = nr_entries;
2206 memcpy(&entry->caller, fstack->calls,
2207 flex_array_size(entry, caller, nr_entries));
2208
2209 __buffer_unlock_commit(buffer, event);
2210
2211 out:
2212 /* Again, don't let gcc optimize things here */
2213 barrier();
2214 __this_cpu_dec(ftrace_stack_reserve);
2215 trace_clear_recursion(bit);
2216}
2217
2218void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
2219 int skip)
2220{
2221 struct trace_buffer *buffer = tr->array_buffer.buffer;
2222
2223 if (rcu_is_watching()) {
2224 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL);
2225 return;
2226 }
2227
2228 if (WARN_ON_ONCE(IS_ENABLED(CONFIG_GENERIC_ENTRY)))
2229 return;
2230
2231 /*
2232 * When an NMI triggers, RCU is enabled via ct_nmi_enter(),
2233 * but if the above rcu_is_watching() failed, then the NMI
2234 * triggered someplace critical, and ct_irq_enter() should
2235 * not be called from NMI.
2236 */
2237 if (unlikely(in_nmi()))
2238 return;
2239
2240 ct_irq_enter_irqson();
2241 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL);
2242 ct_irq_exit_irqson();
2243}
2244
2245/**
2246 * trace_dump_stack - record a stack back trace in the trace buffer
2247 * @skip: Number of functions to skip (helper handlers)
2248 */
2249void trace_dump_stack(int skip)
2250{
2251 if (tracing_disabled || tracing_selftest_running)
2252 return;
2253
2254#ifndef CONFIG_UNWINDER_ORC
2255 /* Skip 1 to skip this function. */
2256 skip++;
2257#endif
2258 __ftrace_trace_stack(printk_trace, printk_trace->array_buffer.buffer,
2259 tracing_gen_ctx(), skip, NULL);
2260}
2261EXPORT_SYMBOL_GPL(trace_dump_stack);
2262
2263#ifdef CONFIG_USER_STACKTRACE_SUPPORT
2264static DEFINE_PER_CPU(int, user_stack_count);
2265
2266static void
2267ftrace_trace_userstack(struct trace_array *tr,
2268 struct trace_buffer *buffer, unsigned int trace_ctx)
2269{
2270 struct ring_buffer_event *event;
2271 struct userstack_entry *entry;
2272
2273 if (!(tr->trace_flags & TRACE_ITER(USERSTACKTRACE)))
2274 return;
2275
2276 /*
2277 * NMIs can not handle page faults, even with fix ups.
2278 * The save user stack can (and often does) fault.
2279 */
2280 if (unlikely(in_nmi()))
2281 return;
2282
2283 /*
2284 * prevent recursion, since the user stack tracing may
2285 * trigger other kernel events.
2286 */
2287 guard(preempt)();
2288 if (__this_cpu_read(user_stack_count))
2289 return;
2290
2291 __this_cpu_inc(user_stack_count);
2292
2293 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2294 sizeof(*entry), trace_ctx);
2295 if (!event)
2296 goto out_drop_count;
2297 entry = ring_buffer_event_data(event);
2298
2299 entry->tgid = current->tgid;
2300 memset(&entry->caller, 0, sizeof(entry->caller));
2301
2302 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
2303 __buffer_unlock_commit(buffer, event);
2304
2305 out_drop_count:
2306 __this_cpu_dec(user_stack_count);
2307}
2308#else /* CONFIG_USER_STACKTRACE_SUPPORT */
2309static void ftrace_trace_userstack(struct trace_array *tr,
2310 struct trace_buffer *buffer,
2311 unsigned int trace_ctx)
2312{
2313}
2314#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
2315
2316#endif /* CONFIG_STACKTRACE */
2317
2318static inline void
2319func_repeats_set_delta_ts(struct func_repeats_entry *entry,
2320 unsigned long long delta)
2321{
2322 entry->bottom_delta_ts = delta & U32_MAX;
2323 entry->top_delta_ts = (delta >> 32);
2324}
2325
2326void trace_last_func_repeats(struct trace_array *tr,
2327 struct trace_func_repeats *last_info,
2328 unsigned int trace_ctx)
2329{
2330 struct trace_buffer *buffer = tr->array_buffer.buffer;
2331 struct func_repeats_entry *entry;
2332 struct ring_buffer_event *event;
2333 u64 delta;
2334
2335 event = __trace_buffer_lock_reserve(buffer, TRACE_FUNC_REPEATS,
2336 sizeof(*entry), trace_ctx);
2337 if (!event)
2338 return;
2339
2340 delta = ring_buffer_event_time_stamp(buffer, event) -
2341 last_info->ts_last_call;
2342
2343 entry = ring_buffer_event_data(event);
2344 entry->ip = last_info->ip;
2345 entry->parent_ip = last_info->parent_ip;
2346 entry->count = last_info->count;
2347 func_repeats_set_delta_ts(entry, delta);
2348
2349 __buffer_unlock_commit(buffer, event);
2350}
2351
2352static struct trace_entry *
2353peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2354 unsigned long *lost_events)
2355{
2356 struct ring_buffer_event *event;
2357 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2358
2359 if (buf_iter) {
2360 event = ring_buffer_iter_peek(buf_iter, ts);
2361 if (lost_events)
2362 *lost_events = ring_buffer_iter_dropped(buf_iter) ?
2363 (unsigned long)-1 : 0;
2364 } else {
2365 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
2366 lost_events);
2367 }
2368
2369 if (event) {
2370 iter->ent_size = ring_buffer_event_length(event);
2371 return ring_buffer_event_data(event);
2372 }
2373 iter->ent_size = 0;
2374 return NULL;
2375}
2376
2377static struct trace_entry *
2378__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2379 unsigned long *missing_events, u64 *ent_ts)
2380{
2381 struct trace_buffer *buffer = iter->array_buffer->buffer;
2382 struct trace_entry *ent, *next = NULL;
2383 unsigned long lost_events = 0, next_lost = 0;
2384 int cpu_file = iter->cpu_file;
2385 u64 next_ts = 0, ts;
2386 int next_cpu = -1;
2387 int next_size = 0;
2388 int cpu;
2389
2390 /*
2391 * If we are in a per_cpu trace file, don't bother by iterating over
2392 * all cpu and peek directly.
2393 */
2394 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2395 if (ring_buffer_empty_cpu(buffer, cpu_file))
2396 return NULL;
2397 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2398 if (ent_cpu)
2399 *ent_cpu = cpu_file;
2400
2401 return ent;
2402 }
2403
2404 for_each_tracing_cpu(cpu) {
2405
2406 if (ring_buffer_empty_cpu(buffer, cpu))
2407 continue;
2408
2409 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2410
2411 /*
2412 * Pick the entry with the smallest timestamp:
2413 */
2414 if (ent && (!next || ts < next_ts)) {
2415 next = ent;
2416 next_cpu = cpu;
2417 next_ts = ts;
2418 next_lost = lost_events;
2419 next_size = iter->ent_size;
2420 }
2421 }
2422
2423 iter->ent_size = next_size;
2424
2425 if (ent_cpu)
2426 *ent_cpu = next_cpu;
2427
2428 if (ent_ts)
2429 *ent_ts = next_ts;
2430
2431 if (missing_events)
2432 *missing_events = next_lost;
2433
2434 return next;
2435}
2436
2437#define STATIC_FMT_BUF_SIZE 128
2438static char static_fmt_buf[STATIC_FMT_BUF_SIZE];
2439
2440char *trace_iter_expand_format(struct trace_iterator *iter)
2441{
2442 char *tmp;
2443
2444 /*
2445 * iter->tr is NULL when used with tp_printk, which makes
2446 * this get called where it is not safe to call krealloc().
2447 */
2448 if (!iter->tr || iter->fmt == static_fmt_buf)
2449 return NULL;
2450
2451 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE,
2452 GFP_KERNEL);
2453 if (tmp) {
2454 iter->fmt_size += STATIC_FMT_BUF_SIZE;
2455 iter->fmt = tmp;
2456 }
2457
2458 return tmp;
2459}
2460
2461/* Returns true if the string is safe to dereference from an event */
2462static bool trace_safe_str(struct trace_iterator *iter, const char *str)
2463{
2464 unsigned long addr = (unsigned long)str;
2465 struct trace_event *trace_event;
2466 struct trace_event_call *event;
2467
2468 /* OK if part of the event data */
2469 if ((addr >= (unsigned long)iter->ent) &&
2470 (addr < (unsigned long)iter->ent + iter->ent_size))
2471 return true;
2472
2473 /* OK if part of the temp seq buffer */
2474 if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&
2475 (addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE))
2476 return true;
2477
2478 /* Core rodata can not be freed */
2479 if (is_kernel_rodata(addr))
2480 return true;
2481
2482 if (trace_is_tracepoint_string(str))
2483 return true;
2484
2485 /*
2486 * Now this could be a module event, referencing core module
2487 * data, which is OK.
2488 */
2489 if (!iter->ent)
2490 return false;
2491
2492 trace_event = ftrace_find_event(iter->ent->type);
2493 if (!trace_event)
2494 return false;
2495
2496 event = container_of(trace_event, struct trace_event_call, event);
2497 if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module)
2498 return false;
2499
2500 /* Would rather have rodata, but this will suffice */
2501 if (within_module_core(addr, event->module))
2502 return true;
2503
2504 return false;
2505}
2506
2507/**
2508 * ignore_event - Check dereferenced fields while writing to the seq buffer
2509 * @iter: The iterator that holds the seq buffer and the event being printed
2510 *
2511 * At boot up, test_event_printk() will flag any event that dereferences
2512 * a string with "%s" that does exist in the ring buffer. It may still
2513 * be valid, as the string may point to a static string in the kernel
2514 * rodata that never gets freed. But if the string pointer is pointing
2515 * to something that was allocated, there's a chance that it can be freed
2516 * by the time the user reads the trace. This would cause a bad memory
2517 * access by the kernel and possibly crash the system.
2518 *
2519 * This function will check if the event has any fields flagged as needing
2520 * to be checked at runtime and perform those checks.
2521 *
2522 * If it is found that a field is unsafe, it will write into the @iter->seq
2523 * a message stating what was found to be unsafe.
2524 *
2525 * @return: true if the event is unsafe and should be ignored,
2526 * false otherwise.
2527 */
2528bool ignore_event(struct trace_iterator *iter)
2529{
2530 struct ftrace_event_field *field;
2531 struct trace_event *trace_event;
2532 struct trace_event_call *event;
2533 struct list_head *head;
2534 struct trace_seq *seq;
2535 const void *ptr;
2536
2537 trace_event = ftrace_find_event(iter->ent->type);
2538
2539 seq = &iter->seq;
2540
2541 if (!trace_event) {
2542 trace_seq_printf(seq, "EVENT ID %d NOT FOUND?\n", iter->ent->type);
2543 return true;
2544 }
2545
2546 event = container_of(trace_event, struct trace_event_call, event);
2547 if (!(event->flags & TRACE_EVENT_FL_TEST_STR))
2548 return false;
2549
2550 head = trace_get_fields(event);
2551 if (!head) {
2552 trace_seq_printf(seq, "FIELDS FOR EVENT '%s' NOT FOUND?\n",
2553 trace_event_name(event));
2554 return true;
2555 }
2556
2557 /* Offsets are from the iter->ent that points to the raw event */
2558 ptr = iter->ent;
2559
2560 list_for_each_entry(field, head, link) {
2561 const char *str;
2562 bool good;
2563
2564 if (!field->needs_test)
2565 continue;
2566
2567 str = *(const char **)(ptr + field->offset);
2568
2569 good = trace_safe_str(iter, str);
2570
2571 /*
2572 * If you hit this warning, it is likely that the
2573 * trace event in question used %s on a string that
2574 * was saved at the time of the event, but may not be
2575 * around when the trace is read. Use __string(),
2576 * __assign_str() and __get_str() helpers in the TRACE_EVENT()
2577 * instead. See samples/trace_events/trace-events-sample.h
2578 * for reference.
2579 */
2580 if (WARN_ONCE(!good, "event '%s' has unsafe pointer field '%s'",
2581 trace_event_name(event), field->name)) {
2582 trace_seq_printf(seq, "EVENT %s: HAS UNSAFE POINTER FIELD '%s'\n",
2583 trace_event_name(event), field->name);
2584 return true;
2585 }
2586 }
2587 return false;
2588}
2589
2590const char *trace_event_format(struct trace_iterator *iter, const char *fmt)
2591{
2592 const char *p, *new_fmt;
2593 char *q;
2594
2595 if (WARN_ON_ONCE(!fmt))
2596 return fmt;
2597
2598 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER(HASH_PTR))
2599 return fmt;
2600
2601 p = fmt;
2602 new_fmt = q = iter->fmt;
2603 while (*p) {
2604 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {
2605 if (!trace_iter_expand_format(iter))
2606 return fmt;
2607
2608 q += iter->fmt - new_fmt;
2609 new_fmt = iter->fmt;
2610 }
2611
2612 *q++ = *p++;
2613
2614 /* Replace %p with %px */
2615 if (p[-1] == '%') {
2616 if (p[0] == '%') {
2617 *q++ = *p++;
2618 } else if (p[0] == 'p' && !isalnum(p[1])) {
2619 *q++ = *p++;
2620 *q++ = 'x';
2621 }
2622 }
2623 }
2624 *q = '\0';
2625
2626 return new_fmt;
2627}
2628
2629#define STATIC_TEMP_BUF_SIZE 128
2630static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
2631
2632/* Find the next real entry, without updating the iterator itself */
2633struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2634 int *ent_cpu, u64 *ent_ts)
2635{
2636 /* __find_next_entry will reset ent_size */
2637 int ent_size = iter->ent_size;
2638 struct trace_entry *entry;
2639
2640 /*
2641 * If called from ftrace_dump(), then the iter->temp buffer
2642 * will be the static_temp_buf and not created from kmalloc.
2643 * If the entry size is greater than the buffer, we can
2644 * not save it. Just return NULL in that case. This is only
2645 * used to add markers when two consecutive events' time
2646 * stamps have a large delta. See trace_print_lat_context()
2647 */
2648 if (iter->temp == static_temp_buf &&
2649 STATIC_TEMP_BUF_SIZE < ent_size)
2650 return NULL;
2651
2652 /*
2653 * The __find_next_entry() may call peek_next_entry(), which may
2654 * call ring_buffer_peek() that may make the contents of iter->ent
2655 * undefined. Need to copy iter->ent now.
2656 */
2657 if (iter->ent && iter->ent != iter->temp) {
2658 if ((!iter->temp || iter->temp_size < iter->ent_size) &&
2659 !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
2660 void *temp;
2661 temp = kmalloc(iter->ent_size, GFP_KERNEL);
2662 if (!temp)
2663 return NULL;
2664 kfree(iter->temp);
2665 iter->temp = temp;
2666 iter->temp_size = iter->ent_size;
2667 }
2668 memcpy(iter->temp, iter->ent, iter->ent_size);
2669 iter->ent = iter->temp;
2670 }
2671 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2672 /* Put back the original ent_size */
2673 iter->ent_size = ent_size;
2674
2675 return entry;
2676}
2677
2678/* Find the next real entry, and increment the iterator to the next entry */
2679void *trace_find_next_entry_inc(struct trace_iterator *iter)
2680{
2681 struct ring_buffer_iter *buf_iter;
2682
2683 iter->ent = __find_next_entry(iter, &iter->cpu,
2684 &iter->lost_events, &iter->ts);
2685
2686 if (iter->ent) {
2687 iter->idx++;
2688 buf_iter = trace_buffer_iter(iter, iter->cpu);
2689 if (buf_iter)
2690 ring_buffer_iter_advance(buf_iter);
2691 }
2692
2693 return iter->ent ? iter : NULL;
2694}
2695
2696static void trace_consume(struct trace_iterator *iter)
2697{
2698 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
2699 &iter->lost_events);
2700}
2701
2702static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2703{
2704 struct trace_iterator *iter = m->private;
2705 int i = (int)*pos;
2706 void *ent;
2707
2708 WARN_ON_ONCE(iter->leftover);
2709
2710 (*pos)++;
2711
2712 /* can't go backwards */
2713 if (iter->idx > i)
2714 return NULL;
2715
2716 if (iter->idx < 0)
2717 ent = trace_find_next_entry_inc(iter);
2718 else
2719 ent = iter;
2720
2721 while (ent && iter->idx < i)
2722 ent = trace_find_next_entry_inc(iter);
2723
2724 iter->pos = *pos;
2725
2726 return ent;
2727}
2728
2729void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2730{
2731 struct ring_buffer_iter *buf_iter;
2732 unsigned long entries = 0;
2733 u64 ts;
2734
2735 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
2736
2737 buf_iter = trace_buffer_iter(iter, cpu);
2738 if (!buf_iter)
2739 return;
2740
2741 ring_buffer_iter_reset(buf_iter);
2742
2743 /*
2744 * We could have the case with the max latency tracers
2745 * that a reset never took place on a cpu. This is evident
2746 * by the timestamp being before the start of the buffer.
2747 */
2748 while (ring_buffer_iter_peek(buf_iter, &ts)) {
2749 if (ts >= iter->array_buffer->time_start)
2750 break;
2751 entries++;
2752 ring_buffer_iter_advance(buf_iter);
2753 /* This could be a big loop */
2754 cond_resched();
2755 }
2756
2757 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
2758}
2759
2760/*
2761 * The current tracer is copied to avoid a global locking
2762 * all around.
2763 */
2764static void *s_start(struct seq_file *m, loff_t *pos)
2765{
2766 struct trace_iterator *iter = m->private;
2767 struct trace_array *tr = iter->tr;
2768 int cpu_file = iter->cpu_file;
2769 void *p = NULL;
2770 loff_t l = 0;
2771 int cpu;
2772
2773 mutex_lock(&trace_types_lock);
2774 if (unlikely(tr->current_trace != iter->trace)) {
2775 /* Close iter->trace before switching to the new current tracer */
2776 if (iter->trace->close)
2777 iter->trace->close(iter);
2778 iter->trace = tr->current_trace;
2779 /* Reopen the new current tracer */
2780 if (iter->trace->open)
2781 iter->trace->open(iter);
2782 }
2783 mutex_unlock(&trace_types_lock);
2784
2785 if (iter->snapshot && tracer_uses_snapshot(iter->trace))
2786 return ERR_PTR(-EBUSY);
2787
2788 if (*pos != iter->pos) {
2789 iter->ent = NULL;
2790 iter->cpu = 0;
2791 iter->idx = -1;
2792
2793 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2794 for_each_tracing_cpu(cpu)
2795 tracing_iter_reset(iter, cpu);
2796 } else
2797 tracing_iter_reset(iter, cpu_file);
2798
2799 iter->leftover = 0;
2800 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2801 ;
2802
2803 } else {
2804 /*
2805 * If we overflowed the seq_file before, then we want
2806 * to just reuse the trace_seq buffer again.
2807 */
2808 if (iter->leftover)
2809 p = iter;
2810 else {
2811 l = *pos - 1;
2812 p = s_next(m, p, &l);
2813 }
2814 }
2815
2816 trace_event_read_lock();
2817 trace_access_lock(cpu_file);
2818 return p;
2819}
2820
2821static void s_stop(struct seq_file *m, void *p)
2822{
2823 struct trace_iterator *iter = m->private;
2824
2825 if (iter->snapshot && tracer_uses_snapshot(iter->trace))
2826 return;
2827
2828 trace_access_unlock(iter->cpu_file);
2829 trace_event_read_unlock();
2830}
2831
2832static void
2833get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
2834 unsigned long *entries, int cpu)
2835{
2836 unsigned long count;
2837
2838 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2839 /*
2840 * If this buffer has skipped entries, then we hold all
2841 * entries for the trace and we need to ignore the
2842 * ones before the time stamp.
2843 */
2844 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2845 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2846 /* total is the same as the entries */
2847 *total = count;
2848 } else
2849 *total = count +
2850 ring_buffer_overrun_cpu(buf->buffer, cpu);
2851 *entries = count;
2852}
2853
2854static void
2855get_total_entries(struct array_buffer *buf,
2856 unsigned long *total, unsigned long *entries)
2857{
2858 unsigned long t, e;
2859 int cpu;
2860
2861 *total = 0;
2862 *entries = 0;
2863
2864 for_each_tracing_cpu(cpu) {
2865 get_total_entries_cpu(buf, &t, &e, cpu);
2866 *total += t;
2867 *entries += e;
2868 }
2869}
2870
2871unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
2872{
2873 unsigned long total, entries;
2874
2875 if (!tr)
2876 tr = &global_trace;
2877
2878 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
2879
2880 return entries;
2881}
2882
2883unsigned long trace_total_entries(struct trace_array *tr)
2884{
2885 unsigned long total, entries;
2886
2887 if (!tr)
2888 tr = &global_trace;
2889
2890 get_total_entries(&tr->array_buffer, &total, &entries);
2891
2892 return entries;
2893}
2894
2895static void print_lat_help_header(struct seq_file *m)
2896{
2897 seq_puts(m, "# _------=> CPU# \n"
2898 "# / _-----=> irqs-off/BH-disabled\n"
2899 "# | / _----=> need-resched \n"
2900 "# || / _---=> hardirq/softirq \n"
2901 "# ||| / _--=> preempt-depth \n"
2902 "# |||| / _-=> migrate-disable \n"
2903 "# ||||| / delay \n"
2904 "# cmd pid |||||| time | caller \n"
2905 "# \\ / |||||| \\ | / \n");
2906}
2907
2908static void print_event_info(struct array_buffer *buf, struct seq_file *m)
2909{
2910 unsigned long total;
2911 unsigned long entries;
2912
2913 get_total_entries(buf, &total, &entries);
2914 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2915 entries, total, num_online_cpus());
2916 seq_puts(m, "#\n");
2917}
2918
2919static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
2920 unsigned int flags)
2921{
2922 bool tgid = flags & TRACE_ITER(RECORD_TGID);
2923
2924 print_event_info(buf, m);
2925
2926 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
2927 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
2928}
2929
2930static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
2931 unsigned int flags)
2932{
2933 bool tgid = flags & TRACE_ITER(RECORD_TGID);
2934 static const char space[] = " ";
2935 int prec = tgid ? 12 : 2;
2936
2937 print_event_info(buf, m);
2938
2939 seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space);
2940 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
2941 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
2942 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
2943 seq_printf(m, "# %.*s||| / _-=> migrate-disable\n", prec, space);
2944 seq_printf(m, "# %.*s|||| / delay\n", prec, space);
2945 seq_printf(m, "# TASK-PID %.*s CPU# ||||| TIMESTAMP FUNCTION\n", prec, " TGID ");
2946 seq_printf(m, "# | | %.*s | ||||| | |\n", prec, " | ");
2947}
2948
2949void
2950print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2951{
2952 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2953 struct array_buffer *buf = iter->array_buffer;
2954 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2955 struct tracer *type = iter->trace;
2956 unsigned long entries;
2957 unsigned long total;
2958 const char *name = type->name;
2959
2960 get_total_entries(buf, &total, &entries);
2961
2962 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2963 name, init_utsname()->release);
2964 seq_puts(m, "# -----------------------------------"
2965 "---------------------------------\n");
2966 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2967 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2968 nsecs_to_usecs(data->saved_latency),
2969 entries,
2970 total,
2971 buf->cpu,
2972 preempt_model_str(),
2973 /* These are reserved for later use */
2974 0, 0, 0, 0);
2975#ifdef CONFIG_SMP
2976 seq_printf(m, " #P:%d)\n", num_online_cpus());
2977#else
2978 seq_puts(m, ")\n");
2979#endif
2980 seq_puts(m, "# -----------------\n");
2981 seq_printf(m, "# | task: %.16s-%d "
2982 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2983 data->comm, data->pid,
2984 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2985 data->policy, data->rt_priority);
2986 seq_puts(m, "# -----------------\n");
2987
2988 if (data->critical_start) {
2989 seq_puts(m, "# => started at: ");
2990 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2991 trace_print_seq(m, &iter->seq);
2992 seq_puts(m, "\n# => ended at: ");
2993 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2994 trace_print_seq(m, &iter->seq);
2995 seq_puts(m, "\n#\n");
2996 }
2997
2998 seq_puts(m, "#\n");
2999}
3000
3001static void test_cpu_buff_start(struct trace_iterator *iter)
3002{
3003 struct trace_seq *s = &iter->seq;
3004 struct trace_array *tr = iter->tr;
3005
3006 if (!(tr->trace_flags & TRACE_ITER(ANNOTATE)))
3007 return;
3008
3009 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3010 return;
3011
3012 if (cpumask_available(iter->started) &&
3013 cpumask_test_cpu(iter->cpu, iter->started))
3014 return;
3015
3016 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
3017 return;
3018
3019 if (cpumask_available(iter->started))
3020 cpumask_set_cpu(iter->cpu, iter->started);
3021
3022 /* Don't print started cpu buffer for the first entry of the trace */
3023 if (iter->idx > 1)
3024 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3025 iter->cpu);
3026}
3027
3028#ifdef CONFIG_FTRACE_SYSCALLS
3029static bool is_syscall_event(struct trace_event *event)
3030{
3031 return (event->funcs == &enter_syscall_print_funcs) ||
3032 (event->funcs == &exit_syscall_print_funcs);
3033
3034}
3035#define syscall_buf_size CONFIG_TRACE_SYSCALL_BUF_SIZE_DEFAULT
3036#else
3037static inline bool is_syscall_event(struct trace_event *event)
3038{
3039 return false;
3040}
3041#define syscall_buf_size 0
3042#endif /* CONFIG_FTRACE_SYSCALLS */
3043
3044static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3045{
3046 struct trace_array *tr = iter->tr;
3047 struct trace_seq *s = &iter->seq;
3048 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3049 struct trace_entry *entry;
3050 struct trace_event *event;
3051
3052 entry = iter->ent;
3053
3054 test_cpu_buff_start(iter);
3055
3056 event = ftrace_find_event(entry->type);
3057
3058 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) {
3059 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3060 trace_print_lat_context(iter);
3061 else
3062 trace_print_context(iter);
3063 }
3064
3065 if (trace_seq_has_overflowed(s))
3066 return TRACE_TYPE_PARTIAL_LINE;
3067
3068 if (event) {
3069 if (tr->trace_flags & TRACE_ITER(FIELDS))
3070 return print_event_fields(iter, event);
3071 /*
3072 * For TRACE_EVENT() events, the print_fmt is not
3073 * safe to use if the array has delta offsets
3074 * Force printing via the fields.
3075 */
3076 if ((tr->text_delta)) {
3077 /* ftrace and system call events are still OK */
3078 if ((event->type > __TRACE_LAST_TYPE) &&
3079 !is_syscall_event(event))
3080 return print_event_fields(iter, event);
3081 }
3082 return event->funcs->trace(iter, sym_flags, event);
3083 }
3084
3085 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3086
3087 return trace_handle_return(s);
3088}
3089
3090static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3091{
3092 struct trace_array *tr = iter->tr;
3093 struct trace_seq *s = &iter->seq;
3094 struct trace_entry *entry;
3095 struct trace_event *event;
3096
3097 entry = iter->ent;
3098
3099 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO))
3100 trace_seq_printf(s, "%d %d %llu ",
3101 entry->pid, iter->cpu, iter->ts);
3102
3103 if (trace_seq_has_overflowed(s))
3104 return TRACE_TYPE_PARTIAL_LINE;
3105
3106 event = ftrace_find_event(entry->type);
3107 if (event)
3108 return event->funcs->raw(iter, 0, event);
3109
3110 trace_seq_printf(s, "%d ?\n", entry->type);
3111
3112 return trace_handle_return(s);
3113}
3114
3115static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3116{
3117 struct trace_array *tr = iter->tr;
3118 struct trace_seq *s = &iter->seq;
3119 unsigned char newline = '\n';
3120 struct trace_entry *entry;
3121 struct trace_event *event;
3122
3123 entry = iter->ent;
3124
3125 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) {
3126 SEQ_PUT_HEX_FIELD(s, entry->pid);
3127 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3128 SEQ_PUT_HEX_FIELD(s, iter->ts);
3129 if (trace_seq_has_overflowed(s))
3130 return TRACE_TYPE_PARTIAL_LINE;
3131 }
3132
3133 event = ftrace_find_event(entry->type);
3134 if (event) {
3135 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3136 if (ret != TRACE_TYPE_HANDLED)
3137 return ret;
3138 }
3139
3140 SEQ_PUT_FIELD(s, newline);
3141
3142 return trace_handle_return(s);
3143}
3144
3145static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3146{
3147 struct trace_array *tr = iter->tr;
3148 struct trace_seq *s = &iter->seq;
3149 struct trace_entry *entry;
3150 struct trace_event *event;
3151
3152 entry = iter->ent;
3153
3154 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) {
3155 SEQ_PUT_FIELD(s, entry->pid);
3156 SEQ_PUT_FIELD(s, iter->cpu);
3157 SEQ_PUT_FIELD(s, iter->ts);
3158 if (trace_seq_has_overflowed(s))
3159 return TRACE_TYPE_PARTIAL_LINE;
3160 }
3161
3162 event = ftrace_find_event(entry->type);
3163 return event ? event->funcs->binary(iter, 0, event) :
3164 TRACE_TYPE_HANDLED;
3165}
3166
3167int trace_empty(struct trace_iterator *iter)
3168{
3169 struct ring_buffer_iter *buf_iter;
3170 int cpu;
3171
3172 /* If we are looking at one CPU buffer, only check that one */
3173 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3174 cpu = iter->cpu_file;
3175 buf_iter = trace_buffer_iter(iter, cpu);
3176 if (buf_iter) {
3177 if (!ring_buffer_iter_empty(buf_iter))
3178 return 0;
3179 } else {
3180 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
3181 return 0;
3182 }
3183 return 1;
3184 }
3185
3186 for_each_tracing_cpu(cpu) {
3187 buf_iter = trace_buffer_iter(iter, cpu);
3188 if (buf_iter) {
3189 if (!ring_buffer_iter_empty(buf_iter))
3190 return 0;
3191 } else {
3192 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
3193 return 0;
3194 }
3195 }
3196
3197 return 1;
3198}
3199
3200/* Called with trace_event_read_lock() held. */
3201enum print_line_t print_trace_line(struct trace_iterator *iter)
3202{
3203 struct trace_array *tr = iter->tr;
3204 unsigned long trace_flags = tr->trace_flags;
3205 enum print_line_t ret;
3206
3207 if (iter->lost_events) {
3208 if (iter->lost_events == (unsigned long)-1)
3209 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
3210 iter->cpu);
3211 else
3212 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3213 iter->cpu, iter->lost_events);
3214 if (trace_seq_has_overflowed(&iter->seq))
3215 return TRACE_TYPE_PARTIAL_LINE;
3216 }
3217
3218 if (iter->trace && iter->trace->print_line) {
3219 ret = iter->trace->print_line(iter);
3220 if (ret != TRACE_TYPE_UNHANDLED)
3221 return ret;
3222 }
3223
3224 if (iter->ent->type == TRACE_BPUTS &&
3225 trace_flags & TRACE_ITER(PRINTK) &&
3226 trace_flags & TRACE_ITER(PRINTK_MSGONLY))
3227 return trace_print_bputs_msg_only(iter);
3228
3229 if (iter->ent->type == TRACE_BPRINT &&
3230 trace_flags & TRACE_ITER(PRINTK) &&
3231 trace_flags & TRACE_ITER(PRINTK_MSGONLY))
3232 return trace_print_bprintk_msg_only(iter);
3233
3234 if (iter->ent->type == TRACE_PRINT &&
3235 trace_flags & TRACE_ITER(PRINTK) &&
3236 trace_flags & TRACE_ITER(PRINTK_MSGONLY))
3237 return trace_print_printk_msg_only(iter);
3238
3239 if (trace_flags & TRACE_ITER(BIN))
3240 return print_bin_fmt(iter);
3241
3242 if (trace_flags & TRACE_ITER(HEX))
3243 return print_hex_fmt(iter);
3244
3245 if (trace_flags & TRACE_ITER(RAW))
3246 return print_raw_fmt(iter);
3247
3248 return print_trace_fmt(iter);
3249}
3250
3251void trace_latency_header(struct seq_file *m)
3252{
3253 struct trace_iterator *iter = m->private;
3254 struct trace_array *tr = iter->tr;
3255
3256 /* print nothing if the buffers are empty */
3257 if (trace_empty(iter))
3258 return;
3259
3260 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3261 print_trace_header(m, iter);
3262
3263 if (!(tr->trace_flags & TRACE_ITER(VERBOSE)))
3264 print_lat_help_header(m);
3265}
3266
3267void trace_default_header(struct seq_file *m)
3268{
3269 struct trace_iterator *iter = m->private;
3270 struct trace_array *tr = iter->tr;
3271 unsigned long trace_flags = tr->trace_flags;
3272
3273 if (!(trace_flags & TRACE_ITER(CONTEXT_INFO)))
3274 return;
3275
3276 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3277 /* print nothing if the buffers are empty */
3278 if (trace_empty(iter))
3279 return;
3280 print_trace_header(m, iter);
3281 if (!(trace_flags & TRACE_ITER(VERBOSE)))
3282 print_lat_help_header(m);
3283 } else {
3284 if (!(trace_flags & TRACE_ITER(VERBOSE))) {
3285 if (trace_flags & TRACE_ITER(IRQ_INFO))
3286 print_func_help_header_irq(iter->array_buffer,
3287 m, trace_flags);
3288 else
3289 print_func_help_header(iter->array_buffer, m,
3290 trace_flags);
3291 }
3292 }
3293}
3294
3295static void test_ftrace_alive(struct seq_file *m)
3296{
3297 if (!ftrace_is_dead())
3298 return;
3299 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3300 "# MAY BE MISSING FUNCTION EVENTS\n");
3301}
3302
3303static int s_show(struct seq_file *m, void *v)
3304{
3305 struct trace_iterator *iter = v;
3306 int ret;
3307
3308 if (iter->ent == NULL) {
3309 if (iter->tr) {
3310 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3311 seq_puts(m, "#\n");
3312 test_ftrace_alive(m);
3313 }
3314 if (iter->snapshot && trace_empty(iter))
3315 print_snapshot_help(m, iter);
3316 else if (iter->trace && iter->trace->print_header)
3317 iter->trace->print_header(m);
3318 else
3319 trace_default_header(m);
3320
3321 } else if (iter->leftover) {
3322 /*
3323 * If we filled the seq_file buffer earlier, we
3324 * want to just show it now.
3325 */
3326 ret = trace_print_seq(m, &iter->seq);
3327
3328 /* ret should this time be zero, but you never know */
3329 iter->leftover = ret;
3330
3331 } else {
3332 ret = print_trace_line(iter);
3333 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3334 iter->seq.full = 0;
3335 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
3336 }
3337 ret = trace_print_seq(m, &iter->seq);
3338 /*
3339 * If we overflow the seq_file buffer, then it will
3340 * ask us for this data again at start up.
3341 * Use that instead.
3342 * ret is 0 if seq_file write succeeded.
3343 * -1 otherwise.
3344 */
3345 iter->leftover = ret;
3346 }
3347
3348 return 0;
3349}
3350
3351static const struct seq_operations tracer_seq_ops = {
3352 .start = s_start,
3353 .next = s_next,
3354 .stop = s_stop,
3355 .show = s_show,
3356};
3357
3358/*
3359 * Note, as iter itself can be allocated and freed in different
3360 * ways, this function is only used to free its content, and not
3361 * the iterator itself. The only requirement to all the allocations
3362 * is that it must zero all fields (kzalloc), as freeing works with
3363 * ethier allocated content or NULL.
3364 */
3365static void free_trace_iter_content(struct trace_iterator *iter)
3366{
3367 /* The fmt is either NULL, allocated or points to static_fmt_buf */
3368 if (iter->fmt != static_fmt_buf)
3369 kfree(iter->fmt);
3370
3371 kfree(iter->temp);
3372 kfree(iter->buffer_iter);
3373 mutex_destroy(&iter->mutex);
3374 free_cpumask_var(iter->started);
3375}
3376
3377struct trace_iterator *
3378__tracing_open(struct inode *inode, struct file *file, bool snapshot)
3379{
3380 struct trace_array *tr = inode->i_private;
3381 struct trace_iterator *iter;
3382 int cpu;
3383
3384 if (tracing_disabled)
3385 return ERR_PTR(-ENODEV);
3386
3387 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3388 if (!iter)
3389 return ERR_PTR(-ENOMEM);
3390
3391 iter->buffer_iter = kzalloc_objs(*iter->buffer_iter, nr_cpu_ids);
3392 if (!iter->buffer_iter)
3393 goto release;
3394
3395 /*
3396 * trace_find_next_entry() may need to save off iter->ent.
3397 * It will place it into the iter->temp buffer. As most
3398 * events are less than 128, allocate a buffer of that size.
3399 * If one is greater, then trace_find_next_entry() will
3400 * allocate a new buffer to adjust for the bigger iter->ent.
3401 * It's not critical if it fails to get allocated here.
3402 */
3403 iter->temp = kmalloc(128, GFP_KERNEL);
3404 if (iter->temp)
3405 iter->temp_size = 128;
3406
3407 /*
3408 * trace_event_printf() may need to modify given format
3409 * string to replace %p with %px so that it shows real address
3410 * instead of hash value. However, that is only for the event
3411 * tracing, other tracer may not need. Defer the allocation
3412 * until it is needed.
3413 */
3414 iter->fmt = NULL;
3415 iter->fmt_size = 0;
3416
3417 mutex_lock(&trace_types_lock);
3418 iter->trace = tr->current_trace;
3419
3420 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3421 goto fail;
3422
3423 iter->tr = tr;
3424
3425#ifdef CONFIG_TRACER_SNAPSHOT
3426 /* Currently only the top directory has a snapshot */
3427 if (tr->current_trace->print_max || snapshot)
3428 iter->array_buffer = &tr->snapshot_buffer;
3429 else
3430#endif
3431 iter->array_buffer = &tr->array_buffer;
3432 iter->snapshot = snapshot;
3433 iter->pos = -1;
3434 iter->cpu_file = tracing_get_cpu(inode);
3435 mutex_init(&iter->mutex);
3436
3437 /* Notify the tracer early; before we stop tracing. */
3438 if (iter->trace->open)
3439 iter->trace->open(iter);
3440
3441 /* Annotate start of buffers if we had overruns */
3442 if (ring_buffer_overruns(iter->array_buffer->buffer))
3443 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3444
3445 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3446 if (trace_clocks[tr->clock_id].in_ns)
3447 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3448
3449 /*
3450 * If pause-on-trace is enabled, then stop the trace while
3451 * dumping, unless this is the "snapshot" file
3452 */
3453 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER(PAUSE_ON_TRACE))) {
3454 iter->iter_flags |= TRACE_FILE_PAUSE;
3455 tracing_stop_tr(tr);
3456 }
3457
3458 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3459 for_each_tracing_cpu(cpu) {
3460 iter->buffer_iter[cpu] =
3461 ring_buffer_read_start(iter->array_buffer->buffer,
3462 cpu, GFP_KERNEL);
3463 tracing_iter_reset(iter, cpu);
3464 }
3465 } else {
3466 cpu = iter->cpu_file;
3467 iter->buffer_iter[cpu] =
3468 ring_buffer_read_start(iter->array_buffer->buffer,
3469 cpu, GFP_KERNEL);
3470 tracing_iter_reset(iter, cpu);
3471 }
3472
3473 mutex_unlock(&trace_types_lock);
3474
3475 return iter;
3476
3477 fail:
3478 mutex_unlock(&trace_types_lock);
3479 free_trace_iter_content(iter);
3480release:
3481 seq_release_private(inode, file);
3482 return ERR_PTR(-ENOMEM);
3483}
3484
3485int tracing_open_generic(struct inode *inode, struct file *filp)
3486{
3487 int ret;
3488
3489 ret = tracing_check_open_get_tr(NULL);
3490 if (ret)
3491 return ret;
3492
3493 filp->private_data = inode->i_private;
3494 return 0;
3495}
3496
3497/*
3498 * Open and update trace_array ref count.
3499 * Must have the current trace_array passed to it.
3500 */
3501int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3502{
3503 struct trace_array *tr = inode->i_private;
3504 int ret;
3505
3506 ret = tracing_check_open_get_tr(tr);
3507 if (ret)
3508 return ret;
3509
3510 if ((filp->f_mode & FMODE_WRITE) && trace_array_is_readonly(tr)) {
3511 trace_array_put(tr);
3512 return -EACCES;
3513 }
3514
3515 filp->private_data = inode->i_private;
3516
3517 return 0;
3518}
3519
3520/*
3521 * The private pointer of the inode is the trace_event_file.
3522 * Update the tr ref count associated to it.
3523 */
3524int tracing_open_file_tr(struct inode *inode, struct file *filp)
3525{
3526 struct trace_event_file *file = inode->i_private;
3527 int ret;
3528
3529 ret = tracing_check_open_get_tr(file->tr);
3530 if (ret)
3531 return ret;
3532
3533 guard(mutex)(&event_mutex);
3534
3535 /* Fail if the file is marked for removal */
3536 if (file->flags & EVENT_FILE_FL_FREED) {
3537 trace_array_put(file->tr);
3538 return -ENODEV;
3539 } else {
3540 event_file_get(file);
3541 }
3542
3543 return 0;
3544}
3545
3546int tracing_release_file_tr(struct inode *inode, struct file *filp)
3547{
3548 struct trace_event_file *file = inode->i_private;
3549
3550 trace_array_put(file->tr);
3551 event_file_put(file);
3552
3553 return 0;
3554}
3555
3556int tracing_single_release_file_tr(struct inode *inode, struct file *filp)
3557{
3558 tracing_release_file_tr(inode, filp);
3559 return single_release(inode, filp);
3560}
3561
3562int tracing_release(struct inode *inode, struct file *file)
3563{
3564 struct trace_array *tr = inode->i_private;
3565 struct seq_file *m = file->private_data;
3566 struct trace_iterator *iter;
3567 int cpu;
3568
3569 if (!(file->f_mode & FMODE_READ)) {
3570 trace_array_put(tr);
3571 return 0;
3572 }
3573
3574 /* Writes do not use seq_file */
3575 iter = m->private;
3576 mutex_lock(&trace_types_lock);
3577
3578 for_each_tracing_cpu(cpu) {
3579 if (iter->buffer_iter[cpu])
3580 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3581 }
3582
3583 if (iter->trace && iter->trace->close)
3584 iter->trace->close(iter);
3585
3586 if (iter->iter_flags & TRACE_FILE_PAUSE)
3587 /* reenable tracing if it was previously enabled */
3588 tracing_start_tr(tr);
3589
3590 __trace_array_put(tr);
3591
3592 mutex_unlock(&trace_types_lock);
3593
3594 free_trace_iter_content(iter);
3595 seq_release_private(inode, file);
3596
3597 return 0;
3598}
3599
3600int tracing_release_generic_tr(struct inode *inode, struct file *file)
3601{
3602 struct trace_array *tr = inode->i_private;
3603
3604 trace_array_put(tr);
3605 return 0;
3606}
3607
3608static int tracing_single_release_tr(struct inode *inode, struct file *file)
3609{
3610 struct trace_array *tr = inode->i_private;
3611
3612 trace_array_put(tr);
3613
3614 return single_release(inode, file);
3615}
3616
3617static bool update_last_data_if_empty(struct trace_array *tr);
3618
3619static int tracing_open(struct inode *inode, struct file *file)
3620{
3621 struct trace_array *tr = inode->i_private;
3622 struct trace_iterator *iter;
3623 int ret;
3624
3625 ret = tracing_check_open_get_tr(tr);
3626 if (ret)
3627 return ret;
3628
3629 /* If this file was open for write, then erase contents */
3630 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3631 int cpu = tracing_get_cpu(inode);
3632 struct array_buffer *trace_buf = &tr->array_buffer;
3633
3634#ifdef CONFIG_TRACER_MAX_TRACE
3635 if (tr->current_trace->print_max)
3636 trace_buf = &tr->snapshot_buffer;
3637#endif
3638
3639 if (cpu == RING_BUFFER_ALL_CPUS)
3640 tracing_reset_online_cpus(trace_buf);
3641 else
3642 tracing_reset_cpu(trace_buf, cpu);
3643
3644 update_last_data_if_empty(tr);
3645 }
3646
3647 if (file->f_mode & FMODE_READ) {
3648 iter = __tracing_open(inode, file, false);
3649 if (IS_ERR(iter))
3650 ret = PTR_ERR(iter);
3651 else if (tr->trace_flags & TRACE_ITER(LATENCY_FMT))
3652 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3653 }
3654
3655 if (ret < 0)
3656 trace_array_put(tr);
3657
3658 return ret;
3659}
3660
3661/*
3662 * Some tracers are not suitable for instance buffers.
3663 * A tracer is always available for the global array (toplevel)
3664 * or if it explicitly states that it is.
3665 */
3666static bool
3667trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3668{
3669 /* arrays with mapped buffer range do not have snapshots */
3670 if (tr->range_addr_start && tracer_uses_snapshot(t))
3671 return false;
3672 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3673}
3674
3675/* Find the next tracer that this trace array may use */
3676static struct tracer *
3677get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3678{
3679 while (t && !trace_ok_for_array(t, tr))
3680 t = t->next;
3681
3682 return t;
3683}
3684
3685static void *
3686t_next(struct seq_file *m, void *v, loff_t *pos)
3687{
3688 struct trace_array *tr = m->private;
3689 struct tracer *t = v;
3690
3691 (*pos)++;
3692
3693 if (t)
3694 t = get_tracer_for_array(tr, t->next);
3695
3696 return t;
3697}
3698
3699static void *t_start(struct seq_file *m, loff_t *pos)
3700{
3701 struct trace_array *tr = m->private;
3702 struct tracer *t;
3703 loff_t l = 0;
3704
3705 mutex_lock(&trace_types_lock);
3706
3707 t = get_tracer_for_array(tr, trace_types);
3708 for (; t && l < *pos; t = t_next(m, t, &l))
3709 ;
3710
3711 return t;
3712}
3713
3714static void t_stop(struct seq_file *m, void *p)
3715{
3716 mutex_unlock(&trace_types_lock);
3717}
3718
3719static int t_show(struct seq_file *m, void *v)
3720{
3721 struct tracer *t = v;
3722
3723 if (!t)
3724 return 0;
3725
3726 seq_puts(m, t->name);
3727 if (t->next)
3728 seq_putc(m, ' ');
3729 else
3730 seq_putc(m, '\n');
3731
3732 return 0;
3733}
3734
3735static const struct seq_operations show_traces_seq_ops = {
3736 .start = t_start,
3737 .next = t_next,
3738 .stop = t_stop,
3739 .show = t_show,
3740};
3741
3742static int show_traces_open(struct inode *inode, struct file *file)
3743{
3744 struct trace_array *tr = inode->i_private;
3745 struct seq_file *m;
3746 int ret;
3747
3748 ret = tracing_check_open_get_tr(tr);
3749 if (ret)
3750 return ret;
3751
3752 ret = seq_open(file, &show_traces_seq_ops);
3753 if (ret) {
3754 trace_array_put(tr);
3755 return ret;
3756 }
3757
3758 m = file->private_data;
3759 m->private = tr;
3760
3761 return 0;
3762}
3763
3764static int tracing_seq_release(struct inode *inode, struct file *file)
3765{
3766 struct trace_array *tr = inode->i_private;
3767
3768 trace_array_put(tr);
3769 return seq_release(inode, file);
3770}
3771
3772static ssize_t
3773tracing_write_stub(struct file *filp, const char __user *ubuf,
3774 size_t count, loff_t *ppos)
3775{
3776 return count;
3777}
3778
3779loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3780{
3781 int ret;
3782
3783 if (file->f_mode & FMODE_READ)
3784 ret = seq_lseek(file, offset, whence);
3785 else
3786 file->f_pos = ret = 0;
3787
3788 return ret;
3789}
3790
3791static const struct file_operations tracing_fops = {
3792 .open = tracing_open,
3793 .read = seq_read,
3794 .read_iter = seq_read_iter,
3795 .splice_read = copy_splice_read,
3796 .write = tracing_write_stub,
3797 .llseek = tracing_lseek,
3798 .release = tracing_release,
3799};
3800
3801static const struct file_operations show_traces_fops = {
3802 .open = show_traces_open,
3803 .read = seq_read,
3804 .llseek = seq_lseek,
3805 .release = tracing_seq_release,
3806};
3807
3808static ssize_t
3809tracing_cpumask_read(struct file *filp, char __user *ubuf,
3810 size_t count, loff_t *ppos)
3811{
3812 struct trace_array *tr = file_inode(filp)->i_private;
3813 char *mask_str __free(kfree) = NULL;
3814 int len;
3815
3816 len = snprintf(NULL, 0, "%*pb\n",
3817 cpumask_pr_args(tr->tracing_cpumask)) + 1;
3818 mask_str = kmalloc(len, GFP_KERNEL);
3819 if (!mask_str)
3820 return -ENOMEM;
3821
3822 len = snprintf(mask_str, len, "%*pb\n",
3823 cpumask_pr_args(tr->tracing_cpumask));
3824 if (len >= count)
3825 return -EINVAL;
3826
3827 return simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
3828}
3829
3830int tracing_set_cpumask(struct trace_array *tr,
3831 cpumask_var_t tracing_cpumask_new)
3832{
3833 int cpu;
3834
3835 if (!tr)
3836 return -EINVAL;
3837
3838 local_irq_disable();
3839 arch_spin_lock(&tr->max_lock);
3840 for_each_tracing_cpu(cpu) {
3841 /*
3842 * Increase/decrease the disabled counter if we are
3843 * about to flip a bit in the cpumask:
3844 */
3845 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3846 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3847 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
3848#ifdef CONFIG_TRACER_SNAPSHOT
3849 ring_buffer_record_disable_cpu(tr->snapshot_buffer.buffer, cpu);
3850#endif
3851 }
3852 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3853 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3854 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
3855#ifdef CONFIG_TRACER_SNAPSHOT
3856 ring_buffer_record_enable_cpu(tr->snapshot_buffer.buffer, cpu);
3857#endif
3858 }
3859 }
3860 arch_spin_unlock(&tr->max_lock);
3861 local_irq_enable();
3862
3863 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3864
3865 return 0;
3866}
3867
3868static ssize_t
3869tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3870 size_t count, loff_t *ppos)
3871{
3872 struct trace_array *tr = file_inode(filp)->i_private;
3873 cpumask_var_t tracing_cpumask_new;
3874 int err;
3875
3876 if (count == 0 || count > KMALLOC_MAX_SIZE)
3877 return -EINVAL;
3878
3879 if (!zalloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3880 return -ENOMEM;
3881
3882 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3883 if (err)
3884 goto err_free;
3885
3886 err = tracing_set_cpumask(tr, tracing_cpumask_new);
3887 if (err)
3888 goto err_free;
3889
3890 free_cpumask_var(tracing_cpumask_new);
3891
3892 return count;
3893
3894err_free:
3895 free_cpumask_var(tracing_cpumask_new);
3896
3897 return err;
3898}
3899
3900static const struct file_operations tracing_cpumask_fops = {
3901 .open = tracing_open_generic_tr,
3902 .read = tracing_cpumask_read,
3903 .write = tracing_cpumask_write,
3904 .release = tracing_release_generic_tr,
3905 .llseek = generic_file_llseek,
3906};
3907
3908static int tracing_trace_options_show(struct seq_file *m, void *v)
3909{
3910 struct tracer_opt *trace_opts;
3911 struct trace_array *tr = m->private;
3912 struct tracer_flags *flags;
3913 u32 tracer_flags;
3914 int i;
3915
3916 guard(mutex)(&trace_types_lock);
3917
3918 for (i = 0; trace_options[i]; i++) {
3919 if (tr->trace_flags & (1ULL << i))
3920 seq_printf(m, "%s\n", trace_options[i]);
3921 else
3922 seq_printf(m, "no%s\n", trace_options[i]);
3923 }
3924
3925 flags = tr->current_trace_flags;
3926 if (!flags || !flags->opts)
3927 return 0;
3928
3929 tracer_flags = flags->val;
3930 trace_opts = flags->opts;
3931
3932 for (i = 0; trace_opts[i].name; i++) {
3933 if (tracer_flags & trace_opts[i].bit)
3934 seq_printf(m, "%s\n", trace_opts[i].name);
3935 else
3936 seq_printf(m, "no%s\n", trace_opts[i].name);
3937 }
3938
3939 return 0;
3940}
3941
3942static int __set_tracer_option(struct trace_array *tr,
3943 struct tracer_flags *tracer_flags,
3944 struct tracer_opt *opts, int neg)
3945{
3946 struct tracer *trace = tracer_flags->trace;
3947 int ret = 0;
3948
3949 if (trace->set_flag)
3950 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3951 if (ret)
3952 return ret;
3953
3954 if (neg)
3955 tracer_flags->val &= ~opts->bit;
3956 else
3957 tracer_flags->val |= opts->bit;
3958 return 0;
3959}
3960
3961/* Try to assign a tracer specific option */
3962static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3963{
3964 struct tracer_flags *tracer_flags = tr->current_trace_flags;
3965 struct tracer_opt *opts = NULL;
3966 int i;
3967
3968 if (!tracer_flags || !tracer_flags->opts)
3969 return 0;
3970
3971 for (i = 0; tracer_flags->opts[i].name; i++) {
3972 opts = &tracer_flags->opts[i];
3973
3974 if (strcmp(cmp, opts->name) == 0)
3975 return __set_tracer_option(tr, tracer_flags, opts, neg);
3976 }
3977
3978 return -EINVAL;
3979}
3980
3981/* Some tracers require overwrite to stay enabled */
3982int trace_keep_overwrite(struct tracer *tracer, u64 mask, int set)
3983{
3984 if (tracer->enabled && (mask & TRACE_ITER(OVERWRITE)) && !set)
3985 return -1;
3986
3987 return 0;
3988}
3989
3990int set_tracer_flag(struct trace_array *tr, u64 mask, int enabled)
3991{
3992 switch (mask) {
3993 case TRACE_ITER(RECORD_TGID):
3994 case TRACE_ITER(RECORD_CMD):
3995 case TRACE_ITER(TRACE_PRINTK):
3996 case TRACE_ITER(COPY_MARKER):
3997 lockdep_assert_held(&event_mutex);
3998 }
3999
4000 /* do nothing if flag is already set */
4001 if (!!(tr->trace_flags & mask) == !!enabled)
4002 return 0;
4003
4004 /* Give the tracer a chance to approve the change */
4005 if (tr->current_trace->flag_changed)
4006 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4007 return -EINVAL;
4008
4009 switch (mask) {
4010 case TRACE_ITER(TRACE_PRINTK):
4011 if (enabled) {
4012 update_printk_trace(tr);
4013 } else {
4014 /*
4015 * The global_trace cannot clear this.
4016 * It's flag only gets cleared if another instance sets it.
4017 */
4018 if (printk_trace == &global_trace)
4019 return -EINVAL;
4020 /*
4021 * An instance must always have it set.
4022 * by default, that's the global_trace instance.
4023 */
4024 if (printk_trace == tr)
4025 update_printk_trace(&global_trace);
4026 }
4027 break;
4028
4029 case TRACE_ITER(COPY_MARKER):
4030 update_marker_trace(tr, enabled);
4031 /* update_marker_trace updates the tr->trace_flags */
4032 return 0;
4033 }
4034
4035 if (enabled)
4036 tr->trace_flags |= mask;
4037 else
4038 tr->trace_flags &= ~mask;
4039
4040 switch (mask) {
4041 case TRACE_ITER(RECORD_CMD):
4042 trace_event_enable_cmd_record(enabled);
4043 break;
4044
4045 case TRACE_ITER(RECORD_TGID):
4046
4047 if (trace_alloc_tgid_map() < 0) {
4048 tr->trace_flags &= ~TRACE_ITER(RECORD_TGID);
4049 return -ENOMEM;
4050 }
4051
4052 trace_event_enable_tgid_record(enabled);
4053 break;
4054
4055 case TRACE_ITER(EVENT_FORK):
4056 trace_event_follow_fork(tr, enabled);
4057 break;
4058
4059 case TRACE_ITER(FUNC_FORK):
4060 ftrace_pid_follow_fork(tr, enabled);
4061 break;
4062
4063 case TRACE_ITER(OVERWRITE):
4064 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
4065#ifdef CONFIG_TRACER_SNAPSHOT
4066 ring_buffer_change_overwrite(tr->snapshot_buffer.buffer, enabled);
4067#endif
4068 break;
4069
4070 case TRACE_ITER(PRINTK):
4071 trace_printk_start_stop_comm(enabled);
4072 trace_printk_control(enabled);
4073 break;
4074
4075#if defined(CONFIG_FUNCTION_PROFILER) && defined(CONFIG_FUNCTION_GRAPH_TRACER)
4076 case TRACE_GRAPH_GRAPH_TIME:
4077 ftrace_graph_graph_time_control(enabled);
4078 break;
4079#endif
4080 }
4081
4082 return 0;
4083}
4084
4085int trace_set_options(struct trace_array *tr, char *option)
4086{
4087 char *cmp;
4088 int neg = 0;
4089 int ret;
4090 size_t orig_len = strlen(option);
4091 int len;
4092
4093 cmp = strstrip(option);
4094
4095 len = str_has_prefix(cmp, "no");
4096 if (len)
4097 neg = 1;
4098
4099 cmp += len;
4100
4101 mutex_lock(&event_mutex);
4102 mutex_lock(&trace_types_lock);
4103
4104 ret = match_string(trace_options, -1, cmp);
4105 /* If no option could be set, test the specific tracer options */
4106 if (ret < 0)
4107 ret = set_tracer_option(tr, cmp, neg);
4108 else
4109 ret = set_tracer_flag(tr, 1ULL << ret, !neg);
4110
4111 mutex_unlock(&trace_types_lock);
4112 mutex_unlock(&event_mutex);
4113
4114 /*
4115 * If the first trailing whitespace is replaced with '\0' by strstrip,
4116 * turn it back into a space.
4117 */
4118 if (orig_len > strlen(option))
4119 option[strlen(option)] = ' ';
4120
4121 return ret;
4122}
4123
4124static void __init apply_trace_boot_options(void)
4125{
4126 char *buf = trace_boot_options_buf;
4127 char *option;
4128
4129 while (true) {
4130 option = strsep(&buf, ",");
4131
4132 if (!option)
4133 break;
4134
4135 if (*option)
4136 trace_set_options(&global_trace, option);
4137
4138 /* Put back the comma to allow this to be called again */
4139 if (buf)
4140 *(buf - 1) = ',';
4141 }
4142}
4143
4144static ssize_t
4145tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4146 size_t cnt, loff_t *ppos)
4147{
4148 struct seq_file *m = filp->private_data;
4149 struct trace_array *tr = m->private;
4150 char buf[64];
4151 int ret;
4152
4153 if (cnt >= sizeof(buf))
4154 return -EINVAL;
4155
4156 if (copy_from_user(buf, ubuf, cnt))
4157 return -EFAULT;
4158
4159 buf[cnt] = 0;
4160
4161 ret = trace_set_options(tr, buf);
4162 if (ret < 0)
4163 return ret;
4164
4165 *ppos += cnt;
4166
4167 return cnt;
4168}
4169
4170static int tracing_trace_options_open(struct inode *inode, struct file *file)
4171{
4172 struct trace_array *tr = inode->i_private;
4173 int ret;
4174
4175 ret = tracing_check_open_get_tr(tr);
4176 if (ret)
4177 return ret;
4178
4179 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4180 if (ret < 0)
4181 trace_array_put(tr);
4182
4183 return ret;
4184}
4185
4186static const struct file_operations tracing_iter_fops = {
4187 .open = tracing_trace_options_open,
4188 .read = seq_read,
4189 .llseek = seq_lseek,
4190 .release = tracing_single_release_tr,
4191 .write = tracing_trace_options_write,
4192};
4193
4194static const char readme_msg[] =
4195 "tracing mini-HOWTO:\n\n"
4196 "By default tracefs removes all OTH file permission bits.\n"
4197 "When mounting tracefs an optional group id can be specified\n"
4198 "which adds the group to every directory and file in tracefs:\n\n"
4199 "\t e.g. mount -t tracefs [-o [gid=<gid>]] nodev /sys/kernel/tracing\n\n"
4200 "# echo 0 > tracing_on : quick way to disable tracing\n"
4201 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4202 " Important files:\n"
4203 " trace\t\t\t- The static contents of the buffer\n"
4204 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4205 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4206 " current_tracer\t- function and latency tracers\n"
4207 " available_tracers\t- list of configured tracers for current_tracer\n"
4208 " error_log\t- error log for failed commands (that support it)\n"
4209 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4210 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4211 " trace_clock\t\t- change the clock used to order events\n"
4212 " local: Per cpu clock but may not be synced across CPUs\n"
4213 " global: Synced across CPUs but slows tracing down.\n"
4214 " counter: Not a clock, but just an increment\n"
4215 " uptime: Jiffy counter from time of boot\n"
4216 " perf: Same clock that perf events use\n"
4217#ifdef CONFIG_X86_64
4218 " x86-tsc: TSC cycle counter\n"
4219#endif
4220 "\n timestamp_mode\t- view the mode used to timestamp events\n"
4221 " delta: Delta difference against a buffer-wide timestamp\n"
4222 " absolute: Absolute (standalone) timestamp\n"
4223 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4224 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4225 " tracing_cpumask\t- Limit which CPUs to trace\n"
4226 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4227 "\t\t\t Remove sub-buffer with rmdir\n"
4228 " trace_options\t\t- Set format or modify how tracing happens\n"
4229 "\t\t\t Disable an option by prefixing 'no' to the\n"
4230 "\t\t\t option name\n"
4231 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4232#ifdef CONFIG_DYNAMIC_FTRACE
4233 "\n available_filter_functions - list of functions that can be filtered on\n"
4234 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4235 "\t\t\t functions\n"
4236 "\t accepts: func_full_name or glob-matching-pattern\n"
4237 "\t modules: Can select a group via module\n"
4238 "\t Format: :mod:<module-name>\n"
4239 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4240 "\t triggers: a command to perform when function is hit\n"
4241 "\t Format: <function>:<trigger>[:count]\n"
4242 "\t trigger: traceon, traceoff\n"
4243 "\t\t enable_event:<system>:<event>\n"
4244 "\t\t disable_event:<system>:<event>\n"
4245#ifdef CONFIG_STACKTRACE
4246 "\t\t stacktrace\n"
4247#endif
4248#ifdef CONFIG_TRACER_SNAPSHOT
4249 "\t\t snapshot\n"
4250#endif
4251 "\t\t dump\n"
4252 "\t\t cpudump\n"
4253 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4254 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4255 "\t The first one will disable tracing every time do_fault is hit\n"
4256 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4257 "\t The first time do trap is hit and it disables tracing, the\n"
4258 "\t counter will decrement to 2. If tracing is already disabled,\n"
4259 "\t the counter will not decrement. It only decrements when the\n"
4260 "\t trigger did work\n"
4261 "\t To remove trigger without count:\n"
4262 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4263 "\t To remove trigger with a count:\n"
4264 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4265 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4266 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4267 "\t modules: Can select a group via module command :mod:\n"
4268 "\t Does not accept triggers\n"
4269#endif /* CONFIG_DYNAMIC_FTRACE */
4270#ifdef CONFIG_FUNCTION_TRACER
4271 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4272 "\t\t (function)\n"
4273 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
4274 "\t\t (function)\n"
4275#endif
4276#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4277 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4278 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4279 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4280#endif
4281#ifdef CONFIG_TRACER_SNAPSHOT
4282 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4283 "\t\t\t snapshot buffer. Read the contents for more\n"
4284 "\t\t\t information\n"
4285#endif
4286#ifdef CONFIG_STACK_TRACER
4287 " stack_trace\t\t- Shows the max stack trace when active\n"
4288 " stack_max_size\t- Shows current max stack size that was traced\n"
4289 "\t\t\t Write into this file to reset the max size (trigger a\n"
4290 "\t\t\t new trace)\n"
4291#ifdef CONFIG_DYNAMIC_FTRACE
4292 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4293 "\t\t\t traces\n"
4294#endif
4295#endif /* CONFIG_STACK_TRACER */
4296#ifdef CONFIG_DYNAMIC_EVENTS
4297 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
4298 "\t\t\t Write into this file to define/undefine new trace events.\n"
4299#endif
4300#ifdef CONFIG_KPROBE_EVENTS
4301 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
4302 "\t\t\t Write into this file to define/undefine new trace events.\n"
4303#endif
4304#ifdef CONFIG_UPROBE_EVENTS
4305 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
4306 "\t\t\t Write into this file to define/undefine new trace events.\n"
4307#endif
4308#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) || \
4309 defined(CONFIG_FPROBE_EVENTS)
4310 "\t accepts: event-definitions (one definition per line)\n"
4311#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
4312 "\t Format: p[:[<group>/][<event>]] <place> [<args>]\n"
4313 "\t r[maxactive][:[<group>/][<event>]] <place> [<args>]\n"
4314#endif
4315#ifdef CONFIG_FPROBE_EVENTS
4316 "\t f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n"
4317 "\t t[:[<group>/][<event>]] <tracepoint> [<args>]\n"
4318#endif
4319#ifdef CONFIG_HIST_TRIGGERS
4320 "\t s:[synthetic/]<event> <field> [<field>]\n"
4321#endif
4322 "\t e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n"
4323 "\t -:[<group>/][<event>]\n"
4324#ifdef CONFIG_KPROBE_EVENTS
4325 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4326 "place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n"
4327#endif
4328#ifdef CONFIG_UPROBE_EVENTS
4329 " place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
4330#endif
4331 "\t args: <name>=fetcharg[:type]\n"
4332 "\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
4333#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
4334 "\t $stack<index>, $stack, $retval, $comm, $arg<N>, $current\n"
4335#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
4336 "\t [(structname[,field])]<argname>[->field[->field|.field...]],\n"
4337 "\t [(structname[,field])](fetcharg)->field[->field|.field...],\n"
4338#endif
4339#else
4340 "\t $stack<index>, $stack, $retval, $comm, $current\n"
4341#endif
4342 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
4343 "\t this_cpu_read(<fetcharg>), this_cpu_ptr(<fetcharg>)\n"
4344 "\t kernel return probes support: $retval, $arg<N>, $comm\n"
4345 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n"
4346 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
4347 "\t symstr, %pd/%pD, <type>\\[<array-size>\\]\n"
4348#ifdef CONFIG_HIST_TRIGGERS
4349 "\t field: <stype> <name>;\n"
4350 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
4351 "\t [unsigned] char/int/long\n"
4352#endif
4353 "\t efield: For event probes ('e' types), the field is on of the fields\n"
4354 "\t of the <attached-group>/<attached-event>.\n"
4355#endif
4356 " set_event\t\t- Enables events by name written into it\n"
4357 "\t\t\t Can enable module events via: :mod:<module>\n"
4358 " events/\t\t- Directory containing all trace event subsystems:\n"
4359 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4360 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4361 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4362 "\t\t\t events\n"
4363 " filter\t\t- If set, only events passing filter are traced\n"
4364 " events/<system>/<event>/\t- Directory containing control files for\n"
4365 "\t\t\t <event>:\n"
4366 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4367 " filter\t\t- If set, only events passing filter are traced\n"
4368 " trigger\t\t- If set, a command to perform when event is hit\n"
4369 "\t Format: <trigger>[:count][if <filter>]\n"
4370 "\t trigger: traceon, traceoff\n"
4371 "\t enable_event:<system>:<event>\n"
4372 "\t disable_event:<system>:<event>\n"
4373#ifdef CONFIG_HIST_TRIGGERS
4374 "\t enable_hist:<system>:<event>\n"
4375 "\t disable_hist:<system>:<event>\n"
4376#endif
4377#ifdef CONFIG_STACKTRACE
4378 "\t\t stacktrace\n"
4379#endif
4380#ifdef CONFIG_TRACER_SNAPSHOT
4381 "\t\t snapshot\n"
4382#endif
4383#ifdef CONFIG_HIST_TRIGGERS
4384 "\t\t hist (see below)\n"
4385#endif
4386 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4387 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4388 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4389 "\t events/block/block_unplug/trigger\n"
4390 "\t The first disables tracing every time block_unplug is hit.\n"
4391 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4392 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4393 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4394 "\t Like function triggers, the counter is only decremented if it\n"
4395 "\t enabled or disabled tracing.\n"
4396 "\t To remove a trigger without a count:\n"
4397 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4398 "\t To remove a trigger with a count:\n"
4399 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4400 "\t Filters can be ignored when removing a trigger.\n"
4401#ifdef CONFIG_HIST_TRIGGERS
4402 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4403 "\t Format: hist:keys=<field1[,field2,...]>\n"
4404 "\t [:<var1>=<field|var_ref|numeric_literal>[,<var2>=...]]\n"
4405 "\t [:values=<field1[,field2,...]>]\n"
4406 "\t [:sort=<field1[,field2,...]>]\n"
4407 "\t [:size=#entries]\n"
4408 "\t [:pause][:continue][:clear]\n"
4409 "\t [:name=histname1]\n"
4410 "\t [:nohitcount]\n"
4411 "\t [:<handler>.<action>]\n"
4412 "\t [if <filter>]\n\n"
4413 "\t Note, special fields can be used as well:\n"
4414 "\t common_timestamp - to record current timestamp\n"
4415 "\t common_cpu - to record the CPU the event happened on\n"
4416 "\n"
4417 "\t A hist trigger variable can be:\n"
4418 "\t - a reference to a field e.g. x=current_timestamp,\n"
4419 "\t - a reference to another variable e.g. y=$x,\n"
4420 "\t - a numeric literal: e.g. ms_per_sec=1000,\n"
4421 "\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
4422 "\n"
4423 "\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
4424 "\t multiplication(*) and division(/) operators. An operand can be either a\n"
4425 "\t variable reference, field or numeric literal.\n"
4426 "\n"
4427 "\t When a matching event is hit, an entry is added to a hash\n"
4428 "\t table using the key(s) and value(s) named, and the value of a\n"
4429 "\t sum called 'hitcount' is incremented. Keys and values\n"
4430 "\t correspond to fields in the event's format description. Keys\n"
4431 "\t can be any field, or the special string 'common_stacktrace'.\n"
4432 "\t Compound keys consisting of up to two fields can be specified\n"
4433 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4434 "\t fields. Sort keys consisting of up to two fields can be\n"
4435 "\t specified using the 'sort' keyword. The sort direction can\n"
4436 "\t be modified by appending '.descending' or '.ascending' to a\n"
4437 "\t sort field. The 'size' parameter can be used to specify more\n"
4438 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4439 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4440 "\t its histogram data will be shared with other triggers of the\n"
4441 "\t same name, and trigger hits will update this common data.\n\n"
4442 "\t Reading the 'hist' file for the event will dump the hash\n"
4443 "\t table in its entirety to stdout. If there are multiple hist\n"
4444 "\t triggers attached to an event, there will be a table for each\n"
4445 "\t trigger in the output. The table displayed for a named\n"
4446 "\t trigger will be the same as any other instance having the\n"
4447 "\t same name. The default format used to display a given field\n"
4448 "\t can be modified by appending any of the following modifiers\n"
4449 "\t to the field name, as applicable:\n\n"
4450 "\t .hex display a number as a hex value\n"
4451 "\t .sym display an address as a symbol\n"
4452 "\t .sym-offset display an address as a symbol and offset\n"
4453 "\t .execname display a common_pid as a program name\n"
4454 "\t .syscall display a syscall id as a syscall name\n"
4455 "\t .log2 display log2 value rather than raw number\n"
4456 "\t .buckets=size display values in groups of size rather than raw number\n"
4457 "\t .usecs display a common_timestamp in microseconds\n"
4458 "\t .percent display a number of percentage value\n"
4459 "\t .graph display a bar-graph of a value\n\n"
4460 "\t The 'pause' parameter can be used to pause an existing hist\n"
4461 "\t trigger or to start a hist trigger but not log any events\n"
4462 "\t until told to do so. 'continue' can be used to start or\n"
4463 "\t restart a paused hist trigger.\n\n"
4464 "\t The 'clear' parameter will clear the contents of a running\n"
4465 "\t hist trigger and leave its current paused/active state\n"
4466 "\t unchanged.\n\n"
4467 "\t The 'nohitcount' (or NOHC) parameter will suppress display of\n"
4468 "\t raw hitcount in the histogram.\n\n"
4469 "\t The enable_hist and disable_hist triggers can be used to\n"
4470 "\t have one event conditionally start and stop another event's\n"
4471 "\t already-attached hist trigger. The syntax is analogous to\n"
4472 "\t the enable_event and disable_event triggers.\n\n"
4473 "\t Hist trigger handlers and actions are executed whenever a\n"
4474 "\t a histogram entry is added or updated. They take the form:\n\n"
4475 "\t <handler>.<action>\n\n"
4476 "\t The available handlers are:\n\n"
4477 "\t onmatch(matching.event) - invoke on addition or update\n"
4478 "\t onmax(var) - invoke if var exceeds current max\n"
4479 "\t onchange(var) - invoke action if var changes\n\n"
4480 "\t The available actions are:\n\n"
4481 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
4482 "\t save(field,...) - save current event fields\n"
4483#ifdef CONFIG_TRACER_SNAPSHOT
4484 "\t snapshot() - snapshot the trace buffer\n\n"
4485#endif
4486#ifdef CONFIG_SYNTH_EVENTS
4487 " synthetic_events\t- Create/append/remove/show synthetic events\n"
4488 "\t Write into this file to define/undefine new synthetic events.\n"
4489 "\t example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n"
4490#endif
4491#endif
4492;
4493
4494static ssize_t
4495tracing_readme_read(struct file *filp, char __user *ubuf,
4496 size_t cnt, loff_t *ppos)
4497{
4498 return simple_read_from_buffer(ubuf, cnt, ppos,
4499 readme_msg, strlen(readme_msg));
4500}
4501
4502static const struct file_operations tracing_readme_fops = {
4503 .open = tracing_open_generic,
4504 .read = tracing_readme_read,
4505 .llseek = generic_file_llseek,
4506};
4507
4508#ifdef CONFIG_TRACE_EVAL_MAP_FILE
4509static union trace_eval_map_item *
4510update_eval_map(union trace_eval_map_item *ptr)
4511{
4512 if (!ptr->map.eval_string) {
4513 if (ptr->tail.next) {
4514 ptr = ptr->tail.next;
4515 /* Set ptr to the next real item (skip head) */
4516 ptr++;
4517 } else
4518 return NULL;
4519 }
4520 return ptr;
4521}
4522
4523static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
4524{
4525 union trace_eval_map_item *ptr = v;
4526
4527 /*
4528 * Paranoid! If ptr points to end, we don't want to increment past it.
4529 * This really should never happen.
4530 */
4531 (*pos)++;
4532 ptr = update_eval_map(ptr);
4533 if (WARN_ON_ONCE(!ptr))
4534 return NULL;
4535
4536 ptr++;
4537 ptr = update_eval_map(ptr);
4538
4539 return ptr;
4540}
4541
4542static void *eval_map_start(struct seq_file *m, loff_t *pos)
4543{
4544 union trace_eval_map_item *v;
4545 loff_t l = 0;
4546
4547 mutex_lock(&trace_eval_mutex);
4548
4549 v = trace_eval_maps;
4550 if (v)
4551 v++;
4552
4553 while (v && l < *pos) {
4554 v = eval_map_next(m, v, &l);
4555 }
4556
4557 return v;
4558}
4559
4560static void eval_map_stop(struct seq_file *m, void *v)
4561{
4562 mutex_unlock(&trace_eval_mutex);
4563}
4564
4565static int eval_map_show(struct seq_file *m, void *v)
4566{
4567 union trace_eval_map_item *ptr = v;
4568
4569 seq_printf(m, "%s %ld (%s)\n",
4570 ptr->map.eval_string, ptr->map.eval_value,
4571 ptr->map.system);
4572
4573 return 0;
4574}
4575
4576static const struct seq_operations tracing_eval_map_seq_ops = {
4577 .start = eval_map_start,
4578 .next = eval_map_next,
4579 .stop = eval_map_stop,
4580 .show = eval_map_show,
4581};
4582
4583static int tracing_eval_map_open(struct inode *inode, struct file *filp)
4584{
4585 int ret;
4586
4587 ret = tracing_check_open_get_tr(NULL);
4588 if (ret)
4589 return ret;
4590
4591 return seq_open(filp, &tracing_eval_map_seq_ops);
4592}
4593
4594static const struct file_operations tracing_eval_map_fops = {
4595 .open = tracing_eval_map_open,
4596 .read = seq_read,
4597 .llseek = seq_lseek,
4598 .release = seq_release,
4599};
4600
4601static inline union trace_eval_map_item *
4602trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
4603{
4604 /* Return tail of array given the head */
4605 return ptr + ptr->head.length + 1;
4606}
4607
4608static void
4609trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
4610 int len)
4611{
4612 struct trace_eval_map **stop;
4613 struct trace_eval_map **map;
4614 union trace_eval_map_item *map_array;
4615 union trace_eval_map_item *ptr;
4616
4617 stop = start + len;
4618
4619 /*
4620 * The trace_eval_maps contains the map plus a head and tail item,
4621 * where the head holds the module and length of array, and the
4622 * tail holds a pointer to the next list.
4623 */
4624 map_array = kmalloc_objs(*map_array, len + 2);
4625 if (!map_array) {
4626 pr_warn("Unable to allocate trace eval mapping\n");
4627 return;
4628 }
4629
4630 guard(mutex)(&trace_eval_mutex);
4631
4632 if (!trace_eval_maps)
4633 trace_eval_maps = map_array;
4634 else {
4635 ptr = trace_eval_maps;
4636 for (;;) {
4637 ptr = trace_eval_jmp_to_tail(ptr);
4638 if (!ptr->tail.next)
4639 break;
4640 ptr = ptr->tail.next;
4641
4642 }
4643 ptr->tail.next = map_array;
4644 }
4645 map_array->head.mod = mod;
4646 map_array->head.length = len;
4647 map_array++;
4648
4649 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4650 map_array->map = **map;
4651 map_array++;
4652 }
4653 memset(map_array, 0, sizeof(*map_array));
4654}
4655
4656static void trace_create_eval_file(struct dentry *d_tracer)
4657{
4658 trace_create_file("eval_map", TRACE_MODE_READ, d_tracer,
4659 NULL, &tracing_eval_map_fops);
4660}
4661
4662#else /* CONFIG_TRACE_EVAL_MAP_FILE */
4663static inline void trace_create_eval_file(struct dentry *d_tracer) { }
4664static inline void trace_insert_eval_map_file(struct module *mod,
4665 struct trace_eval_map **start, int len) { }
4666#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
4667
4668static void
4669trace_event_update_with_eval_map(struct module *mod,
4670 struct trace_eval_map **start,
4671 int len)
4672{
4673 struct trace_eval_map **map;
4674
4675 /* Always run sanitizer only if btf_type_tag attr exists. */
4676 if (len <= 0) {
4677 if (!(IS_ENABLED(CONFIG_DEBUG_INFO_BTF) &&
4678 IS_ENABLED(CONFIG_PAHOLE_HAS_BTF_TAG) &&
4679 __has_attribute(btf_type_tag)))
4680 return;
4681 }
4682
4683 map = start;
4684
4685 trace_event_update_all(map, len, mod);
4686
4687 if (len <= 0)
4688 return;
4689
4690 trace_insert_eval_map_file(mod, start, len);
4691}
4692
4693static ssize_t
4694tracing_set_trace_read(struct file *filp, char __user *ubuf,
4695 size_t cnt, loff_t *ppos)
4696{
4697 struct trace_array *tr = filp->private_data;
4698 char buf[MAX_TRACER_SIZE+2];
4699 int r;
4700
4701 scoped_guard(mutex, &trace_types_lock) {
4702 r = sprintf(buf, "%s\n", tr->current_trace->name);
4703 }
4704
4705 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4706}
4707
4708int tracer_init(struct tracer *t, struct trace_array *tr)
4709{
4710 tracing_reset_online_cpus(&tr->array_buffer);
4711 update_last_data_if_empty(tr);
4712 return t->init(tr);
4713}
4714
4715void trace_set_buffer_entries(struct array_buffer *buf, unsigned long val)
4716{
4717 int cpu;
4718
4719 for_each_tracing_cpu(cpu)
4720 per_cpu_ptr(buf->data, cpu)->entries = val;
4721}
4722
4723static void update_buffer_entries(struct array_buffer *buf, int cpu)
4724{
4725 if (cpu == RING_BUFFER_ALL_CPUS) {
4726 trace_set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0));
4727 } else {
4728 per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu);
4729 }
4730}
4731
4732static int __tracing_resize_ring_buffer(struct trace_array *tr,
4733 unsigned long size, int cpu)
4734{
4735 int ret;
4736
4737 /*
4738 * If kernel or user changes the size of the ring buffer
4739 * we use the size that was given, and we can forget about
4740 * expanding it later.
4741 */
4742 trace_set_ring_buffer_expanded(tr);
4743
4744 /* May be called before buffers are initialized */
4745 if (!tr->array_buffer.buffer)
4746 return 0;
4747
4748 /* Do not allow tracing while resizing ring buffer */
4749 tracing_stop_tr(tr);
4750
4751 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
4752 if (ret < 0)
4753 goto out_start;
4754
4755#ifdef CONFIG_TRACER_SNAPSHOT
4756 if (!tr->allocated_snapshot)
4757 goto out;
4758
4759 ret = ring_buffer_resize(tr->snapshot_buffer.buffer, size, cpu);
4760 if (ret < 0) {
4761 int r = resize_buffer_duplicate_size(&tr->array_buffer,
4762 &tr->array_buffer, cpu);
4763 if (r < 0) {
4764 /*
4765 * AARGH! We are left with different
4766 * size max buffer!!!!
4767 * The max buffer is our "snapshot" buffer.
4768 * When a tracer needs a snapshot (one of the
4769 * latency tracers), it swaps the max buffer
4770 * with the saved snap shot. We succeeded to
4771 * update the size of the main buffer, but failed to
4772 * update the size of the max buffer. But when we tried
4773 * to reset the main buffer to the original size, we
4774 * failed there too. This is very unlikely to
4775 * happen, but if it does, warn and kill all
4776 * tracing.
4777 */
4778 WARN_ON(1);
4779 tracing_disabled = 1;
4780 }
4781 goto out_start;
4782 }
4783
4784 update_buffer_entries(&tr->snapshot_buffer, cpu);
4785
4786 out:
4787#endif /* CONFIG_TRACER_SNAPSHOT */
4788
4789 update_buffer_entries(&tr->array_buffer, cpu);
4790 out_start:
4791 tracing_start_tr(tr);
4792 return ret;
4793}
4794
4795ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4796 unsigned long size, int cpu_id)
4797{
4798 guard(mutex)(&trace_types_lock);
4799
4800 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4801 /* make sure, this cpu is enabled in the mask */
4802 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask))
4803 return -EINVAL;
4804 }
4805
4806 return __tracing_resize_ring_buffer(tr, size, cpu_id);
4807}
4808
4809struct trace_mod_entry {
4810 unsigned long mod_addr;
4811 char mod_name[MODULE_NAME_LEN];
4812};
4813
4814struct trace_scratch {
4815 unsigned int clock_id;
4816 unsigned long text_addr;
4817 unsigned long nr_entries;
4818 struct trace_mod_entry entries[];
4819};
4820
4821static DEFINE_MUTEX(scratch_mutex);
4822
4823static int cmp_mod_entry(const void *key, const void *pivot)
4824{
4825 unsigned long addr = (unsigned long)key;
4826 const struct trace_mod_entry *ent = pivot;
4827
4828 if (addr < ent[0].mod_addr)
4829 return -1;
4830
4831 return addr >= ent[1].mod_addr;
4832}
4833
4834/**
4835 * trace_adjust_address() - Adjust prev boot address to current address.
4836 * @tr: Persistent ring buffer's trace_array.
4837 * @addr: Address in @tr which is adjusted.
4838 */
4839unsigned long trace_adjust_address(struct trace_array *tr, unsigned long addr)
4840{
4841 struct trace_module_delta *module_delta;
4842 struct trace_scratch *tscratch;
4843 struct trace_mod_entry *entry;
4844 unsigned long raddr;
4845 int idx = 0, nr_entries;
4846
4847 /* If we don't have last boot delta, return the address */
4848 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
4849 return addr;
4850
4851 /* tr->module_delta must be protected by rcu. */
4852 guard(rcu)();
4853 tscratch = tr->scratch;
4854 /* if there is no tscrach, module_delta must be NULL. */
4855 module_delta = READ_ONCE(tr->module_delta);
4856 if (!module_delta || !tscratch->nr_entries ||
4857 tscratch->entries[0].mod_addr > addr) {
4858 raddr = addr + tr->text_delta;
4859 return __is_kernel(raddr) || is_kernel_core_data(raddr) ||
4860 is_kernel_rodata(raddr) ? raddr : addr;
4861 }
4862
4863 /* Note that entries must be sorted. */
4864 nr_entries = tscratch->nr_entries;
4865 if (nr_entries == 1 ||
4866 tscratch->entries[nr_entries - 1].mod_addr < addr)
4867 idx = nr_entries - 1;
4868 else {
4869 entry = __inline_bsearch((void *)addr,
4870 tscratch->entries,
4871 nr_entries - 1,
4872 sizeof(tscratch->entries[0]),
4873 cmp_mod_entry);
4874 if (entry)
4875 idx = entry - tscratch->entries;
4876 }
4877
4878 return addr + module_delta->delta[idx];
4879}
4880
4881#ifdef CONFIG_MODULES
4882static int save_mod(struct module *mod, void *data)
4883{
4884 struct trace_array *tr = data;
4885 struct trace_scratch *tscratch;
4886 struct trace_mod_entry *entry;
4887 unsigned int size;
4888
4889 tscratch = tr->scratch;
4890 if (!tscratch)
4891 return -1;
4892 size = tr->scratch_size;
4893
4894 if (struct_size(tscratch, entries, tscratch->nr_entries + 1) > size)
4895 return -1;
4896
4897 entry = &tscratch->entries[tscratch->nr_entries];
4898
4899 tscratch->nr_entries++;
4900
4901 entry->mod_addr = (unsigned long)mod->mem[MOD_TEXT].base;
4902 strscpy(entry->mod_name, mod->name);
4903
4904 return 0;
4905}
4906#else
4907static int save_mod(struct module *mod, void *data)
4908{
4909 return 0;
4910}
4911#endif
4912
4913static void update_last_data(struct trace_array *tr)
4914{
4915 struct trace_module_delta *module_delta;
4916 struct trace_scratch *tscratch;
4917
4918 if (!(tr->flags & TRACE_ARRAY_FL_BOOT))
4919 return;
4920
4921 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
4922 return;
4923
4924 /* Only if the buffer has previous boot data clear and update it. */
4925 tr->flags &= ~TRACE_ARRAY_FL_LAST_BOOT;
4926
4927 /* If this is a backup instance, mark it for autoremove. */
4928 if (tr->flags & TRACE_ARRAY_FL_VMALLOC)
4929 tr->free_on_close = true;
4930
4931 /* Reset the module list and reload them */
4932 if (tr->scratch) {
4933 struct trace_scratch *tscratch = tr->scratch;
4934
4935 tscratch->clock_id = tr->clock_id;
4936 memset(tscratch->entries, 0,
4937 flex_array_size(tscratch, entries, tscratch->nr_entries));
4938 tscratch->nr_entries = 0;
4939
4940 guard(mutex)(&scratch_mutex);
4941 module_for_each_mod(save_mod, tr);
4942 }
4943
4944 /*
4945 * Need to clear all CPU buffers as there cannot be events
4946 * from the previous boot mixed with events with this boot
4947 * as that will cause a confusing trace. Need to clear all
4948 * CPU buffers, even for those that may currently be offline.
4949 */
4950 tracing_reset_all_cpus(&tr->array_buffer);
4951
4952 /* Using current data now */
4953 tr->text_delta = 0;
4954
4955 if (!tr->scratch)
4956 return;
4957
4958 tscratch = tr->scratch;
4959 module_delta = READ_ONCE(tr->module_delta);
4960 WRITE_ONCE(tr->module_delta, NULL);
4961 kfree_rcu(module_delta, rcu);
4962
4963 /* Set the persistent ring buffer meta data to this address */
4964 tscratch->text_addr = (unsigned long)_text;
4965}
4966
4967/**
4968 * tracing_update_buffers - used by tracing facility to expand ring buffers
4969 * @tr: The tracing instance
4970 *
4971 * To save on memory when the tracing is never used on a system with it
4972 * configured in. The ring buffers are set to a minimum size. But once
4973 * a user starts to use the tracing facility, then they need to grow
4974 * to their default size.
4975 *
4976 * This function is to be called when a tracer is about to be used.
4977 */
4978int tracing_update_buffers(struct trace_array *tr)
4979{
4980 int ret = 0;
4981
4982 if (!tr)
4983 tr = &global_trace;
4984
4985 guard(mutex)(&trace_types_lock);
4986
4987 update_last_data(tr);
4988
4989 if (!tr->ring_buffer_expanded)
4990 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4991 RING_BUFFER_ALL_CPUS);
4992 return ret;
4993}
4994
4995/*
4996 * Used to clear out the tracer before deletion of an instance.
4997 * Must have trace_types_lock held.
4998 */
4999static void tracing_set_nop(struct trace_array *tr)
5000{
5001 if (tr->current_trace == &nop_trace)
5002 return;
5003
5004 tr->current_trace->enabled--;
5005
5006 if (tr->current_trace->reset)
5007 tr->current_trace->reset(tr);
5008
5009 tr->current_trace = &nop_trace;
5010 tr->current_trace_flags = nop_trace.flags;
5011}
5012
5013static bool tracer_options_updated;
5014
5015int tracing_set_tracer(struct trace_array *tr, const char *buf)
5016{
5017 struct tracer *trace = NULL;
5018 struct tracers *t;
5019 bool had_max_tr;
5020 int ret;
5021
5022 guard(mutex)(&trace_types_lock);
5023
5024 update_last_data(tr);
5025
5026 if (!tr->ring_buffer_expanded) {
5027 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5028 RING_BUFFER_ALL_CPUS);
5029 if (ret < 0)
5030 return ret;
5031 }
5032
5033 list_for_each_entry(t, &tr->tracers, list) {
5034 if (strcmp(t->tracer->name, buf) == 0) {
5035 trace = t->tracer;
5036 break;
5037 }
5038 }
5039 if (!trace)
5040 return -EINVAL;
5041
5042 if (trace == tr->current_trace)
5043 return 0;
5044
5045#ifdef CONFIG_TRACER_SNAPSHOT
5046 if (tracer_uses_snapshot(trace)) {
5047 local_irq_disable();
5048 arch_spin_lock(&tr->max_lock);
5049 ret = tr->cond_snapshot ? -EBUSY : 0;
5050 arch_spin_unlock(&tr->max_lock);
5051 local_irq_enable();
5052 if (ret)
5053 return ret;
5054 }
5055#endif
5056 /* Some tracers won't work on kernel command line */
5057 if (system_state < SYSTEM_RUNNING && trace->noboot) {
5058 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5059 trace->name);
5060 return -EINVAL;
5061 }
5062
5063 /* Some tracers are only allowed for the top level buffer */
5064 if (!trace_ok_for_array(trace, tr))
5065 return -EINVAL;
5066
5067 /* If trace pipe files are being read, we can't change the tracer */
5068 if (tr->trace_ref)
5069 return -EBUSY;
5070
5071 trace_branch_disable();
5072
5073 tr->current_trace->enabled--;
5074
5075 if (tr->current_trace->reset)
5076 tr->current_trace->reset(tr);
5077
5078 had_max_tr = tracer_uses_snapshot(tr->current_trace);
5079
5080 /* Current trace needs to be nop_trace before synchronize_rcu */
5081 tr->current_trace = &nop_trace;
5082 tr->current_trace_flags = nop_trace.flags;
5083
5084 if (had_max_tr && !tracer_uses_snapshot(trace)) {
5085 /*
5086 * We need to make sure that the update_max_tr sees that
5087 * current_trace changed to nop_trace to keep it from
5088 * swapping the buffers after we resize it.
5089 * The update_max_tr is called from interrupts disabled
5090 * so a synchronized_sched() is sufficient.
5091 */
5092 synchronize_rcu();
5093 free_snapshot(tr);
5094 tracing_disarm_snapshot(tr);
5095 }
5096
5097 if (!had_max_tr && tracer_uses_snapshot(trace)) {
5098 ret = tracing_arm_snapshot_locked(tr);
5099 if (ret)
5100 return ret;
5101 }
5102
5103 if (t->flags)
5104 tr->current_trace_flags = t->flags;
5105 else if (t->tracer->flags)
5106 tr->current_trace_flags = t->tracer->flags;
5107 else
5108 tr->current_trace_flags = &no_tracer_flags;
5109
5110 if (trace->init) {
5111 ret = tracer_init(trace, tr);
5112 if (ret) {
5113 if (tracer_uses_snapshot(trace))
5114 tracing_disarm_snapshot(tr);
5115 tr->current_trace_flags = nop_trace.flags;
5116 return ret;
5117 }
5118 }
5119
5120 tr->current_trace = trace;
5121 tr->current_trace->enabled++;
5122 trace_branch_enable(tr);
5123
5124 return 0;
5125}
5126
5127static ssize_t
5128tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5129 size_t cnt, loff_t *ppos)
5130{
5131 struct trace_array *tr = filp->private_data;
5132 char buf[MAX_TRACER_SIZE+1];
5133 char *name;
5134 size_t ret;
5135 int err;
5136
5137 ret = cnt;
5138
5139 if (cnt > MAX_TRACER_SIZE)
5140 cnt = MAX_TRACER_SIZE;
5141
5142 if (copy_from_user(buf, ubuf, cnt))
5143 return -EFAULT;
5144
5145 buf[cnt] = 0;
5146
5147 name = strim(buf);
5148
5149 err = tracing_set_tracer(tr, name);
5150 if (err)
5151 return err;
5152
5153 *ppos += ret;
5154
5155 return ret;
5156}
5157
5158ssize_t tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5159 size_t cnt, loff_t *ppos)
5160{
5161 char buf[64];
5162 int r;
5163
5164 r = snprintf(buf, sizeof(buf), "%ld\n",
5165 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5166 if (r > sizeof(buf))
5167 r = sizeof(buf);
5168 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5169}
5170
5171ssize_t tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5172 size_t cnt, loff_t *ppos)
5173{
5174 unsigned long val;
5175 int ret;
5176
5177 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5178 if (ret)
5179 return ret;
5180
5181 *ptr = val * 1000;
5182
5183 return cnt;
5184}
5185
5186static ssize_t
5187tracing_thresh_read(struct file *filp, char __user *ubuf,
5188 size_t cnt, loff_t *ppos)
5189{
5190 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5191}
5192
5193static ssize_t
5194tracing_thresh_write(struct file *filp, const char __user *ubuf,
5195 size_t cnt, loff_t *ppos)
5196{
5197 struct trace_array *tr = filp->private_data;
5198 int ret;
5199
5200 guard(mutex)(&trace_types_lock);
5201 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5202 if (ret < 0)
5203 return ret;
5204
5205 if (tr->current_trace->update_thresh) {
5206 ret = tr->current_trace->update_thresh(tr);
5207 if (ret < 0)
5208 return ret;
5209 }
5210
5211 return cnt;
5212}
5213
5214static int open_pipe_on_cpu(struct trace_array *tr, int cpu)
5215{
5216 if (cpu == RING_BUFFER_ALL_CPUS) {
5217 if (cpumask_empty(tr->pipe_cpumask)) {
5218 cpumask_setall(tr->pipe_cpumask);
5219 return 0;
5220 }
5221 } else if (!cpumask_test_cpu(cpu, tr->pipe_cpumask)) {
5222 cpumask_set_cpu(cpu, tr->pipe_cpumask);
5223 return 0;
5224 }
5225 return -EBUSY;
5226}
5227
5228static void close_pipe_on_cpu(struct trace_array *tr, int cpu)
5229{
5230 if (cpu == RING_BUFFER_ALL_CPUS) {
5231 WARN_ON(!cpumask_full(tr->pipe_cpumask));
5232 cpumask_clear(tr->pipe_cpumask);
5233 } else {
5234 WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask));
5235 cpumask_clear_cpu(cpu, tr->pipe_cpumask);
5236 }
5237}
5238
5239static int tracing_open_pipe(struct inode *inode, struct file *filp)
5240{
5241 struct trace_array *tr = inode->i_private;
5242 struct trace_iterator *iter;
5243 int cpu;
5244 int ret;
5245
5246 ret = tracing_check_open_get_tr(tr);
5247 if (ret)
5248 return ret;
5249
5250 guard(mutex)(&trace_types_lock);
5251 cpu = tracing_get_cpu(inode);
5252 ret = open_pipe_on_cpu(tr, cpu);
5253 if (ret)
5254 goto fail_pipe_on_cpu;
5255
5256 /* create a buffer to store the information to pass to userspace */
5257 iter = kzalloc_obj(*iter);
5258 if (!iter) {
5259 ret = -ENOMEM;
5260 goto fail_alloc_iter;
5261 }
5262
5263 trace_seq_init(&iter->seq);
5264 iter->trace = tr->current_trace;
5265
5266 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5267 ret = -ENOMEM;
5268 goto fail;
5269 }
5270
5271 /* trace pipe does not show start of buffer */
5272 cpumask_setall(iter->started);
5273
5274 if (tr->trace_flags & TRACE_ITER(LATENCY_FMT))
5275 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5276
5277 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5278 if (trace_clocks[tr->clock_id].in_ns)
5279 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5280
5281 iter->tr = tr;
5282 iter->array_buffer = &tr->array_buffer;
5283 iter->cpu_file = cpu;
5284 mutex_init(&iter->mutex);
5285 filp->private_data = iter;
5286
5287 if (iter->trace->pipe_open)
5288 iter->trace->pipe_open(iter);
5289
5290 nonseekable_open(inode, filp);
5291
5292 tr->trace_ref++;
5293
5294 return ret;
5295
5296fail:
5297 kfree(iter);
5298fail_alloc_iter:
5299 close_pipe_on_cpu(tr, cpu);
5300fail_pipe_on_cpu:
5301 __trace_array_put(tr);
5302 return ret;
5303}
5304
5305static int tracing_release_pipe(struct inode *inode, struct file *file)
5306{
5307 struct trace_iterator *iter = file->private_data;
5308 struct trace_array *tr = inode->i_private;
5309
5310 scoped_guard(mutex, &trace_types_lock) {
5311 tr->trace_ref--;
5312
5313 if (iter->trace->pipe_close)
5314 iter->trace->pipe_close(iter);
5315 close_pipe_on_cpu(tr, iter->cpu_file);
5316 }
5317
5318 free_trace_iter_content(iter);
5319 kfree(iter);
5320
5321 trace_array_put(tr);
5322
5323 return 0;
5324}
5325
5326static __poll_t
5327trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
5328{
5329 struct trace_array *tr = iter->tr;
5330
5331 /* Iterators are static, they should be filled or empty */
5332 if (trace_buffer_iter(iter, iter->cpu_file))
5333 return EPOLLIN | EPOLLRDNORM;
5334
5335 if (tr->trace_flags & TRACE_ITER(BLOCK))
5336 /*
5337 * Always select as readable when in blocking mode
5338 */
5339 return EPOLLIN | EPOLLRDNORM;
5340 else
5341 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
5342 filp, poll_table, iter->tr->buffer_percent);
5343}
5344
5345static __poll_t
5346tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5347{
5348 struct trace_iterator *iter = filp->private_data;
5349
5350 return trace_poll(iter, filp, poll_table);
5351}
5352
5353/* Must be called with iter->mutex held. */
5354static int tracing_wait_pipe(struct file *filp)
5355{
5356 struct trace_iterator *iter = filp->private_data;
5357 int ret;
5358
5359 while (trace_empty(iter)) {
5360
5361 if ((filp->f_flags & O_NONBLOCK)) {
5362 return -EAGAIN;
5363 }
5364
5365 /*
5366 * We block until we read something and tracing is disabled.
5367 * We still block if tracing is disabled, but we have never
5368 * read anything. This allows a user to cat this file, and
5369 * then enable tracing. But after we have read something,
5370 * we give an EOF when tracing is again disabled.
5371 *
5372 * iter->pos will be 0 if we haven't read anything.
5373 */
5374 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
5375 break;
5376
5377 mutex_unlock(&iter->mutex);
5378
5379 ret = wait_on_pipe(iter, 0);
5380
5381 mutex_lock(&iter->mutex);
5382
5383 if (ret)
5384 return ret;
5385 }
5386
5387 return 1;
5388}
5389
5390static bool update_last_data_if_empty(struct trace_array *tr)
5391{
5392 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
5393 return false;
5394
5395 if (!ring_buffer_empty(tr->array_buffer.buffer))
5396 return false;
5397
5398 /*
5399 * If the buffer contains the last boot data and all per-cpu
5400 * buffers are empty, reset it from the kernel side.
5401 */
5402 update_last_data(tr);
5403 return true;
5404}
5405
5406/*
5407 * Consumer reader.
5408 */
5409static ssize_t
5410tracing_read_pipe(struct file *filp, char __user *ubuf,
5411 size_t cnt, loff_t *ppos)
5412{
5413 struct trace_iterator *iter = filp->private_data;
5414 ssize_t sret;
5415
5416 /*
5417 * Avoid more than one consumer on a single file descriptor
5418 * This is just a matter of traces coherency, the ring buffer itself
5419 * is protected.
5420 */
5421 guard(mutex)(&iter->mutex);
5422
5423 /* return any leftover data */
5424 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5425 if (sret != -EBUSY)
5426 return sret;
5427
5428 trace_seq_init(&iter->seq);
5429
5430 if (iter->trace->read) {
5431 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5432 if (sret)
5433 return sret;
5434 }
5435
5436waitagain:
5437 if (update_last_data_if_empty(iter->tr))
5438 return 0;
5439
5440 sret = tracing_wait_pipe(filp);
5441 if (sret <= 0)
5442 return sret;
5443
5444 /* stop when tracing is finished */
5445 if (trace_empty(iter))
5446 return 0;
5447
5448 if (cnt >= TRACE_SEQ_BUFFER_SIZE)
5449 cnt = TRACE_SEQ_BUFFER_SIZE - 1;
5450
5451 /* reset all but tr, trace, and overruns */
5452 trace_iterator_reset(iter);
5453 cpumask_clear(iter->started);
5454 trace_seq_init(&iter->seq);
5455
5456 trace_event_read_lock();
5457 trace_access_lock(iter->cpu_file);
5458 while (trace_find_next_entry_inc(iter) != NULL) {
5459 enum print_line_t ret;
5460 int save_len = iter->seq.seq.len;
5461
5462 ret = print_trace_line(iter);
5463 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5464 /*
5465 * If one print_trace_line() fills entire trace_seq in one shot,
5466 * trace_seq_to_user() will returns -EBUSY because save_len == 0,
5467 * In this case, we need to consume it, otherwise, loop will peek
5468 * this event next time, resulting in an infinite loop.
5469 */
5470 if (save_len == 0) {
5471 iter->seq.full = 0;
5472 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
5473 trace_consume(iter);
5474 break;
5475 }
5476
5477 /* In other cases, don't print partial lines */
5478 iter->seq.seq.len = save_len;
5479 break;
5480 }
5481 if (ret != TRACE_TYPE_NO_CONSUME)
5482 trace_consume(iter);
5483
5484 if (trace_seq_used(&iter->seq) >= cnt)
5485 break;
5486
5487 /*
5488 * Setting the full flag means we reached the trace_seq buffer
5489 * size and we should leave by partial output condition above.
5490 * One of the trace_seq_* functions is not used properly.
5491 */
5492 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
5493 iter->ent->type);
5494 }
5495 trace_access_unlock(iter->cpu_file);
5496 trace_event_read_unlock();
5497
5498 /* Now copy what we have to the user */
5499 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5500 if (iter->seq.readpos >= trace_seq_used(&iter->seq))
5501 trace_seq_init(&iter->seq);
5502
5503 /*
5504 * If there was nothing to send to user, in spite of consuming trace
5505 * entries, go back to wait for more entries.
5506 */
5507 if (sret == -EBUSY)
5508 goto waitagain;
5509
5510 return sret;
5511}
5512
5513static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
5514 unsigned int idx)
5515{
5516 __free_page(spd->pages[idx]);
5517}
5518
5519static size_t
5520tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5521{
5522 size_t count;
5523 int save_len;
5524 int ret;
5525
5526 /* Seq buffer is page-sized, exactly what we need. */
5527 for (;;) {
5528 save_len = iter->seq.seq.len;
5529 ret = print_trace_line(iter);
5530
5531 if (trace_seq_has_overflowed(&iter->seq)) {
5532 iter->seq.seq.len = save_len;
5533 break;
5534 }
5535
5536 /*
5537 * This should not be hit, because it should only
5538 * be set if the iter->seq overflowed. But check it
5539 * anyway to be safe.
5540 */
5541 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5542 iter->seq.seq.len = save_len;
5543 break;
5544 }
5545
5546 count = trace_seq_used(&iter->seq) - save_len;
5547 if (rem < count) {
5548 rem = 0;
5549 iter->seq.seq.len = save_len;
5550 break;
5551 }
5552
5553 if (ret != TRACE_TYPE_NO_CONSUME)
5554 trace_consume(iter);
5555 rem -= count;
5556 if (!trace_find_next_entry_inc(iter)) {
5557 rem = 0;
5558 iter->ent = NULL;
5559 break;
5560 }
5561 }
5562
5563 return rem;
5564}
5565
5566static ssize_t tracing_splice_read_pipe(struct file *filp,
5567 loff_t *ppos,
5568 struct pipe_inode_info *pipe,
5569 size_t len,
5570 unsigned int flags)
5571{
5572 struct page *pages_def[PIPE_DEF_BUFFERS];
5573 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5574 struct trace_iterator *iter = filp->private_data;
5575 struct splice_pipe_desc spd = {
5576 .pages = pages_def,
5577 .partial = partial_def,
5578 .nr_pages = 0, /* This gets updated below. */
5579 .nr_pages_max = PIPE_DEF_BUFFERS,
5580 .ops = &default_pipe_buf_ops,
5581 .spd_release = tracing_spd_release_pipe,
5582 };
5583 ssize_t ret;
5584 size_t rem;
5585 unsigned int i;
5586
5587 if (splice_grow_spd(pipe, &spd))
5588 return -ENOMEM;
5589
5590 mutex_lock(&iter->mutex);
5591
5592 if (iter->trace->splice_read) {
5593 ret = iter->trace->splice_read(iter, filp,
5594 ppos, pipe, len, flags);
5595 if (ret)
5596 goto out_err;
5597 }
5598
5599 ret = tracing_wait_pipe(filp);
5600 if (ret <= 0)
5601 goto out_err;
5602
5603 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
5604 ret = -EFAULT;
5605 goto out_err;
5606 }
5607
5608 trace_event_read_lock();
5609 trace_access_lock(iter->cpu_file);
5610
5611 /* Fill as many pages as possible. */
5612 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
5613 spd.pages[i] = alloc_page(GFP_KERNEL);
5614 if (!spd.pages[i])
5615 break;
5616
5617 rem = tracing_fill_pipe_page(rem, iter);
5618
5619 /* Copy the data into the page, so we can start over. */
5620 ret = trace_seq_to_buffer(&iter->seq,
5621 page_address(spd.pages[i]),
5622 min((size_t)trace_seq_used(&iter->seq),
5623 (size_t)PAGE_SIZE));
5624 if (ret < 0) {
5625 __free_page(spd.pages[i]);
5626 break;
5627 }
5628 spd.partial[i].offset = 0;
5629 spd.partial[i].len = ret;
5630
5631 trace_seq_init(&iter->seq);
5632 }
5633
5634 trace_access_unlock(iter->cpu_file);
5635 trace_event_read_unlock();
5636 mutex_unlock(&iter->mutex);
5637
5638 spd.nr_pages = i;
5639
5640 if (i)
5641 ret = splice_to_pipe(pipe, &spd);
5642 else
5643 ret = 0;
5644out:
5645 splice_shrink_spd(&spd);
5646 return ret;
5647
5648out_err:
5649 mutex_unlock(&iter->mutex);
5650 goto out;
5651}
5652
5653static ssize_t
5654tracing_syscall_buf_read(struct file *filp, char __user *ubuf,
5655 size_t cnt, loff_t *ppos)
5656{
5657 struct inode *inode = file_inode(filp);
5658 struct trace_array *tr = inode->i_private;
5659 char buf[64];
5660 int r;
5661
5662 r = snprintf(buf, 64, "%d\n", tr->syscall_buf_sz);
5663
5664 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5665}
5666
5667static ssize_t
5668tracing_syscall_buf_write(struct file *filp, const char __user *ubuf,
5669 size_t cnt, loff_t *ppos)
5670{
5671 struct inode *inode = file_inode(filp);
5672 struct trace_array *tr = inode->i_private;
5673 unsigned long val;
5674 int ret;
5675
5676 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5677 if (ret)
5678 return ret;
5679
5680 if (val > SYSCALL_FAULT_USER_MAX)
5681 val = SYSCALL_FAULT_USER_MAX;
5682
5683 tr->syscall_buf_sz = val;
5684
5685 *ppos += cnt;
5686
5687 return cnt;
5688}
5689
5690static ssize_t
5691tracing_entries_read(struct file *filp, char __user *ubuf,
5692 size_t cnt, loff_t *ppos)
5693{
5694 struct inode *inode = file_inode(filp);
5695 struct trace_array *tr = inode->i_private;
5696 int cpu = tracing_get_cpu(inode);
5697 char buf[64];
5698 int r = 0;
5699 ssize_t ret;
5700
5701 mutex_lock(&trace_types_lock);
5702
5703 if (cpu == RING_BUFFER_ALL_CPUS) {
5704 int cpu, buf_size_same;
5705 unsigned long size;
5706
5707 size = 0;
5708 buf_size_same = 1;
5709 /* check if all cpu sizes are same */
5710 for_each_tracing_cpu(cpu) {
5711 /* fill in the size from first enabled cpu */
5712 if (size == 0)
5713 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
5714 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
5715 buf_size_same = 0;
5716 break;
5717 }
5718 }
5719
5720 if (buf_size_same) {
5721 if (!tr->ring_buffer_expanded)
5722 r = sprintf(buf, "%lu (expanded: %lu)\n",
5723 size >> 10,
5724 trace_buf_size >> 10);
5725 else
5726 r = sprintf(buf, "%lu\n", size >> 10);
5727 } else
5728 r = sprintf(buf, "X\n");
5729 } else
5730 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
5731
5732 mutex_unlock(&trace_types_lock);
5733
5734 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5735 return ret;
5736}
5737
5738static ssize_t
5739tracing_entries_write(struct file *filp, const char __user *ubuf,
5740 size_t cnt, loff_t *ppos)
5741{
5742 struct inode *inode = file_inode(filp);
5743 struct trace_array *tr = inode->i_private;
5744 unsigned long val;
5745 int ret;
5746
5747 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5748 if (ret)
5749 return ret;
5750
5751 /* must have at least 1 entry */
5752 if (!val)
5753 return -EINVAL;
5754
5755 /* value is in KB */
5756 val <<= 10;
5757 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5758 if (ret < 0)
5759 return ret;
5760
5761 *ppos += cnt;
5762
5763 return cnt;
5764}
5765
5766static ssize_t
5767tracing_total_entries_read(struct file *filp, char __user *ubuf,
5768 size_t cnt, loff_t *ppos)
5769{
5770 struct trace_array *tr = filp->private_data;
5771 char buf[64];
5772 int r, cpu;
5773 unsigned long size = 0, expanded_size = 0;
5774
5775 mutex_lock(&trace_types_lock);
5776 for_each_tracing_cpu(cpu) {
5777 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
5778 if (!tr->ring_buffer_expanded)
5779 expanded_size += trace_buf_size >> 10;
5780 }
5781 if (tr->ring_buffer_expanded)
5782 r = sprintf(buf, "%lu\n", size);
5783 else
5784 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5785 mutex_unlock(&trace_types_lock);
5786
5787 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5788}
5789
5790#define LAST_BOOT_HEADER ((void *)1)
5791
5792static void *l_next(struct seq_file *m, void *v, loff_t *pos)
5793{
5794 struct trace_array *tr = m->private;
5795 struct trace_scratch *tscratch = tr->scratch;
5796 unsigned int index = *pos;
5797
5798 (*pos)++;
5799
5800 if (*pos == 1)
5801 return LAST_BOOT_HEADER;
5802
5803 /* Only show offsets of the last boot data */
5804 if (!tscratch || !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
5805 return NULL;
5806
5807 /* *pos 0 is for the header, 1 is for the first module */
5808 index--;
5809
5810 if (index >= tscratch->nr_entries)
5811 return NULL;
5812
5813 return &tscratch->entries[index];
5814}
5815
5816static void *l_start(struct seq_file *m, loff_t *pos)
5817{
5818 mutex_lock(&scratch_mutex);
5819
5820 return l_next(m, NULL, pos);
5821}
5822
5823static void l_stop(struct seq_file *m, void *p)
5824{
5825 mutex_unlock(&scratch_mutex);
5826}
5827
5828static void show_last_boot_header(struct seq_file *m, struct trace_array *tr)
5829{
5830 struct trace_scratch *tscratch = tr->scratch;
5831
5832 /*
5833 * Do not leak KASLR address. This only shows the KASLR address of
5834 * the last boot. When the ring buffer is started, the LAST_BOOT
5835 * flag gets cleared, and this should only report "current".
5836 * Otherwise it shows the KASLR address from the previous boot which
5837 * should not be the same as the current boot.
5838 */
5839 if (tscratch && (tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
5840 seq_printf(m, "%lx\t[kernel]\n", tscratch->text_addr);
5841 else
5842 seq_puts(m, "# Current\n");
5843}
5844
5845static int l_show(struct seq_file *m, void *v)
5846{
5847 struct trace_array *tr = m->private;
5848 struct trace_mod_entry *entry = v;
5849
5850 if (v == LAST_BOOT_HEADER) {
5851 show_last_boot_header(m, tr);
5852 return 0;
5853 }
5854
5855 seq_printf(m, "%lx\t%s\n", entry->mod_addr, entry->mod_name);
5856 return 0;
5857}
5858
5859static const struct seq_operations last_boot_seq_ops = {
5860 .start = l_start,
5861 .next = l_next,
5862 .stop = l_stop,
5863 .show = l_show,
5864};
5865
5866static int tracing_last_boot_open(struct inode *inode, struct file *file)
5867{
5868 struct trace_array *tr = inode->i_private;
5869 struct seq_file *m;
5870 int ret;
5871
5872 ret = tracing_check_open_get_tr(tr);
5873 if (ret)
5874 return ret;
5875
5876 ret = seq_open(file, &last_boot_seq_ops);
5877 if (ret) {
5878 trace_array_put(tr);
5879 return ret;
5880 }
5881
5882 m = file->private_data;
5883 m->private = tr;
5884
5885 return 0;
5886}
5887
5888static int tracing_buffer_meta_open(struct inode *inode, struct file *filp)
5889{
5890 struct trace_array *tr = inode->i_private;
5891 int cpu = tracing_get_cpu(inode);
5892 int ret;
5893
5894 ret = tracing_check_open_get_tr(tr);
5895 if (ret)
5896 return ret;
5897
5898 ret = ring_buffer_meta_seq_init(filp, tr->array_buffer.buffer, cpu);
5899 if (ret < 0)
5900 trace_array_put(tr);
5901 return ret;
5902}
5903
5904static ssize_t
5905tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5906 size_t cnt, loff_t *ppos)
5907{
5908 /*
5909 * There is no need to read what the user has written, this function
5910 * is just to make sure that there is no error when "echo" is used
5911 */
5912
5913 *ppos += cnt;
5914
5915 return cnt;
5916}
5917
5918static int
5919tracing_free_buffer_release(struct inode *inode, struct file *filp)
5920{
5921 struct trace_array *tr = inode->i_private;
5922
5923 /* disable tracing ? */
5924 if (tr->trace_flags & TRACE_ITER(STOP_ON_FREE))
5925 tracer_tracing_off(tr);
5926 /* resize the ring buffer to 0 */
5927 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5928
5929 trace_array_put(tr);
5930
5931 return 0;
5932}
5933
5934#define TRACE_MARKER_MAX_SIZE 4096
5935
5936static ssize_t write_marker_to_buffer(struct trace_array *tr, const char *buf,
5937 size_t cnt, unsigned long ip)
5938{
5939 struct ring_buffer_event *event;
5940 enum event_trigger_type tt = ETT_NONE;
5941 struct trace_buffer *buffer;
5942 struct print_entry *entry;
5943 int meta_size;
5944 ssize_t written;
5945 size_t size;
5946
5947 meta_size = sizeof(*entry) + 2; /* add '\0' and possible '\n' */
5948 again:
5949 size = cnt + meta_size;
5950
5951 buffer = tr->array_buffer.buffer;
5952 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5953 tracing_gen_ctx());
5954 if (unlikely(!event)) {
5955 /*
5956 * If the size was greater than what was allowed, then
5957 * make it smaller and try again.
5958 */
5959 if (size > ring_buffer_max_event_size(buffer)) {
5960 cnt = ring_buffer_max_event_size(buffer) - meta_size;
5961 /* The above should only happen once */
5962 if (WARN_ON_ONCE(cnt + meta_size == size))
5963 return -EBADF;
5964 goto again;
5965 }
5966
5967 /* Ring buffer disabled, return as if not open for write */
5968 return -EBADF;
5969 }
5970
5971 entry = ring_buffer_event_data(event);
5972 entry->ip = ip;
5973 memcpy(&entry->buf, buf, cnt);
5974 written = cnt;
5975
5976 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
5977 /* do not add \n before testing triggers, but add \0 */
5978 entry->buf[cnt] = '\0';
5979 tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event);
5980 }
5981
5982 if (entry->buf[cnt - 1] != '\n') {
5983 entry->buf[cnt] = '\n';
5984 entry->buf[cnt + 1] = '\0';
5985 } else
5986 entry->buf[cnt] = '\0';
5987
5988 if (static_branch_unlikely(&trace_marker_exports_enabled))
5989 ftrace_exports(event, TRACE_EXPORT_MARKER);
5990 __buffer_unlock_commit(buffer, event);
5991
5992 if (tt)
5993 event_triggers_post_call(tr->trace_marker_file, tt);
5994
5995 return written;
5996}
5997
5998struct trace_user_buf {
5999 char *buf;
6000};
6001
6002static DEFINE_MUTEX(trace_user_buffer_mutex);
6003static struct trace_user_buf_info *trace_user_buffer;
6004
6005/**
6006 * trace_user_fault_destroy - free up allocated memory of a trace user buffer
6007 * @tinfo: The descriptor to free up
6008 *
6009 * Frees any data allocated in the trace info dsecriptor.
6010 */
6011void trace_user_fault_destroy(struct trace_user_buf_info *tinfo)
6012{
6013 char *buf;
6014 int cpu;
6015
6016 if (!tinfo || !tinfo->tbuf)
6017 return;
6018
6019 for_each_possible_cpu(cpu) {
6020 buf = per_cpu_ptr(tinfo->tbuf, cpu)->buf;
6021 kfree(buf);
6022 }
6023 free_percpu(tinfo->tbuf);
6024}
6025
6026static int user_fault_buffer_enable(struct trace_user_buf_info *tinfo, size_t size)
6027{
6028 char *buf;
6029 int cpu;
6030
6031 lockdep_assert_held(&trace_user_buffer_mutex);
6032
6033 tinfo->tbuf = alloc_percpu(struct trace_user_buf);
6034 if (!tinfo->tbuf)
6035 return -ENOMEM;
6036
6037 tinfo->ref = 1;
6038 tinfo->size = size;
6039
6040 /* Clear each buffer in case of error */
6041 for_each_possible_cpu(cpu) {
6042 per_cpu_ptr(tinfo->tbuf, cpu)->buf = NULL;
6043 }
6044
6045 for_each_possible_cpu(cpu) {
6046 buf = kmalloc_node(size, GFP_KERNEL,
6047 cpu_to_node(cpu));
6048 if (!buf)
6049 return -ENOMEM;
6050 per_cpu_ptr(tinfo->tbuf, cpu)->buf = buf;
6051 }
6052
6053 return 0;
6054}
6055
6056/* For internal use. Free and reinitialize */
6057static void user_buffer_free(struct trace_user_buf_info **tinfo)
6058{
6059 lockdep_assert_held(&trace_user_buffer_mutex);
6060
6061 trace_user_fault_destroy(*tinfo);
6062 kfree(*tinfo);
6063 *tinfo = NULL;
6064}
6065
6066/* For internal use. Initialize and allocate */
6067static int user_buffer_init(struct trace_user_buf_info **tinfo, size_t size)
6068{
6069 bool alloc = false;
6070 int ret;
6071
6072 lockdep_assert_held(&trace_user_buffer_mutex);
6073
6074 if (!*tinfo) {
6075 alloc = true;
6076 *tinfo = kzalloc_obj(**tinfo);
6077 if (!*tinfo)
6078 return -ENOMEM;
6079 }
6080
6081 ret = user_fault_buffer_enable(*tinfo, size);
6082 if (ret < 0 && alloc)
6083 user_buffer_free(tinfo);
6084
6085 return ret;
6086}
6087
6088/* For internal use, derefrence and free if necessary */
6089static void user_buffer_put(struct trace_user_buf_info **tinfo)
6090{
6091 guard(mutex)(&trace_user_buffer_mutex);
6092
6093 if (WARN_ON_ONCE(!*tinfo || !(*tinfo)->ref))
6094 return;
6095
6096 if (--(*tinfo)->ref)
6097 return;
6098
6099 user_buffer_free(tinfo);
6100}
6101
6102/**
6103 * trace_user_fault_init - Allocated or reference a per CPU buffer
6104 * @tinfo: A pointer to the trace buffer descriptor
6105 * @size: The size to allocate each per CPU buffer
6106 *
6107 * Create a per CPU buffer that can be used to copy from user space
6108 * in a task context. When calling trace_user_fault_read(), preemption
6109 * must be disabled, and it will enable preemption and copy user
6110 * space data to the buffer. If any schedule switches occur, it will
6111 * retry until it succeeds without a schedule switch knowing the buffer
6112 * is still valid.
6113 *
6114 * Returns 0 on success, negative on failure.
6115 */
6116int trace_user_fault_init(struct trace_user_buf_info *tinfo, size_t size)
6117{
6118 int ret;
6119
6120 if (!tinfo)
6121 return -EINVAL;
6122
6123 guard(mutex)(&trace_user_buffer_mutex);
6124
6125 ret = user_buffer_init(&tinfo, size);
6126 if (ret < 0)
6127 trace_user_fault_destroy(tinfo);
6128
6129 return ret;
6130}
6131
6132/**
6133 * trace_user_fault_get - up the ref count for the user buffer
6134 * @tinfo: A pointer to a pointer to the trace buffer descriptor
6135 *
6136 * Ups the ref count of the trace buffer.
6137 *
6138 * Returns the new ref count.
6139 */
6140int trace_user_fault_get(struct trace_user_buf_info *tinfo)
6141{
6142 if (!tinfo)
6143 return -1;
6144
6145 guard(mutex)(&trace_user_buffer_mutex);
6146
6147 tinfo->ref++;
6148 return tinfo->ref;
6149}
6150
6151/**
6152 * trace_user_fault_put - dereference a per cpu trace buffer
6153 * @tinfo: The @tinfo that was passed to trace_user_fault_get()
6154 *
6155 * Decrement the ref count of @tinfo.
6156 *
6157 * Returns the new refcount (negative on error).
6158 */
6159int trace_user_fault_put(struct trace_user_buf_info *tinfo)
6160{
6161 guard(mutex)(&trace_user_buffer_mutex);
6162
6163 if (WARN_ON_ONCE(!tinfo || !tinfo->ref))
6164 return -1;
6165
6166 --tinfo->ref;
6167 return tinfo->ref;
6168}
6169
6170/**
6171 * trace_user_fault_read - Read user space into a per CPU buffer
6172 * @tinfo: The @tinfo allocated by trace_user_fault_get()
6173 * @ptr: The user space pointer to read
6174 * @size: The size of user space to read.
6175 * @copy_func: Optional function to use to copy from user space
6176 * @data: Data to pass to copy_func if it was supplied
6177 *
6178 * Preemption must be disabled when this is called, and must not
6179 * be enabled while using the returned buffer.
6180 * This does the copying from user space into a per CPU buffer.
6181 *
6182 * The @size must not be greater than the size passed in to
6183 * trace_user_fault_init().
6184 *
6185 * If @copy_func is NULL, trace_user_fault_read() will use copy_from_user(),
6186 * otherwise it will call @copy_func. It will call @copy_func with:
6187 *
6188 * buffer: the per CPU buffer of the @tinfo.
6189 * ptr: The pointer @ptr to user space to read
6190 * size: The @size of the ptr to read
6191 * data: The @data parameter
6192 *
6193 * It is expected that @copy_func will return 0 on success and non zero
6194 * if there was a fault.
6195 *
6196 * Returns a pointer to the buffer with the content read from @ptr.
6197 * Preemption must remain disabled while the caller accesses the
6198 * buffer returned by this function.
6199 * Returns NULL if there was a fault, or the size passed in is
6200 * greater than the size passed to trace_user_fault_init().
6201 */
6202char *trace_user_fault_read(struct trace_user_buf_info *tinfo,
6203 const char __user *ptr, size_t size,
6204 trace_user_buf_copy copy_func, void *data)
6205{
6206 int cpu = smp_processor_id();
6207 char *buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf;
6208 unsigned long long cnt;
6209 int trys = 0;
6210 int ret;
6211
6212 lockdep_assert_preemption_disabled();
6213
6214 /*
6215 * It's up to the caller to not try to copy more than it said
6216 * it would.
6217 */
6218 if (size > tinfo->size)
6219 return NULL;
6220
6221 /*
6222 * This acts similar to a seqcount. The per CPU context switches are
6223 * recorded, migration is disabled and preemption is enabled. The
6224 * read of the user space memory is copied into the per CPU buffer.
6225 * Preemption is disabled again, and if the per CPU context switches count
6226 * is still the same, it means the buffer has not been corrupted.
6227 * If the count is different, it is assumed the buffer is corrupted
6228 * and reading must be tried again.
6229 */
6230
6231 do {
6232 /*
6233 * It is possible that something is trying to migrate this
6234 * task. What happens then, is when preemption is enabled,
6235 * the migration thread will preempt this task, try to
6236 * migrate it, fail, then let it run again. That will
6237 * cause this to loop again and never succeed.
6238 * On failures, enabled and disable preemption with
6239 * migration enabled, to allow the migration thread to
6240 * migrate this task.
6241 */
6242 if (trys) {
6243 preempt_enable_notrace();
6244 preempt_disable_notrace();
6245 cpu = smp_processor_id();
6246 buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf;
6247 }
6248
6249 /*
6250 * If for some reason, copy_from_user() always causes a context
6251 * switch, this would then cause an infinite loop.
6252 * If this task is preempted by another user space task, it
6253 * will cause this task to try again. But just in case something
6254 * changes where the copying from user space causes another task
6255 * to run, prevent this from going into an infinite loop.
6256 * 100 tries should be plenty.
6257 */
6258 if (WARN_ONCE(trys++ > 100, "Error: Too many tries to read user space"))
6259 return NULL;
6260
6261 /* Read the current CPU context switch counter */
6262 cnt = nr_context_switches_cpu(cpu);
6263
6264 /*
6265 * Preemption is going to be enabled, but this task must
6266 * remain on this CPU.
6267 */
6268 migrate_disable();
6269
6270 /*
6271 * Now preemption is being enabled and another task can come in
6272 * and use the same buffer and corrupt our data.
6273 */
6274 preempt_enable_notrace();
6275
6276 /* Make sure preemption is enabled here */
6277 lockdep_assert_preemption_enabled();
6278
6279 if (copy_func) {
6280 ret = copy_func(buffer, ptr, size, data);
6281 } else {
6282 ret = __copy_from_user(buffer, ptr, size);
6283 }
6284
6285 preempt_disable_notrace();
6286 migrate_enable();
6287
6288 /* if it faulted, no need to test if the buffer was corrupted */
6289 if (ret)
6290 return NULL;
6291
6292 /*
6293 * Preemption is disabled again, now check the per CPU context
6294 * switch counter. If it doesn't match, then another user space
6295 * process may have schedule in and corrupted our buffer. In that
6296 * case the copying must be retried.
6297 */
6298 } while (nr_context_switches_cpu(cpu) != cnt);
6299
6300 return buffer;
6301}
6302
6303static ssize_t
6304tracing_mark_write(struct file *filp, const char __user *ubuf,
6305 size_t cnt, loff_t *fpos)
6306{
6307 struct trace_array *tr = filp->private_data;
6308 ssize_t written = -ENODEV;
6309 unsigned long ip;
6310 char *buf;
6311
6312 if (unlikely(tracing_disabled))
6313 return -EINVAL;
6314
6315 if (!(tr->trace_flags & TRACE_ITER(MARKERS)))
6316 return -EINVAL;
6317
6318 if ((ssize_t)cnt < 0)
6319 return -EINVAL;
6320
6321 if (cnt > TRACE_MARKER_MAX_SIZE)
6322 cnt = TRACE_MARKER_MAX_SIZE;
6323
6324 /* Must have preemption disabled while having access to the buffer */
6325 guard(preempt_notrace)();
6326
6327 buf = trace_user_fault_read(trace_user_buffer, ubuf, cnt, NULL, NULL);
6328 if (!buf)
6329 return -EFAULT;
6330
6331 /* The selftests expect this function to be the IP address */
6332 ip = _THIS_IP_;
6333
6334 /* The global trace_marker can go to multiple instances */
6335 if (tr == &global_trace) {
6336 guard(rcu)();
6337 list_for_each_entry_rcu(tr, &marker_copies, marker_list) {
6338 written = write_marker_to_buffer(tr, buf, cnt, ip);
6339 if (written < 0)
6340 break;
6341 }
6342 } else {
6343 written = write_marker_to_buffer(tr, buf, cnt, ip);
6344 }
6345
6346 return written;
6347}
6348
6349static ssize_t write_raw_marker_to_buffer(struct trace_array *tr,
6350 const char *buf, size_t cnt)
6351{
6352 struct ring_buffer_event *event;
6353 struct trace_buffer *buffer;
6354 struct raw_data_entry *entry;
6355 ssize_t written;
6356 size_t size;
6357
6358 /* cnt includes both the entry->id and the data behind it. */
6359 size = struct_offset(entry, id) + cnt;
6360
6361 buffer = tr->array_buffer.buffer;
6362
6363 if (size > ring_buffer_max_event_size(buffer))
6364 return -EINVAL;
6365
6366 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6367 tracing_gen_ctx());
6368 if (!event)
6369 /* Ring buffer disabled, return as if not open for write */
6370 return -EBADF;
6371
6372 entry = ring_buffer_event_data(event);
6373 unsafe_memcpy(&entry->id, buf, cnt,
6374 "id and content already reserved on ring buffer"
6375 "'buf' includes the 'id' and the data."
6376 "'entry' was allocated with cnt from 'id'.");
6377 written = cnt;
6378
6379 __buffer_unlock_commit(buffer, event);
6380
6381 return written;
6382}
6383
6384static ssize_t
6385tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6386 size_t cnt, loff_t *fpos)
6387{
6388 struct trace_array *tr = filp->private_data;
6389 ssize_t written = -ENODEV;
6390 char *buf;
6391
6392 if (unlikely(tracing_disabled))
6393 return -EINVAL;
6394
6395 if (!(tr->trace_flags & TRACE_ITER(MARKERS)))
6396 return -EINVAL;
6397
6398 /* The marker must at least have a tag id */
6399 if (cnt < sizeof(unsigned int))
6400 return -EINVAL;
6401
6402 /* raw write is all or nothing */
6403 if (cnt > TRACE_MARKER_MAX_SIZE)
6404 return -EINVAL;
6405
6406 /* Must have preemption disabled while having access to the buffer */
6407 guard(preempt_notrace)();
6408
6409 buf = trace_user_fault_read(trace_user_buffer, ubuf, cnt, NULL, NULL);
6410 if (!buf)
6411 return -EFAULT;
6412
6413 /* The global trace_marker_raw can go to multiple instances */
6414 if (tr == &global_trace) {
6415 guard(rcu)();
6416 list_for_each_entry_rcu(tr, &marker_copies, marker_list) {
6417 written = write_raw_marker_to_buffer(tr, buf, cnt);
6418 if (written < 0)
6419 break;
6420 }
6421 } else {
6422 written = write_raw_marker_to_buffer(tr, buf, cnt);
6423 }
6424
6425 return written;
6426}
6427
6428static int tracing_mark_open(struct inode *inode, struct file *filp)
6429{
6430 int ret;
6431
6432 scoped_guard(mutex, &trace_user_buffer_mutex) {
6433 if (!trace_user_buffer) {
6434 ret = user_buffer_init(&trace_user_buffer, TRACE_MARKER_MAX_SIZE);
6435 if (ret < 0)
6436 return ret;
6437 } else {
6438 trace_user_buffer->ref++;
6439 }
6440 }
6441
6442 stream_open(inode, filp);
6443 ret = tracing_open_generic_tr(inode, filp);
6444 if (ret < 0)
6445 user_buffer_put(&trace_user_buffer);
6446 return ret;
6447}
6448
6449static int tracing_mark_release(struct inode *inode, struct file *file)
6450{
6451 user_buffer_put(&trace_user_buffer);
6452 return tracing_release_generic_tr(inode, file);
6453}
6454
6455static int tracing_clock_show(struct seq_file *m, void *v)
6456{
6457 struct trace_array *tr = m->private;
6458 int i;
6459
6460 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6461 seq_printf(m,
6462 "%s%s%s%s", i ? " " : "",
6463 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6464 i == tr->clock_id ? "]" : "");
6465 seq_putc(m, '\n');
6466
6467 return 0;
6468}
6469
6470int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6471{
6472 int i;
6473
6474 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6475 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6476 break;
6477 }
6478 if (i == ARRAY_SIZE(trace_clocks))
6479 return -EINVAL;
6480
6481 guard(mutex)(&trace_types_lock);
6482
6483 tr->clock_id = i;
6484
6485 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
6486
6487 /*
6488 * New clock may not be consistent with the previous clock.
6489 * Reset the buffer so that it doesn't have incomparable timestamps.
6490 */
6491 tracing_reset_online_cpus(&tr->array_buffer);
6492
6493#ifdef CONFIG_TRACER_SNAPSHOT
6494 if (tr->snapshot_buffer.buffer)
6495 ring_buffer_set_clock(tr->snapshot_buffer.buffer, trace_clocks[i].func);
6496 tracing_reset_online_cpus(&tr->snapshot_buffer);
6497#endif
6498 update_last_data_if_empty(tr);
6499
6500 if (tr->scratch && !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) {
6501 struct trace_scratch *tscratch = tr->scratch;
6502
6503 tscratch->clock_id = i;
6504 }
6505
6506 return 0;
6507}
6508
6509static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6510 size_t cnt, loff_t *fpos)
6511{
6512 struct seq_file *m = filp->private_data;
6513 struct trace_array *tr = m->private;
6514 char buf[64];
6515 const char *clockstr;
6516 int ret;
6517
6518 if (cnt >= sizeof(buf))
6519 return -EINVAL;
6520
6521 if (copy_from_user(buf, ubuf, cnt))
6522 return -EFAULT;
6523
6524 buf[cnt] = 0;
6525
6526 clockstr = strstrip(buf);
6527
6528 ret = tracing_set_clock(tr, clockstr);
6529 if (ret)
6530 return ret;
6531
6532 *fpos += cnt;
6533
6534 return cnt;
6535}
6536
6537static int tracing_clock_open(struct inode *inode, struct file *file)
6538{
6539 struct trace_array *tr = inode->i_private;
6540 int ret;
6541
6542 ret = tracing_check_open_get_tr(tr);
6543 if (ret)
6544 return ret;
6545
6546 if ((file->f_mode & FMODE_WRITE) && trace_array_is_readonly(tr)) {
6547 trace_array_put(tr);
6548 return -EACCES;
6549 }
6550
6551 ret = single_open(file, tracing_clock_show, inode->i_private);
6552 if (ret < 0)
6553 trace_array_put(tr);
6554
6555 return ret;
6556}
6557
6558static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6559{
6560 struct trace_array *tr = m->private;
6561
6562 guard(mutex)(&trace_types_lock);
6563
6564 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
6565 seq_puts(m, "delta [absolute]\n");
6566 else
6567 seq_puts(m, "[delta] absolute\n");
6568
6569 return 0;
6570}
6571
6572static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6573{
6574 struct trace_array *tr = inode->i_private;
6575 int ret;
6576
6577 ret = tracing_check_open_get_tr(tr);
6578 if (ret)
6579 return ret;
6580
6581 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6582 if (ret < 0)
6583 trace_array_put(tr);
6584
6585 return ret;
6586}
6587
6588u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_event *rbe)
6589{
6590 if (rbe == this_cpu_read(trace_buffered_event))
6591 return ring_buffer_time_stamp(buffer);
6592
6593 return ring_buffer_event_time_stamp(buffer, rbe);
6594}
6595
6596static const struct file_operations tracing_thresh_fops = {
6597 .open = tracing_open_generic,
6598 .read = tracing_thresh_read,
6599 .write = tracing_thresh_write,
6600 .llseek = generic_file_llseek,
6601};
6602
6603static const struct file_operations set_tracer_fops = {
6604 .open = tracing_open_generic_tr,
6605 .read = tracing_set_trace_read,
6606 .write = tracing_set_trace_write,
6607 .llseek = generic_file_llseek,
6608 .release = tracing_release_generic_tr,
6609};
6610
6611static const struct file_operations tracing_pipe_fops = {
6612 .open = tracing_open_pipe,
6613 .poll = tracing_poll_pipe,
6614 .read = tracing_read_pipe,
6615 .splice_read = tracing_splice_read_pipe,
6616 .release = tracing_release_pipe,
6617};
6618
6619static const struct file_operations tracing_entries_fops = {
6620 .open = tracing_open_generic_tr,
6621 .read = tracing_entries_read,
6622 .write = tracing_entries_write,
6623 .llseek = generic_file_llseek,
6624 .release = tracing_release_generic_tr,
6625};
6626
6627static const struct file_operations tracing_syscall_buf_fops = {
6628 .open = tracing_open_generic_tr,
6629 .read = tracing_syscall_buf_read,
6630 .write = tracing_syscall_buf_write,
6631 .llseek = generic_file_llseek,
6632 .release = tracing_release_generic_tr,
6633};
6634
6635static const struct file_operations tracing_buffer_meta_fops = {
6636 .open = tracing_buffer_meta_open,
6637 .read = seq_read,
6638 .llseek = seq_lseek,
6639 .release = tracing_seq_release,
6640};
6641
6642static const struct file_operations tracing_total_entries_fops = {
6643 .open = tracing_open_generic_tr,
6644 .read = tracing_total_entries_read,
6645 .llseek = generic_file_llseek,
6646 .release = tracing_release_generic_tr,
6647};
6648
6649static const struct file_operations tracing_free_buffer_fops = {
6650 .open = tracing_open_generic_tr,
6651 .write = tracing_free_buffer_write,
6652 .release = tracing_free_buffer_release,
6653};
6654
6655static const struct file_operations tracing_mark_fops = {
6656 .open = tracing_mark_open,
6657 .write = tracing_mark_write,
6658 .release = tracing_mark_release,
6659};
6660
6661static const struct file_operations tracing_mark_raw_fops = {
6662 .open = tracing_mark_open,
6663 .write = tracing_mark_raw_write,
6664 .release = tracing_mark_release,
6665};
6666
6667static const struct file_operations trace_clock_fops = {
6668 .open = tracing_clock_open,
6669 .read = seq_read,
6670 .llseek = seq_lseek,
6671 .release = tracing_single_release_tr,
6672 .write = tracing_clock_write,
6673};
6674
6675static const struct file_operations trace_time_stamp_mode_fops = {
6676 .open = tracing_time_stamp_mode_open,
6677 .read = seq_read,
6678 .llseek = seq_lseek,
6679 .release = tracing_single_release_tr,
6680};
6681
6682static const struct file_operations last_boot_fops = {
6683 .open = tracing_last_boot_open,
6684 .read = seq_read,
6685 .llseek = seq_lseek,
6686 .release = tracing_seq_release,
6687};
6688
6689/*
6690 * trace_min_max_write - Write a u64 value to a trace_min_max_param struct
6691 * @filp: The active open file structure
6692 * @ubuf: The userspace provided buffer to read value into
6693 * @cnt: The maximum number of bytes to read
6694 * @ppos: The current "file" position
6695 *
6696 * This function implements the write interface for a struct trace_min_max_param.
6697 * The filp->private_data must point to a trace_min_max_param structure that
6698 * defines where to write the value, the min and the max acceptable values,
6699 * and a lock to protect the write.
6700 */
6701static ssize_t
6702trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
6703{
6704 struct trace_min_max_param *param = filp->private_data;
6705 u64 val;
6706 int err;
6707
6708 if (!param)
6709 return -EFAULT;
6710
6711 err = kstrtoull_from_user(ubuf, cnt, 10, &val);
6712 if (err)
6713 return err;
6714
6715 if (param->lock)
6716 mutex_lock(param->lock);
6717
6718 if (param->min && val < *param->min)
6719 err = -EINVAL;
6720
6721 if (param->max && val > *param->max)
6722 err = -EINVAL;
6723
6724 if (!err)
6725 *param->val = val;
6726
6727 if (param->lock)
6728 mutex_unlock(param->lock);
6729
6730 if (err)
6731 return err;
6732
6733 return cnt;
6734}
6735
6736/*
6737 * trace_min_max_read - Read a u64 value from a trace_min_max_param struct
6738 * @filp: The active open file structure
6739 * @ubuf: The userspace provided buffer to read value into
6740 * @cnt: The maximum number of bytes to read
6741 * @ppos: The current "file" position
6742 *
6743 * This function implements the read interface for a struct trace_min_max_param.
6744 * The filp->private_data must point to a trace_min_max_param struct with valid
6745 * data.
6746 */
6747static ssize_t
6748trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
6749{
6750 struct trace_min_max_param *param = filp->private_data;
6751 char buf[U64_STR_SIZE];
6752 int len;
6753 u64 val;
6754
6755 if (!param)
6756 return -EFAULT;
6757
6758 val = *param->val;
6759
6760 if (cnt > sizeof(buf))
6761 cnt = sizeof(buf);
6762
6763 len = snprintf(buf, sizeof(buf), "%llu\n", val);
6764
6765 return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
6766}
6767
6768const struct file_operations trace_min_max_fops = {
6769 .open = tracing_open_generic,
6770 .read = trace_min_max_read,
6771 .write = trace_min_max_write,
6772};
6773
6774#define TRACING_LOG_ERRS_MAX 8
6775#define TRACING_LOG_LOC_MAX 128
6776
6777#define CMD_PREFIX " Command: "
6778
6779struct err_info {
6780 const char **errs; /* ptr to loc-specific array of err strings */
6781 u8 type; /* index into errs -> specific err string */
6782 u16 pos; /* caret position */
6783 u64 ts;
6784};
6785
6786struct tracing_log_err {
6787 struct list_head list;
6788 struct err_info info;
6789 char loc[TRACING_LOG_LOC_MAX]; /* err location */
6790 char *cmd; /* what caused err */
6791};
6792
6793static DEFINE_MUTEX(tracing_err_log_lock);
6794
6795static struct tracing_log_err *alloc_tracing_log_err(int len)
6796{
6797 struct tracing_log_err *err;
6798
6799 err = kzalloc_obj(*err);
6800 if (!err)
6801 return ERR_PTR(-ENOMEM);
6802
6803 err->cmd = kzalloc(len, GFP_KERNEL);
6804 if (!err->cmd) {
6805 kfree(err);
6806 return ERR_PTR(-ENOMEM);
6807 }
6808
6809 return err;
6810}
6811
6812static void free_tracing_log_err(struct tracing_log_err *err)
6813{
6814 kfree(err->cmd);
6815 kfree(err);
6816}
6817
6818static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr,
6819 int len)
6820{
6821 struct tracing_log_err *err;
6822 char *cmd;
6823
6824 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
6825 err = alloc_tracing_log_err(len);
6826 if (PTR_ERR(err) != -ENOMEM)
6827 tr->n_err_log_entries++;
6828
6829 return err;
6830 }
6831 cmd = kzalloc(len, GFP_KERNEL);
6832 if (!cmd)
6833 return ERR_PTR(-ENOMEM);
6834 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
6835 kfree(err->cmd);
6836 err->cmd = cmd;
6837 list_del(&err->list);
6838
6839 return err;
6840}
6841
6842/**
6843 * err_pos - find the position of a string within a command for error careting
6844 * @cmd: The tracing command that caused the error
6845 * @str: The string to position the caret at within @cmd
6846 *
6847 * Finds the position of the first occurrence of @str within @cmd. The
6848 * return value can be passed to tracing_log_err() for caret placement
6849 * within @cmd.
6850 *
6851 * Returns the index within @cmd of the first occurrence of @str or 0
6852 * if @str was not found.
6853 */
6854unsigned int err_pos(char *cmd, const char *str)
6855{
6856 char *found;
6857
6858 if (WARN_ON(!strlen(cmd)))
6859 return 0;
6860
6861 found = strstr(cmd, str);
6862 if (found)
6863 return found - cmd;
6864
6865 return 0;
6866}
6867
6868/**
6869 * tracing_log_err - write an error to the tracing error log
6870 * @tr: The associated trace array for the error (NULL for top level array)
6871 * @loc: A string describing where the error occurred
6872 * @cmd: The tracing command that caused the error
6873 * @errs: The array of loc-specific static error strings
6874 * @type: The index into errs[], which produces the specific static err string
6875 * @pos: The position the caret should be placed in the cmd
6876 *
6877 * Writes an error into tracing/error_log of the form:
6878 *
6879 * <loc>: error: <text>
6880 * Command: <cmd>
6881 * ^
6882 *
6883 * tracing/error_log is a small log file containing the last
6884 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
6885 * unless there has been a tracing error, and the error log can be
6886 * cleared and have its memory freed by writing the empty string in
6887 * truncation mode to it i.e. echo > tracing/error_log.
6888 *
6889 * NOTE: the @errs array along with the @type param are used to
6890 * produce a static error string - this string is not copied and saved
6891 * when the error is logged - only a pointer to it is saved. See
6892 * existing callers for examples of how static strings are typically
6893 * defined for use with tracing_log_err().
6894 */
6895void tracing_log_err(struct trace_array *tr,
6896 const char *loc, const char *cmd,
6897 const char **errs, u8 type, u16 pos)
6898{
6899 struct tracing_log_err *err;
6900 int len = 0;
6901
6902 if (!tr)
6903 tr = &global_trace;
6904
6905 len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1;
6906
6907 guard(mutex)(&tracing_err_log_lock);
6908
6909 err = get_tracing_log_err(tr, len);
6910 if (PTR_ERR(err) == -ENOMEM)
6911 return;
6912
6913 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
6914 snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd);
6915
6916 err->info.errs = errs;
6917 err->info.type = type;
6918 err->info.pos = pos;
6919 err->info.ts = local_clock();
6920
6921 list_add_tail(&err->list, &tr->err_log);
6922}
6923
6924static void clear_tracing_err_log(struct trace_array *tr)
6925{
6926 struct tracing_log_err *err, *next;
6927
6928 guard(mutex)(&tracing_err_log_lock);
6929
6930 list_for_each_entry_safe(err, next, &tr->err_log, list) {
6931 list_del(&err->list);
6932 free_tracing_log_err(err);
6933 }
6934
6935 tr->n_err_log_entries = 0;
6936}
6937
6938static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
6939{
6940 struct trace_array *tr = m->private;
6941
6942 mutex_lock(&tracing_err_log_lock);
6943
6944 return seq_list_start(&tr->err_log, *pos);
6945}
6946
6947static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
6948{
6949 struct trace_array *tr = m->private;
6950
6951 return seq_list_next(v, &tr->err_log, pos);
6952}
6953
6954static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
6955{
6956 mutex_unlock(&tracing_err_log_lock);
6957}
6958
6959static void tracing_err_log_show_pos(struct seq_file *m, u16 pos)
6960{
6961 u16 i;
6962
6963 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
6964 seq_putc(m, ' ');
6965 for (i = 0; i < pos; i++)
6966 seq_putc(m, ' ');
6967 seq_puts(m, "^\n");
6968}
6969
6970static int tracing_err_log_seq_show(struct seq_file *m, void *v)
6971{
6972 struct tracing_log_err *err = v;
6973
6974 if (err) {
6975 const char *err_text = err->info.errs[err->info.type];
6976 u64 sec = err->info.ts;
6977 u32 nsec;
6978
6979 nsec = do_div(sec, NSEC_PER_SEC);
6980 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
6981 err->loc, err_text);
6982 seq_printf(m, "%s", err->cmd);
6983 tracing_err_log_show_pos(m, err->info.pos);
6984 }
6985
6986 return 0;
6987}
6988
6989static const struct seq_operations tracing_err_log_seq_ops = {
6990 .start = tracing_err_log_seq_start,
6991 .next = tracing_err_log_seq_next,
6992 .stop = tracing_err_log_seq_stop,
6993 .show = tracing_err_log_seq_show
6994};
6995
6996static int tracing_err_log_open(struct inode *inode, struct file *file)
6997{
6998 struct trace_array *tr = inode->i_private;
6999 int ret = 0;
7000
7001 ret = tracing_check_open_get_tr(tr);
7002 if (ret)
7003 return ret;
7004
7005 /* If this file was opened for write, then erase contents */
7006 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7007 clear_tracing_err_log(tr);
7008
7009 if (file->f_mode & FMODE_READ) {
7010 ret = seq_open(file, &tracing_err_log_seq_ops);
7011 if (!ret) {
7012 struct seq_file *m = file->private_data;
7013 m->private = tr;
7014 } else {
7015 trace_array_put(tr);
7016 }
7017 }
7018 return ret;
7019}
7020
7021static ssize_t tracing_err_log_write(struct file *file,
7022 const char __user *buffer,
7023 size_t count, loff_t *ppos)
7024{
7025 return count;
7026}
7027
7028static int tracing_err_log_release(struct inode *inode, struct file *file)
7029{
7030 struct trace_array *tr = inode->i_private;
7031
7032 trace_array_put(tr);
7033
7034 if (file->f_mode & FMODE_READ)
7035 seq_release(inode, file);
7036
7037 return 0;
7038}
7039
7040static const struct file_operations tracing_err_log_fops = {
7041 .open = tracing_err_log_open,
7042 .write = tracing_err_log_write,
7043 .read = seq_read,
7044 .llseek = tracing_lseek,
7045 .release = tracing_err_log_release,
7046};
7047
7048int tracing_buffers_open(struct inode *inode, struct file *filp)
7049{
7050 struct trace_array *tr = inode->i_private;
7051 struct ftrace_buffer_info *info;
7052 int ret;
7053
7054 ret = tracing_check_open_get_tr(tr);
7055 if (ret)
7056 return ret;
7057
7058 info = kvzalloc_obj(*info);
7059 if (!info) {
7060 trace_array_put(tr);
7061 return -ENOMEM;
7062 }
7063
7064 mutex_lock(&trace_types_lock);
7065
7066 info->iter.tr = tr;
7067 info->iter.cpu_file = tracing_get_cpu(inode);
7068 info->iter.trace = tr->current_trace;
7069 info->iter.array_buffer = &tr->array_buffer;
7070 info->spare = NULL;
7071 /* Force reading ring buffer for first read */
7072 info->read = (unsigned int)-1;
7073
7074 filp->private_data = info;
7075
7076 tr->trace_ref++;
7077
7078 mutex_unlock(&trace_types_lock);
7079
7080 ret = nonseekable_open(inode, filp);
7081 if (ret < 0)
7082 trace_array_put(tr);
7083
7084 return ret;
7085}
7086
7087static __poll_t
7088tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7089{
7090 struct ftrace_buffer_info *info = filp->private_data;
7091 struct trace_iterator *iter = &info->iter;
7092
7093 return trace_poll(iter, filp, poll_table);
7094}
7095
7096ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7097 size_t count, loff_t *ppos)
7098{
7099 struct ftrace_buffer_info *info = filp->private_data;
7100 struct trace_iterator *iter = &info->iter;
7101 unsigned int spare_size;
7102 void *trace_data;
7103 ssize_t ret = 0;
7104 ssize_t size;
7105
7106 if (!count)
7107 return 0;
7108
7109 if (iter->snapshot && tracer_uses_snapshot(iter->tr->current_trace))
7110 return -EBUSY;
7111
7112 spare_size = ring_buffer_read_page_size(info->spare);
7113
7114again:
7115 /* Do we have previous read data to read? */
7116 if (info->read < spare_size)
7117 goto read;
7118
7119 ret = ring_buffer_alloc_read_page(iter->array_buffer->buffer, iter->cpu_file,
7120 &info->spare);
7121 if (ret)
7122 return ret;
7123
7124 spare_size = ring_buffer_read_page_size(info->spare);
7125 info->read = spare_size;
7126 info->spare_cpu = iter->cpu_file;
7127
7128 trace_access_lock(iter->cpu_file);
7129 ret = ring_buffer_read_page(iter->array_buffer->buffer,
7130 info->spare,
7131 count,
7132 iter->cpu_file, 0);
7133 trace_access_unlock(iter->cpu_file);
7134
7135 if (ret < 0) {
7136 if (trace_empty(iter) && !iter->closed) {
7137 if (update_last_data_if_empty(iter->tr))
7138 return 0;
7139
7140 if ((filp->f_flags & O_NONBLOCK))
7141 return -EAGAIN;
7142
7143 ret = wait_on_pipe(iter, 0);
7144 if (ret)
7145 return ret;
7146
7147 goto again;
7148 }
7149 return 0;
7150 }
7151
7152 info->read = 0;
7153
7154 read:
7155 size = spare_size - info->read;
7156 if (size > count)
7157 size = count;
7158 trace_data = ring_buffer_read_page_data(info->spare);
7159 ret = copy_to_user(ubuf, trace_data + info->read, size);
7160 if (ret == size)
7161 return -EFAULT;
7162
7163 size -= ret;
7164
7165 *ppos += size;
7166 info->read += size;
7167
7168 return size;
7169}
7170
7171static int tracing_buffers_flush(struct file *file, fl_owner_t id)
7172{
7173 struct ftrace_buffer_info *info = file->private_data;
7174 struct trace_iterator *iter = &info->iter;
7175
7176 iter->closed = true;
7177 /* Make sure the waiters see the new wait_index */
7178 (void)atomic_fetch_inc_release(&iter->wait_index);
7179
7180 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
7181
7182 return 0;
7183}
7184
7185int tracing_buffers_release(struct inode *inode, struct file *file)
7186{
7187 struct ftrace_buffer_info *info = file->private_data;
7188 struct trace_iterator *iter = &info->iter;
7189
7190 guard(mutex)(&trace_types_lock);
7191
7192 iter->tr->trace_ref--;
7193
7194 __trace_array_put(iter->tr);
7195
7196 ring_buffer_free_read_page(iter->array_buffer->buffer, info->spare_cpu, info->spare);
7197 kvfree(info);
7198
7199 return 0;
7200}
7201
7202struct buffer_ref {
7203 struct trace_buffer *buffer;
7204 struct buffer_data_read_page *rpage;
7205 int cpu;
7206 refcount_t refcount;
7207};
7208
7209static void buffer_ref_release(struct buffer_ref *ref)
7210{
7211 if (!refcount_dec_and_test(&ref->refcount))
7212 return;
7213 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->rpage);
7214 kfree(ref);
7215}
7216
7217static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7218 struct pipe_buffer *buf)
7219{
7220 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7221
7222 buffer_ref_release(ref);
7223 buf->private = 0;
7224}
7225
7226static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
7227 struct pipe_buffer *buf)
7228{
7229 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7230
7231 if (refcount_read(&ref->refcount) > INT_MAX/2)
7232 return false;
7233
7234 refcount_inc(&ref->refcount);
7235 return true;
7236}
7237
7238/* Pipe buffer operations for a buffer. */
7239static const struct pipe_buf_operations buffer_pipe_buf_ops = {
7240 .release = buffer_pipe_buf_release,
7241 .get = buffer_pipe_buf_get,
7242};
7243
7244/*
7245 * Callback from splice_to_pipe(), if we need to release some pages
7246 * at the end of the spd in case we error'ed out in filling the pipe.
7247 */
7248static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7249{
7250 struct buffer_ref *ref =
7251 (struct buffer_ref *)spd->partial[i].private;
7252
7253 buffer_ref_release(ref);
7254 spd->partial[i].private = 0;
7255}
7256
7257ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7258 struct pipe_inode_info *pipe, size_t len,
7259 unsigned int flags)
7260{
7261 struct ftrace_buffer_info *info = file->private_data;
7262 struct trace_iterator *iter = &info->iter;
7263 struct partial_page partial_def[PIPE_DEF_BUFFERS];
7264 struct page *pages_def[PIPE_DEF_BUFFERS];
7265 struct splice_pipe_desc spd = {
7266 .pages = pages_def,
7267 .partial = partial_def,
7268 .nr_pages_max = PIPE_DEF_BUFFERS,
7269 .ops = &buffer_pipe_buf_ops,
7270 .spd_release = buffer_spd_release,
7271 };
7272 unsigned int page_size = 0;
7273 struct buffer_ref *ref;
7274 bool woken = false;
7275 int entries, i;
7276 ssize_t ret = 0;
7277
7278 if (iter->snapshot && tracer_uses_snapshot(iter->tr->current_trace))
7279 return -EBUSY;
7280
7281 if (splice_grow_spd(pipe, &spd))
7282 return -ENOMEM;
7283
7284 again:
7285 trace_access_lock(iter->cpu_file);
7286 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7287
7288 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) {
7289 struct page *page;
7290 int r;
7291
7292 ref = kzalloc_obj(*ref);
7293 if (!ref) {
7294 ret = -ENOMEM;
7295 break;
7296 }
7297
7298 refcount_set(&ref->refcount, 1);
7299 ref->buffer = iter->array_buffer->buffer;
7300
7301 ret = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file, &ref->rpage);
7302 if (ret) {
7303 kfree(ref);
7304 break;
7305 }
7306 ref->cpu = iter->cpu_file;
7307
7308 page_size = ring_buffer_read_page_size(ref->rpage);
7309
7310 r = -EINVAL;
7311 if (IS_ALIGNED(*ppos, page_size) && len >= page_size) {
7312 r = ring_buffer_read_page(ref->buffer, ref->rpage, len, iter->cpu_file, 1);
7313 } else if (!i) {
7314 /*
7315 * If this fails to read on the first iteration, it
7316 * means the length was too small and an error should
7317 * be returned to user space. Otherwise, at least
7318 * one sub-buffer was successfully read but this failed
7319 * due to either the length was unaligned or the
7320 * subbuf order changed. Either case, do not report
7321 * an error.
7322 */
7323 ret = -EINVAL;
7324 }
7325
7326 if (r < 0) {
7327 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->rpage);
7328 kfree(ref);
7329 break;
7330 }
7331
7332 page = virt_to_page(ring_buffer_read_page_data(ref->rpage));
7333
7334 spd.pages[i] = page;
7335 spd.partial[i].len = page_size;
7336 spd.partial[i].offset = 0;
7337 spd.partial[i].private = (unsigned long)ref;
7338 spd.nr_pages++;
7339 *ppos += page_size;
7340
7341 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7342 }
7343
7344 trace_access_unlock(iter->cpu_file);
7345 spd.nr_pages = i;
7346
7347 /* did we read anything? */
7348 if (!spd.nr_pages) {
7349
7350 if (ret)
7351 goto out;
7352
7353 if (woken)
7354 goto out;
7355
7356 ret = -EAGAIN;
7357 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
7358 goto out;
7359
7360 ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent);
7361 if (ret)
7362 goto out;
7363
7364 /* No need to wait after waking up when tracing is off */
7365 if (!tracer_tracing_is_on(iter->tr))
7366 goto out;
7367
7368 /* Iterate one more time to collect any new data then exit */
7369 woken = true;
7370
7371 goto again;
7372 }
7373
7374 ret = splice_to_pipe(pipe, &spd);
7375out:
7376 splice_shrink_spd(&spd);
7377
7378 return ret;
7379}
7380
7381static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
7382{
7383 struct ftrace_buffer_info *info = file->private_data;
7384 struct trace_iterator *iter = &info->iter;
7385 int err;
7386
7387 if (cmd == TRACE_MMAP_IOCTL_GET_READER) {
7388 if (!(file->f_flags & O_NONBLOCK)) {
7389 err = ring_buffer_wait(iter->array_buffer->buffer,
7390 iter->cpu_file,
7391 iter->tr->buffer_percent,
7392 NULL, NULL);
7393 if (err)
7394 return err;
7395 }
7396
7397 return ring_buffer_map_get_reader(iter->array_buffer->buffer,
7398 iter->cpu_file);
7399 } else if (cmd) {
7400 return -ENOTTY;
7401 }
7402
7403 /*
7404 * An ioctl call with cmd 0 to the ring buffer file will wake up all
7405 * waiters
7406 */
7407 guard(mutex)(&trace_types_lock);
7408
7409 /* Make sure the waiters see the new wait_index */
7410 (void)atomic_fetch_inc_release(&iter->wait_index);
7411
7412 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
7413
7414 return 0;
7415}
7416
7417/*
7418 * This is called when a VMA is duplicated (e.g., on fork()) to increment
7419 * the user_mapped counter without remapping pages.
7420 */
7421static void tracing_buffers_mmap_open(struct vm_area_struct *vma)
7422{
7423 struct ftrace_buffer_info *info = vma->vm_file->private_data;
7424 struct trace_iterator *iter = &info->iter;
7425
7426 ring_buffer_map_dup(iter->array_buffer->buffer, iter->cpu_file);
7427}
7428
7429static void tracing_buffers_mmap_close(struct vm_area_struct *vma)
7430{
7431 struct ftrace_buffer_info *info = vma->vm_file->private_data;
7432 struct trace_iterator *iter = &info->iter;
7433
7434 WARN_ON(ring_buffer_unmap(iter->array_buffer->buffer, iter->cpu_file));
7435 put_snapshot_map(iter->tr);
7436}
7437
7438static int tracing_buffers_may_split(struct vm_area_struct *vma, unsigned long addr)
7439{
7440 /*
7441 * Trace buffer mappings require the complete buffer including
7442 * the meta page. Partial mappings are not supported.
7443 */
7444 return -EINVAL;
7445}
7446
7447static const struct vm_operations_struct tracing_buffers_vmops = {
7448 .open = tracing_buffers_mmap_open,
7449 .close = tracing_buffers_mmap_close,
7450 .may_split = tracing_buffers_may_split,
7451};
7452
7453static int tracing_buffers_mmap(struct file *filp, struct vm_area_struct *vma)
7454{
7455 struct ftrace_buffer_info *info = filp->private_data;
7456 struct trace_iterator *iter = &info->iter;
7457 int ret = 0;
7458
7459 /* A memmap'ed and backup buffers are not supported for user space mmap */
7460 if (iter->tr->flags & (TRACE_ARRAY_FL_MEMMAP | TRACE_ARRAY_FL_VMALLOC))
7461 return -ENODEV;
7462
7463 ret = get_snapshot_map(iter->tr);
7464 if (ret)
7465 return ret;
7466
7467 ret = ring_buffer_map(iter->array_buffer->buffer, iter->cpu_file, vma);
7468 if (ret)
7469 put_snapshot_map(iter->tr);
7470
7471 vma->vm_ops = &tracing_buffers_vmops;
7472
7473 return ret;
7474}
7475
7476static const struct file_operations tracing_buffers_fops = {
7477 .open = tracing_buffers_open,
7478 .read = tracing_buffers_read,
7479 .poll = tracing_buffers_poll,
7480 .release = tracing_buffers_release,
7481 .flush = tracing_buffers_flush,
7482 .splice_read = tracing_buffers_splice_read,
7483 .unlocked_ioctl = tracing_buffers_ioctl,
7484 .mmap = tracing_buffers_mmap,
7485};
7486
7487static ssize_t
7488tracing_stats_read(struct file *filp, char __user *ubuf,
7489 size_t count, loff_t *ppos)
7490{
7491 struct inode *inode = file_inode(filp);
7492 struct trace_array *tr = inode->i_private;
7493 struct array_buffer *trace_buf = &tr->array_buffer;
7494 int cpu = tracing_get_cpu(inode);
7495 struct trace_seq *s;
7496 unsigned long cnt;
7497 unsigned long long t;
7498 unsigned long usec_rem;
7499
7500 s = kmalloc_obj(*s);
7501 if (!s)
7502 return -ENOMEM;
7503
7504 trace_seq_init(s);
7505
7506 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
7507 trace_seq_printf(s, "entries: %ld\n", cnt);
7508
7509 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
7510 trace_seq_printf(s, "overrun: %ld\n", cnt);
7511
7512 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
7513 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7514
7515 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
7516 trace_seq_printf(s, "bytes: %ld\n", cnt);
7517
7518 if (trace_clocks[tr->clock_id].in_ns) {
7519 /* local or global for trace_clock */
7520 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7521 usec_rem = do_div(t, USEC_PER_SEC);
7522 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7523 t, usec_rem);
7524
7525 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer));
7526 usec_rem = do_div(t, USEC_PER_SEC);
7527 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7528 } else {
7529 /* counter or tsc mode for trace_clock */
7530 trace_seq_printf(s, "oldest event ts: %llu\n",
7531 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7532
7533 trace_seq_printf(s, "now ts: %llu\n",
7534 ring_buffer_time_stamp(trace_buf->buffer));
7535 }
7536
7537 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7538 trace_seq_printf(s, "dropped events: %ld\n", cnt);
7539
7540 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7541 trace_seq_printf(s, "read events: %ld\n", cnt);
7542
7543 count = simple_read_from_buffer(ubuf, count, ppos,
7544 s->buffer, trace_seq_used(s));
7545
7546 kfree(s);
7547
7548 return count;
7549}
7550
7551static const struct file_operations tracing_stats_fops = {
7552 .open = tracing_open_generic_tr,
7553 .read = tracing_stats_read,
7554 .llseek = generic_file_llseek,
7555 .release = tracing_release_generic_tr,
7556};
7557
7558#ifdef CONFIG_DYNAMIC_FTRACE
7559
7560static ssize_t
7561tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7562 size_t cnt, loff_t *ppos)
7563{
7564 ssize_t ret;
7565 char *buf;
7566 int r;
7567
7568 /* 512 should be plenty to hold the amount needed */
7569#define DYN_INFO_BUF_SIZE 512
7570
7571 buf = kmalloc(DYN_INFO_BUF_SIZE, GFP_KERNEL);
7572 if (!buf)
7573 return -ENOMEM;
7574
7575 r = scnprintf(buf, DYN_INFO_BUF_SIZE,
7576 "%ld pages:%ld groups: %ld\n"
7577 "ftrace boot update time = %llu (ns)\n"
7578 "ftrace module total update time = %llu (ns)\n",
7579 ftrace_update_tot_cnt,
7580 ftrace_number_of_pages,
7581 ftrace_number_of_groups,
7582 ftrace_update_time,
7583 ftrace_total_mod_time);
7584
7585 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7586 kfree(buf);
7587 return ret;
7588}
7589
7590static const struct file_operations tracing_dyn_info_fops = {
7591 .open = tracing_open_generic,
7592 .read = tracing_read_dyn_info,
7593 .llseek = generic_file_llseek,
7594};
7595#endif /* CONFIG_DYNAMIC_FTRACE */
7596
7597static struct dentry *tracing_get_dentry(struct trace_array *tr)
7598{
7599 /* Top directory uses NULL as the parent */
7600 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7601 return NULL;
7602
7603 if (WARN_ON(!tr->dir))
7604 return ERR_PTR(-ENODEV);
7605
7606 /* All sub buffers have a descriptor */
7607 return tr->dir;
7608}
7609
7610static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7611{
7612 struct dentry *d_tracer;
7613
7614 if (tr->percpu_dir)
7615 return tr->percpu_dir;
7616
7617 d_tracer = tracing_get_dentry(tr);
7618 if (IS_ERR(d_tracer))
7619 return NULL;
7620
7621 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
7622
7623 MEM_FAIL(!tr->percpu_dir,
7624 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
7625
7626 return tr->percpu_dir;
7627}
7628
7629struct dentry *
7630trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
7631 void *data, long cpu, const struct file_operations *fops)
7632{
7633 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
7634
7635 if (ret) /* See tracing_get_cpu() */
7636 d_inode(ret)->i_cdev = (void *)(cpu + 1);
7637 return ret;
7638}
7639
7640static void
7641tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
7642{
7643 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
7644 struct dentry *d_cpu;
7645 char cpu_dir[30]; /* 30 characters should be more than enough */
7646
7647 if (!d_percpu)
7648 return;
7649
7650 snprintf(cpu_dir, 30, "cpu%ld", cpu);
7651 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
7652 if (!d_cpu) {
7653 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
7654 return;
7655 }
7656
7657 /* per cpu trace_pipe */
7658 trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu,
7659 tr, cpu, &tracing_pipe_fops);
7660
7661 /* per cpu trace */
7662 trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu,
7663 tr, cpu, &tracing_fops);
7664
7665 trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu,
7666 tr, cpu, &tracing_buffers_fops);
7667
7668 trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu,
7669 tr, cpu, &tracing_stats_fops);
7670
7671 trace_create_cpu_file("buffer_size_kb", TRACE_MODE_WRITE, d_cpu,
7672 tr, cpu, &tracing_entries_fops);
7673
7674 if (tr->range_addr_start)
7675 trace_create_cpu_file("buffer_meta", TRACE_MODE_READ, d_cpu,
7676 tr, cpu, &tracing_buffer_meta_fops);
7677#ifdef CONFIG_TRACER_SNAPSHOT
7678 if (!tr->range_addr_start) {
7679 trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu,
7680 tr, cpu, &snapshot_fops);
7681
7682 trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu,
7683 tr, cpu, &snapshot_raw_fops);
7684 }
7685#endif
7686}
7687
7688#ifdef CONFIG_FTRACE_SELFTEST
7689/* Let selftest have access to static functions in this file */
7690#include "trace_selftest.c"
7691#endif
7692
7693static ssize_t
7694trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
7695 loff_t *ppos)
7696{
7697 struct trace_option_dentry *topt = filp->private_data;
7698 char *buf;
7699
7700 if (topt->flags->val & topt->opt->bit)
7701 buf = "1\n";
7702 else
7703 buf = "0\n";
7704
7705 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7706}
7707
7708static ssize_t
7709trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
7710 loff_t *ppos)
7711{
7712 struct trace_option_dentry *topt = filp->private_data;
7713 unsigned long val;
7714 int ret;
7715
7716 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7717 if (ret)
7718 return ret;
7719
7720 if (val != 0 && val != 1)
7721 return -EINVAL;
7722
7723 if (!!(topt->flags->val & topt->opt->bit) != val) {
7724 guard(mutex)(&trace_types_lock);
7725 ret = __set_tracer_option(topt->tr, topt->flags,
7726 topt->opt, !val);
7727 if (ret)
7728 return ret;
7729 }
7730
7731 *ppos += cnt;
7732
7733 return cnt;
7734}
7735
7736static bool tr_option_match(struct trace_array *tr, void *topt)
7737{
7738 for (int i = 0; i < tr->nr_topts; i++) {
7739 struct trace_options *tr_topts = &tr->topts[i];
7740
7741 if (topt >= (void *)&tr_topts->topts[0] &&
7742 topt < (void *)&tr_topts->topts[tr_topts->nr_topts])
7743 return true;
7744 }
7745 return false;
7746}
7747
7748/*
7749 * The topt is the address of a trace_array->topts[] element that holds the
7750 * the tracer options descriptor. But since the trace_array reference has not
7751 * been taken yet, it cannot be dereferenced as it could have been freed by
7752 * a rmdir of the instance the trace_array represents.
7753 *
7754 * Search the list of trace_arrays and compare the topt to the address of
7755 * the entire trace_array topts array for each trace_array in the list.
7756 * If one is matched, then take the reference and return it. If not, the
7757 * trace_array no longer exits.
7758 */
7759static int trace_array_tracer_options_get(void *topt)
7760{
7761 struct trace_array *tr;
7762 int ret;
7763
7764 ret = security_locked_down(LOCKDOWN_TRACEFS);
7765 if (ret)
7766 return ret;
7767
7768 if (tracing_disabled)
7769 return -ENODEV;
7770
7771 guard(mutex)(&trace_types_lock);
7772 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7773 if (tr_option_match(tr, topt))
7774 return __trace_array_get(tr);
7775 }
7776 return -ENODEV;
7777}
7778
7779static int tracing_open_options(struct inode *inode, struct file *filp)
7780{
7781 struct trace_option_dentry *topt = inode->i_private;
7782 int ret;
7783
7784 ret = trace_array_tracer_options_get(topt);
7785 if (ret)
7786 return ret;
7787
7788 filp->private_data = inode->i_private;
7789 return 0;
7790}
7791
7792static int tracing_release_options(struct inode *inode, struct file *file)
7793{
7794 struct trace_option_dentry *topt = file->private_data;
7795
7796 trace_array_put(topt->tr);
7797 return 0;
7798}
7799
7800static const struct file_operations trace_options_fops = {
7801 .open = tracing_open_options,
7802 .read = trace_options_read,
7803 .write = trace_options_write,
7804 .llseek = generic_file_llseek,
7805 .release = tracing_release_options,
7806};
7807
7808/*
7809 * In order to pass in both the trace_array descriptor as well as the index
7810 * to the flag that the trace option file represents, the trace_array
7811 * has a character array of trace_flags_index[], which holds the index
7812 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
7813 * The address of this character array is passed to the flag option file
7814 * read/write callbacks.
7815 *
7816 * In order to extract both the index and the trace_array descriptor,
7817 * get_tr_index() uses the following algorithm.
7818 *
7819 * idx = *ptr;
7820 *
7821 * As the pointer itself contains the address of the index (remember
7822 * index[1] == 1).
7823 *
7824 * Then to get the trace_array descriptor, by subtracting that index
7825 * from the ptr, we get to the start of the index itself.
7826 *
7827 * ptr - idx == &index[0]
7828 *
7829 * Then a simple container_of() from that pointer gets us to the
7830 * trace_array descriptor.
7831 */
7832static void get_tr_index(void *data, struct trace_array **ptr,
7833 unsigned int *pindex)
7834{
7835 *pindex = *(unsigned char *)data;
7836
7837 *ptr = container_of(data - *pindex, struct trace_array,
7838 trace_flags_index);
7839}
7840
7841static ssize_t
7842trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
7843 loff_t *ppos)
7844{
7845 void *tr_index = filp->private_data;
7846 struct trace_array *tr;
7847 unsigned int index;
7848 char *buf;
7849
7850 get_tr_index(tr_index, &tr, &index);
7851
7852 if (tr->trace_flags & (1ULL << index))
7853 buf = "1\n";
7854 else
7855 buf = "0\n";
7856
7857 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7858}
7859
7860static ssize_t
7861trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
7862 loff_t *ppos)
7863{
7864 void *tr_index = filp->private_data;
7865 struct trace_array *tr;
7866 unsigned int index;
7867 unsigned long val;
7868 int ret;
7869
7870 get_tr_index(tr_index, &tr, &index);
7871
7872 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7873 if (ret)
7874 return ret;
7875
7876 if (val != 0 && val != 1)
7877 return -EINVAL;
7878
7879 mutex_lock(&event_mutex);
7880 mutex_lock(&trace_types_lock);
7881 ret = set_tracer_flag(tr, 1ULL << index, val);
7882 mutex_unlock(&trace_types_lock);
7883 mutex_unlock(&event_mutex);
7884
7885 if (ret < 0)
7886 return ret;
7887
7888 *ppos += cnt;
7889
7890 return cnt;
7891}
7892
7893/*
7894 * The tr_index is the address of a trace_array->trace_flags_index[]
7895 * element that holds the index of the trace flag. But since the
7896 * trace_array reference has not been taken yet, it cannot be referenced
7897 * as it could have been freed by a rmdir of the instance the trace_array
7898 * represents.
7899 *
7900 * Search the list of trace_arrays and compare the tr_index to the
7901 * address of the entire trace_array trace_flags_index array for each
7902 * trace_array in the list. If one is matched, then take the reference
7903 * and return it. If not, the trace_array no longer exits.
7904 */
7905static int trace_array_options_get(void *tr_index)
7906{
7907 struct trace_array *tr;
7908 int ret;
7909
7910 ret = security_locked_down(LOCKDOWN_TRACEFS);
7911 if (ret)
7912 return ret;
7913
7914 if (tracing_disabled)
7915 return -ENODEV;
7916
7917 guard(mutex)(&trace_types_lock);
7918 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7919 if (tr_index >= (void *)&tr->trace_flags_index[0] &&
7920 tr_index < (void *)&tr->trace_flags_index[TRACE_FLAGS_MAX_SIZE])
7921 return __trace_array_get(tr);
7922 }
7923 return -ENODEV;
7924}
7925
7926static int trace_options_open(struct inode *inode, struct file *filp)
7927{
7928 void *tr_index = inode->i_private;
7929
7930 if (trace_array_options_get(tr_index) < 0)
7931 return -ENODEV;
7932
7933 filp->private_data = tr_index;
7934
7935 return 0;
7936}
7937
7938static int trace_options_release(struct inode *inode, struct file *filp)
7939{
7940 void *tr_index = filp->private_data;
7941 struct trace_array *tr;
7942 unsigned int index;
7943
7944 get_tr_index(tr_index, &tr, &index);
7945
7946 trace_array_put(tr);
7947
7948 return 0;
7949}
7950
7951static const struct file_operations trace_options_core_fops = {
7952 .open = trace_options_open,
7953 .read = trace_options_core_read,
7954 .write = trace_options_core_write,
7955 .llseek = generic_file_llseek,
7956 .release = trace_options_release,
7957};
7958
7959struct dentry *trace_create_file(const char *name,
7960 umode_t mode,
7961 struct dentry *parent,
7962 void *data,
7963 const struct file_operations *fops)
7964{
7965 struct dentry *ret;
7966
7967 ret = tracefs_create_file(name, mode, parent, data, fops);
7968 if (!ret)
7969 pr_warn("Could not create tracefs '%s' entry\n", name);
7970
7971 return ret;
7972}
7973
7974
7975static struct dentry *trace_options_init_dentry(struct trace_array *tr)
7976{
7977 struct dentry *d_tracer;
7978
7979 if (tr->options)
7980 return tr->options;
7981
7982 d_tracer = tracing_get_dentry(tr);
7983 if (IS_ERR(d_tracer))
7984 return NULL;
7985
7986 tr->options = tracefs_create_dir("options", d_tracer);
7987 if (!tr->options) {
7988 pr_warn("Could not create tracefs directory 'options'\n");
7989 return NULL;
7990 }
7991
7992 return tr->options;
7993}
7994
7995static void
7996create_trace_option_file(struct trace_array *tr,
7997 struct trace_option_dentry *topt,
7998 struct tracer_flags *flags,
7999 struct tracer_opt *opt)
8000{
8001 struct dentry *t_options;
8002
8003 t_options = trace_options_init_dentry(tr);
8004 if (!t_options)
8005 return;
8006
8007 topt->flags = flags;
8008 topt->opt = opt;
8009 topt->tr = tr;
8010
8011 topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,
8012 t_options, topt, &trace_options_fops);
8013}
8014
8015static int
8016create_trace_option_files(struct trace_array *tr, struct tracer *tracer,
8017 struct tracer_flags *flags)
8018{
8019 struct trace_option_dentry *topts;
8020 struct trace_options *tr_topts;
8021 struct tracer_opt *opts;
8022 int cnt;
8023
8024 if (!flags || !flags->opts)
8025 return 0;
8026
8027 opts = flags->opts;
8028
8029 for (cnt = 0; opts[cnt].name; cnt++)
8030 ;
8031
8032 topts = kzalloc_objs(*topts, cnt + 1);
8033 if (!topts)
8034 return 0;
8035
8036 tr_topts = krealloc_array(tr->topts, tr->nr_topts + 1, sizeof(*tr->topts),
8037 GFP_KERNEL);
8038 if (!tr_topts) {
8039 kfree(topts);
8040 return -ENOMEM;
8041 }
8042
8043 tr->topts = tr_topts;
8044 tr->topts[tr->nr_topts].tracer = tracer;
8045 tr->topts[tr->nr_topts].topts = topts;
8046 tr->topts[tr->nr_topts].nr_topts = cnt;
8047 tr->nr_topts++;
8048
8049 for (cnt = 0; opts[cnt].name; cnt++) {
8050 create_trace_option_file(tr, &topts[cnt], flags,
8051 &opts[cnt]);
8052 MEM_FAIL(topts[cnt].entry == NULL,
8053 "Failed to create trace option: %s",
8054 opts[cnt].name);
8055 }
8056 return 0;
8057}
8058
8059static int get_global_flags_val(struct tracer *tracer)
8060{
8061 struct tracers *t;
8062
8063 list_for_each_entry(t, &global_trace.tracers, list) {
8064 if (t->tracer != tracer)
8065 continue;
8066 if (!t->flags)
8067 return -1;
8068 return t->flags->val;
8069 }
8070 return -1;
8071}
8072
8073static int add_tracer_options(struct trace_array *tr, struct tracers *t)
8074{
8075 struct tracer *tracer = t->tracer;
8076 struct tracer_flags *flags = t->flags ?: tracer->flags;
8077
8078 if (!flags)
8079 return 0;
8080
8081 /* Only add tracer options after update_tracer_options finish */
8082 if (!tracer_options_updated)
8083 return 0;
8084
8085 return create_trace_option_files(tr, tracer, flags);
8086}
8087
8088static int add_tracer(struct trace_array *tr, struct tracer *tracer)
8089{
8090 struct tracer_flags *flags;
8091 struct tracers *t;
8092 int ret;
8093
8094 /* Only enable if the directory has been created already. */
8095 if (!tr->dir && !(tr->flags & TRACE_ARRAY_FL_GLOBAL))
8096 return 0;
8097
8098 /*
8099 * If this is an instance, only create flags for tracers
8100 * the instance may have.
8101 */
8102 if (!trace_ok_for_array(tracer, tr))
8103 return 0;
8104
8105 t = kmalloc_obj(*t);
8106 if (!t)
8107 return -ENOMEM;
8108
8109 t->tracer = tracer;
8110 t->flags = NULL;
8111 list_add(&t->list, &tr->tracers);
8112
8113 flags = tracer->flags;
8114 if (!flags) {
8115 if (!tracer->default_flags)
8116 return 0;
8117
8118 /*
8119 * If the tracer defines default flags, it means the flags are
8120 * per trace instance.
8121 */
8122 flags = kmalloc_obj(*flags);
8123 if (!flags)
8124 return -ENOMEM;
8125
8126 *flags = *tracer->default_flags;
8127 flags->trace = tracer;
8128
8129 t->flags = flags;
8130
8131 /* If this is an instance, inherit the global_trace flags */
8132 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL)) {
8133 int val = get_global_flags_val(tracer);
8134 if (!WARN_ON_ONCE(val < 0))
8135 flags->val = val;
8136 }
8137 }
8138
8139 ret = add_tracer_options(tr, t);
8140 if (ret < 0) {
8141 list_del(&t->list);
8142 kfree(t->flags);
8143 kfree(t);
8144 }
8145
8146 return ret;
8147}
8148
8149static struct dentry *
8150create_trace_option_core_file(struct trace_array *tr,
8151 const char *option, long index)
8152{
8153 struct dentry *t_options;
8154
8155 t_options = trace_options_init_dentry(tr);
8156 if (!t_options)
8157 return NULL;
8158
8159 return trace_create_file(option, TRACE_MODE_WRITE, t_options,
8160 (void *)&tr->trace_flags_index[index],
8161 &trace_options_core_fops);
8162}
8163
8164static void create_trace_options_dir(struct trace_array *tr)
8165{
8166 struct dentry *t_options;
8167 bool top_level = tr == &global_trace;
8168 int i;
8169
8170 t_options = trace_options_init_dentry(tr);
8171 if (!t_options)
8172 return;
8173
8174 for (i = 0; trace_options[i]; i++) {
8175 if (top_level ||
8176 !((1ULL << i) & TOP_LEVEL_TRACE_FLAGS)) {
8177 create_trace_option_core_file(tr, trace_options[i], i);
8178 }
8179 }
8180}
8181
8182static ssize_t
8183rb_simple_read(struct file *filp, char __user *ubuf,
8184 size_t cnt, loff_t *ppos)
8185{
8186 struct trace_array *tr = filp->private_data;
8187 char buf[64];
8188 int r;
8189
8190 r = tracer_tracing_is_on(tr);
8191 r = sprintf(buf, "%d\n", r);
8192
8193 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8194}
8195
8196static ssize_t
8197rb_simple_write(struct file *filp, const char __user *ubuf,
8198 size_t cnt, loff_t *ppos)
8199{
8200 struct trace_array *tr = filp->private_data;
8201 struct trace_buffer *buffer = tr->array_buffer.buffer;
8202 unsigned long val;
8203 int ret;
8204
8205 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8206 if (ret)
8207 return ret;
8208
8209 if (buffer) {
8210 guard(mutex)(&trace_types_lock);
8211 if (!!val == tracer_tracing_is_on(tr)) {
8212 val = 0; /* do nothing */
8213 } else if (val) {
8214 tracer_tracing_on(tr);
8215 if (tr->current_trace->start)
8216 tr->current_trace->start(tr);
8217 } else {
8218 tracer_tracing_off(tr);
8219 if (tr->current_trace->stop)
8220 tr->current_trace->stop(tr);
8221 /* Wake up any waiters */
8222 ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS);
8223 }
8224 }
8225
8226 (*ppos)++;
8227
8228 return cnt;
8229}
8230
8231static const struct file_operations rb_simple_fops = {
8232 .open = tracing_open_generic_tr,
8233 .read = rb_simple_read,
8234 .write = rb_simple_write,
8235 .release = tracing_release_generic_tr,
8236 .llseek = default_llseek,
8237};
8238
8239static ssize_t
8240buffer_percent_read(struct file *filp, char __user *ubuf,
8241 size_t cnt, loff_t *ppos)
8242{
8243 struct trace_array *tr = filp->private_data;
8244 char buf[64];
8245 int r;
8246
8247 r = tr->buffer_percent;
8248 r = sprintf(buf, "%d\n", r);
8249
8250 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8251}
8252
8253static ssize_t
8254buffer_percent_write(struct file *filp, const char __user *ubuf,
8255 size_t cnt, loff_t *ppos)
8256{
8257 struct trace_array *tr = filp->private_data;
8258 unsigned long val;
8259 int ret;
8260
8261 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8262 if (ret)
8263 return ret;
8264
8265 if (val > 100)
8266 return -EINVAL;
8267
8268 tr->buffer_percent = val;
8269
8270 (*ppos)++;
8271
8272 return cnt;
8273}
8274
8275static const struct file_operations buffer_percent_fops = {
8276 .open = tracing_open_generic_tr,
8277 .read = buffer_percent_read,
8278 .write = buffer_percent_write,
8279 .release = tracing_release_generic_tr,
8280 .llseek = default_llseek,
8281};
8282
8283static ssize_t
8284buffer_subbuf_size_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
8285{
8286 struct trace_array *tr = filp->private_data;
8287 size_t size;
8288 char buf[64];
8289 int order;
8290 int r;
8291
8292 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
8293 size = (PAGE_SIZE << order) / 1024;
8294
8295 r = sprintf(buf, "%zd\n", size);
8296
8297 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8298}
8299
8300static ssize_t
8301buffer_subbuf_size_write(struct file *filp, const char __user *ubuf,
8302 size_t cnt, loff_t *ppos)
8303{
8304 struct trace_array *tr = filp->private_data;
8305 unsigned long val;
8306 int old_order;
8307 int order;
8308 int pages;
8309 int ret;
8310
8311 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8312 if (ret)
8313 return ret;
8314
8315 val *= 1024; /* value passed in is in KB */
8316
8317 pages = DIV_ROUND_UP(val, PAGE_SIZE);
8318 order = fls(pages - 1);
8319
8320 /* limit between 1 and 128 system pages */
8321 if (order < 0 || order > 7)
8322 return -EINVAL;
8323
8324 /* Do not allow tracing while changing the order of the ring buffer */
8325 tracing_stop_tr(tr);
8326
8327 trace_access_lock(RING_BUFFER_ALL_CPUS);
8328
8329 old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
8330 if (old_order == order)
8331 goto out;
8332
8333 ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order);
8334 if (ret)
8335 goto out;
8336
8337#ifdef CONFIG_TRACER_SNAPSHOT
8338
8339 if (!tr->allocated_snapshot)
8340 goto out_max;
8341
8342 ret = ring_buffer_subbuf_order_set(tr->snapshot_buffer.buffer, order);
8343 if (ret) {
8344 /* Put back the old order */
8345 cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order);
8346 if (WARN_ON_ONCE(cnt)) {
8347 /*
8348 * AARGH! We are left with different orders!
8349 * The max buffer is our "snapshot" buffer.
8350 * When a tracer needs a snapshot (one of the
8351 * latency tracers), it swaps the max buffer
8352 * with the saved snap shot. We succeeded to
8353 * update the order of the main buffer, but failed to
8354 * update the order of the max buffer. But when we tried
8355 * to reset the main buffer to the original size, we
8356 * failed there too. This is very unlikely to
8357 * happen, but if it does, warn and kill all
8358 * tracing.
8359 */
8360 tracing_disabled = 1;
8361 }
8362 goto out;
8363 }
8364 out_max:
8365#endif
8366 (*ppos)++;
8367 out:
8368 trace_access_unlock(RING_BUFFER_ALL_CPUS);
8369 if (ret)
8370 cnt = ret;
8371 tracing_start_tr(tr);
8372 return cnt;
8373}
8374
8375static const struct file_operations buffer_subbuf_size_fops = {
8376 .open = tracing_open_generic_tr,
8377 .read = buffer_subbuf_size_read,
8378 .write = buffer_subbuf_size_write,
8379 .release = tracing_release_generic_tr,
8380 .llseek = default_llseek,
8381};
8382
8383static struct dentry *trace_instance_dir;
8384
8385static void
8386init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
8387
8388#ifdef CONFIG_MODULES
8389static int make_mod_delta(struct module *mod, void *data)
8390{
8391 struct trace_module_delta *module_delta;
8392 struct trace_scratch *tscratch;
8393 struct trace_mod_entry *entry;
8394 struct trace_array *tr = data;
8395 int i;
8396
8397 tscratch = tr->scratch;
8398 module_delta = READ_ONCE(tr->module_delta);
8399 for (i = 0; i < tscratch->nr_entries; i++) {
8400 entry = &tscratch->entries[i];
8401 if (strcmp(mod->name, entry->mod_name))
8402 continue;
8403 if (mod->state == MODULE_STATE_GOING)
8404 module_delta->delta[i] = 0;
8405 else
8406 module_delta->delta[i] = (unsigned long)mod->mem[MOD_TEXT].base
8407 - entry->mod_addr;
8408 break;
8409 }
8410 return 0;
8411}
8412#else
8413static int make_mod_delta(struct module *mod, void *data)
8414{
8415 return 0;
8416}
8417#endif
8418
8419static int mod_addr_comp(const void *a, const void *b, const void *data)
8420{
8421 const struct trace_mod_entry *e1 = a;
8422 const struct trace_mod_entry *e2 = b;
8423
8424 return e1->mod_addr > e2->mod_addr ? 1 : -1;
8425}
8426
8427static void setup_trace_scratch(struct trace_array *tr,
8428 struct trace_scratch *tscratch, unsigned int size)
8429{
8430 struct trace_module_delta *module_delta;
8431 struct trace_mod_entry *entry;
8432 int i, nr_entries;
8433
8434 if (!tscratch)
8435 return;
8436
8437 tr->scratch = tscratch;
8438 tr->scratch_size = size;
8439
8440 if (tscratch->text_addr)
8441 tr->text_delta = (unsigned long)_text - tscratch->text_addr;
8442
8443 if (struct_size(tscratch, entries, tscratch->nr_entries) > size)
8444 goto reset;
8445
8446 /* Check if each module name is a valid string */
8447 for (i = 0; i < tscratch->nr_entries; i++) {
8448 int n;
8449
8450 entry = &tscratch->entries[i];
8451
8452 for (n = 0; n < MODULE_NAME_LEN; n++) {
8453 if (entry->mod_name[n] == '\0')
8454 break;
8455 if (!isprint(entry->mod_name[n]))
8456 goto reset;
8457 }
8458 if (n == MODULE_NAME_LEN)
8459 goto reset;
8460 }
8461
8462 /* Sort the entries so that we can find appropriate module from address. */
8463 nr_entries = tscratch->nr_entries;
8464 sort_r(tscratch->entries, nr_entries, sizeof(struct trace_mod_entry),
8465 mod_addr_comp, NULL, NULL);
8466
8467 if (IS_ENABLED(CONFIG_MODULES)) {
8468 module_delta = kzalloc_flex(*module_delta, delta, nr_entries);
8469 if (!module_delta) {
8470 pr_info("module_delta allocation failed. Not able to decode module address.");
8471 goto reset;
8472 }
8473 init_rcu_head(&module_delta->rcu);
8474 } else
8475 module_delta = NULL;
8476 WRITE_ONCE(tr->module_delta, module_delta);
8477
8478 /* Scan modules to make text delta for modules. */
8479 module_for_each_mod(make_mod_delta, tr);
8480
8481 /* Set trace_clock as the same of the previous boot. */
8482 if (tscratch->clock_id != tr->clock_id) {
8483 if (tscratch->clock_id >= ARRAY_SIZE(trace_clocks) ||
8484 tracing_set_clock(tr, trace_clocks[tscratch->clock_id].name) < 0) {
8485 pr_info("the previous trace_clock info is not valid.");
8486 goto reset;
8487 }
8488 }
8489 return;
8490 reset:
8491 /* Invalid trace modules */
8492 memset(tscratch, 0, size);
8493}
8494
8495#define TRACE_TEST_PTRACING_NAME "ptracingtest"
8496
8497int allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
8498{
8499 enum ring_buffer_flags rb_flags;
8500 struct trace_scratch *tscratch;
8501 unsigned int scratch_size = 0;
8502
8503 rb_flags = tr->trace_flags & TRACE_ITER(OVERWRITE) ? RB_FL_OVERWRITE : 0;
8504
8505 buf->tr = tr;
8506
8507 if (tr->range_addr_start && tr->range_addr_size) {
8508 if (tr->name && !strcmp(tr->name, TRACE_TEST_PTRACING_NAME))
8509 rb_flags |= RB_FL_TESTING;
8510 /* Add scratch buffer to handle 128 modules */
8511 buf->buffer = ring_buffer_alloc_range(size, rb_flags, 0,
8512 tr->range_addr_start,
8513 tr->range_addr_size,
8514 struct_size(tscratch, entries, 128));
8515
8516 tscratch = ring_buffer_meta_scratch(buf->buffer, &scratch_size);
8517 setup_trace_scratch(tr, tscratch, scratch_size);
8518
8519 /*
8520 * This is basically the same as a mapped buffer,
8521 * with the same restrictions.
8522 */
8523 tr->mapped++;
8524 } else {
8525 buf->buffer = ring_buffer_alloc(size, rb_flags);
8526 }
8527 if (!buf->buffer)
8528 return -ENOMEM;
8529
8530 buf->data = alloc_percpu(struct trace_array_cpu);
8531 if (!buf->data) {
8532 ring_buffer_free(buf->buffer);
8533 buf->buffer = NULL;
8534 return -ENOMEM;
8535 }
8536
8537 /* Allocate the first page for all buffers */
8538 trace_set_buffer_entries(&tr->array_buffer,
8539 ring_buffer_size(tr->array_buffer.buffer, 0));
8540
8541 return 0;
8542}
8543
8544static void free_trace_buffer(struct array_buffer *buf)
8545{
8546 if (buf->buffer) {
8547 ring_buffer_free(buf->buffer);
8548 buf->buffer = NULL;
8549 free_percpu(buf->data);
8550 buf->data = NULL;
8551 }
8552}
8553
8554static int allocate_trace_buffers(struct trace_array *tr, unsigned long size)
8555{
8556 int ret;
8557
8558 ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
8559 if (ret)
8560 return ret;
8561
8562 ret = trace_allocate_snapshot(tr, size);
8563 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n"))
8564 free_trace_buffer(&tr->array_buffer);
8565
8566 return ret;
8567}
8568
8569static void free_trace_buffers(struct trace_array *tr)
8570{
8571 if (!tr)
8572 return;
8573
8574 free_trace_buffer(&tr->array_buffer);
8575 kfree(tr->module_delta);
8576
8577#ifdef CONFIG_TRACER_SNAPSHOT
8578 free_trace_buffer(&tr->snapshot_buffer);
8579#endif
8580}
8581
8582static void init_trace_flags_index(struct trace_array *tr)
8583{
8584 int i;
8585
8586 /* Used by the trace options files */
8587 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8588 tr->trace_flags_index[i] = i;
8589}
8590
8591static int __update_tracer(struct trace_array *tr)
8592{
8593 struct tracer *t;
8594 int ret = 0;
8595
8596 for (t = trace_types; t && !ret; t = t->next)
8597 ret = add_tracer(tr, t);
8598
8599 return ret;
8600}
8601
8602static __init int __update_tracer_options(struct trace_array *tr)
8603{
8604 struct tracers *t;
8605 int ret = 0;
8606
8607 list_for_each_entry(t, &tr->tracers, list) {
8608 ret = add_tracer_options(tr, t);
8609 if (ret < 0)
8610 break;
8611 }
8612
8613 return ret;
8614}
8615
8616static __init void update_tracer_options(void)
8617{
8618 struct trace_array *tr;
8619
8620 guard(mutex)(&trace_types_lock);
8621 tracer_options_updated = true;
8622 list_for_each_entry(tr, &ftrace_trace_arrays, list)
8623 __update_tracer_options(tr);
8624}
8625
8626/* Must have trace_types_lock held */
8627struct trace_array *trace_array_find(const char *instance)
8628{
8629 struct trace_array *tr, *found = NULL;
8630
8631 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8632 if (tr->name && strcmp(tr->name, instance) == 0) {
8633 found = tr;
8634 break;
8635 }
8636 }
8637
8638 return found;
8639}
8640
8641struct trace_array *trace_array_find_get(const char *instance)
8642{
8643 struct trace_array *tr;
8644
8645 guard(mutex)(&trace_types_lock);
8646 tr = trace_array_find(instance);
8647 if (tr && __trace_array_get(tr) < 0)
8648 tr = NULL;
8649
8650 return tr;
8651}
8652
8653static int trace_array_create_dir(struct trace_array *tr)
8654{
8655 int ret;
8656
8657 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
8658 if (!tr->dir)
8659 return -EINVAL;
8660
8661 ret = event_trace_add_tracer(tr->dir, tr);
8662 if (ret) {
8663 tracefs_remove(tr->dir);
8664 return ret;
8665 }
8666
8667 init_tracer_tracefs(tr, tr->dir);
8668 ret = __update_tracer(tr);
8669 if (ret) {
8670 event_trace_del_tracer(tr);
8671 tracefs_remove(tr->dir);
8672 return ret;
8673 }
8674 return 0;
8675}
8676
8677static struct trace_array *
8678trace_array_create_systems(const char *name, const char *systems,
8679 unsigned long range_addr_start,
8680 unsigned long range_addr_size)
8681{
8682 struct trace_array *tr;
8683 int ret;
8684
8685 ret = -ENOMEM;
8686 tr = kzalloc_obj(*tr);
8687 if (!tr)
8688 return ERR_PTR(ret);
8689
8690 tr->name = kstrdup(name, GFP_KERNEL);
8691 if (!tr->name)
8692 goto out_free_tr;
8693
8694 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8695 goto out_free_tr;
8696
8697 if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL))
8698 goto out_free_tr;
8699
8700 if (systems) {
8701 tr->system_names = kstrdup_const(systems, GFP_KERNEL);
8702 if (!tr->system_names)
8703 goto out_free_tr;
8704 }
8705
8706 /* Only for boot up memory mapped ring buffers */
8707 tr->range_addr_start = range_addr_start;
8708 tr->range_addr_size = range_addr_size;
8709
8710 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
8711
8712 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8713
8714 raw_spin_lock_init(&tr->start_lock);
8715
8716 tr->syscall_buf_sz = global_trace.syscall_buf_sz;
8717
8718 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8719#ifdef CONFIG_TRACER_SNAPSHOT
8720 spin_lock_init(&tr->snapshot_trigger_lock);
8721#endif
8722 tr->current_trace = &nop_trace;
8723 tr->current_trace_flags = nop_trace.flags;
8724
8725 INIT_LIST_HEAD(&tr->systems);
8726 INIT_LIST_HEAD(&tr->events);
8727 INIT_LIST_HEAD(&tr->hist_vars);
8728 INIT_LIST_HEAD(&tr->err_log);
8729 INIT_LIST_HEAD(&tr->tracers);
8730 INIT_LIST_HEAD(&tr->marker_list);
8731
8732#ifdef CONFIG_MODULES
8733 INIT_LIST_HEAD(&tr->mod_events);
8734#endif
8735
8736 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
8737 goto out_free_tr;
8738
8739 /* The ring buffer is defaultly expanded */
8740 trace_set_ring_buffer_expanded(tr);
8741
8742 if (ftrace_allocate_ftrace_ops(tr) < 0)
8743 goto out_free_tr;
8744
8745 trace_array_init_autoremove(tr);
8746
8747 ftrace_init_trace_array(tr);
8748
8749 init_trace_flags_index(tr);
8750
8751 if (trace_instance_dir) {
8752 ret = trace_array_create_dir(tr);
8753 if (ret)
8754 goto out_free_tr;
8755 } else
8756 __trace_early_add_events(tr);
8757
8758 list_add(&tr->list, &ftrace_trace_arrays);
8759
8760 tr->ref++;
8761
8762 return tr;
8763
8764 out_free_tr:
8765 ftrace_free_ftrace_ops(tr);
8766 free_trace_buffers(tr);
8767 free_cpumask_var(tr->pipe_cpumask);
8768 free_cpumask_var(tr->tracing_cpumask);
8769 kfree_const(tr->system_names);
8770 kfree(tr->range_name);
8771 kfree(tr->name);
8772 kfree(tr);
8773
8774 return ERR_PTR(ret);
8775}
8776
8777static struct trace_array *trace_array_create(const char *name)
8778{
8779 return trace_array_create_systems(name, NULL, 0, 0);
8780}
8781
8782static int instance_mkdir(const char *name)
8783{
8784 struct trace_array *tr;
8785 int ret;
8786
8787 guard(mutex)(&event_mutex);
8788 guard(mutex)(&trace_types_lock);
8789
8790 ret = -EEXIST;
8791 if (trace_array_find(name))
8792 return -EEXIST;
8793
8794 tr = trace_array_create(name);
8795
8796 ret = PTR_ERR_OR_ZERO(tr);
8797
8798 return ret;
8799}
8800
8801#ifdef CONFIG_MMU
8802static u64 map_pages(unsigned long start, unsigned long size)
8803{
8804 unsigned long vmap_start, vmap_end;
8805 struct vm_struct *area;
8806 int ret;
8807
8808 area = get_vm_area(size, VM_IOREMAP);
8809 if (!area)
8810 return 0;
8811
8812 vmap_start = (unsigned long) area->addr;
8813 vmap_end = vmap_start + size;
8814
8815 ret = vmap_page_range(vmap_start, vmap_end,
8816 start, pgprot_nx(PAGE_KERNEL));
8817 if (ret < 0) {
8818 free_vm_area(area);
8819 return 0;
8820 }
8821
8822 return (u64)vmap_start;
8823}
8824#else
8825static inline u64 map_pages(unsigned long start, unsigned long size)
8826{
8827 return 0;
8828}
8829#endif
8830
8831/**
8832 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
8833 * @name: The name of the trace array to be looked up/created.
8834 * @systems: A list of systems to create event directories for (NULL for all)
8835 *
8836 * Returns pointer to trace array with given name.
8837 * NULL, if it cannot be created.
8838 *
8839 * NOTE: This function increments the reference counter associated with the
8840 * trace array returned. This makes sure it cannot be freed while in use.
8841 * Use trace_array_put() once the trace array is no longer needed.
8842 * If the trace_array is to be freed, trace_array_destroy() needs to
8843 * be called after the trace_array_put(), or simply let user space delete
8844 * it from the tracefs instances directory. But until the
8845 * trace_array_put() is called, user space can not delete it.
8846 *
8847 */
8848struct trace_array *trace_array_get_by_name(const char *name, const char *systems)
8849{
8850 struct trace_array *tr;
8851
8852 guard(mutex)(&event_mutex);
8853 guard(mutex)(&trace_types_lock);
8854
8855 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8856 if (tr->name && strcmp(tr->name, name) == 0) {
8857 /* if this fails, @tr is going to be removed. */
8858 if (__trace_array_get(tr) < 0)
8859 tr = NULL;
8860 return tr;
8861 }
8862 }
8863
8864 tr = trace_array_create_systems(name, systems, 0, 0);
8865
8866 if (IS_ERR(tr))
8867 tr = NULL;
8868 else
8869 tr->ref++;
8870
8871 return tr;
8872}
8873EXPORT_SYMBOL_GPL(trace_array_get_by_name);
8874
8875static int __remove_instance(struct trace_array *tr)
8876{
8877 int i;
8878
8879 /* Reference counter for a newly created trace array = 1. */
8880 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))
8881 return -EBUSY;
8882
8883 list_del(&tr->list);
8884
8885 if (printk_trace == tr)
8886 update_printk_trace(&global_trace);
8887
8888 /* Must be done before disabling all the flags */
8889 if (update_marker_trace(tr, 0))
8890 synchronize_rcu();
8891
8892 /* Disable all the flags that were enabled coming in */
8893 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8894 if ((1ULL << i) & ZEROED_TRACE_FLAGS)
8895 set_tracer_flag(tr, 1ULL << i, 0);
8896 }
8897
8898 trace_array_cancel_autoremove(tr);
8899 tracing_set_nop(tr);
8900 clear_ftrace_function_probes(tr);
8901 event_trace_del_tracer(tr);
8902 ftrace_clear_pids(tr);
8903 ftrace_destroy_function_files(tr);
8904 tracefs_remove(tr->dir);
8905 free_percpu(tr->last_func_repeats);
8906 free_trace_buffers(tr);
8907 clear_tracing_err_log(tr);
8908 free_tracers(tr);
8909
8910 if (tr->range_name) {
8911 reserve_mem_release_by_name(tr->range_name);
8912 kfree(tr->range_name);
8913 }
8914 if (tr->flags & TRACE_ARRAY_FL_VMALLOC)
8915 vfree((void *)tr->range_addr_start);
8916
8917 for (i = 0; i < tr->nr_topts; i++) {
8918 kfree(tr->topts[i].topts);
8919 }
8920 kfree(tr->topts);
8921
8922 free_cpumask_var(tr->pipe_cpumask);
8923 free_cpumask_var(tr->tracing_cpumask);
8924 kfree_const(tr->system_names);
8925 kfree(tr->name);
8926 kfree(tr);
8927
8928 return 0;
8929}
8930
8931int trace_array_destroy(struct trace_array *this_tr)
8932{
8933 struct trace_array *tr;
8934
8935 if (!this_tr)
8936 return -EINVAL;
8937
8938 guard(mutex)(&event_mutex);
8939 guard(mutex)(&trace_types_lock);
8940
8941
8942 /* Making sure trace array exists before destroying it. */
8943 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8944 if (tr == this_tr)
8945 return __remove_instance(tr);
8946 }
8947
8948 return -ENODEV;
8949}
8950EXPORT_SYMBOL_GPL(trace_array_destroy);
8951
8952static int instance_rmdir(const char *name)
8953{
8954 struct trace_array *tr;
8955
8956 guard(mutex)(&event_mutex);
8957 guard(mutex)(&trace_types_lock);
8958
8959 tr = trace_array_find(name);
8960 if (!tr)
8961 return -ENODEV;
8962
8963 return __remove_instance(tr);
8964}
8965
8966static __init void create_trace_instances(struct dentry *d_tracer)
8967{
8968 struct trace_array *tr;
8969
8970 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8971 instance_mkdir,
8972 instance_rmdir);
8973 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
8974 return;
8975
8976 guard(mutex)(&event_mutex);
8977 guard(mutex)(&trace_types_lock);
8978
8979 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8980 if (!tr->name)
8981 continue;
8982 if (MEM_FAIL(trace_array_create_dir(tr) < 0,
8983 "Failed to create instance directory\n"))
8984 return;
8985 }
8986}
8987
8988static void
8989init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8990{
8991 umode_t writable_mode = TRACE_MODE_WRITE;
8992 int cpu;
8993
8994 if (trace_array_is_readonly(tr))
8995 writable_mode = TRACE_MODE_READ;
8996
8997 trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer,
8998 tr, &show_traces_fops);
8999
9000 trace_create_file("current_tracer", writable_mode, d_tracer,
9001 tr, &set_tracer_fops);
9002
9003 trace_create_file("tracing_cpumask", writable_mode, d_tracer,
9004 tr, &tracing_cpumask_fops);
9005
9006 /* Options are used for changing print-format even for readonly instance. */
9007 trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer,
9008 tr, &tracing_iter_fops);
9009
9010 trace_create_file("trace", TRACE_MODE_WRITE, d_tracer,
9011 tr, &tracing_fops);
9012
9013 trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer,
9014 tr, &tracing_pipe_fops);
9015
9016 trace_create_file("buffer_size_kb", writable_mode, d_tracer,
9017 tr, &tracing_entries_fops);
9018
9019 trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer,
9020 tr, &tracing_total_entries_fops);
9021
9022 trace_create_file("trace_clock", writable_mode, d_tracer, tr,
9023 &trace_clock_fops);
9024
9025 trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr,
9026 &trace_time_stamp_mode_fops);
9027
9028 tr->buffer_percent = 50;
9029
9030 trace_create_file("buffer_subbuf_size_kb", writable_mode, d_tracer,
9031 tr, &buffer_subbuf_size_fops);
9032
9033 create_trace_options_dir(tr);
9034
9035 if (tr->range_addr_start)
9036 trace_create_file("last_boot_info", TRACE_MODE_READ, d_tracer,
9037 tr, &last_boot_fops);
9038
9039 for_each_tracing_cpu(cpu)
9040 tracing_init_tracefs_percpu(tr, cpu);
9041
9042 /* Read-only instance has above files only. */
9043 if (trace_array_is_readonly(tr))
9044 return;
9045
9046 trace_create_file("free_buffer", 0200, d_tracer,
9047 tr, &tracing_free_buffer_fops);
9048
9049 trace_create_file("trace_marker", 0220, d_tracer,
9050 tr, &tracing_mark_fops);
9051
9052 tr->trace_marker_file = __find_event_file(tr, "ftrace", "print");
9053
9054 trace_create_file("trace_marker_raw", 0220, d_tracer,
9055 tr, &tracing_mark_raw_fops);
9056
9057 trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer,
9058 tr, &buffer_percent_fops);
9059
9060 trace_create_file("syscall_user_buf_size", TRACE_MODE_WRITE, d_tracer,
9061 tr, &tracing_syscall_buf_fops);
9062
9063 trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer,
9064 tr, &rb_simple_fops);
9065
9066 trace_create_maxlat_file(tr, d_tracer);
9067
9068 if (ftrace_create_function_files(tr, d_tracer))
9069 MEM_FAIL(1, "Could not allocate function filter files");
9070
9071#ifdef CONFIG_TRACER_SNAPSHOT
9072 if (!tr->range_addr_start)
9073 trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer,
9074 tr, &snapshot_fops);
9075#endif
9076
9077 trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer,
9078 tr, &tracing_err_log_fops);
9079
9080 ftrace_init_tracefs(tr, d_tracer);
9081}
9082
9083#ifdef CONFIG_TRACEFS_AUTOMOUNT_DEPRECATED
9084static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
9085{
9086 struct vfsmount *mnt;
9087 struct file_system_type *type;
9088 struct fs_context *fc;
9089 int ret;
9090
9091 /*
9092 * To maintain backward compatibility for tools that mount
9093 * debugfs to get to the tracing facility, tracefs is automatically
9094 * mounted to the debugfs/tracing directory.
9095 */
9096 type = get_fs_type("tracefs");
9097 if (!type)
9098 return NULL;
9099
9100 fc = fs_context_for_submount(type, mntpt);
9101 put_filesystem(type);
9102 if (IS_ERR(fc))
9103 return ERR_CAST(fc);
9104
9105 pr_warn("NOTICE: Automounting of tracing to debugfs is deprecated and will be removed in 2030\n");
9106
9107 ret = vfs_parse_fs_string(fc, "source", "tracefs");
9108 if (!ret)
9109 mnt = fc_mount(fc);
9110 else
9111 mnt = ERR_PTR(ret);
9112
9113 put_fs_context(fc);
9114 return mnt;
9115}
9116#endif
9117
9118/**
9119 * tracing_init_dentry - initialize top level trace array
9120 *
9121 * This is called when creating files or directories in the tracing
9122 * directory. It is called via fs_initcall() by any of the boot up code
9123 * and expects to return the dentry of the top level tracing directory.
9124 */
9125int tracing_init_dentry(void)
9126{
9127 struct trace_array *tr = &global_trace;
9128
9129 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9130 pr_warn("Tracing disabled due to lockdown\n");
9131 return -EPERM;
9132 }
9133
9134 /* The top level trace array uses NULL as parent */
9135 if (tr->dir)
9136 return 0;
9137
9138 if (WARN_ON(!tracefs_initialized()))
9139 return -ENODEV;
9140
9141#ifdef CONFIG_TRACEFS_AUTOMOUNT_DEPRECATED
9142 /*
9143 * As there may still be users that expect the tracing
9144 * files to exist in debugfs/tracing, we must automount
9145 * the tracefs file system there, so older tools still
9146 * work with the newer kernel.
9147 */
9148 tr->dir = debugfs_create_automount("tracing", NULL,
9149 trace_automount, NULL);
9150#endif
9151
9152 return 0;
9153}
9154
9155extern struct trace_eval_map *__start_ftrace_eval_maps[];
9156extern struct trace_eval_map *__stop_ftrace_eval_maps[];
9157
9158struct workqueue_struct *trace_init_wq __initdata;
9159static struct work_struct eval_map_work __initdata;
9160static struct work_struct tracerfs_init_work __initdata;
9161
9162static void __init eval_map_work_func(struct work_struct *work)
9163{
9164 int len;
9165
9166 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
9167 trace_event_update_with_eval_map(NULL, __start_ftrace_eval_maps, len);
9168}
9169
9170static int __init trace_eval_init(void)
9171{
9172 INIT_WORK(&eval_map_work, eval_map_work_func);
9173
9174 trace_init_wq = alloc_workqueue("trace_init_wq", WQ_UNBOUND, 0);
9175 if (!trace_init_wq) {
9176 pr_err("Unable to allocate trace_init_wq\n");
9177 /* Do work here */
9178 eval_map_work_func(&eval_map_work);
9179 return -ENOMEM;
9180 }
9181
9182 queue_work(trace_init_wq, &eval_map_work);
9183 return 0;
9184}
9185
9186subsys_initcall(trace_eval_init);
9187
9188static int __init trace_eval_sync(void)
9189{
9190 /* Make sure the eval map updates are finished */
9191 if (trace_init_wq)
9192 destroy_workqueue(trace_init_wq);
9193 return 0;
9194}
9195
9196late_initcall_sync(trace_eval_sync);
9197
9198
9199#ifdef CONFIG_MODULES
9200
9201bool module_exists(const char *module)
9202{
9203 /* All modules have the symbol __this_module */
9204 static const char this_mod[] = "__this_module";
9205 char modname[MODULE_NAME_LEN + sizeof(this_mod) + 2];
9206 unsigned long val;
9207 int n;
9208
9209 n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
9210
9211 if (n > sizeof(modname) - 1)
9212 return false;
9213
9214 val = module_kallsyms_lookup_name(modname);
9215 return val != 0;
9216}
9217
9218static void trace_module_add_evals(struct module *mod)
9219{
9220 /*
9221 * Modules with bad taint do not have events created, do
9222 * not bother with enums either.
9223 */
9224 if (trace_module_has_bad_taint(mod))
9225 return;
9226
9227 /* Even if no trace_evals, this need to sanitize field types. */
9228 trace_event_update_with_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
9229}
9230
9231#ifdef CONFIG_TRACE_EVAL_MAP_FILE
9232static void trace_module_remove_evals(struct module *mod)
9233{
9234 union trace_eval_map_item *map;
9235 union trace_eval_map_item **last = &trace_eval_maps;
9236
9237 if (!mod->num_trace_evals)
9238 return;
9239
9240 guard(mutex)(&trace_eval_mutex);
9241
9242 map = trace_eval_maps;
9243
9244 while (map) {
9245 if (map->head.mod == mod)
9246 break;
9247 map = trace_eval_jmp_to_tail(map);
9248 last = &map->tail.next;
9249 map = map->tail.next;
9250 }
9251 if (!map)
9252 return;
9253
9254 *last = trace_eval_jmp_to_tail(map)->tail.next;
9255 kfree(map);
9256}
9257#else
9258static inline void trace_module_remove_evals(struct module *mod) { }
9259#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9260
9261static void trace_module_record(struct module *mod, bool add)
9262{
9263 struct trace_array *tr;
9264 unsigned long flags;
9265
9266 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9267 flags = tr->flags & (TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT);
9268 /* Update any persistent trace array that has already been started */
9269 if (flags == TRACE_ARRAY_FL_BOOT && add) {
9270 guard(mutex)(&scratch_mutex);
9271 save_mod(mod, tr);
9272 } else if (flags & TRACE_ARRAY_FL_LAST_BOOT) {
9273 /* Update delta if the module loaded in previous boot */
9274 make_mod_delta(mod, tr);
9275 }
9276 }
9277}
9278
9279static int trace_module_notify(struct notifier_block *self,
9280 unsigned long val, void *data)
9281{
9282 struct module *mod = data;
9283
9284 switch (val) {
9285 case MODULE_STATE_COMING:
9286 trace_module_add_evals(mod);
9287 trace_module_record(mod, true);
9288 break;
9289 case MODULE_STATE_GOING:
9290 trace_module_remove_evals(mod);
9291 trace_module_record(mod, false);
9292 break;
9293 }
9294
9295 return NOTIFY_OK;
9296}
9297
9298static struct notifier_block trace_module_nb = {
9299 .notifier_call = trace_module_notify,
9300 .priority = 0,
9301};
9302#endif /* CONFIG_MODULES */
9303
9304static __init void tracer_init_tracefs_work_func(struct work_struct *work)
9305{
9306 /*
9307 * Do not attempt to initialize tracefs if tracing was disabled or
9308 * failed to allocate
9309 */
9310 if (tracing_disabled)
9311 return;
9312
9313 event_trace_init();
9314
9315 init_tracer_tracefs(&global_trace, NULL);
9316 ftrace_init_tracefs_toplevel(&global_trace, NULL);
9317
9318 trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL,
9319 &global_trace, &tracing_thresh_fops);
9320
9321 trace_create_file("README", TRACE_MODE_READ, NULL,
9322 NULL, &tracing_readme_fops);
9323
9324 trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL,
9325 NULL, &tracing_saved_cmdlines_fops);
9326
9327 trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
9328 NULL, &tracing_saved_cmdlines_size_fops);
9329
9330 trace_create_file("saved_tgids", TRACE_MODE_READ, NULL,
9331 NULL, &tracing_saved_tgids_fops);
9332
9333 trace_create_eval_file(NULL);
9334
9335#ifdef CONFIG_MODULES
9336 register_module_notifier(&trace_module_nb);
9337#endif
9338
9339#ifdef CONFIG_DYNAMIC_FTRACE
9340 trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
9341 NULL, &tracing_dyn_info_fops);
9342#endif
9343
9344 create_trace_instances(NULL);
9345
9346 update_tracer_options();
9347}
9348
9349static __init int tracer_init_tracefs(void)
9350{
9351 int ret;
9352
9353 trace_access_lock_init();
9354
9355 ret = tracing_init_dentry();
9356 if (ret)
9357 return 0;
9358
9359 if (trace_init_wq) {
9360 INIT_WORK(&tracerfs_init_work, tracer_init_tracefs_work_func);
9361 queue_work(trace_init_wq, &tracerfs_init_work);
9362 } else {
9363 tracer_init_tracefs_work_func(NULL);
9364 }
9365
9366 if (rv_init_interface())
9367 pr_err("RV: Error while creating the RV interface\n");
9368
9369 return 0;
9370}
9371
9372fs_initcall(tracer_init_tracefs);
9373
9374static int trace_die_panic_handler(struct notifier_block *self,
9375 unsigned long ev, void *unused);
9376
9377static struct notifier_block trace_panic_notifier = {
9378 .notifier_call = trace_die_panic_handler,
9379 .priority = INT_MAX - 1,
9380};
9381
9382static struct notifier_block trace_die_notifier = {
9383 .notifier_call = trace_die_panic_handler,
9384 .priority = INT_MAX - 1,
9385};
9386
9387/*
9388 * The idea is to execute the following die/panic callback early, in order
9389 * to avoid showing irrelevant information in the trace (like other panic
9390 * notifier functions); we are the 2nd to run, after hung_task/rcu_stall
9391 * warnings get disabled (to prevent potential log flooding).
9392 */
9393static int trace_die_panic_handler(struct notifier_block *self,
9394 unsigned long ev, void *unused)
9395{
9396 if (!ftrace_dump_on_oops_enabled())
9397 return NOTIFY_DONE;
9398
9399 /* The die notifier requires DIE_OOPS to trigger */
9400 if (self == &trace_die_notifier && ev != DIE_OOPS)
9401 return NOTIFY_DONE;
9402
9403 ftrace_dump(DUMP_PARAM);
9404
9405 return NOTIFY_DONE;
9406}
9407
9408/*
9409 * printk is set to max of 1024, we really don't need it that big.
9410 * Nothing should be printing 1000 characters anyway.
9411 */
9412#define TRACE_MAX_PRINT 1000
9413
9414/*
9415 * Define here KERN_TRACE so that we have one place to modify
9416 * it if we decide to change what log level the ftrace dump
9417 * should be at.
9418 */
9419#define KERN_TRACE KERN_EMERG
9420
9421void
9422trace_printk_seq(struct trace_seq *s)
9423{
9424 /* Probably should print a warning here. */
9425 if (s->seq.len >= TRACE_MAX_PRINT)
9426 s->seq.len = TRACE_MAX_PRINT;
9427
9428 /*
9429 * More paranoid code. Although the buffer size is set to
9430 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9431 * an extra layer of protection.
9432 */
9433 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9434 s->seq.len = s->seq.size - 1;
9435
9436 /* should be zero ended, but we are paranoid. */
9437 s->buffer[s->seq.len] = 0;
9438
9439 printk(KERN_TRACE "%s", s->buffer);
9440
9441 trace_seq_init(s);
9442}
9443
9444static void trace_init_iter(struct trace_iterator *iter, struct trace_array *tr)
9445{
9446 iter->tr = tr;
9447 iter->trace = iter->tr->current_trace;
9448 iter->cpu_file = RING_BUFFER_ALL_CPUS;
9449 iter->array_buffer = &tr->array_buffer;
9450
9451 if (iter->trace && iter->trace->open)
9452 iter->trace->open(iter);
9453
9454 /* Annotate start of buffers if we had overruns */
9455 if (ring_buffer_overruns(iter->array_buffer->buffer))
9456 iter->iter_flags |= TRACE_FILE_ANNOTATE;
9457
9458 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
9459 if (trace_clocks[iter->tr->clock_id].in_ns)
9460 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9461
9462 /* Can not use kmalloc for iter.temp and iter.fmt */
9463 iter->temp = static_temp_buf;
9464 iter->temp_size = STATIC_TEMP_BUF_SIZE;
9465 iter->fmt = static_fmt_buf;
9466 iter->fmt_size = STATIC_FMT_BUF_SIZE;
9467}
9468
9469void trace_init_global_iter(struct trace_iterator *iter)
9470{
9471 trace_init_iter(iter, &global_trace);
9472}
9473
9474static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_mode)
9475{
9476 /* use static because iter can be a bit big for the stack */
9477 static struct trace_iterator iter;
9478 unsigned int old_userobj;
9479 unsigned long flags;
9480 int cnt = 0;
9481
9482 /*
9483 * Always turn off tracing when we dump.
9484 * We don't need to show trace output of what happens
9485 * between multiple crashes.
9486 *
9487 * If the user does a sysrq-z, then they can re-enable
9488 * tracing with echo 1 > tracing_on.
9489 */
9490 tracer_tracing_off(tr);
9491
9492 local_irq_save(flags);
9493
9494 /* Simulate the iterator */
9495 trace_init_iter(&iter, tr);
9496
9497 /* While dumping, do not allow the buffer to be enable */
9498 tracer_tracing_disable(tr);
9499
9500 old_userobj = tr->trace_flags & TRACE_ITER(SYM_USEROBJ);
9501
9502 /* don't look at user memory in panic mode */
9503 tr->trace_flags &= ~TRACE_ITER(SYM_USEROBJ);
9504
9505 if (dump_mode == DUMP_ORIG)
9506 iter.cpu_file = raw_smp_processor_id();
9507 else
9508 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9509
9510 if (tr == &global_trace)
9511 printk(KERN_TRACE "Dumping ftrace buffer:\n");
9512 else
9513 printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name);
9514
9515 /* Did function tracer already get disabled? */
9516 if (ftrace_is_dead()) {
9517 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9518 printk("# MAY BE MISSING FUNCTION EVENTS\n");
9519 }
9520
9521 /*
9522 * We need to stop all tracing on all CPUS to read
9523 * the next buffer. This is a bit expensive, but is
9524 * not done often. We fill all what we can read,
9525 * and then release the locks again.
9526 */
9527
9528 while (!trace_empty(&iter)) {
9529
9530 if (!cnt)
9531 printk(KERN_TRACE "---------------------------------\n");
9532
9533 cnt++;
9534
9535 trace_iterator_reset(&iter);
9536 iter.iter_flags |= TRACE_FILE_LAT_FMT;
9537
9538 if (trace_find_next_entry_inc(&iter) != NULL) {
9539 int ret;
9540
9541 ret = print_trace_line(&iter);
9542 if (ret != TRACE_TYPE_NO_CONSUME)
9543 trace_consume(&iter);
9544
9545 trace_printk_seq(&iter.seq);
9546 }
9547 touch_nmi_watchdog();
9548 }
9549
9550 if (!cnt)
9551 printk(KERN_TRACE " (ftrace buffer empty)\n");
9552 else
9553 printk(KERN_TRACE "---------------------------------\n");
9554
9555 tr->trace_flags |= old_userobj;
9556
9557 tracer_tracing_enable(tr);
9558 local_irq_restore(flags);
9559}
9560
9561static void ftrace_dump_by_param(void)
9562{
9563 bool first_param = true;
9564 char dump_param[MAX_TRACER_SIZE];
9565 char *buf, *token, *inst_name;
9566 struct trace_array *tr;
9567
9568 strscpy(dump_param, ftrace_dump_on_oops, MAX_TRACER_SIZE);
9569 buf = dump_param;
9570
9571 while ((token = strsep(&buf, ",")) != NULL) {
9572 if (first_param) {
9573 first_param = false;
9574 if (!strcmp("0", token))
9575 continue;
9576 else if (!strcmp("1", token)) {
9577 ftrace_dump_one(&global_trace, DUMP_ALL);
9578 continue;
9579 }
9580 else if (!strcmp("2", token) ||
9581 !strcmp("orig_cpu", token)) {
9582 ftrace_dump_one(&global_trace, DUMP_ORIG);
9583 continue;
9584 }
9585 }
9586
9587 inst_name = strsep(&token, "=");
9588 tr = trace_array_find(inst_name);
9589 if (!tr) {
9590 printk(KERN_TRACE "Instance %s not found\n", inst_name);
9591 continue;
9592 }
9593
9594 if (token && (!strcmp("2", token) ||
9595 !strcmp("orig_cpu", token)))
9596 ftrace_dump_one(tr, DUMP_ORIG);
9597 else
9598 ftrace_dump_one(tr, DUMP_ALL);
9599 }
9600}
9601
9602void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
9603{
9604 static atomic_t dump_running;
9605
9606 /* Only allow one dump user at a time. */
9607 if (atomic_inc_return(&dump_running) != 1) {
9608 atomic_dec(&dump_running);
9609 return;
9610 }
9611
9612 switch (oops_dump_mode) {
9613 case DUMP_ALL:
9614 ftrace_dump_one(&global_trace, DUMP_ALL);
9615 break;
9616 case DUMP_ORIG:
9617 ftrace_dump_one(&global_trace, DUMP_ORIG);
9618 break;
9619 case DUMP_PARAM:
9620 ftrace_dump_by_param();
9621 break;
9622 case DUMP_NONE:
9623 break;
9624 default:
9625 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
9626 ftrace_dump_one(&global_trace, DUMP_ALL);
9627 }
9628
9629 atomic_dec(&dump_running);
9630}
9631EXPORT_SYMBOL_GPL(ftrace_dump);
9632
9633#define WRITE_BUFSIZE 4096
9634
9635ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9636 size_t count, loff_t *ppos,
9637 int (*createfn)(const char *))
9638{
9639 char *kbuf __free(kfree) = NULL;
9640 char *buf, *tmp;
9641 int ret = 0;
9642 size_t done = 0;
9643 size_t size;
9644
9645 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9646 if (!kbuf)
9647 return -ENOMEM;
9648
9649 while (done < count) {
9650 size = count - done;
9651
9652 if (size >= WRITE_BUFSIZE)
9653 size = WRITE_BUFSIZE - 1;
9654
9655 if (copy_from_user(kbuf, buffer + done, size))
9656 return -EFAULT;
9657
9658 kbuf[size] = '\0';
9659 buf = kbuf;
9660 do {
9661 tmp = strchr(buf, '\n');
9662 if (tmp) {
9663 *tmp = '\0';
9664 size = tmp - buf + 1;
9665 } else {
9666 size = strlen(buf);
9667 if (done + size < count) {
9668 if (buf != kbuf)
9669 break;
9670 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9671 pr_warn("Line length is too long: Should be less than %d\n",
9672 WRITE_BUFSIZE - 2);
9673 return -EINVAL;
9674 }
9675 }
9676 done += size;
9677
9678 /* Remove comments */
9679 tmp = strchr(buf, '#');
9680
9681 if (tmp)
9682 *tmp = '\0';
9683
9684 ret = createfn(buf);
9685 if (ret)
9686 return ret;
9687 buf += size;
9688
9689 } while (done < count);
9690 }
9691 return done;
9692}
9693
9694__init static int backup_instance_area(const char *backup,
9695 unsigned long *addr, phys_addr_t *size)
9696{
9697 struct trace_array *backup_tr;
9698 void *allocated_vaddr = NULL;
9699
9700 backup_tr = trace_array_get_by_name(backup, NULL);
9701 if (!backup_tr) {
9702 pr_warn("Tracing: Instance %s is not found.\n", backup);
9703 return -ENOENT;
9704 }
9705
9706 if (!(backup_tr->flags & TRACE_ARRAY_FL_BOOT)) {
9707 pr_warn("Tracing: Instance %s is not boot mapped.\n", backup);
9708 trace_array_put(backup_tr);
9709 return -EINVAL;
9710 }
9711
9712 *size = backup_tr->range_addr_size;
9713
9714 allocated_vaddr = vzalloc(*size);
9715 if (!allocated_vaddr) {
9716 pr_warn("Tracing: Failed to allocate memory for copying instance %s (size 0x%lx)\n",
9717 backup, (unsigned long)*size);
9718 trace_array_put(backup_tr);
9719 return -ENOMEM;
9720 }
9721
9722 memcpy(allocated_vaddr,
9723 (void *)backup_tr->range_addr_start, (size_t)*size);
9724 *addr = (unsigned long)allocated_vaddr;
9725
9726 trace_array_put(backup_tr);
9727 return 0;
9728}
9729
9730__init static void enable_instances(void)
9731{
9732 struct trace_array *tr;
9733 bool memmap_area = false;
9734 char *curr_str;
9735 char *name;
9736 char *str;
9737 char *tok;
9738
9739 /* A tab is always appended */
9740 boot_instance_info[boot_instance_index - 1] = '\0';
9741 str = boot_instance_info;
9742
9743 while ((curr_str = strsep(&str, "\t"))) {
9744 phys_addr_t start = 0;
9745 phys_addr_t size = 0;
9746 unsigned long addr = 0;
9747 bool traceprintk = false;
9748 bool traceoff = false;
9749 char *flag_delim;
9750 char *addr_delim;
9751 char *rname __free(kfree) = NULL;
9752 char *backup;
9753
9754 tok = strsep(&curr_str, ",");
9755
9756 name = strsep(&tok, "=");
9757 backup = tok;
9758
9759 flag_delim = strchr(name, '^');
9760 addr_delim = strchr(name, '@');
9761
9762 if (addr_delim)
9763 *addr_delim++ = '\0';
9764
9765 if (flag_delim)
9766 *flag_delim++ = '\0';
9767
9768 if (trace_array_find(name)) {
9769 pr_warn("Tracing: Instance %s already exists\n", name);
9770 continue;
9771 }
9772
9773 if (backup) {
9774 if (backup_instance_area(backup, &addr, &size) < 0)
9775 continue;
9776 }
9777
9778 if (flag_delim) {
9779 char *flag;
9780
9781 while ((flag = strsep(&flag_delim, "^"))) {
9782 if (strcmp(flag, "traceoff") == 0) {
9783 traceoff = true;
9784 } else if ((strcmp(flag, "printk") == 0) ||
9785 (strcmp(flag, "traceprintk") == 0) ||
9786 (strcmp(flag, "trace_printk") == 0)) {
9787 traceprintk = true;
9788 } else {
9789 pr_info("Tracing: Invalid instance flag '%s' for %s\n",
9790 flag, name);
9791 }
9792 }
9793 }
9794
9795 tok = addr_delim;
9796 if (tok && isdigit(*tok)) {
9797 start = memparse(tok, &tok);
9798 if (!start) {
9799 pr_warn("Tracing: Invalid boot instance address for %s\n",
9800 name);
9801 continue;
9802 }
9803 if (*tok != ':') {
9804 pr_warn("Tracing: No size specified for instance %s\n", name);
9805 continue;
9806 }
9807 tok++;
9808 size = memparse(tok, &tok);
9809 if (!size) {
9810 pr_warn("Tracing: Invalid boot instance size for %s\n",
9811 name);
9812 continue;
9813 }
9814 memmap_area = true;
9815 } else if (tok) {
9816 if (!reserve_mem_find_by_name(tok, &start, &size)) {
9817 start = 0;
9818 pr_warn("Failed to map boot instance %s to %s\n", name, tok);
9819 continue;
9820 }
9821 rname = kstrdup(tok, GFP_KERNEL);
9822 }
9823
9824 if (start) {
9825 /* Start and size must be page aligned */
9826 if (start & ~PAGE_MASK) {
9827 pr_warn("Tracing: mapping start addr %pa is not page aligned\n", &start);
9828 continue;
9829 }
9830 if (size & ~PAGE_MASK) {
9831 pr_warn("Tracing: mapping size %pa is not page aligned\n", &size);
9832 continue;
9833 }
9834
9835 if (memmap_area)
9836 addr = map_pages(start, size);
9837 else
9838 addr = (unsigned long)phys_to_virt(start);
9839 if (addr) {
9840 pr_info("Tracing: mapped boot instance %s at physical memory %pa of size 0x%lx\n",
9841 name, &start, (unsigned long)size);
9842 } else {
9843 pr_warn("Tracing: Failed to map boot instance %s\n", name);
9844 continue;
9845 }
9846 } else {
9847 /* Only non mapped buffers have snapshot buffers */
9848 do_allocate_snapshot(name);
9849 }
9850
9851 tr = trace_array_create_systems(name, NULL, addr, size);
9852 if (IS_ERR(tr)) {
9853 pr_warn("Tracing: Failed to create instance buffer '%s' (%ld)\n", name,
9854 PTR_ERR(tr));
9855 continue;
9856 }
9857
9858 if (traceoff)
9859 tracer_tracing_off(tr);
9860
9861 if (traceprintk)
9862 update_printk_trace(tr);
9863
9864 /*
9865 * memmap'd buffers can not be freed.
9866 */
9867 if (memmap_area) {
9868 tr->flags |= TRACE_ARRAY_FL_MEMMAP;
9869 tr->ref++;
9870 }
9871
9872 /*
9873 * Backup buffers can be freed but need vfree().
9874 */
9875 if (backup) {
9876 tr->flags |= TRACE_ARRAY_FL_VMALLOC | TRACE_ARRAY_FL_RDONLY;
9877 trace_array_start_autoremove();
9878 }
9879
9880 if (start || backup) {
9881 tr->flags |= TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT;
9882 tr->range_name = no_free_ptr(rname);
9883 }
9884
9885 /*
9886 * Save the events to start and enabled them after all boot instances
9887 * have been created.
9888 */
9889 tr->boot_events = curr_str;
9890 }
9891
9892 /* Enable the events after all boot instances have been created */
9893 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9894
9895 if (!tr->boot_events || !(*tr->boot_events)) {
9896 tr->boot_events = NULL;
9897 continue;
9898 }
9899
9900 curr_str = tr->boot_events;
9901
9902 /* Clear the instance if this is a persistent buffer */
9903 if (tr->flags & TRACE_ARRAY_FL_LAST_BOOT)
9904 update_last_data(tr);
9905
9906 while ((tok = strsep(&curr_str, ","))) {
9907 early_enable_events(tr, tok, true);
9908 }
9909 tr->boot_events = NULL;
9910 }
9911}
9912
9913__init static int tracer_alloc_buffers(void)
9914{
9915 unsigned long ring_buf_size;
9916 int ret = -ENOMEM;
9917
9918
9919 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9920 pr_warn("Tracing disabled due to lockdown\n");
9921 return -EPERM;
9922 }
9923
9924 /*
9925 * Make sure we don't accidentally add more trace options
9926 * than we have bits for.
9927 */
9928 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
9929
9930 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
9931 return -ENOMEM;
9932
9933 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9934 goto out_free_buffer_mask;
9935
9936 /* Only allocate trace_printk buffers if a trace_printk exists */
9937 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
9938 /* Must be called before global_trace.buffer is allocated */
9939 trace_printk_init_buffers();
9940
9941 /* To save memory, keep the ring buffer size to its minimum */
9942 if (global_trace.ring_buffer_expanded)
9943 ring_buf_size = trace_buf_size;
9944 else
9945 ring_buf_size = 1;
9946
9947 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
9948 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9949
9950 raw_spin_lock_init(&global_trace.start_lock);
9951
9952 /*
9953 * The prepare callbacks allocates some memory for the ring buffer. We
9954 * don't free the buffer if the CPU goes down. If we were to free
9955 * the buffer, then the user would lose any trace that was in the
9956 * buffer. The memory will be removed once the "instance" is removed.
9957 */
9958 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
9959 "trace/RB:prepare", trace_rb_cpu_prepare,
9960 NULL);
9961 if (ret < 0)
9962 goto out_free_cpumask;
9963 /* Used for event triggers */
9964 ret = -ENOMEM;
9965 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
9966 if (!temp_buffer)
9967 goto out_rm_hp_state;
9968
9969 if (trace_create_savedcmd() < 0)
9970 goto out_free_temp_buffer;
9971
9972 if (!zalloc_cpumask_var(&global_trace.pipe_cpumask, GFP_KERNEL))
9973 goto out_free_savedcmd;
9974
9975 /* TODO: make the number of buffers hot pluggable with CPUS */
9976 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
9977 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
9978 goto out_free_pipe_cpumask;
9979 }
9980 if (global_trace.buffer_disabled)
9981 tracing_off();
9982
9983 if (trace_boot_clock) {
9984 ret = tracing_set_clock(&global_trace, trace_boot_clock);
9985 if (ret < 0)
9986 pr_warn("Trace clock %s not defined, going back to default\n",
9987 trace_boot_clock);
9988 }
9989
9990 /*
9991 * register_tracer() might reference current_trace, so it
9992 * needs to be set before we register anything. This is
9993 * just a bootstrap of current_trace anyway.
9994 */
9995 global_trace.current_trace = &nop_trace;
9996 global_trace.current_trace_flags = nop_trace.flags;
9997
9998 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9999#ifdef CONFIG_TRACER_SNAPSHOT
10000 spin_lock_init(&global_trace.snapshot_trigger_lock);
10001#endif
10002 ftrace_init_global_array_ops(&global_trace);
10003
10004#ifdef CONFIG_MODULES
10005 INIT_LIST_HEAD(&global_trace.mod_events);
10006#endif
10007
10008 init_trace_flags_index(&global_trace);
10009
10010 INIT_LIST_HEAD(&global_trace.tracers);
10011
10012 /* All seems OK, enable tracing */
10013 tracing_disabled = 0;
10014
10015 atomic_notifier_chain_register(&panic_notifier_list,
10016 &trace_panic_notifier);
10017
10018 register_die_notifier(&trace_die_notifier);
10019
10020 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
10021
10022 global_trace.syscall_buf_sz = syscall_buf_size;
10023
10024 INIT_LIST_HEAD(&global_trace.systems);
10025 INIT_LIST_HEAD(&global_trace.events);
10026 INIT_LIST_HEAD(&global_trace.hist_vars);
10027 INIT_LIST_HEAD(&global_trace.err_log);
10028 list_add(&global_trace.marker_list, &marker_copies);
10029 list_add(&global_trace.list, &ftrace_trace_arrays);
10030
10031 register_tracer(&nop_trace);
10032
10033 /* Function tracing may start here (via kernel command line) */
10034 init_function_trace();
10035
10036 apply_trace_boot_options();
10037
10038 register_snapshot_cmd();
10039
10040 return 0;
10041
10042out_free_pipe_cpumask:
10043 free_cpumask_var(global_trace.pipe_cpumask);
10044out_free_savedcmd:
10045 trace_free_saved_cmdlines_buffer();
10046out_free_temp_buffer:
10047 ring_buffer_free(temp_buffer);
10048out_rm_hp_state:
10049 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
10050out_free_cpumask:
10051 free_cpumask_var(global_trace.tracing_cpumask);
10052out_free_buffer_mask:
10053 free_cpumask_var(tracing_buffer_mask);
10054 return ret;
10055}
10056
10057#ifdef CONFIG_FUNCTION_TRACER
10058/* Used to set module cached ftrace filtering at boot up */
10059struct trace_array *trace_get_global_array(void)
10060{
10061 return &global_trace;
10062}
10063#endif
10064
10065void __init early_trace_init(void)
10066{
10067 if (tracepoint_printk) {
10068 tracepoint_print_iter = kzalloc_obj(*tracepoint_print_iter);
10069 if (MEM_FAIL(!tracepoint_print_iter,
10070 "Failed to allocate trace iterator\n"))
10071 tracepoint_printk = 0;
10072 else
10073 static_key_enable(&tracepoint_printk_key.key);
10074 }
10075 tracer_alloc_buffers();
10076
10077 init_events();
10078}
10079
10080void __init trace_init(void)
10081{
10082 trace_event_init();
10083
10084 if (boot_instance_index)
10085 enable_instances();
10086}
10087
10088__init static void clear_boot_tracer(void)
10089{
10090 /*
10091 * The default tracer at boot buffer is an init section.
10092 * This function is called in lateinit. If we did not
10093 * find the boot tracer, then clear it out, to prevent
10094 * later registration from accessing the buffer that is
10095 * about to be freed.
10096 */
10097 if (!default_bootup_tracer)
10098 return;
10099
10100 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
10101 default_bootup_tracer);
10102 default_bootup_tracer = NULL;
10103}
10104
10105#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
10106__init static void tracing_set_default_clock(void)
10107{
10108 /* sched_clock_stable() is determined in late_initcall */
10109 if (!trace_boot_clock && !sched_clock_stable()) {
10110 if (security_locked_down(LOCKDOWN_TRACEFS)) {
10111 pr_warn("Can not set tracing clock due to lockdown\n");
10112 return;
10113 }
10114
10115 printk(KERN_WARNING
10116 "Unstable clock detected, switching default tracing clock to \"global\"\n"
10117 "If you want to keep using the local clock, then add:\n"
10118 " \"trace_clock=local\"\n"
10119 "on the kernel command line\n");
10120 tracing_set_clock(&global_trace, "global");
10121 }
10122}
10123#else
10124static inline void tracing_set_default_clock(void) { }
10125#endif
10126
10127__init static int late_trace_init(void)
10128{
10129 if (tracepoint_printk && tracepoint_printk_stop_on_boot) {
10130 static_key_disable(&tracepoint_printk_key.key);
10131 tracepoint_printk = 0;
10132 }
10133
10134 if (traceoff_after_boot)
10135 tracing_off();
10136
10137 tracing_set_default_clock();
10138 clear_boot_tracer();
10139 return 0;
10140}
10141
10142late_initcall_sync(late_trace_init);