projects/linux·blob v5.17 · refs

10195 lines · 248.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * ring buffer based function tracer
4 *
5 * Copyright (C) 2007-2012 Steven Rostedt <[email protected]>
6 * Copyright (C) 2008 Ingo Molnar <[email protected]>
7 *
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <[email protected]>
10 *
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
14 */
15#include <linux/ring_buffer.h>
16#include <generated/utsrelease.h>
17#include <linux/stacktrace.h>
18#include <linux/writeback.h>
19#include <linux/kallsyms.h>
20#include <linux/security.h>
21#include <linux/seq_file.h>
22#include <linux/notifier.h>
23#include <linux/irqflags.h>
24#include <linux/debugfs.h>
25#include <linux/tracefs.h>
26#include <linux/pagemap.h>
27#include <linux/hardirq.h>
28#include <linux/linkage.h>
29#include <linux/uaccess.h>
30#include <linux/vmalloc.h>
31#include <linux/ftrace.h>
32#include <linux/module.h>
33#include <linux/percpu.h>
34#include <linux/splice.h>
35#include <linux/kdebug.h>
36#include <linux/string.h>
37#include <linux/mount.h>
38#include <linux/rwsem.h>
39#include <linux/slab.h>
40#include <linux/ctype.h>
41#include <linux/init.h>
42#include <linux/panic_notifier.h>
43#include <linux/poll.h>
44#include <linux/nmi.h>
45#include <linux/fs.h>
46#include <linux/trace.h>
47#include <linux/sched/clock.h>
48#include <linux/sched/rt.h>
49#include <linux/fsnotify.h>
50#include <linux/irq_work.h>
51#include <linux/workqueue.h>
52
53#include "trace.h"
54#include "trace_output.h"
55
56/*
57 * On boot up, the ring buffer is set to the minimum size, so that
58 * we do not waste memory on systems that are not using tracing.
59 */
60bool ring_buffer_expanded;
61
62/*
63 * We need to change this state when a selftest is running.
64 * A selftest will lurk into the ring-buffer to count the
65 * entries inserted during the selftest although some concurrent
66 * insertions into the ring-buffer such as trace_printk could occurred
67 * at the same time, giving false positive or negative results.
68 */
69static bool __read_mostly tracing_selftest_running;
70
71/*
72 * If boot-time tracing including tracers/events via kernel cmdline
73 * is running, we do not want to run SELFTEST.
74 */
75bool __read_mostly tracing_selftest_disabled;
76
77#ifdef CONFIG_FTRACE_STARTUP_TEST
78void __init disable_tracing_selftest(const char *reason)
79{
80 if (!tracing_selftest_disabled) {
81 tracing_selftest_disabled = true;
82 pr_info("Ftrace startup test is disabled due to %s\n", reason);
83 }
84}
85#endif
86
87/* Pipe tracepoints to printk */
88struct trace_iterator *tracepoint_print_iter;
89int tracepoint_printk;
90static bool tracepoint_printk_stop_on_boot __initdata;
91static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
92
93/* For tracers that don't implement custom flags */
94static struct tracer_opt dummy_tracer_opt[] = {
95 { }
96};
97
98static int
99dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
100{
101 return 0;
102}
103
104/*
105 * To prevent the comm cache from being overwritten when no
106 * tracing is active, only save the comm when a trace event
107 * occurred.
108 */
109static DEFINE_PER_CPU(bool, trace_taskinfo_save);
110
111/*
112 * Kill all tracing for good (never come back).
113 * It is initialized to 1 but will turn to zero if the initialization
114 * of the tracer is successful. But that is the only place that sets
115 * this back to zero.
116 */
117static int tracing_disabled = 1;
118
119cpumask_var_t __read_mostly tracing_buffer_mask;
120
121/*
122 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
123 *
124 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
125 * is set, then ftrace_dump is called. This will output the contents
126 * of the ftrace buffers to the console. This is very useful for
127 * capturing traces that lead to crashes and outputing it to a
128 * serial console.
129 *
130 * It is default off, but you can enable it with either specifying
131 * "ftrace_dump_on_oops" in the kernel command line, or setting
132 * /proc/sys/kernel/ftrace_dump_on_oops
133 * Set 1 if you want to dump buffers of all CPUs
134 * Set 2 if you want to dump the buffer of the CPU that triggered oops
135 */
136
137enum ftrace_dump_mode ftrace_dump_on_oops;
138
139/* When set, tracing will stop when a WARN*() is hit */
140int __disable_trace_on_warning;
141
142#ifdef CONFIG_TRACE_EVAL_MAP_FILE
143/* Map of enums to their values, for "eval_map" file */
144struct trace_eval_map_head {
145 struct module *mod;
146 unsigned long length;
147};
148
149union trace_eval_map_item;
150
151struct trace_eval_map_tail {
152 /*
153 * "end" is first and points to NULL as it must be different
154 * than "mod" or "eval_string"
155 */
156 union trace_eval_map_item *next;
157 const char *end; /* points to NULL */
158};
159
160static DEFINE_MUTEX(trace_eval_mutex);
161
162/*
163 * The trace_eval_maps are saved in an array with two extra elements,
164 * one at the beginning, and one at the end. The beginning item contains
165 * the count of the saved maps (head.length), and the module they
166 * belong to if not built in (head.mod). The ending item contains a
167 * pointer to the next array of saved eval_map items.
168 */
169union trace_eval_map_item {
170 struct trace_eval_map map;
171 struct trace_eval_map_head head;
172 struct trace_eval_map_tail tail;
173};
174
175static union trace_eval_map_item *trace_eval_maps;
176#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
177
178int tracing_set_tracer(struct trace_array *tr, const char *buf);
179static void ftrace_trace_userstack(struct trace_array *tr,
180 struct trace_buffer *buffer,
181 unsigned int trace_ctx);
182
183#define MAX_TRACER_SIZE 100
184static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
185static char *default_bootup_tracer;
186
187static bool allocate_snapshot;
188
189static int __init set_cmdline_ftrace(char *str)
190{
191 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
192 default_bootup_tracer = bootup_tracer_buf;
193 /* We are using ftrace early, expand it */
194 ring_buffer_expanded = true;
195 return 1;
196}
197__setup("ftrace=", set_cmdline_ftrace);
198
199static int __init set_ftrace_dump_on_oops(char *str)
200{
201 if (*str++ != '=' || !*str || !strcmp("1", str)) {
202 ftrace_dump_on_oops = DUMP_ALL;
203 return 1;
204 }
205
206 if (!strcmp("orig_cpu", str) || !strcmp("2", str)) {
207 ftrace_dump_on_oops = DUMP_ORIG;
208 return 1;
209 }
210
211 return 0;
212}
213__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
214
215static int __init stop_trace_on_warning(char *str)
216{
217 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
218 __disable_trace_on_warning = 1;
219 return 1;
220}
221__setup("traceoff_on_warning", stop_trace_on_warning);
222
223static int __init boot_alloc_snapshot(char *str)
224{
225 allocate_snapshot = true;
226 /* We also need the main ring buffer expanded */
227 ring_buffer_expanded = true;
228 return 1;
229}
230__setup("alloc_snapshot", boot_alloc_snapshot);
231
232
233static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
234
235static int __init set_trace_boot_options(char *str)
236{
237 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
238 return 1;
239}
240__setup("trace_options=", set_trace_boot_options);
241
242static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
243static char *trace_boot_clock __initdata;
244
245static int __init set_trace_boot_clock(char *str)
246{
247 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
248 trace_boot_clock = trace_boot_clock_buf;
249 return 1;
250}
251__setup("trace_clock=", set_trace_boot_clock);
252
253static int __init set_tracepoint_printk(char *str)
254{
255 /* Ignore the "tp_printk_stop_on_boot" param */
256 if (*str == '_')
257 return 0;
258
259 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
260 tracepoint_printk = 1;
261 return 1;
262}
263__setup("tp_printk", set_tracepoint_printk);
264
265static int __init set_tracepoint_printk_stop(char *str)
266{
267 tracepoint_printk_stop_on_boot = true;
268 return 1;
269}
270__setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);
271
272unsigned long long ns2usecs(u64 nsec)
273{
274 nsec += 500;
275 do_div(nsec, 1000);
276 return nsec;
277}
278
279static void
280trace_process_export(struct trace_export *export,
281 struct ring_buffer_event *event, int flag)
282{
283 struct trace_entry *entry;
284 unsigned int size = 0;
285
286 if (export->flags & flag) {
287 entry = ring_buffer_event_data(event);
288 size = ring_buffer_event_length(event);
289 export->write(export, entry, size);
290 }
291}
292
293static DEFINE_MUTEX(ftrace_export_lock);
294
295static struct trace_export __rcu *ftrace_exports_list __read_mostly;
296
297static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
298static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
299static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
300
301static inline void ftrace_exports_enable(struct trace_export *export)
302{
303 if (export->flags & TRACE_EXPORT_FUNCTION)
304 static_branch_inc(&trace_function_exports_enabled);
305
306 if (export->flags & TRACE_EXPORT_EVENT)
307 static_branch_inc(&trace_event_exports_enabled);
308
309 if (export->flags & TRACE_EXPORT_MARKER)
310 static_branch_inc(&trace_marker_exports_enabled);
311}
312
313static inline void ftrace_exports_disable(struct trace_export *export)
314{
315 if (export->flags & TRACE_EXPORT_FUNCTION)
316 static_branch_dec(&trace_function_exports_enabled);
317
318 if (export->flags & TRACE_EXPORT_EVENT)
319 static_branch_dec(&trace_event_exports_enabled);
320
321 if (export->flags & TRACE_EXPORT_MARKER)
322 static_branch_dec(&trace_marker_exports_enabled);
323}
324
325static void ftrace_exports(struct ring_buffer_event *event, int flag)
326{
327 struct trace_export *export;
328
329 preempt_disable_notrace();
330
331 export = rcu_dereference_raw_check(ftrace_exports_list);
332 while (export) {
333 trace_process_export(export, event, flag);
334 export = rcu_dereference_raw_check(export->next);
335 }
336
337 preempt_enable_notrace();
338}
339
340static inline void
341add_trace_export(struct trace_export **list, struct trace_export *export)
342{
343 rcu_assign_pointer(export->next, *list);
344 /*
345 * We are entering export into the list but another
346 * CPU might be walking that list. We need to make sure
347 * the export->next pointer is valid before another CPU sees
348 * the export pointer included into the list.
349 */
350 rcu_assign_pointer(*list, export);
351}
352
353static inline int
354rm_trace_export(struct trace_export **list, struct trace_export *export)
355{
356 struct trace_export **p;
357
358 for (p = list; *p != NULL; p = &(*p)->next)
359 if (*p == export)
360 break;
361
362 if (*p != export)
363 return -1;
364
365 rcu_assign_pointer(*p, (*p)->next);
366
367 return 0;
368}
369
370static inline void
371add_ftrace_export(struct trace_export **list, struct trace_export *export)
372{
373 ftrace_exports_enable(export);
374
375 add_trace_export(list, export);
376}
377
378static inline int
379rm_ftrace_export(struct trace_export **list, struct trace_export *export)
380{
381 int ret;
382
383 ret = rm_trace_export(list, export);
384 ftrace_exports_disable(export);
385
386 return ret;
387}
388
389int register_ftrace_export(struct trace_export *export)
390{
391 if (WARN_ON_ONCE(!export->write))
392 return -1;
393
394 mutex_lock(&ftrace_export_lock);
395
396 add_ftrace_export(&ftrace_exports_list, export);
397
398 mutex_unlock(&ftrace_export_lock);
399
400 return 0;
401}
402EXPORT_SYMBOL_GPL(register_ftrace_export);
403
404int unregister_ftrace_export(struct trace_export *export)
405{
406 int ret;
407
408 mutex_lock(&ftrace_export_lock);
409
410 ret = rm_ftrace_export(&ftrace_exports_list, export);
411
412 mutex_unlock(&ftrace_export_lock);
413
414 return ret;
415}
416EXPORT_SYMBOL_GPL(unregister_ftrace_export);
417
418/* trace_flags holds trace_options default values */
419#define TRACE_DEFAULT_FLAGS \
420 (FUNCTION_DEFAULT_FLAGS | \
421 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
422 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
423 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
424 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | \
425 TRACE_ITER_HASH_PTR)
426
427/* trace_options that are only supported by global_trace */
428#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
429 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
430
431/* trace_flags that are default zero for instances */
432#define ZEROED_TRACE_FLAGS \
433 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
434
435/*
436 * The global_trace is the descriptor that holds the top-level tracing
437 * buffers for the live tracing.
438 */
439static struct trace_array global_trace = {
440 .trace_flags = TRACE_DEFAULT_FLAGS,
441};
442
443LIST_HEAD(ftrace_trace_arrays);
444
445int trace_array_get(struct trace_array *this_tr)
446{
447 struct trace_array *tr;
448 int ret = -ENODEV;
449
450 mutex_lock(&trace_types_lock);
451 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
452 if (tr == this_tr) {
453 tr->ref++;
454 ret = 0;
455 break;
456 }
457 }
458 mutex_unlock(&trace_types_lock);
459
460 return ret;
461}
462
463static void __trace_array_put(struct trace_array *this_tr)
464{
465 WARN_ON(!this_tr->ref);
466 this_tr->ref--;
467}
468
469/**
470 * trace_array_put - Decrement the reference counter for this trace array.
471 * @this_tr : pointer to the trace array
472 *
473 * NOTE: Use this when we no longer need the trace array returned by
474 * trace_array_get_by_name(). This ensures the trace array can be later
475 * destroyed.
476 *
477 */
478void trace_array_put(struct trace_array *this_tr)
479{
480 if (!this_tr)
481 return;
482
483 mutex_lock(&trace_types_lock);
484 __trace_array_put(this_tr);
485 mutex_unlock(&trace_types_lock);
486}
487EXPORT_SYMBOL_GPL(trace_array_put);
488
489int tracing_check_open_get_tr(struct trace_array *tr)
490{
491 int ret;
492
493 ret = security_locked_down(LOCKDOWN_TRACEFS);
494 if (ret)
495 return ret;
496
497 if (tracing_disabled)
498 return -ENODEV;
499
500 if (tr && trace_array_get(tr) < 0)
501 return -ENODEV;
502
503 return 0;
504}
505
506int call_filter_check_discard(struct trace_event_call *call, void *rec,
507 struct trace_buffer *buffer,
508 struct ring_buffer_event *event)
509{
510 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
511 !filter_match_preds(call->filter, rec)) {
512 __trace_event_discard_commit(buffer, event);
513 return 1;
514 }
515
516 return 0;
517}
518
519/**
520 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
521 * @filtered_pids: The list of pids to check
522 * @search_pid: The PID to find in @filtered_pids
523 *
524 * Returns true if @search_pid is found in @filtered_pids, and false otherwise.
525 */
526bool
527trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
528{
529 return trace_pid_list_is_set(filtered_pids, search_pid);
530}
531
532/**
533 * trace_ignore_this_task - should a task be ignored for tracing
534 * @filtered_pids: The list of pids to check
535 * @filtered_no_pids: The list of pids not to be traced
536 * @task: The task that should be ignored if not filtered
537 *
538 * Checks if @task should be traced or not from @filtered_pids.
539 * Returns true if @task should *NOT* be traced.
540 * Returns false if @task should be traced.
541 */
542bool
543trace_ignore_this_task(struct trace_pid_list *filtered_pids,
544 struct trace_pid_list *filtered_no_pids,
545 struct task_struct *task)
546{
547 /*
548 * If filtered_no_pids is not empty, and the task's pid is listed
549 * in filtered_no_pids, then return true.
550 * Otherwise, if filtered_pids is empty, that means we can
551 * trace all tasks. If it has content, then only trace pids
552 * within filtered_pids.
553 */
554
555 return (filtered_pids &&
556 !trace_find_filtered_pid(filtered_pids, task->pid)) ||
557 (filtered_no_pids &&
558 trace_find_filtered_pid(filtered_no_pids, task->pid));
559}
560
561/**
562 * trace_filter_add_remove_task - Add or remove a task from a pid_list
563 * @pid_list: The list to modify
564 * @self: The current task for fork or NULL for exit
565 * @task: The task to add or remove
566 *
567 * If adding a task, if @self is defined, the task is only added if @self
568 * is also included in @pid_list. This happens on fork and tasks should
569 * only be added when the parent is listed. If @self is NULL, then the
570 * @task pid will be removed from the list, which would happen on exit
571 * of a task.
572 */
573void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
574 struct task_struct *self,
575 struct task_struct *task)
576{
577 if (!pid_list)
578 return;
579
580 /* For forks, we only add if the forking task is listed */
581 if (self) {
582 if (!trace_find_filtered_pid(pid_list, self->pid))
583 return;
584 }
585
586 /* "self" is set for forks, and NULL for exits */
587 if (self)
588 trace_pid_list_set(pid_list, task->pid);
589 else
590 trace_pid_list_clear(pid_list, task->pid);
591}
592
593/**
594 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
595 * @pid_list: The pid list to show
596 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
597 * @pos: The position of the file
598 *
599 * This is used by the seq_file "next" operation to iterate the pids
600 * listed in a trace_pid_list structure.
601 *
602 * Returns the pid+1 as we want to display pid of zero, but NULL would
603 * stop the iteration.
604 */
605void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
606{
607 long pid = (unsigned long)v;
608 unsigned int next;
609
610 (*pos)++;
611
612 /* pid already is +1 of the actual previous bit */
613 if (trace_pid_list_next(pid_list, pid, &next) < 0)
614 return NULL;
615
616 pid = next;
617
618 /* Return pid + 1 to allow zero to be represented */
619 return (void *)(pid + 1);
620}
621
622/**
623 * trace_pid_start - Used for seq_file to start reading pid lists
624 * @pid_list: The pid list to show
625 * @pos: The position of the file
626 *
627 * This is used by seq_file "start" operation to start the iteration
628 * of listing pids.
629 *
630 * Returns the pid+1 as we want to display pid of zero, but NULL would
631 * stop the iteration.
632 */
633void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
634{
635 unsigned long pid;
636 unsigned int first;
637 loff_t l = 0;
638
639 if (trace_pid_list_first(pid_list, &first) < 0)
640 return NULL;
641
642 pid = first;
643
644 /* Return pid + 1 so that zero can be the exit value */
645 for (pid++; pid && l < *pos;
646 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
647 ;
648 return (void *)pid;
649}
650
651/**
652 * trace_pid_show - show the current pid in seq_file processing
653 * @m: The seq_file structure to write into
654 * @v: A void pointer of the pid (+1) value to display
655 *
656 * Can be directly used by seq_file operations to display the current
657 * pid value.
658 */
659int trace_pid_show(struct seq_file *m, void *v)
660{
661 unsigned long pid = (unsigned long)v - 1;
662
663 seq_printf(m, "%lu\n", pid);
664 return 0;
665}
666
667/* 128 should be much more than enough */
668#define PID_BUF_SIZE 127
669
670int trace_pid_write(struct trace_pid_list *filtered_pids,
671 struct trace_pid_list **new_pid_list,
672 const char __user *ubuf, size_t cnt)
673{
674 struct trace_pid_list *pid_list;
675 struct trace_parser parser;
676 unsigned long val;
677 int nr_pids = 0;
678 ssize_t read = 0;
679 ssize_t ret;
680 loff_t pos;
681 pid_t pid;
682
683 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
684 return -ENOMEM;
685
686 /*
687 * Always recreate a new array. The write is an all or nothing
688 * operation. Always create a new array when adding new pids by
689 * the user. If the operation fails, then the current list is
690 * not modified.
691 */
692 pid_list = trace_pid_list_alloc();
693 if (!pid_list) {
694 trace_parser_put(&parser);
695 return -ENOMEM;
696 }
697
698 if (filtered_pids) {
699 /* copy the current bits to the new max */
700 ret = trace_pid_list_first(filtered_pids, &pid);
701 while (!ret) {
702 trace_pid_list_set(pid_list, pid);
703 ret = trace_pid_list_next(filtered_pids, pid + 1, &pid);
704 nr_pids++;
705 }
706 }
707
708 ret = 0;
709 while (cnt > 0) {
710
711 pos = 0;
712
713 ret = trace_get_user(&parser, ubuf, cnt, &pos);
714 if (ret < 0 || !trace_parser_loaded(&parser))
715 break;
716
717 read += ret;
718 ubuf += ret;
719 cnt -= ret;
720
721 ret = -EINVAL;
722 if (kstrtoul(parser.buffer, 0, &val))
723 break;
724
725 pid = (pid_t)val;
726
727 if (trace_pid_list_set(pid_list, pid) < 0) {
728 ret = -1;
729 break;
730 }
731 nr_pids++;
732
733 trace_parser_clear(&parser);
734 ret = 0;
735 }
736 trace_parser_put(&parser);
737
738 if (ret < 0) {
739 trace_pid_list_free(pid_list);
740 return ret;
741 }
742
743 if (!nr_pids) {
744 /* Cleared the list of pids */
745 trace_pid_list_free(pid_list);
746 read = ret;
747 pid_list = NULL;
748 }
749
750 *new_pid_list = pid_list;
751
752 return read;
753}
754
755static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
756{
757 u64 ts;
758
759 /* Early boot up does not have a buffer yet */
760 if (!buf->buffer)
761 return trace_clock_local();
762
763 ts = ring_buffer_time_stamp(buf->buffer);
764 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
765
766 return ts;
767}
768
769u64 ftrace_now(int cpu)
770{
771 return buffer_ftrace_now(&global_trace.array_buffer, cpu);
772}
773
774/**
775 * tracing_is_enabled - Show if global_trace has been enabled
776 *
777 * Shows if the global trace has been enabled or not. It uses the
778 * mirror flag "buffer_disabled" to be used in fast paths such as for
779 * the irqsoff tracer. But it may be inaccurate due to races. If you
780 * need to know the accurate state, use tracing_is_on() which is a little
781 * slower, but accurate.
782 */
783int tracing_is_enabled(void)
784{
785 /*
786 * For quick access (irqsoff uses this in fast path), just
787 * return the mirror variable of the state of the ring buffer.
788 * It's a little racy, but we don't really care.
789 */
790 smp_rmb();
791 return !global_trace.buffer_disabled;
792}
793
794/*
795 * trace_buf_size is the size in bytes that is allocated
796 * for a buffer. Note, the number of bytes is always rounded
797 * to page size.
798 *
799 * This number is purposely set to a low number of 16384.
800 * If the dump on oops happens, it will be much appreciated
801 * to not have to wait for all that output. Anyway this can be
802 * boot time and run time configurable.
803 */
804#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
805
806static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
807
808/* trace_types holds a link list of available tracers. */
809static struct tracer *trace_types __read_mostly;
810
811/*
812 * trace_types_lock is used to protect the trace_types list.
813 */
814DEFINE_MUTEX(trace_types_lock);
815
816/*
817 * serialize the access of the ring buffer
818 *
819 * ring buffer serializes readers, but it is low level protection.
820 * The validity of the events (which returns by ring_buffer_peek() ..etc)
821 * are not protected by ring buffer.
822 *
823 * The content of events may become garbage if we allow other process consumes
824 * these events concurrently:
825 * A) the page of the consumed events may become a normal page
826 * (not reader page) in ring buffer, and this page will be rewritten
827 * by events producer.
828 * B) The page of the consumed events may become a page for splice_read,
829 * and this page will be returned to system.
830 *
831 * These primitives allow multi process access to different cpu ring buffer
832 * concurrently.
833 *
834 * These primitives don't distinguish read-only and read-consume access.
835 * Multi read-only access are also serialized.
836 */
837
838#ifdef CONFIG_SMP
839static DECLARE_RWSEM(all_cpu_access_lock);
840static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
841
842static inline void trace_access_lock(int cpu)
843{
844 if (cpu == RING_BUFFER_ALL_CPUS) {
845 /* gain it for accessing the whole ring buffer. */
846 down_write(&all_cpu_access_lock);
847 } else {
848 /* gain it for accessing a cpu ring buffer. */
849
850 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
851 down_read(&all_cpu_access_lock);
852
853 /* Secondly block other access to this @cpu ring buffer. */
854 mutex_lock(&per_cpu(cpu_access_lock, cpu));
855 }
856}
857
858static inline void trace_access_unlock(int cpu)
859{
860 if (cpu == RING_BUFFER_ALL_CPUS) {
861 up_write(&all_cpu_access_lock);
862 } else {
863 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
864 up_read(&all_cpu_access_lock);
865 }
866}
867
868static inline void trace_access_lock_init(void)
869{
870 int cpu;
871
872 for_each_possible_cpu(cpu)
873 mutex_init(&per_cpu(cpu_access_lock, cpu));
874}
875
876#else
877
878static DEFINE_MUTEX(access_lock);
879
880static inline void trace_access_lock(int cpu)
881{
882 (void)cpu;
883 mutex_lock(&access_lock);
884}
885
886static inline void trace_access_unlock(int cpu)
887{
888 (void)cpu;
889 mutex_unlock(&access_lock);
890}
891
892static inline void trace_access_lock_init(void)
893{
894}
895
896#endif
897
898#ifdef CONFIG_STACKTRACE
899static void __ftrace_trace_stack(struct trace_buffer *buffer,
900 unsigned int trace_ctx,
901 int skip, struct pt_regs *regs);
902static inline void ftrace_trace_stack(struct trace_array *tr,
903 struct trace_buffer *buffer,
904 unsigned int trace_ctx,
905 int skip, struct pt_regs *regs);
906
907#else
908static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
909 unsigned int trace_ctx,
910 int skip, struct pt_regs *regs)
911{
912}
913static inline void ftrace_trace_stack(struct trace_array *tr,
914 struct trace_buffer *buffer,
915 unsigned long trace_ctx,
916 int skip, struct pt_regs *regs)
917{
918}
919
920#endif
921
922static __always_inline void
923trace_event_setup(struct ring_buffer_event *event,
924 int type, unsigned int trace_ctx)
925{
926 struct trace_entry *ent = ring_buffer_event_data(event);
927
928 tracing_generic_entry_update(ent, type, trace_ctx);
929}
930
931static __always_inline struct ring_buffer_event *
932__trace_buffer_lock_reserve(struct trace_buffer *buffer,
933 int type,
934 unsigned long len,
935 unsigned int trace_ctx)
936{
937 struct ring_buffer_event *event;
938
939 event = ring_buffer_lock_reserve(buffer, len);
940 if (event != NULL)
941 trace_event_setup(event, type, trace_ctx);
942
943 return event;
944}
945
946void tracer_tracing_on(struct trace_array *tr)
947{
948 if (tr->array_buffer.buffer)
949 ring_buffer_record_on(tr->array_buffer.buffer);
950 /*
951 * This flag is looked at when buffers haven't been allocated
952 * yet, or by some tracers (like irqsoff), that just want to
953 * know if the ring buffer has been disabled, but it can handle
954 * races of where it gets disabled but we still do a record.
955 * As the check is in the fast path of the tracers, it is more
956 * important to be fast than accurate.
957 */
958 tr->buffer_disabled = 0;
959 /* Make the flag seen by readers */
960 smp_wmb();
961}
962
963/**
964 * tracing_on - enable tracing buffers
965 *
966 * This function enables tracing buffers that may have been
967 * disabled with tracing_off.
968 */
969void tracing_on(void)
970{
971 tracer_tracing_on(&global_trace);
972}
973EXPORT_SYMBOL_GPL(tracing_on);
974
975
976static __always_inline void
977__buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)
978{
979 __this_cpu_write(trace_taskinfo_save, true);
980
981 /* If this is the temp buffer, we need to commit fully */
982 if (this_cpu_read(trace_buffered_event) == event) {
983 /* Length is in event->array[0] */
984 ring_buffer_write(buffer, event->array[0], &event->array[1]);
985 /* Release the temp buffer */
986 this_cpu_dec(trace_buffered_event_cnt);
987 /* ring_buffer_unlock_commit() enables preemption */
988 preempt_enable_notrace();
989 } else
990 ring_buffer_unlock_commit(buffer, event);
991}
992
993/**
994 * __trace_puts - write a constant string into the trace buffer.
995 * @ip: The address of the caller
996 * @str: The constant string to write
997 * @size: The size of the string.
998 */
999int __trace_puts(unsigned long ip, const char *str, int size)
1000{
1001 struct ring_buffer_event *event;
1002 struct trace_buffer *buffer;
1003 struct print_entry *entry;
1004 unsigned int trace_ctx;
1005 int alloc;
1006
1007 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
1008 return 0;
1009
1010 if (unlikely(tracing_selftest_running || tracing_disabled))
1011 return 0;
1012
1013 alloc = sizeof(*entry) + size + 2; /* possible \n added */
1014
1015 trace_ctx = tracing_gen_ctx();
1016 buffer = global_trace.array_buffer.buffer;
1017 ring_buffer_nest_start(buffer);
1018 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
1019 trace_ctx);
1020 if (!event) {
1021 size = 0;
1022 goto out;
1023 }
1024
1025 entry = ring_buffer_event_data(event);
1026 entry->ip = ip;
1027
1028 memcpy(&entry->buf, str, size);
1029
1030 /* Add a newline if necessary */
1031 if (entry->buf[size - 1] != '\n') {
1032 entry->buf[size] = '\n';
1033 entry->buf[size + 1] = '\0';
1034 } else
1035 entry->buf[size] = '\0';
1036
1037 __buffer_unlock_commit(buffer, event);
1038 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
1039 out:
1040 ring_buffer_nest_end(buffer);
1041 return size;
1042}
1043EXPORT_SYMBOL_GPL(__trace_puts);
1044
1045/**
1046 * __trace_bputs - write the pointer to a constant string into trace buffer
1047 * @ip: The address of the caller
1048 * @str: The constant string to write to the buffer to
1049 */
1050int __trace_bputs(unsigned long ip, const char *str)
1051{
1052 struct ring_buffer_event *event;
1053 struct trace_buffer *buffer;
1054 struct bputs_entry *entry;
1055 unsigned int trace_ctx;
1056 int size = sizeof(struct bputs_entry);
1057 int ret = 0;
1058
1059 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
1060 return 0;
1061
1062 if (unlikely(tracing_selftest_running || tracing_disabled))
1063 return 0;
1064
1065 trace_ctx = tracing_gen_ctx();
1066 buffer = global_trace.array_buffer.buffer;
1067
1068 ring_buffer_nest_start(buffer);
1069 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
1070 trace_ctx);
1071 if (!event)
1072 goto out;
1073
1074 entry = ring_buffer_event_data(event);
1075 entry->ip = ip;
1076 entry->str = str;
1077
1078 __buffer_unlock_commit(buffer, event);
1079 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
1080
1081 ret = 1;
1082 out:
1083 ring_buffer_nest_end(buffer);
1084 return ret;
1085}
1086EXPORT_SYMBOL_GPL(__trace_bputs);
1087
1088#ifdef CONFIG_TRACER_SNAPSHOT
1089static void tracing_snapshot_instance_cond(struct trace_array *tr,
1090 void *cond_data)
1091{
1092 struct tracer *tracer = tr->current_trace;
1093 unsigned long flags;
1094
1095 if (in_nmi()) {
1096 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
1097 internal_trace_puts("*** snapshot is being ignored ***\n");
1098 return;
1099 }
1100
1101 if (!tr->allocated_snapshot) {
1102 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
1103 internal_trace_puts("*** stopping trace here! ***\n");
1104 tracing_off();
1105 return;
1106 }
1107
1108 /* Note, snapshot can not be used when the tracer uses it */
1109 if (tracer->use_max_tr) {
1110 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
1111 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
1112 return;
1113 }
1114
1115 local_irq_save(flags);
1116 update_max_tr(tr, current, smp_processor_id(), cond_data);
1117 local_irq_restore(flags);
1118}
1119
1120void tracing_snapshot_instance(struct trace_array *tr)
1121{
1122 tracing_snapshot_instance_cond(tr, NULL);
1123}
1124
1125/**
1126 * tracing_snapshot - take a snapshot of the current buffer.
1127 *
1128 * This causes a swap between the snapshot buffer and the current live
1129 * tracing buffer. You can use this to take snapshots of the live
1130 * trace when some condition is triggered, but continue to trace.
1131 *
1132 * Note, make sure to allocate the snapshot with either
1133 * a tracing_snapshot_alloc(), or by doing it manually
1134 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
1135 *
1136 * If the snapshot buffer is not allocated, it will stop tracing.
1137 * Basically making a permanent snapshot.
1138 */
1139void tracing_snapshot(void)
1140{
1141 struct trace_array *tr = &global_trace;
1142
1143 tracing_snapshot_instance(tr);
1144}
1145EXPORT_SYMBOL_GPL(tracing_snapshot);
1146
1147/**
1148 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1149 * @tr: The tracing instance to snapshot
1150 * @cond_data: The data to be tested conditionally, and possibly saved
1151 *
1152 * This is the same as tracing_snapshot() except that the snapshot is
1153 * conditional - the snapshot will only happen if the
1154 * cond_snapshot.update() implementation receiving the cond_data
1155 * returns true, which means that the trace array's cond_snapshot
1156 * update() operation used the cond_data to determine whether the
1157 * snapshot should be taken, and if it was, presumably saved it along
1158 * with the snapshot.
1159 */
1160void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1161{
1162 tracing_snapshot_instance_cond(tr, cond_data);
1163}
1164EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1165
1166/**
1167 * tracing_snapshot_cond_data - get the user data associated with a snapshot
1168 * @tr: The tracing instance
1169 *
1170 * When the user enables a conditional snapshot using
1171 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1172 * with the snapshot. This accessor is used to retrieve it.
1173 *
1174 * Should not be called from cond_snapshot.update(), since it takes
1175 * the tr->max_lock lock, which the code calling
1176 * cond_snapshot.update() has already done.
1177 *
1178 * Returns the cond_data associated with the trace array's snapshot.
1179 */
1180void *tracing_cond_snapshot_data(struct trace_array *tr)
1181{
1182 void *cond_data = NULL;
1183
1184 arch_spin_lock(&tr->max_lock);
1185
1186 if (tr->cond_snapshot)
1187 cond_data = tr->cond_snapshot->cond_data;
1188
1189 arch_spin_unlock(&tr->max_lock);
1190
1191 return cond_data;
1192}
1193EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1194
1195static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1196 struct array_buffer *size_buf, int cpu_id);
1197static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1198
1199int tracing_alloc_snapshot_instance(struct trace_array *tr)
1200{
1201 int ret;
1202
1203 if (!tr->allocated_snapshot) {
1204
1205 /* allocate spare buffer */
1206 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1207 &tr->array_buffer, RING_BUFFER_ALL_CPUS);
1208 if (ret < 0)
1209 return ret;
1210
1211 tr->allocated_snapshot = true;
1212 }
1213
1214 return 0;
1215}
1216
1217static void free_snapshot(struct trace_array *tr)
1218{
1219 /*
1220 * We don't free the ring buffer. instead, resize it because
1221 * The max_tr ring buffer has some state (e.g. ring->clock) and
1222 * we want preserve it.
1223 */
1224 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1225 set_buffer_entries(&tr->max_buffer, 1);
1226 tracing_reset_online_cpus(&tr->max_buffer);
1227 tr->allocated_snapshot = false;
1228}
1229
1230/**
1231 * tracing_alloc_snapshot - allocate snapshot buffer.
1232 *
1233 * This only allocates the snapshot buffer if it isn't already
1234 * allocated - it doesn't also take a snapshot.
1235 *
1236 * This is meant to be used in cases where the snapshot buffer needs
1237 * to be set up for events that can't sleep but need to be able to
1238 * trigger a snapshot.
1239 */
1240int tracing_alloc_snapshot(void)
1241{
1242 struct trace_array *tr = &global_trace;
1243 int ret;
1244
1245 ret = tracing_alloc_snapshot_instance(tr);
1246 WARN_ON(ret < 0);
1247
1248 return ret;
1249}
1250EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1251
1252/**
1253 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1254 *
1255 * This is similar to tracing_snapshot(), but it will allocate the
1256 * snapshot buffer if it isn't already allocated. Use this only
1257 * where it is safe to sleep, as the allocation may sleep.
1258 *
1259 * This causes a swap between the snapshot buffer and the current live
1260 * tracing buffer. You can use this to take snapshots of the live
1261 * trace when some condition is triggered, but continue to trace.
1262 */
1263void tracing_snapshot_alloc(void)
1264{
1265 int ret;
1266
1267 ret = tracing_alloc_snapshot();
1268 if (ret < 0)
1269 return;
1270
1271 tracing_snapshot();
1272}
1273EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1274
1275/**
1276 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1277 * @tr: The tracing instance
1278 * @cond_data: User data to associate with the snapshot
1279 * @update: Implementation of the cond_snapshot update function
1280 *
1281 * Check whether the conditional snapshot for the given instance has
1282 * already been enabled, or if the current tracer is already using a
1283 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1284 * save the cond_data and update function inside.
1285 *
1286 * Returns 0 if successful, error otherwise.
1287 */
1288int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1289 cond_update_fn_t update)
1290{
1291 struct cond_snapshot *cond_snapshot;
1292 int ret = 0;
1293
1294 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1295 if (!cond_snapshot)
1296 return -ENOMEM;
1297
1298 cond_snapshot->cond_data = cond_data;
1299 cond_snapshot->update = update;
1300
1301 mutex_lock(&trace_types_lock);
1302
1303 ret = tracing_alloc_snapshot_instance(tr);
1304 if (ret)
1305 goto fail_unlock;
1306
1307 if (tr->current_trace->use_max_tr) {
1308 ret = -EBUSY;
1309 goto fail_unlock;
1310 }
1311
1312 /*
1313 * The cond_snapshot can only change to NULL without the
1314 * trace_types_lock. We don't care if we race with it going
1315 * to NULL, but we want to make sure that it's not set to
1316 * something other than NULL when we get here, which we can
1317 * do safely with only holding the trace_types_lock and not
1318 * having to take the max_lock.
1319 */
1320 if (tr->cond_snapshot) {
1321 ret = -EBUSY;
1322 goto fail_unlock;
1323 }
1324
1325 arch_spin_lock(&tr->max_lock);
1326 tr->cond_snapshot = cond_snapshot;
1327 arch_spin_unlock(&tr->max_lock);
1328
1329 mutex_unlock(&trace_types_lock);
1330
1331 return ret;
1332
1333 fail_unlock:
1334 mutex_unlock(&trace_types_lock);
1335 kfree(cond_snapshot);
1336 return ret;
1337}
1338EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1339
1340/**
1341 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1342 * @tr: The tracing instance
1343 *
1344 * Check whether the conditional snapshot for the given instance is
1345 * enabled; if so, free the cond_snapshot associated with it,
1346 * otherwise return -EINVAL.
1347 *
1348 * Returns 0 if successful, error otherwise.
1349 */
1350int tracing_snapshot_cond_disable(struct trace_array *tr)
1351{
1352 int ret = 0;
1353
1354 arch_spin_lock(&tr->max_lock);
1355
1356 if (!tr->cond_snapshot)
1357 ret = -EINVAL;
1358 else {
1359 kfree(tr->cond_snapshot);
1360 tr->cond_snapshot = NULL;
1361 }
1362
1363 arch_spin_unlock(&tr->max_lock);
1364
1365 return ret;
1366}
1367EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1368#else
1369void tracing_snapshot(void)
1370{
1371 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1372}
1373EXPORT_SYMBOL_GPL(tracing_snapshot);
1374void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1375{
1376 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1377}
1378EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1379int tracing_alloc_snapshot(void)
1380{
1381 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1382 return -ENODEV;
1383}
1384EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1385void tracing_snapshot_alloc(void)
1386{
1387 /* Give warning */
1388 tracing_snapshot();
1389}
1390EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1391void *tracing_cond_snapshot_data(struct trace_array *tr)
1392{
1393 return NULL;
1394}
1395EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1396int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1397{
1398 return -ENODEV;
1399}
1400EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1401int tracing_snapshot_cond_disable(struct trace_array *tr)
1402{
1403 return false;
1404}
1405EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1406#endif /* CONFIG_TRACER_SNAPSHOT */
1407
1408void tracer_tracing_off(struct trace_array *tr)
1409{
1410 if (tr->array_buffer.buffer)
1411 ring_buffer_record_off(tr->array_buffer.buffer);
1412 /*
1413 * This flag is looked at when buffers haven't been allocated
1414 * yet, or by some tracers (like irqsoff), that just want to
1415 * know if the ring buffer has been disabled, but it can handle
1416 * races of where it gets disabled but we still do a record.
1417 * As the check is in the fast path of the tracers, it is more
1418 * important to be fast than accurate.
1419 */
1420 tr->buffer_disabled = 1;
1421 /* Make the flag seen by readers */
1422 smp_wmb();
1423}
1424
1425/**
1426 * tracing_off - turn off tracing buffers
1427 *
1428 * This function stops the tracing buffers from recording data.
1429 * It does not disable any overhead the tracers themselves may
1430 * be causing. This function simply causes all recording to
1431 * the ring buffers to fail.
1432 */
1433void tracing_off(void)
1434{
1435 tracer_tracing_off(&global_trace);
1436}
1437EXPORT_SYMBOL_GPL(tracing_off);
1438
1439void disable_trace_on_warning(void)
1440{
1441 if (__disable_trace_on_warning) {
1442 trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
1443 "Disabling tracing due to warning\n");
1444 tracing_off();
1445 }
1446}
1447
1448/**
1449 * tracer_tracing_is_on - show real state of ring buffer enabled
1450 * @tr : the trace array to know if ring buffer is enabled
1451 *
1452 * Shows real state of the ring buffer if it is enabled or not.
1453 */
1454bool tracer_tracing_is_on(struct trace_array *tr)
1455{
1456 if (tr->array_buffer.buffer)
1457 return ring_buffer_record_is_on(tr->array_buffer.buffer);
1458 return !tr->buffer_disabled;
1459}
1460
1461/**
1462 * tracing_is_on - show state of ring buffers enabled
1463 */
1464int tracing_is_on(void)
1465{
1466 return tracer_tracing_is_on(&global_trace);
1467}
1468EXPORT_SYMBOL_GPL(tracing_is_on);
1469
1470static int __init set_buf_size(char *str)
1471{
1472 unsigned long buf_size;
1473
1474 if (!str)
1475 return 0;
1476 buf_size = memparse(str, &str);
1477 /*
1478 * nr_entries can not be zero and the startup
1479 * tests require some buffer space. Therefore
1480 * ensure we have at least 4096 bytes of buffer.
1481 */
1482 trace_buf_size = max(4096UL, buf_size);
1483 return 1;
1484}
1485__setup("trace_buf_size=", set_buf_size);
1486
1487static int __init set_tracing_thresh(char *str)
1488{
1489 unsigned long threshold;
1490 int ret;
1491
1492 if (!str)
1493 return 0;
1494 ret = kstrtoul(str, 0, &threshold);
1495 if (ret < 0)
1496 return 0;
1497 tracing_thresh = threshold * 1000;
1498 return 1;
1499}
1500__setup("tracing_thresh=", set_tracing_thresh);
1501
1502unsigned long nsecs_to_usecs(unsigned long nsecs)
1503{
1504 return nsecs / 1000;
1505}
1506
1507/*
1508 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1509 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1510 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1511 * of strings in the order that the evals (enum) were defined.
1512 */
1513#undef C
1514#define C(a, b) b
1515
1516/* These must match the bit positions in trace_iterator_flags */
1517static const char *trace_options[] = {
1518 TRACE_FLAGS
1519 NULL
1520};
1521
1522static struct {
1523 u64 (*func)(void);
1524 const char *name;
1525 int in_ns; /* is this clock in nanoseconds? */
1526} trace_clocks[] = {
1527 { trace_clock_local, "local", 1 },
1528 { trace_clock_global, "global", 1 },
1529 { trace_clock_counter, "counter", 0 },
1530 { trace_clock_jiffies, "uptime", 0 },
1531 { trace_clock, "perf", 1 },
1532 { ktime_get_mono_fast_ns, "mono", 1 },
1533 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1534 { ktime_get_boot_fast_ns, "boot", 1 },
1535 ARCH_TRACE_CLOCKS
1536};
1537
1538bool trace_clock_in_ns(struct trace_array *tr)
1539{
1540 if (trace_clocks[tr->clock_id].in_ns)
1541 return true;
1542
1543 return false;
1544}
1545
1546/*
1547 * trace_parser_get_init - gets the buffer for trace parser
1548 */
1549int trace_parser_get_init(struct trace_parser *parser, int size)
1550{
1551 memset(parser, 0, sizeof(*parser));
1552
1553 parser->buffer = kmalloc(size, GFP_KERNEL);
1554 if (!parser->buffer)
1555 return 1;
1556
1557 parser->size = size;
1558 return 0;
1559}
1560
1561/*
1562 * trace_parser_put - frees the buffer for trace parser
1563 */
1564void trace_parser_put(struct trace_parser *parser)
1565{
1566 kfree(parser->buffer);
1567 parser->buffer = NULL;
1568}
1569
1570/*
1571 * trace_get_user - reads the user input string separated by space
1572 * (matched by isspace(ch))
1573 *
1574 * For each string found the 'struct trace_parser' is updated,
1575 * and the function returns.
1576 *
1577 * Returns number of bytes read.
1578 *
1579 * See kernel/trace/trace.h for 'struct trace_parser' details.
1580 */
1581int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1582 size_t cnt, loff_t *ppos)
1583{
1584 char ch;
1585 size_t read = 0;
1586 ssize_t ret;
1587
1588 if (!*ppos)
1589 trace_parser_clear(parser);
1590
1591 ret = get_user(ch, ubuf++);
1592 if (ret)
1593 goto out;
1594
1595 read++;
1596 cnt--;
1597
1598 /*
1599 * The parser is not finished with the last write,
1600 * continue reading the user input without skipping spaces.
1601 */
1602 if (!parser->cont) {
1603 /* skip white space */
1604 while (cnt && isspace(ch)) {
1605 ret = get_user(ch, ubuf++);
1606 if (ret)
1607 goto out;
1608 read++;
1609 cnt--;
1610 }
1611
1612 parser->idx = 0;
1613
1614 /* only spaces were written */
1615 if (isspace(ch) || !ch) {
1616 *ppos += read;
1617 ret = read;
1618 goto out;
1619 }
1620 }
1621
1622 /* read the non-space input */
1623 while (cnt && !isspace(ch) && ch) {
1624 if (parser->idx < parser->size - 1)
1625 parser->buffer[parser->idx++] = ch;
1626 else {
1627 ret = -EINVAL;
1628 goto out;
1629 }
1630 ret = get_user(ch, ubuf++);
1631 if (ret)
1632 goto out;
1633 read++;
1634 cnt--;
1635 }
1636
1637 /* We either got finished input or we have to wait for another call. */
1638 if (isspace(ch) || !ch) {
1639 parser->buffer[parser->idx] = 0;
1640 parser->cont = false;
1641 } else if (parser->idx < parser->size - 1) {
1642 parser->cont = true;
1643 parser->buffer[parser->idx++] = ch;
1644 /* Make sure the parsed string always terminates with '\0'. */
1645 parser->buffer[parser->idx] = 0;
1646 } else {
1647 ret = -EINVAL;
1648 goto out;
1649 }
1650
1651 *ppos += read;
1652 ret = read;
1653
1654out:
1655 return ret;
1656}
1657
1658/* TODO add a seq_buf_to_buffer() */
1659static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1660{
1661 int len;
1662
1663 if (trace_seq_used(s) <= s->seq.readpos)
1664 return -EBUSY;
1665
1666 len = trace_seq_used(s) - s->seq.readpos;
1667 if (cnt > len)
1668 cnt = len;
1669 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1670
1671 s->seq.readpos += cnt;
1672 return cnt;
1673}
1674
1675unsigned long __read_mostly tracing_thresh;
1676static const struct file_operations tracing_max_lat_fops;
1677
1678#ifdef LATENCY_FS_NOTIFY
1679
1680static struct workqueue_struct *fsnotify_wq;
1681
1682static void latency_fsnotify_workfn(struct work_struct *work)
1683{
1684 struct trace_array *tr = container_of(work, struct trace_array,
1685 fsnotify_work);
1686 fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);
1687}
1688
1689static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1690{
1691 struct trace_array *tr = container_of(iwork, struct trace_array,
1692 fsnotify_irqwork);
1693 queue_work(fsnotify_wq, &tr->fsnotify_work);
1694}
1695
1696static void trace_create_maxlat_file(struct trace_array *tr,
1697 struct dentry *d_tracer)
1698{
1699 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1700 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1701 tr->d_max_latency = trace_create_file("tracing_max_latency",
1702 TRACE_MODE_WRITE,
1703 d_tracer, &tr->max_latency,
1704 &tracing_max_lat_fops);
1705}
1706
1707__init static int latency_fsnotify_init(void)
1708{
1709 fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1710 WQ_UNBOUND | WQ_HIGHPRI, 0);
1711 if (!fsnotify_wq) {
1712 pr_err("Unable to allocate tr_max_lat_wq\n");
1713 return -ENOMEM;
1714 }
1715 return 0;
1716}
1717
1718late_initcall_sync(latency_fsnotify_init);
1719
1720void latency_fsnotify(struct trace_array *tr)
1721{
1722 if (!fsnotify_wq)
1723 return;
1724 /*
1725 * We cannot call queue_work(&tr->fsnotify_work) from here because it's
1726 * possible that we are called from __schedule() or do_idle(), which
1727 * could cause a deadlock.
1728 */
1729 irq_work_queue(&tr->fsnotify_irqwork);
1730}
1731
1732#elif defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) \
1733 || defined(CONFIG_OSNOISE_TRACER)
1734
1735#define trace_create_maxlat_file(tr, d_tracer) \
1736 trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \
1737 d_tracer, &tr->max_latency, &tracing_max_lat_fops)
1738
1739#else
1740#define trace_create_maxlat_file(tr, d_tracer) do { } while (0)
1741#endif
1742
1743#ifdef CONFIG_TRACER_MAX_TRACE
1744/*
1745 * Copy the new maximum trace into the separate maximum-trace
1746 * structure. (this way the maximum trace is permanently saved,
1747 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1748 */
1749static void
1750__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1751{
1752 struct array_buffer *trace_buf = &tr->array_buffer;
1753 struct array_buffer *max_buf = &tr->max_buffer;
1754 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1755 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1756
1757 max_buf->cpu = cpu;
1758 max_buf->time_start = data->preempt_timestamp;
1759
1760 max_data->saved_latency = tr->max_latency;
1761 max_data->critical_start = data->critical_start;
1762 max_data->critical_end = data->critical_end;
1763
1764 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1765 max_data->pid = tsk->pid;
1766 /*
1767 * If tsk == current, then use current_uid(), as that does not use
1768 * RCU. The irq tracer can be called out of RCU scope.
1769 */
1770 if (tsk == current)
1771 max_data->uid = current_uid();
1772 else
1773 max_data->uid = task_uid(tsk);
1774
1775 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1776 max_data->policy = tsk->policy;
1777 max_data->rt_priority = tsk->rt_priority;
1778
1779 /* record this tasks comm */
1780 tracing_record_cmdline(tsk);
1781 latency_fsnotify(tr);
1782}
1783
1784/**
1785 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1786 * @tr: tracer
1787 * @tsk: the task with the latency
1788 * @cpu: The cpu that initiated the trace.
1789 * @cond_data: User data associated with a conditional snapshot
1790 *
1791 * Flip the buffers between the @tr and the max_tr and record information
1792 * about which task was the cause of this latency.
1793 */
1794void
1795update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1796 void *cond_data)
1797{
1798 if (tr->stop_count)
1799 return;
1800
1801 WARN_ON_ONCE(!irqs_disabled());
1802
1803 if (!tr->allocated_snapshot) {
1804 /* Only the nop tracer should hit this when disabling */
1805 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1806 return;
1807 }
1808
1809 arch_spin_lock(&tr->max_lock);
1810
1811 /* Inherit the recordable setting from array_buffer */
1812 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1813 ring_buffer_record_on(tr->max_buffer.buffer);
1814 else
1815 ring_buffer_record_off(tr->max_buffer.buffer);
1816
1817#ifdef CONFIG_TRACER_SNAPSHOT
1818 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1819 goto out_unlock;
1820#endif
1821 swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1822
1823 __update_max_tr(tr, tsk, cpu);
1824
1825 out_unlock:
1826 arch_spin_unlock(&tr->max_lock);
1827}
1828
1829/**
1830 * update_max_tr_single - only copy one trace over, and reset the rest
1831 * @tr: tracer
1832 * @tsk: task with the latency
1833 * @cpu: the cpu of the buffer to copy.
1834 *
1835 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1836 */
1837void
1838update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1839{
1840 int ret;
1841
1842 if (tr->stop_count)
1843 return;
1844
1845 WARN_ON_ONCE(!irqs_disabled());
1846 if (!tr->allocated_snapshot) {
1847 /* Only the nop tracer should hit this when disabling */
1848 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1849 return;
1850 }
1851
1852 arch_spin_lock(&tr->max_lock);
1853
1854 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
1855
1856 if (ret == -EBUSY) {
1857 /*
1858 * We failed to swap the buffer due to a commit taking
1859 * place on this CPU. We fail to record, but we reset
1860 * the max trace buffer (no one writes directly to it)
1861 * and flag that it failed.
1862 */
1863 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1864 "Failed to swap buffers due to commit in progress\n");
1865 }
1866
1867 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1868
1869 __update_max_tr(tr, tsk, cpu);
1870 arch_spin_unlock(&tr->max_lock);
1871}
1872#endif /* CONFIG_TRACER_MAX_TRACE */
1873
1874static int wait_on_pipe(struct trace_iterator *iter, int full)
1875{
1876 /* Iterators are static, they should be filled or empty */
1877 if (trace_buffer_iter(iter, iter->cpu_file))
1878 return 0;
1879
1880 return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file,
1881 full);
1882}
1883
1884#ifdef CONFIG_FTRACE_STARTUP_TEST
1885static bool selftests_can_run;
1886
1887struct trace_selftests {
1888 struct list_head list;
1889 struct tracer *type;
1890};
1891
1892static LIST_HEAD(postponed_selftests);
1893
1894static int save_selftest(struct tracer *type)
1895{
1896 struct trace_selftests *selftest;
1897
1898 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1899 if (!selftest)
1900 return -ENOMEM;
1901
1902 selftest->type = type;
1903 list_add(&selftest->list, &postponed_selftests);
1904 return 0;
1905}
1906
1907static int run_tracer_selftest(struct tracer *type)
1908{
1909 struct trace_array *tr = &global_trace;
1910 struct tracer *saved_tracer = tr->current_trace;
1911 int ret;
1912
1913 if (!type->selftest || tracing_selftest_disabled)
1914 return 0;
1915
1916 /*
1917 * If a tracer registers early in boot up (before scheduling is
1918 * initialized and such), then do not run its selftests yet.
1919 * Instead, run it a little later in the boot process.
1920 */
1921 if (!selftests_can_run)
1922 return save_selftest(type);
1923
1924 if (!tracing_is_on()) {
1925 pr_warn("Selftest for tracer %s skipped due to tracing disabled\n",
1926 type->name);
1927 return 0;
1928 }
1929
1930 /*
1931 * Run a selftest on this tracer.
1932 * Here we reset the trace buffer, and set the current
1933 * tracer to be this tracer. The tracer can then run some
1934 * internal tracing to verify that everything is in order.
1935 * If we fail, we do not register this tracer.
1936 */
1937 tracing_reset_online_cpus(&tr->array_buffer);
1938
1939 tr->current_trace = type;
1940
1941#ifdef CONFIG_TRACER_MAX_TRACE
1942 if (type->use_max_tr) {
1943 /* If we expanded the buffers, make sure the max is expanded too */
1944 if (ring_buffer_expanded)
1945 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1946 RING_BUFFER_ALL_CPUS);
1947 tr->allocated_snapshot = true;
1948 }
1949#endif
1950
1951 /* the test is responsible for initializing and enabling */
1952 pr_info("Testing tracer %s: ", type->name);
1953 ret = type->selftest(type, tr);
1954 /* the test is responsible for resetting too */
1955 tr->current_trace = saved_tracer;
1956 if (ret) {
1957 printk(KERN_CONT "FAILED!\n");
1958 /* Add the warning after printing 'FAILED' */
1959 WARN_ON(1);
1960 return -1;
1961 }
1962 /* Only reset on passing, to avoid touching corrupted buffers */
1963 tracing_reset_online_cpus(&tr->array_buffer);
1964
1965#ifdef CONFIG_TRACER_MAX_TRACE
1966 if (type->use_max_tr) {
1967 tr->allocated_snapshot = false;
1968
1969 /* Shrink the max buffer again */
1970 if (ring_buffer_expanded)
1971 ring_buffer_resize(tr->max_buffer.buffer, 1,
1972 RING_BUFFER_ALL_CPUS);
1973 }
1974#endif
1975
1976 printk(KERN_CONT "PASSED\n");
1977 return 0;
1978}
1979
1980static __init int init_trace_selftests(void)
1981{
1982 struct trace_selftests *p, *n;
1983 struct tracer *t, **last;
1984 int ret;
1985
1986 selftests_can_run = true;
1987
1988 mutex_lock(&trace_types_lock);
1989
1990 if (list_empty(&postponed_selftests))
1991 goto out;
1992
1993 pr_info("Running postponed tracer tests:\n");
1994
1995 tracing_selftest_running = true;
1996 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1997 /* This loop can take minutes when sanitizers are enabled, so
1998 * lets make sure we allow RCU processing.
1999 */
2000 cond_resched();
2001 ret = run_tracer_selftest(p->type);
2002 /* If the test fails, then warn and remove from available_tracers */
2003 if (ret < 0) {
2004 WARN(1, "tracer: %s failed selftest, disabling\n",
2005 p->type->name);
2006 last = &trace_types;
2007 for (t = trace_types; t; t = t->next) {
2008 if (t == p->type) {
2009 *last = t->next;
2010 break;
2011 }
2012 last = &t->next;
2013 }
2014 }
2015 list_del(&p->list);
2016 kfree(p);
2017 }
2018 tracing_selftest_running = false;
2019
2020 out:
2021 mutex_unlock(&trace_types_lock);
2022
2023 return 0;
2024}
2025core_initcall(init_trace_selftests);
2026#else
2027static inline int run_tracer_selftest(struct tracer *type)
2028{
2029 return 0;
2030}
2031#endif /* CONFIG_FTRACE_STARTUP_TEST */
2032
2033static void add_tracer_options(struct trace_array *tr, struct tracer *t);
2034
2035static void __init apply_trace_boot_options(void);
2036
2037/**
2038 * register_tracer - register a tracer with the ftrace system.
2039 * @type: the plugin for the tracer
2040 *
2041 * Register a new plugin tracer.
2042 */
2043int __init register_tracer(struct tracer *type)
2044{
2045 struct tracer *t;
2046 int ret = 0;
2047
2048 if (!type->name) {
2049 pr_info("Tracer must have a name\n");
2050 return -1;
2051 }
2052
2053 if (strlen(type->name) >= MAX_TRACER_SIZE) {
2054 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
2055 return -1;
2056 }
2057
2058 if (security_locked_down(LOCKDOWN_TRACEFS)) {
2059 pr_warn("Can not register tracer %s due to lockdown\n",
2060 type->name);
2061 return -EPERM;
2062 }
2063
2064 mutex_lock(&trace_types_lock);
2065
2066 tracing_selftest_running = true;
2067
2068 for (t = trace_types; t; t = t->next) {
2069 if (strcmp(type->name, t->name) == 0) {
2070 /* already found */
2071 pr_info("Tracer %s already registered\n",
2072 type->name);
2073 ret = -1;
2074 goto out;
2075 }
2076 }
2077
2078 if (!type->set_flag)
2079 type->set_flag = &dummy_set_flag;
2080 if (!type->flags) {
2081 /*allocate a dummy tracer_flags*/
2082 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
2083 if (!type->flags) {
2084 ret = -ENOMEM;
2085 goto out;
2086 }
2087 type->flags->val = 0;
2088 type->flags->opts = dummy_tracer_opt;
2089 } else
2090 if (!type->flags->opts)
2091 type->flags->opts = dummy_tracer_opt;
2092
2093 /* store the tracer for __set_tracer_option */
2094 type->flags->trace = type;
2095
2096 ret = run_tracer_selftest(type);
2097 if (ret < 0)
2098 goto out;
2099
2100 type->next = trace_types;
2101 trace_types = type;
2102 add_tracer_options(&global_trace, type);
2103
2104 out:
2105 tracing_selftest_running = false;
2106 mutex_unlock(&trace_types_lock);
2107
2108 if (ret || !default_bootup_tracer)
2109 goto out_unlock;
2110
2111 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
2112 goto out_unlock;
2113
2114 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
2115 /* Do we want this tracer to start on bootup? */
2116 tracing_set_tracer(&global_trace, type->name);
2117 default_bootup_tracer = NULL;
2118
2119 apply_trace_boot_options();
2120
2121 /* disable other selftests, since this will break it. */
2122 disable_tracing_selftest("running a tracer");
2123
2124 out_unlock:
2125 return ret;
2126}
2127
2128static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
2129{
2130 struct trace_buffer *buffer = buf->buffer;
2131
2132 if (!buffer)
2133 return;
2134
2135 ring_buffer_record_disable(buffer);
2136
2137 /* Make sure all commits have finished */
2138 synchronize_rcu();
2139 ring_buffer_reset_cpu(buffer, cpu);
2140
2141 ring_buffer_record_enable(buffer);
2142}
2143
2144void tracing_reset_online_cpus(struct array_buffer *buf)
2145{
2146 struct trace_buffer *buffer = buf->buffer;
2147
2148 if (!buffer)
2149 return;
2150
2151 ring_buffer_record_disable(buffer);
2152
2153 /* Make sure all commits have finished */
2154 synchronize_rcu();
2155
2156 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2157
2158 ring_buffer_reset_online_cpus(buffer);
2159
2160 ring_buffer_record_enable(buffer);
2161}
2162
2163/* Must have trace_types_lock held */
2164void tracing_reset_all_online_cpus(void)
2165{
2166 struct trace_array *tr;
2167
2168 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2169 if (!tr->clear_trace)
2170 continue;
2171 tr->clear_trace = false;
2172 tracing_reset_online_cpus(&tr->array_buffer);
2173#ifdef CONFIG_TRACER_MAX_TRACE
2174 tracing_reset_online_cpus(&tr->max_buffer);
2175#endif
2176 }
2177}
2178
2179/*
2180 * The tgid_map array maps from pid to tgid; i.e. the value stored at index i
2181 * is the tgid last observed corresponding to pid=i.
2182 */
2183static int *tgid_map;
2184
2185/* The maximum valid index into tgid_map. */
2186static size_t tgid_map_max;
2187
2188#define SAVED_CMDLINES_DEFAULT 128
2189#define NO_CMDLINE_MAP UINT_MAX
2190static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2191struct saved_cmdlines_buffer {
2192 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
2193 unsigned *map_cmdline_to_pid;
2194 unsigned cmdline_num;
2195 int cmdline_idx;
2196 char *saved_cmdlines;
2197};
2198static struct saved_cmdlines_buffer *savedcmd;
2199
2200static inline char *get_saved_cmdlines(int idx)
2201{
2202 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
2203}
2204
2205static inline void set_cmdline(int idx, const char *cmdline)
2206{
2207 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
2208}
2209
2210static int allocate_cmdlines_buffer(unsigned int val,
2211 struct saved_cmdlines_buffer *s)
2212{
2213 s->map_cmdline_to_pid = kmalloc_array(val,
2214 sizeof(*s->map_cmdline_to_pid),
2215 GFP_KERNEL);
2216 if (!s->map_cmdline_to_pid)
2217 return -ENOMEM;
2218
2219 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
2220 if (!s->saved_cmdlines) {
2221 kfree(s->map_cmdline_to_pid);
2222 return -ENOMEM;
2223 }
2224
2225 s->cmdline_idx = 0;
2226 s->cmdline_num = val;
2227 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
2228 sizeof(s->map_pid_to_cmdline));
2229 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
2230 val * sizeof(*s->map_cmdline_to_pid));
2231
2232 return 0;
2233}
2234
2235static int trace_create_savedcmd(void)
2236{
2237 int ret;
2238
2239 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
2240 if (!savedcmd)
2241 return -ENOMEM;
2242
2243 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
2244 if (ret < 0) {
2245 kfree(savedcmd);
2246 savedcmd = NULL;
2247 return -ENOMEM;
2248 }
2249
2250 return 0;
2251}
2252
2253int is_tracing_stopped(void)
2254{
2255 return global_trace.stop_count;
2256}
2257
2258/**
2259 * tracing_start - quick start of the tracer
2260 *
2261 * If tracing is enabled but was stopped by tracing_stop,
2262 * this will start the tracer back up.
2263 */
2264void tracing_start(void)
2265{
2266 struct trace_buffer *buffer;
2267 unsigned long flags;
2268
2269 if (tracing_disabled)
2270 return;
2271
2272 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2273 if (--global_trace.stop_count) {
2274 if (global_trace.stop_count < 0) {
2275 /* Someone screwed up their debugging */
2276 WARN_ON_ONCE(1);
2277 global_trace.stop_count = 0;
2278 }
2279 goto out;
2280 }
2281
2282 /* Prevent the buffers from switching */
2283 arch_spin_lock(&global_trace.max_lock);
2284
2285 buffer = global_trace.array_buffer.buffer;
2286 if (buffer)
2287 ring_buffer_record_enable(buffer);
2288
2289#ifdef CONFIG_TRACER_MAX_TRACE
2290 buffer = global_trace.max_buffer.buffer;
2291 if (buffer)
2292 ring_buffer_record_enable(buffer);
2293#endif
2294
2295 arch_spin_unlock(&global_trace.max_lock);
2296
2297 out:
2298 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2299}
2300
2301static void tracing_start_tr(struct trace_array *tr)
2302{
2303 struct trace_buffer *buffer;
2304 unsigned long flags;
2305
2306 if (tracing_disabled)
2307 return;
2308
2309 /* If global, we need to also start the max tracer */
2310 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2311 return tracing_start();
2312
2313 raw_spin_lock_irqsave(&tr->start_lock, flags);
2314
2315 if (--tr->stop_count) {
2316 if (tr->stop_count < 0) {
2317 /* Someone screwed up their debugging */
2318 WARN_ON_ONCE(1);
2319 tr->stop_count = 0;
2320 }
2321 goto out;
2322 }
2323
2324 buffer = tr->array_buffer.buffer;
2325 if (buffer)
2326 ring_buffer_record_enable(buffer);
2327
2328 out:
2329 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2330}
2331
2332/**
2333 * tracing_stop - quick stop of the tracer
2334 *
2335 * Light weight way to stop tracing. Use in conjunction with
2336 * tracing_start.
2337 */
2338void tracing_stop(void)
2339{
2340 struct trace_buffer *buffer;
2341 unsigned long flags;
2342
2343 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2344 if (global_trace.stop_count++)
2345 goto out;
2346
2347 /* Prevent the buffers from switching */
2348 arch_spin_lock(&global_trace.max_lock);
2349
2350 buffer = global_trace.array_buffer.buffer;
2351 if (buffer)
2352 ring_buffer_record_disable(buffer);
2353
2354#ifdef CONFIG_TRACER_MAX_TRACE
2355 buffer = global_trace.max_buffer.buffer;
2356 if (buffer)
2357 ring_buffer_record_disable(buffer);
2358#endif
2359
2360 arch_spin_unlock(&global_trace.max_lock);
2361
2362 out:
2363 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2364}
2365
2366static void tracing_stop_tr(struct trace_array *tr)
2367{
2368 struct trace_buffer *buffer;
2369 unsigned long flags;
2370
2371 /* If global, we need to also stop the max tracer */
2372 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2373 return tracing_stop();
2374
2375 raw_spin_lock_irqsave(&tr->start_lock, flags);
2376 if (tr->stop_count++)
2377 goto out;
2378
2379 buffer = tr->array_buffer.buffer;
2380 if (buffer)
2381 ring_buffer_record_disable(buffer);
2382
2383 out:
2384 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2385}
2386
2387static int trace_save_cmdline(struct task_struct *tsk)
2388{
2389 unsigned tpid, idx;
2390
2391 /* treat recording of idle task as a success */
2392 if (!tsk->pid)
2393 return 1;
2394
2395 tpid = tsk->pid & (PID_MAX_DEFAULT - 1);
2396
2397 /*
2398 * It's not the end of the world if we don't get
2399 * the lock, but we also don't want to spin
2400 * nor do we want to disable interrupts,
2401 * so if we miss here, then better luck next time.
2402 */
2403 if (!arch_spin_trylock(&trace_cmdline_lock))
2404 return 0;
2405
2406 idx = savedcmd->map_pid_to_cmdline[tpid];
2407 if (idx == NO_CMDLINE_MAP) {
2408 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2409
2410 savedcmd->map_pid_to_cmdline[tpid] = idx;
2411 savedcmd->cmdline_idx = idx;
2412 }
2413
2414 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2415 set_cmdline(idx, tsk->comm);
2416
2417 arch_spin_unlock(&trace_cmdline_lock);
2418
2419 return 1;
2420}
2421
2422static void __trace_find_cmdline(int pid, char comm[])
2423{
2424 unsigned map;
2425 int tpid;
2426
2427 if (!pid) {
2428 strcpy(comm, "<idle>");
2429 return;
2430 }
2431
2432 if (WARN_ON_ONCE(pid < 0)) {
2433 strcpy(comm, "<XXX>");
2434 return;
2435 }
2436
2437 tpid = pid & (PID_MAX_DEFAULT - 1);
2438 map = savedcmd->map_pid_to_cmdline[tpid];
2439 if (map != NO_CMDLINE_MAP) {
2440 tpid = savedcmd->map_cmdline_to_pid[map];
2441 if (tpid == pid) {
2442 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2443 return;
2444 }
2445 }
2446 strcpy(comm, "<...>");
2447}
2448
2449void trace_find_cmdline(int pid, char comm[])
2450{
2451 preempt_disable();
2452 arch_spin_lock(&trace_cmdline_lock);
2453
2454 __trace_find_cmdline(pid, comm);
2455
2456 arch_spin_unlock(&trace_cmdline_lock);
2457 preempt_enable();
2458}
2459
2460static int *trace_find_tgid_ptr(int pid)
2461{
2462 /*
2463 * Pairs with the smp_store_release in set_tracer_flag() to ensure that
2464 * if we observe a non-NULL tgid_map then we also observe the correct
2465 * tgid_map_max.
2466 */
2467 int *map = smp_load_acquire(&tgid_map);
2468
2469 if (unlikely(!map || pid > tgid_map_max))
2470 return NULL;
2471
2472 return &map[pid];
2473}
2474
2475int trace_find_tgid(int pid)
2476{
2477 int *ptr = trace_find_tgid_ptr(pid);
2478
2479 return ptr ? *ptr : 0;
2480}
2481
2482static int trace_save_tgid(struct task_struct *tsk)
2483{
2484 int *ptr;
2485
2486 /* treat recording of idle task as a success */
2487 if (!tsk->pid)
2488 return 1;
2489
2490 ptr = trace_find_tgid_ptr(tsk->pid);
2491 if (!ptr)
2492 return 0;
2493
2494 *ptr = tsk->tgid;
2495 return 1;
2496}
2497
2498static bool tracing_record_taskinfo_skip(int flags)
2499{
2500 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2501 return true;
2502 if (!__this_cpu_read(trace_taskinfo_save))
2503 return true;
2504 return false;
2505}
2506
2507/**
2508 * tracing_record_taskinfo - record the task info of a task
2509 *
2510 * @task: task to record
2511 * @flags: TRACE_RECORD_CMDLINE for recording comm
2512 * TRACE_RECORD_TGID for recording tgid
2513 */
2514void tracing_record_taskinfo(struct task_struct *task, int flags)
2515{
2516 bool done;
2517
2518 if (tracing_record_taskinfo_skip(flags))
2519 return;
2520
2521 /*
2522 * Record as much task information as possible. If some fail, continue
2523 * to try to record the others.
2524 */
2525 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2526 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2527
2528 /* If recording any information failed, retry again soon. */
2529 if (!done)
2530 return;
2531
2532 __this_cpu_write(trace_taskinfo_save, false);
2533}
2534
2535/**
2536 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2537 *
2538 * @prev: previous task during sched_switch
2539 * @next: next task during sched_switch
2540 * @flags: TRACE_RECORD_CMDLINE for recording comm
2541 * TRACE_RECORD_TGID for recording tgid
2542 */
2543void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2544 struct task_struct *next, int flags)
2545{
2546 bool done;
2547
2548 if (tracing_record_taskinfo_skip(flags))
2549 return;
2550
2551 /*
2552 * Record as much task information as possible. If some fail, continue
2553 * to try to record the others.
2554 */
2555 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2556 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2557 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2558 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2559
2560 /* If recording any information failed, retry again soon. */
2561 if (!done)
2562 return;
2563
2564 __this_cpu_write(trace_taskinfo_save, false);
2565}
2566
2567/* Helpers to record a specific task information */
2568void tracing_record_cmdline(struct task_struct *task)
2569{
2570 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2571}
2572
2573void tracing_record_tgid(struct task_struct *task)
2574{
2575 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2576}
2577
2578/*
2579 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2580 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2581 * simplifies those functions and keeps them in sync.
2582 */
2583enum print_line_t trace_handle_return(struct trace_seq *s)
2584{
2585 return trace_seq_has_overflowed(s) ?
2586 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2587}
2588EXPORT_SYMBOL_GPL(trace_handle_return);
2589
2590static unsigned short migration_disable_value(void)
2591{
2592#if defined(CONFIG_SMP)
2593 return current->migration_disabled;
2594#else
2595 return 0;
2596#endif
2597}
2598
2599unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
2600{
2601 unsigned int trace_flags = irqs_status;
2602 unsigned int pc;
2603
2604 pc = preempt_count();
2605
2606 if (pc & NMI_MASK)
2607 trace_flags |= TRACE_FLAG_NMI;
2608 if (pc & HARDIRQ_MASK)
2609 trace_flags |= TRACE_FLAG_HARDIRQ;
2610 if (in_serving_softirq())
2611 trace_flags |= TRACE_FLAG_SOFTIRQ;
2612 if (softirq_count() >> (SOFTIRQ_SHIFT + 1))
2613 trace_flags |= TRACE_FLAG_BH_OFF;
2614
2615 if (tif_need_resched())
2616 trace_flags |= TRACE_FLAG_NEED_RESCHED;
2617 if (test_preempt_need_resched())
2618 trace_flags |= TRACE_FLAG_PREEMPT_RESCHED;
2619 return (trace_flags << 16) | (min_t(unsigned int, pc & 0xff, 0xf)) |
2620 (min_t(unsigned int, migration_disable_value(), 0xf)) << 4;
2621}
2622
2623struct ring_buffer_event *
2624trace_buffer_lock_reserve(struct trace_buffer *buffer,
2625 int type,
2626 unsigned long len,
2627 unsigned int trace_ctx)
2628{
2629 return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
2630}
2631
2632DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2633DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2634static int trace_buffered_event_ref;
2635
2636/**
2637 * trace_buffered_event_enable - enable buffering events
2638 *
2639 * When events are being filtered, it is quicker to use a temporary
2640 * buffer to write the event data into if there's a likely chance
2641 * that it will not be committed. The discard of the ring buffer
2642 * is not as fast as committing, and is much slower than copying
2643 * a commit.
2644 *
2645 * When an event is to be filtered, allocate per cpu buffers to
2646 * write the event data into, and if the event is filtered and discarded
2647 * it is simply dropped, otherwise, the entire data is to be committed
2648 * in one shot.
2649 */
2650void trace_buffered_event_enable(void)
2651{
2652 struct ring_buffer_event *event;
2653 struct page *page;
2654 int cpu;
2655
2656 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2657
2658 if (trace_buffered_event_ref++)
2659 return;
2660
2661 for_each_tracing_cpu(cpu) {
2662 page = alloc_pages_node(cpu_to_node(cpu),
2663 GFP_KERNEL | __GFP_NORETRY, 0);
2664 if (!page)
2665 goto failed;
2666
2667 event = page_address(page);
2668 memset(event, 0, sizeof(*event));
2669
2670 per_cpu(trace_buffered_event, cpu) = event;
2671
2672 preempt_disable();
2673 if (cpu == smp_processor_id() &&
2674 __this_cpu_read(trace_buffered_event) !=
2675 per_cpu(trace_buffered_event, cpu))
2676 WARN_ON_ONCE(1);
2677 preempt_enable();
2678 }
2679
2680 return;
2681 failed:
2682 trace_buffered_event_disable();
2683}
2684
2685static void enable_trace_buffered_event(void *data)
2686{
2687 /* Probably not needed, but do it anyway */
2688 smp_rmb();
2689 this_cpu_dec(trace_buffered_event_cnt);
2690}
2691
2692static void disable_trace_buffered_event(void *data)
2693{
2694 this_cpu_inc(trace_buffered_event_cnt);
2695}
2696
2697/**
2698 * trace_buffered_event_disable - disable buffering events
2699 *
2700 * When a filter is removed, it is faster to not use the buffered
2701 * events, and to commit directly into the ring buffer. Free up
2702 * the temp buffers when there are no more users. This requires
2703 * special synchronization with current events.
2704 */
2705void trace_buffered_event_disable(void)
2706{
2707 int cpu;
2708
2709 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2710
2711 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2712 return;
2713
2714 if (--trace_buffered_event_ref)
2715 return;
2716
2717 preempt_disable();
2718 /* For each CPU, set the buffer as used. */
2719 smp_call_function_many(tracing_buffer_mask,
2720 disable_trace_buffered_event, NULL, 1);
2721 preempt_enable();
2722
2723 /* Wait for all current users to finish */
2724 synchronize_rcu();
2725
2726 for_each_tracing_cpu(cpu) {
2727 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2728 per_cpu(trace_buffered_event, cpu) = NULL;
2729 }
2730 /*
2731 * Make sure trace_buffered_event is NULL before clearing
2732 * trace_buffered_event_cnt.
2733 */
2734 smp_wmb();
2735
2736 preempt_disable();
2737 /* Do the work on each cpu */
2738 smp_call_function_many(tracing_buffer_mask,
2739 enable_trace_buffered_event, NULL, 1);
2740 preempt_enable();
2741}
2742
2743static struct trace_buffer *temp_buffer;
2744
2745struct ring_buffer_event *
2746trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2747 struct trace_event_file *trace_file,
2748 int type, unsigned long len,
2749 unsigned int trace_ctx)
2750{
2751 struct ring_buffer_event *entry;
2752 struct trace_array *tr = trace_file->tr;
2753 int val;
2754
2755 *current_rb = tr->array_buffer.buffer;
2756
2757 if (!tr->no_filter_buffering_ref &&
2758 (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) {
2759 preempt_disable_notrace();
2760 /*
2761 * Filtering is on, so try to use the per cpu buffer first.
2762 * This buffer will simulate a ring_buffer_event,
2763 * where the type_len is zero and the array[0] will
2764 * hold the full length.
2765 * (see include/linux/ring-buffer.h for details on
2766 * how the ring_buffer_event is structured).
2767 *
2768 * Using a temp buffer during filtering and copying it
2769 * on a matched filter is quicker than writing directly
2770 * into the ring buffer and then discarding it when
2771 * it doesn't match. That is because the discard
2772 * requires several atomic operations to get right.
2773 * Copying on match and doing nothing on a failed match
2774 * is still quicker than no copy on match, but having
2775 * to discard out of the ring buffer on a failed match.
2776 */
2777 if ((entry = __this_cpu_read(trace_buffered_event))) {
2778 int max_len = PAGE_SIZE - struct_size(entry, array, 1);
2779
2780 val = this_cpu_inc_return(trace_buffered_event_cnt);
2781
2782 /*
2783 * Preemption is disabled, but interrupts and NMIs
2784 * can still come in now. If that happens after
2785 * the above increment, then it will have to go
2786 * back to the old method of allocating the event
2787 * on the ring buffer, and if the filter fails, it
2788 * will have to call ring_buffer_discard_commit()
2789 * to remove it.
2790 *
2791 * Need to also check the unlikely case that the
2792 * length is bigger than the temp buffer size.
2793 * If that happens, then the reserve is pretty much
2794 * guaranteed to fail, as the ring buffer currently
2795 * only allows events less than a page. But that may
2796 * change in the future, so let the ring buffer reserve
2797 * handle the failure in that case.
2798 */
2799 if (val == 1 && likely(len <= max_len)) {
2800 trace_event_setup(entry, type, trace_ctx);
2801 entry->array[0] = len;
2802 /* Return with preemption disabled */
2803 return entry;
2804 }
2805 this_cpu_dec(trace_buffered_event_cnt);
2806 }
2807 /* __trace_buffer_lock_reserve() disables preemption */
2808 preempt_enable_notrace();
2809 }
2810
2811 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2812 trace_ctx);
2813 /*
2814 * If tracing is off, but we have triggers enabled
2815 * we still need to look at the event data. Use the temp_buffer
2816 * to store the trace event for the trigger to use. It's recursive
2817 * safe and will not be recorded anywhere.
2818 */
2819 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2820 *current_rb = temp_buffer;
2821 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2822 trace_ctx);
2823 }
2824 return entry;
2825}
2826EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2827
2828static DEFINE_SPINLOCK(tracepoint_iter_lock);
2829static DEFINE_MUTEX(tracepoint_printk_mutex);
2830
2831static void output_printk(struct trace_event_buffer *fbuffer)
2832{
2833 struct trace_event_call *event_call;
2834 struct trace_event_file *file;
2835 struct trace_event *event;
2836 unsigned long flags;
2837 struct trace_iterator *iter = tracepoint_print_iter;
2838
2839 /* We should never get here if iter is NULL */
2840 if (WARN_ON_ONCE(!iter))
2841 return;
2842
2843 event_call = fbuffer->trace_file->event_call;
2844 if (!event_call || !event_call->event.funcs ||
2845 !event_call->event.funcs->trace)
2846 return;
2847
2848 file = fbuffer->trace_file;
2849 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
2850 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2851 !filter_match_preds(file->filter, fbuffer->entry)))
2852 return;
2853
2854 event = &fbuffer->trace_file->event_call->event;
2855
2856 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2857 trace_seq_init(&iter->seq);
2858 iter->ent = fbuffer->entry;
2859 event_call->event.funcs->trace(iter, 0, event);
2860 trace_seq_putc(&iter->seq, 0);
2861 printk("%s", iter->seq.buffer);
2862
2863 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2864}
2865
2866int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2867 void *buffer, size_t *lenp,
2868 loff_t *ppos)
2869{
2870 int save_tracepoint_printk;
2871 int ret;
2872
2873 mutex_lock(&tracepoint_printk_mutex);
2874 save_tracepoint_printk = tracepoint_printk;
2875
2876 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2877
2878 /*
2879 * This will force exiting early, as tracepoint_printk
2880 * is always zero when tracepoint_printk_iter is not allocated
2881 */
2882 if (!tracepoint_print_iter)
2883 tracepoint_printk = 0;
2884
2885 if (save_tracepoint_printk == tracepoint_printk)
2886 goto out;
2887
2888 if (tracepoint_printk)
2889 static_key_enable(&tracepoint_printk_key.key);
2890 else
2891 static_key_disable(&tracepoint_printk_key.key);
2892
2893 out:
2894 mutex_unlock(&tracepoint_printk_mutex);
2895
2896 return ret;
2897}
2898
2899void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2900{
2901 enum event_trigger_type tt = ETT_NONE;
2902 struct trace_event_file *file = fbuffer->trace_file;
2903
2904 if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event,
2905 fbuffer->entry, &tt))
2906 goto discard;
2907
2908 if (static_key_false(&tracepoint_printk_key.key))
2909 output_printk(fbuffer);
2910
2911 if (static_branch_unlikely(&trace_event_exports_enabled))
2912 ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
2913
2914 trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer,
2915 fbuffer->event, fbuffer->trace_ctx, fbuffer->regs);
2916
2917discard:
2918 if (tt)
2919 event_triggers_post_call(file, tt);
2920
2921}
2922EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2923
2924/*
2925 * Skip 3:
2926 *
2927 * trace_buffer_unlock_commit_regs()
2928 * trace_event_buffer_commit()
2929 * trace_event_raw_event_xxx()
2930 */
2931# define STACK_SKIP 3
2932
2933void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2934 struct trace_buffer *buffer,
2935 struct ring_buffer_event *event,
2936 unsigned int trace_ctx,
2937 struct pt_regs *regs)
2938{
2939 __buffer_unlock_commit(buffer, event);
2940
2941 /*
2942 * If regs is not set, then skip the necessary functions.
2943 * Note, we can still get here via blktrace, wakeup tracer
2944 * and mmiotrace, but that's ok if they lose a function or
2945 * two. They are not that meaningful.
2946 */
2947 ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs);
2948 ftrace_trace_userstack(tr, buffer, trace_ctx);
2949}
2950
2951/*
2952 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2953 */
2954void
2955trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2956 struct ring_buffer_event *event)
2957{
2958 __buffer_unlock_commit(buffer, event);
2959}
2960
2961void
2962trace_function(struct trace_array *tr, unsigned long ip, unsigned long
2963 parent_ip, unsigned int trace_ctx)
2964{
2965 struct trace_event_call *call = &event_function;
2966 struct trace_buffer *buffer = tr->array_buffer.buffer;
2967 struct ring_buffer_event *event;
2968 struct ftrace_entry *entry;
2969
2970 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2971 trace_ctx);
2972 if (!event)
2973 return;
2974 entry = ring_buffer_event_data(event);
2975 entry->ip = ip;
2976 entry->parent_ip = parent_ip;
2977
2978 if (!call_filter_check_discard(call, entry, buffer, event)) {
2979 if (static_branch_unlikely(&trace_function_exports_enabled))
2980 ftrace_exports(event, TRACE_EXPORT_FUNCTION);
2981 __buffer_unlock_commit(buffer, event);
2982 }
2983}
2984
2985#ifdef CONFIG_STACKTRACE
2986
2987/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2988#define FTRACE_KSTACK_NESTING 4
2989
2990#define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2991
2992struct ftrace_stack {
2993 unsigned long calls[FTRACE_KSTACK_ENTRIES];
2994};
2995
2996
2997struct ftrace_stacks {
2998 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2999};
3000
3001static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
3002static DEFINE_PER_CPU(int, ftrace_stack_reserve);
3003
3004static void __ftrace_trace_stack(struct trace_buffer *buffer,
3005 unsigned int trace_ctx,
3006 int skip, struct pt_regs *regs)
3007{
3008 struct trace_event_call *call = &event_kernel_stack;
3009 struct ring_buffer_event *event;
3010 unsigned int size, nr_entries;
3011 struct ftrace_stack *fstack;
3012 struct stack_entry *entry;
3013 int stackidx;
3014
3015 /*
3016 * Add one, for this function and the call to save_stack_trace()
3017 * If regs is set, then these functions will not be in the way.
3018 */
3019#ifndef CONFIG_UNWINDER_ORC
3020 if (!regs)
3021 skip++;
3022#endif
3023
3024 preempt_disable_notrace();
3025
3026 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
3027
3028 /* This should never happen. If it does, yell once and skip */
3029 if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
3030 goto out;
3031
3032 /*
3033 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
3034 * interrupt will either see the value pre increment or post
3035 * increment. If the interrupt happens pre increment it will have
3036 * restored the counter when it returns. We just need a barrier to
3037 * keep gcc from moving things around.
3038 */
3039 barrier();
3040
3041 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
3042 size = ARRAY_SIZE(fstack->calls);
3043
3044 if (regs) {
3045 nr_entries = stack_trace_save_regs(regs, fstack->calls,
3046 size, skip);
3047 } else {
3048 nr_entries = stack_trace_save(fstack->calls, size, skip);
3049 }
3050
3051 size = nr_entries * sizeof(unsigned long);
3052 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
3053 (sizeof(*entry) - sizeof(entry->caller)) + size,
3054 trace_ctx);
3055 if (!event)
3056 goto out;
3057 entry = ring_buffer_event_data(event);
3058
3059 memcpy(&entry->caller, fstack->calls, size);
3060 entry->size = nr_entries;
3061
3062 if (!call_filter_check_discard(call, entry, buffer, event))
3063 __buffer_unlock_commit(buffer, event);
3064
3065 out:
3066 /* Again, don't let gcc optimize things here */
3067 barrier();
3068 __this_cpu_dec(ftrace_stack_reserve);
3069 preempt_enable_notrace();
3070
3071}
3072
3073static inline void ftrace_trace_stack(struct trace_array *tr,
3074 struct trace_buffer *buffer,
3075 unsigned int trace_ctx,
3076 int skip, struct pt_regs *regs)
3077{
3078 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
3079 return;
3080
3081 __ftrace_trace_stack(buffer, trace_ctx, skip, regs);
3082}
3083
3084void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
3085 int skip)
3086{
3087 struct trace_buffer *buffer = tr->array_buffer.buffer;
3088
3089 if (rcu_is_watching()) {
3090 __ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3091 return;
3092 }
3093
3094 /*
3095 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
3096 * but if the above rcu_is_watching() failed, then the NMI
3097 * triggered someplace critical, and rcu_irq_enter() should
3098 * not be called from NMI.
3099 */
3100 if (unlikely(in_nmi()))
3101 return;
3102
3103 rcu_irq_enter_irqson();
3104 __ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3105 rcu_irq_exit_irqson();
3106}
3107
3108/**
3109 * trace_dump_stack - record a stack back trace in the trace buffer
3110 * @skip: Number of functions to skip (helper handlers)
3111 */
3112void trace_dump_stack(int skip)
3113{
3114 if (tracing_disabled || tracing_selftest_running)
3115 return;
3116
3117#ifndef CONFIG_UNWINDER_ORC
3118 /* Skip 1 to skip this function. */
3119 skip++;
3120#endif
3121 __ftrace_trace_stack(global_trace.array_buffer.buffer,
3122 tracing_gen_ctx(), skip, NULL);
3123}
3124EXPORT_SYMBOL_GPL(trace_dump_stack);
3125
3126#ifdef CONFIG_USER_STACKTRACE_SUPPORT
3127static DEFINE_PER_CPU(int, user_stack_count);
3128
3129static void
3130ftrace_trace_userstack(struct trace_array *tr,
3131 struct trace_buffer *buffer, unsigned int trace_ctx)
3132{
3133 struct trace_event_call *call = &event_user_stack;
3134 struct ring_buffer_event *event;
3135 struct userstack_entry *entry;
3136
3137 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
3138 return;
3139
3140 /*
3141 * NMIs can not handle page faults, even with fix ups.
3142 * The save user stack can (and often does) fault.
3143 */
3144 if (unlikely(in_nmi()))
3145 return;
3146
3147 /*
3148 * prevent recursion, since the user stack tracing may
3149 * trigger other kernel events.
3150 */
3151 preempt_disable();
3152 if (__this_cpu_read(user_stack_count))
3153 goto out;
3154
3155 __this_cpu_inc(user_stack_count);
3156
3157 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3158 sizeof(*entry), trace_ctx);
3159 if (!event)
3160 goto out_drop_count;
3161 entry = ring_buffer_event_data(event);
3162
3163 entry->tgid = current->tgid;
3164 memset(&entry->caller, 0, sizeof(entry->caller));
3165
3166 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3167 if (!call_filter_check_discard(call, entry, buffer, event))
3168 __buffer_unlock_commit(buffer, event);
3169
3170 out_drop_count:
3171 __this_cpu_dec(user_stack_count);
3172 out:
3173 preempt_enable();
3174}
3175#else /* CONFIG_USER_STACKTRACE_SUPPORT */
3176static void ftrace_trace_userstack(struct trace_array *tr,
3177 struct trace_buffer *buffer,
3178 unsigned int trace_ctx)
3179{
3180}
3181#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3182
3183#endif /* CONFIG_STACKTRACE */
3184
3185static inline void
3186func_repeats_set_delta_ts(struct func_repeats_entry *entry,
3187 unsigned long long delta)
3188{
3189 entry->bottom_delta_ts = delta & U32_MAX;
3190 entry->top_delta_ts = (delta >> 32);
3191}
3192
3193void trace_last_func_repeats(struct trace_array *tr,
3194 struct trace_func_repeats *last_info,
3195 unsigned int trace_ctx)
3196{
3197 struct trace_buffer *buffer = tr->array_buffer.buffer;
3198 struct func_repeats_entry *entry;
3199 struct ring_buffer_event *event;
3200 u64 delta;
3201
3202 event = __trace_buffer_lock_reserve(buffer, TRACE_FUNC_REPEATS,
3203 sizeof(*entry), trace_ctx);
3204 if (!event)
3205 return;
3206
3207 delta = ring_buffer_event_time_stamp(buffer, event) -
3208 last_info->ts_last_call;
3209
3210 entry = ring_buffer_event_data(event);
3211 entry->ip = last_info->ip;
3212 entry->parent_ip = last_info->parent_ip;
3213 entry->count = last_info->count;
3214 func_repeats_set_delta_ts(entry, delta);
3215
3216 __buffer_unlock_commit(buffer, event);
3217}
3218
3219/* created for use with alloc_percpu */
3220struct trace_buffer_struct {
3221 int nesting;
3222 char buffer[4][TRACE_BUF_SIZE];
3223};
3224
3225static struct trace_buffer_struct __percpu *trace_percpu_buffer;
3226
3227/*
3228 * This allows for lockless recording. If we're nested too deeply, then
3229 * this returns NULL.
3230 */
3231static char *get_trace_buf(void)
3232{
3233 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3234
3235 if (!trace_percpu_buffer || buffer->nesting >= 4)
3236 return NULL;
3237
3238 buffer->nesting++;
3239
3240 /* Interrupts must see nesting incremented before we use the buffer */
3241 barrier();
3242 return &buffer->buffer[buffer->nesting - 1][0];
3243}
3244
3245static void put_trace_buf(void)
3246{
3247 /* Don't let the decrement of nesting leak before this */
3248 barrier();
3249 this_cpu_dec(trace_percpu_buffer->nesting);
3250}
3251
3252static int alloc_percpu_trace_buffer(void)
3253{
3254 struct trace_buffer_struct __percpu *buffers;
3255
3256 if (trace_percpu_buffer)
3257 return 0;
3258
3259 buffers = alloc_percpu(struct trace_buffer_struct);
3260 if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3261 return -ENOMEM;
3262
3263 trace_percpu_buffer = buffers;
3264 return 0;
3265}
3266
3267static int buffers_allocated;
3268
3269void trace_printk_init_buffers(void)
3270{
3271 if (buffers_allocated)
3272 return;
3273
3274 if (alloc_percpu_trace_buffer())
3275 return;
3276
3277 /* trace_printk() is for debug use only. Don't use it in production. */
3278
3279 pr_warn("\n");
3280 pr_warn("**********************************************************\n");
3281 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3282 pr_warn("** **\n");
3283 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3284 pr_warn("** **\n");
3285 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3286 pr_warn("** unsafe for production use. **\n");
3287 pr_warn("** **\n");
3288 pr_warn("** If you see this message and you are not debugging **\n");
3289 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3290 pr_warn("** **\n");
3291 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3292 pr_warn("**********************************************************\n");
3293
3294 /* Expand the buffers to set size */
3295 tracing_update_buffers();
3296
3297 buffers_allocated = 1;
3298
3299 /*
3300 * trace_printk_init_buffers() can be called by modules.
3301 * If that happens, then we need to start cmdline recording
3302 * directly here. If the global_trace.buffer is already
3303 * allocated here, then this was called by module code.
3304 */
3305 if (global_trace.array_buffer.buffer)
3306 tracing_start_cmdline_record();
3307}
3308EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3309
3310void trace_printk_start_comm(void)
3311{
3312 /* Start tracing comms if trace printk is set */
3313 if (!buffers_allocated)
3314 return;
3315 tracing_start_cmdline_record();
3316}
3317
3318static void trace_printk_start_stop_comm(int enabled)
3319{
3320 if (!buffers_allocated)
3321 return;
3322
3323 if (enabled)
3324 tracing_start_cmdline_record();
3325 else
3326 tracing_stop_cmdline_record();
3327}
3328
3329/**
3330 * trace_vbprintk - write binary msg to tracing buffer
3331 * @ip: The address of the caller
3332 * @fmt: The string format to write to the buffer
3333 * @args: Arguments for @fmt
3334 */
3335int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3336{
3337 struct trace_event_call *call = &event_bprint;
3338 struct ring_buffer_event *event;
3339 struct trace_buffer *buffer;
3340 struct trace_array *tr = &global_trace;
3341 struct bprint_entry *entry;
3342 unsigned int trace_ctx;
3343 char *tbuffer;
3344 int len = 0, size;
3345
3346 if (unlikely(tracing_selftest_running || tracing_disabled))
3347 return 0;
3348
3349 /* Don't pollute graph traces with trace_vprintk internals */
3350 pause_graph_tracing();
3351
3352 trace_ctx = tracing_gen_ctx();
3353 preempt_disable_notrace();
3354
3355 tbuffer = get_trace_buf();
3356 if (!tbuffer) {
3357 len = 0;
3358 goto out_nobuffer;
3359 }
3360
3361 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3362
3363 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3364 goto out_put;
3365
3366 size = sizeof(*entry) + sizeof(u32) * len;
3367 buffer = tr->array_buffer.buffer;
3368 ring_buffer_nest_start(buffer);
3369 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3370 trace_ctx);
3371 if (!event)
3372 goto out;
3373 entry = ring_buffer_event_data(event);
3374 entry->ip = ip;
3375 entry->fmt = fmt;
3376
3377 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3378 if (!call_filter_check_discard(call, entry, buffer, event)) {
3379 __buffer_unlock_commit(buffer, event);
3380 ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL);
3381 }
3382
3383out:
3384 ring_buffer_nest_end(buffer);
3385out_put:
3386 put_trace_buf();
3387
3388out_nobuffer:
3389 preempt_enable_notrace();
3390 unpause_graph_tracing();
3391
3392 return len;
3393}
3394EXPORT_SYMBOL_GPL(trace_vbprintk);
3395
3396__printf(3, 0)
3397static int
3398__trace_array_vprintk(struct trace_buffer *buffer,
3399 unsigned long ip, const char *fmt, va_list args)
3400{
3401 struct trace_event_call *call = &event_print;
3402 struct ring_buffer_event *event;
3403 int len = 0, size;
3404 struct print_entry *entry;
3405 unsigned int trace_ctx;
3406 char *tbuffer;
3407
3408 if (tracing_disabled || tracing_selftest_running)
3409 return 0;
3410
3411 /* Don't pollute graph traces with trace_vprintk internals */
3412 pause_graph_tracing();
3413
3414 trace_ctx = tracing_gen_ctx();
3415 preempt_disable_notrace();
3416
3417
3418 tbuffer = get_trace_buf();
3419 if (!tbuffer) {
3420 len = 0;
3421 goto out_nobuffer;
3422 }
3423
3424 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3425
3426 size = sizeof(*entry) + len + 1;
3427 ring_buffer_nest_start(buffer);
3428 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3429 trace_ctx);
3430 if (!event)
3431 goto out;
3432 entry = ring_buffer_event_data(event);
3433 entry->ip = ip;
3434
3435 memcpy(&entry->buf, tbuffer, len + 1);
3436 if (!call_filter_check_discard(call, entry, buffer, event)) {
3437 __buffer_unlock_commit(buffer, event);
3438 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 6, NULL);
3439 }
3440
3441out:
3442 ring_buffer_nest_end(buffer);
3443 put_trace_buf();
3444
3445out_nobuffer:
3446 preempt_enable_notrace();
3447 unpause_graph_tracing();
3448
3449 return len;
3450}
3451
3452__printf(3, 0)
3453int trace_array_vprintk(struct trace_array *tr,
3454 unsigned long ip, const char *fmt, va_list args)
3455{
3456 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3457}
3458
3459/**
3460 * trace_array_printk - Print a message to a specific instance
3461 * @tr: The instance trace_array descriptor
3462 * @ip: The instruction pointer that this is called from.
3463 * @fmt: The format to print (printf format)
3464 *
3465 * If a subsystem sets up its own instance, they have the right to
3466 * printk strings into their tracing instance buffer using this
3467 * function. Note, this function will not write into the top level
3468 * buffer (use trace_printk() for that), as writing into the top level
3469 * buffer should only have events that can be individually disabled.
3470 * trace_printk() is only used for debugging a kernel, and should not
3471 * be ever incorporated in normal use.
3472 *
3473 * trace_array_printk() can be used, as it will not add noise to the
3474 * top level tracing buffer.
3475 *
3476 * Note, trace_array_init_printk() must be called on @tr before this
3477 * can be used.
3478 */
3479__printf(3, 0)
3480int trace_array_printk(struct trace_array *tr,
3481 unsigned long ip, const char *fmt, ...)
3482{
3483 int ret;
3484 va_list ap;
3485
3486 if (!tr)
3487 return -ENOENT;
3488
3489 /* This is only allowed for created instances */
3490 if (tr == &global_trace)
3491 return 0;
3492
3493 if (!(tr->trace_flags & TRACE_ITER_PRINTK))
3494 return 0;
3495
3496 va_start(ap, fmt);
3497 ret = trace_array_vprintk(tr, ip, fmt, ap);
3498 va_end(ap);
3499 return ret;
3500}
3501EXPORT_SYMBOL_GPL(trace_array_printk);
3502
3503/**
3504 * trace_array_init_printk - Initialize buffers for trace_array_printk()
3505 * @tr: The trace array to initialize the buffers for
3506 *
3507 * As trace_array_printk() only writes into instances, they are OK to
3508 * have in the kernel (unlike trace_printk()). This needs to be called
3509 * before trace_array_printk() can be used on a trace_array.
3510 */
3511int trace_array_init_printk(struct trace_array *tr)
3512{
3513 if (!tr)
3514 return -ENOENT;
3515
3516 /* This is only allowed for created instances */
3517 if (tr == &global_trace)
3518 return -EINVAL;
3519
3520 return alloc_percpu_trace_buffer();
3521}
3522EXPORT_SYMBOL_GPL(trace_array_init_printk);
3523
3524__printf(3, 4)
3525int trace_array_printk_buf(struct trace_buffer *buffer,
3526 unsigned long ip, const char *fmt, ...)
3527{
3528 int ret;
3529 va_list ap;
3530
3531 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3532 return 0;
3533
3534 va_start(ap, fmt);
3535 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3536 va_end(ap);
3537 return ret;
3538}
3539
3540__printf(2, 0)
3541int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3542{
3543 return trace_array_vprintk(&global_trace, ip, fmt, args);
3544}
3545EXPORT_SYMBOL_GPL(trace_vprintk);
3546
3547static void trace_iterator_increment(struct trace_iterator *iter)
3548{
3549 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3550
3551 iter->idx++;
3552 if (buf_iter)
3553 ring_buffer_iter_advance(buf_iter);
3554}
3555
3556static struct trace_entry *
3557peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3558 unsigned long *lost_events)
3559{
3560 struct ring_buffer_event *event;
3561 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3562
3563 if (buf_iter) {
3564 event = ring_buffer_iter_peek(buf_iter, ts);
3565 if (lost_events)
3566 *lost_events = ring_buffer_iter_dropped(buf_iter) ?
3567 (unsigned long)-1 : 0;
3568 } else {
3569 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3570 lost_events);
3571 }
3572
3573 if (event) {
3574 iter->ent_size = ring_buffer_event_length(event);
3575 return ring_buffer_event_data(event);
3576 }
3577 iter->ent_size = 0;
3578 return NULL;
3579}
3580
3581static struct trace_entry *
3582__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3583 unsigned long *missing_events, u64 *ent_ts)
3584{
3585 struct trace_buffer *buffer = iter->array_buffer->buffer;
3586 struct trace_entry *ent, *next = NULL;
3587 unsigned long lost_events = 0, next_lost = 0;
3588 int cpu_file = iter->cpu_file;
3589 u64 next_ts = 0, ts;
3590 int next_cpu = -1;
3591 int next_size = 0;
3592 int cpu;
3593
3594 /*
3595 * If we are in a per_cpu trace file, don't bother by iterating over
3596 * all cpu and peek directly.
3597 */
3598 if (cpu_file > RING_BUFFER_ALL_CPUS) {
3599 if (ring_buffer_empty_cpu(buffer, cpu_file))
3600 return NULL;
3601 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3602 if (ent_cpu)
3603 *ent_cpu = cpu_file;
3604
3605 return ent;
3606 }
3607
3608 for_each_tracing_cpu(cpu) {
3609
3610 if (ring_buffer_empty_cpu(buffer, cpu))
3611 continue;
3612
3613 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3614
3615 /*
3616 * Pick the entry with the smallest timestamp:
3617 */
3618 if (ent && (!next || ts < next_ts)) {
3619 next = ent;
3620 next_cpu = cpu;
3621 next_ts = ts;
3622 next_lost = lost_events;
3623 next_size = iter->ent_size;
3624 }
3625 }
3626
3627 iter->ent_size = next_size;
3628
3629 if (ent_cpu)
3630 *ent_cpu = next_cpu;
3631
3632 if (ent_ts)
3633 *ent_ts = next_ts;
3634
3635 if (missing_events)
3636 *missing_events = next_lost;
3637
3638 return next;
3639}
3640
3641#define STATIC_FMT_BUF_SIZE 128
3642static char static_fmt_buf[STATIC_FMT_BUF_SIZE];
3643
3644static char *trace_iter_expand_format(struct trace_iterator *iter)
3645{
3646 char *tmp;
3647
3648 /*
3649 * iter->tr is NULL when used with tp_printk, which makes
3650 * this get called where it is not safe to call krealloc().
3651 */
3652 if (!iter->tr || iter->fmt == static_fmt_buf)
3653 return NULL;
3654
3655 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE,
3656 GFP_KERNEL);
3657 if (tmp) {
3658 iter->fmt_size += STATIC_FMT_BUF_SIZE;
3659 iter->fmt = tmp;
3660 }
3661
3662 return tmp;
3663}
3664
3665/* Returns true if the string is safe to dereference from an event */
3666static bool trace_safe_str(struct trace_iterator *iter, const char *str)
3667{
3668 unsigned long addr = (unsigned long)str;
3669 struct trace_event *trace_event;
3670 struct trace_event_call *event;
3671
3672 /* OK if part of the event data */
3673 if ((addr >= (unsigned long)iter->ent) &&
3674 (addr < (unsigned long)iter->ent + iter->ent_size))
3675 return true;
3676
3677 /* OK if part of the temp seq buffer */
3678 if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&
3679 (addr < (unsigned long)iter->tmp_seq.buffer + PAGE_SIZE))
3680 return true;
3681
3682 /* Core rodata can not be freed */
3683 if (is_kernel_rodata(addr))
3684 return true;
3685
3686 if (trace_is_tracepoint_string(str))
3687 return true;
3688
3689 /*
3690 * Now this could be a module event, referencing core module
3691 * data, which is OK.
3692 */
3693 if (!iter->ent)
3694 return false;
3695
3696 trace_event = ftrace_find_event(iter->ent->type);
3697 if (!trace_event)
3698 return false;
3699
3700 event = container_of(trace_event, struct trace_event_call, event);
3701 if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module)
3702 return false;
3703
3704 /* Would rather have rodata, but this will suffice */
3705 if (within_module_core(addr, event->module))
3706 return true;
3707
3708 return false;
3709}
3710
3711static const char *show_buffer(struct trace_seq *s)
3712{
3713 struct seq_buf *seq = &s->seq;
3714
3715 seq_buf_terminate(seq);
3716
3717 return seq->buffer;
3718}
3719
3720static DEFINE_STATIC_KEY_FALSE(trace_no_verify);
3721
3722static int test_can_verify_check(const char *fmt, ...)
3723{
3724 char buf[16];
3725 va_list ap;
3726 int ret;
3727
3728 /*
3729 * The verifier is dependent on vsnprintf() modifies the va_list
3730 * passed to it, where it is sent as a reference. Some architectures
3731 * (like x86_32) passes it by value, which means that vsnprintf()
3732 * does not modify the va_list passed to it, and the verifier
3733 * would then need to be able to understand all the values that
3734 * vsnprintf can use. If it is passed by value, then the verifier
3735 * is disabled.
3736 */
3737 va_start(ap, fmt);
3738 vsnprintf(buf, 16, "%d", ap);
3739 ret = va_arg(ap, int);
3740 va_end(ap);
3741
3742 return ret;
3743}
3744
3745static void test_can_verify(void)
3746{
3747 if (!test_can_verify_check("%d %d", 0, 1)) {
3748 pr_info("trace event string verifier disabled\n");
3749 static_branch_inc(&trace_no_verify);
3750 }
3751}
3752
3753/**
3754 * trace_check_vprintf - Check dereferenced strings while writing to the seq buffer
3755 * @iter: The iterator that holds the seq buffer and the event being printed
3756 * @fmt: The format used to print the event
3757 * @ap: The va_list holding the data to print from @fmt.
3758 *
3759 * This writes the data into the @iter->seq buffer using the data from
3760 * @fmt and @ap. If the format has a %s, then the source of the string
3761 * is examined to make sure it is safe to print, otherwise it will
3762 * warn and print "[UNSAFE MEMORY]" in place of the dereferenced string
3763 * pointer.
3764 */
3765void trace_check_vprintf(struct trace_iterator *iter, const char *fmt,
3766 va_list ap)
3767{
3768 const char *p = fmt;
3769 const char *str;
3770 int i, j;
3771
3772 if (WARN_ON_ONCE(!fmt))
3773 return;
3774
3775 if (static_branch_unlikely(&trace_no_verify))
3776 goto print;
3777
3778 /* Don't bother checking when doing a ftrace_dump() */
3779 if (iter->fmt == static_fmt_buf)
3780 goto print;
3781
3782 while (*p) {
3783 bool star = false;
3784 int len = 0;
3785
3786 j = 0;
3787
3788 /* We only care about %s and variants */
3789 for (i = 0; p[i]; i++) {
3790 if (i + 1 >= iter->fmt_size) {
3791 /*
3792 * If we can't expand the copy buffer,
3793 * just print it.
3794 */
3795 if (!trace_iter_expand_format(iter))
3796 goto print;
3797 }
3798
3799 if (p[i] == '\\' && p[i+1]) {
3800 i++;
3801 continue;
3802 }
3803 if (p[i] == '%') {
3804 /* Need to test cases like %08.*s */
3805 for (j = 1; p[i+j]; j++) {
3806 if (isdigit(p[i+j]) ||
3807 p[i+j] == '.')
3808 continue;
3809 if (p[i+j] == '*') {
3810 star = true;
3811 continue;
3812 }
3813 break;
3814 }
3815 if (p[i+j] == 's')
3816 break;
3817 star = false;
3818 }
3819 j = 0;
3820 }
3821 /* If no %s found then just print normally */
3822 if (!p[i])
3823 break;
3824
3825 /* Copy up to the %s, and print that */
3826 strncpy(iter->fmt, p, i);
3827 iter->fmt[i] = '\0';
3828 trace_seq_vprintf(&iter->seq, iter->fmt, ap);
3829
3830 /*
3831 * If iter->seq is full, the above call no longer guarantees
3832 * that ap is in sync with fmt processing, and further calls
3833 * to va_arg() can return wrong positional arguments.
3834 *
3835 * Ensure that ap is no longer used in this case.
3836 */
3837 if (iter->seq.full) {
3838 p = "";
3839 break;
3840 }
3841
3842 if (star)
3843 len = va_arg(ap, int);
3844
3845 /* The ap now points to the string data of the %s */
3846 str = va_arg(ap, const char *);
3847
3848 /*
3849 * If you hit this warning, it is likely that the
3850 * trace event in question used %s on a string that
3851 * was saved at the time of the event, but may not be
3852 * around when the trace is read. Use __string(),
3853 * __assign_str() and __get_str() helpers in the TRACE_EVENT()
3854 * instead. See samples/trace_events/trace-events-sample.h
3855 * for reference.
3856 */
3857 if (WARN_ONCE(!trace_safe_str(iter, str),
3858 "fmt: '%s' current_buffer: '%s'",
3859 fmt, show_buffer(&iter->seq))) {
3860 int ret;
3861
3862 /* Try to safely read the string */
3863 if (star) {
3864 if (len + 1 > iter->fmt_size)
3865 len = iter->fmt_size - 1;
3866 if (len < 0)
3867 len = 0;
3868 ret = copy_from_kernel_nofault(iter->fmt, str, len);
3869 iter->fmt[len] = 0;
3870 star = false;
3871 } else {
3872 ret = strncpy_from_kernel_nofault(iter->fmt, str,
3873 iter->fmt_size);
3874 }
3875 if (ret < 0)
3876 trace_seq_printf(&iter->seq, "(0x%px)", str);
3877 else
3878 trace_seq_printf(&iter->seq, "(0x%px:%s)",
3879 str, iter->fmt);
3880 str = "[UNSAFE-MEMORY]";
3881 strcpy(iter->fmt, "%s");
3882 } else {
3883 strncpy(iter->fmt, p + i, j + 1);
3884 iter->fmt[j+1] = '\0';
3885 }
3886 if (star)
3887 trace_seq_printf(&iter->seq, iter->fmt, len, str);
3888 else
3889 trace_seq_printf(&iter->seq, iter->fmt, str);
3890
3891 p += i + j + 1;
3892 }
3893 print:
3894 if (*p)
3895 trace_seq_vprintf(&iter->seq, p, ap);
3896}
3897
3898const char *trace_event_format(struct trace_iterator *iter, const char *fmt)
3899{
3900 const char *p, *new_fmt;
3901 char *q;
3902
3903 if (WARN_ON_ONCE(!fmt))
3904 return fmt;
3905
3906 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR)
3907 return fmt;
3908
3909 p = fmt;
3910 new_fmt = q = iter->fmt;
3911 while (*p) {
3912 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {
3913 if (!trace_iter_expand_format(iter))
3914 return fmt;
3915
3916 q += iter->fmt - new_fmt;
3917 new_fmt = iter->fmt;
3918 }
3919
3920 *q++ = *p++;
3921
3922 /* Replace %p with %px */
3923 if (p[-1] == '%') {
3924 if (p[0] == '%') {
3925 *q++ = *p++;
3926 } else if (p[0] == 'p' && !isalnum(p[1])) {
3927 *q++ = *p++;
3928 *q++ = 'x';
3929 }
3930 }
3931 }
3932 *q = '\0';
3933
3934 return new_fmt;
3935}
3936
3937#define STATIC_TEMP_BUF_SIZE 128
3938static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
3939
3940/* Find the next real entry, without updating the iterator itself */
3941struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3942 int *ent_cpu, u64 *ent_ts)
3943{
3944 /* __find_next_entry will reset ent_size */
3945 int ent_size = iter->ent_size;
3946 struct trace_entry *entry;
3947
3948 /*
3949 * If called from ftrace_dump(), then the iter->temp buffer
3950 * will be the static_temp_buf and not created from kmalloc.
3951 * If the entry size is greater than the buffer, we can
3952 * not save it. Just return NULL in that case. This is only
3953 * used to add markers when two consecutive events' time
3954 * stamps have a large delta. See trace_print_lat_context()
3955 */
3956 if (iter->temp == static_temp_buf &&
3957 STATIC_TEMP_BUF_SIZE < ent_size)
3958 return NULL;
3959
3960 /*
3961 * The __find_next_entry() may call peek_next_entry(), which may
3962 * call ring_buffer_peek() that may make the contents of iter->ent
3963 * undefined. Need to copy iter->ent now.
3964 */
3965 if (iter->ent && iter->ent != iter->temp) {
3966 if ((!iter->temp || iter->temp_size < iter->ent_size) &&
3967 !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
3968 void *temp;
3969 temp = kmalloc(iter->ent_size, GFP_KERNEL);
3970 if (!temp)
3971 return NULL;
3972 kfree(iter->temp);
3973 iter->temp = temp;
3974 iter->temp_size = iter->ent_size;
3975 }
3976 memcpy(iter->temp, iter->ent, iter->ent_size);
3977 iter->ent = iter->temp;
3978 }
3979 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3980 /* Put back the original ent_size */
3981 iter->ent_size = ent_size;
3982
3983 return entry;
3984}
3985
3986/* Find the next real entry, and increment the iterator to the next entry */
3987void *trace_find_next_entry_inc(struct trace_iterator *iter)
3988{
3989 iter->ent = __find_next_entry(iter, &iter->cpu,
3990 &iter->lost_events, &iter->ts);
3991
3992 if (iter->ent)
3993 trace_iterator_increment(iter);
3994
3995 return iter->ent ? iter : NULL;
3996}
3997
3998static void trace_consume(struct trace_iterator *iter)
3999{
4000 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
4001 &iter->lost_events);
4002}
4003
4004static void *s_next(struct seq_file *m, void *v, loff_t *pos)
4005{
4006 struct trace_iterator *iter = m->private;
4007 int i = (int)*pos;
4008 void *ent;
4009
4010 WARN_ON_ONCE(iter->leftover);
4011
4012 (*pos)++;
4013
4014 /* can't go backwards */
4015 if (iter->idx > i)
4016 return NULL;
4017
4018 if (iter->idx < 0)
4019 ent = trace_find_next_entry_inc(iter);
4020 else
4021 ent = iter;
4022
4023 while (ent && iter->idx < i)
4024 ent = trace_find_next_entry_inc(iter);
4025
4026 iter->pos = *pos;
4027
4028 return ent;
4029}
4030
4031void tracing_iter_reset(struct trace_iterator *iter, int cpu)
4032{
4033 struct ring_buffer_iter *buf_iter;
4034 unsigned long entries = 0;
4035 u64 ts;
4036
4037 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
4038
4039 buf_iter = trace_buffer_iter(iter, cpu);
4040 if (!buf_iter)
4041 return;
4042
4043 ring_buffer_iter_reset(buf_iter);
4044
4045 /*
4046 * We could have the case with the max latency tracers
4047 * that a reset never took place on a cpu. This is evident
4048 * by the timestamp being before the start of the buffer.
4049 */
4050 while (ring_buffer_iter_peek(buf_iter, &ts)) {
4051 if (ts >= iter->array_buffer->time_start)
4052 break;
4053 entries++;
4054 ring_buffer_iter_advance(buf_iter);
4055 }
4056
4057 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
4058}
4059
4060/*
4061 * The current tracer is copied to avoid a global locking
4062 * all around.
4063 */
4064static void *s_start(struct seq_file *m, loff_t *pos)
4065{
4066 struct trace_iterator *iter = m->private;
4067 struct trace_array *tr = iter->tr;
4068 int cpu_file = iter->cpu_file;
4069 void *p = NULL;
4070 loff_t l = 0;
4071 int cpu;
4072
4073 /*
4074 * copy the tracer to avoid using a global lock all around.
4075 * iter->trace is a copy of current_trace, the pointer to the
4076 * name may be used instead of a strcmp(), as iter->trace->name
4077 * will point to the same string as current_trace->name.
4078 */
4079 mutex_lock(&trace_types_lock);
4080 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
4081 *iter->trace = *tr->current_trace;
4082 mutex_unlock(&trace_types_lock);
4083
4084#ifdef CONFIG_TRACER_MAX_TRACE
4085 if (iter->snapshot && iter->trace->use_max_tr)
4086 return ERR_PTR(-EBUSY);
4087#endif
4088
4089 if (*pos != iter->pos) {
4090 iter->ent = NULL;
4091 iter->cpu = 0;
4092 iter->idx = -1;
4093
4094 if (cpu_file == RING_BUFFER_ALL_CPUS) {
4095 for_each_tracing_cpu(cpu)
4096 tracing_iter_reset(iter, cpu);
4097 } else
4098 tracing_iter_reset(iter, cpu_file);
4099
4100 iter->leftover = 0;
4101 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
4102 ;
4103
4104 } else {
4105 /*
4106 * If we overflowed the seq_file before, then we want
4107 * to just reuse the trace_seq buffer again.
4108 */
4109 if (iter->leftover)
4110 p = iter;
4111 else {
4112 l = *pos - 1;
4113 p = s_next(m, p, &l);
4114 }
4115 }
4116
4117 trace_event_read_lock();
4118 trace_access_lock(cpu_file);
4119 return p;
4120}
4121
4122static void s_stop(struct seq_file *m, void *p)
4123{
4124 struct trace_iterator *iter = m->private;
4125
4126#ifdef CONFIG_TRACER_MAX_TRACE
4127 if (iter->snapshot && iter->trace->use_max_tr)
4128 return;
4129#endif
4130
4131 trace_access_unlock(iter->cpu_file);
4132 trace_event_read_unlock();
4133}
4134
4135static void
4136get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
4137 unsigned long *entries, int cpu)
4138{
4139 unsigned long count;
4140
4141 count = ring_buffer_entries_cpu(buf->buffer, cpu);
4142 /*
4143 * If this buffer has skipped entries, then we hold all
4144 * entries for the trace and we need to ignore the
4145 * ones before the time stamp.
4146 */
4147 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
4148 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
4149 /* total is the same as the entries */
4150 *total = count;
4151 } else
4152 *total = count +
4153 ring_buffer_overrun_cpu(buf->buffer, cpu);
4154 *entries = count;
4155}
4156
4157static void
4158get_total_entries(struct array_buffer *buf,
4159 unsigned long *total, unsigned long *entries)
4160{
4161 unsigned long t, e;
4162 int cpu;
4163
4164 *total = 0;
4165 *entries = 0;
4166
4167 for_each_tracing_cpu(cpu) {
4168 get_total_entries_cpu(buf, &t, &e, cpu);
4169 *total += t;
4170 *entries += e;
4171 }
4172}
4173
4174unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
4175{
4176 unsigned long total, entries;
4177
4178 if (!tr)
4179 tr = &global_trace;
4180
4181 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
4182
4183 return entries;
4184}
4185
4186unsigned long trace_total_entries(struct trace_array *tr)
4187{
4188 unsigned long total, entries;
4189
4190 if (!tr)
4191 tr = &global_trace;
4192
4193 get_total_entries(&tr->array_buffer, &total, &entries);
4194
4195 return entries;
4196}
4197
4198static void print_lat_help_header(struct seq_file *m)
4199{
4200 seq_puts(m, "# _------=> CPU# \n"
4201 "# / _-----=> irqs-off/BH-disabled\n"
4202 "# | / _----=> need-resched \n"
4203 "# || / _---=> hardirq/softirq \n"
4204 "# ||| / _--=> preempt-depth \n"
4205 "# |||| / _-=> migrate-disable \n"
4206 "# ||||| / delay \n"
4207 "# cmd pid |||||| time | caller \n"
4208 "# \\ / |||||| \\ | / \n");
4209}
4210
4211static void print_event_info(struct array_buffer *buf, struct seq_file *m)
4212{
4213 unsigned long total;
4214 unsigned long entries;
4215
4216 get_total_entries(buf, &total, &entries);
4217 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
4218 entries, total, num_online_cpus());
4219 seq_puts(m, "#\n");
4220}
4221
4222static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
4223 unsigned int flags)
4224{
4225 bool tgid = flags & TRACE_ITER_RECORD_TGID;
4226
4227 print_event_info(buf, m);
4228
4229 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
4230 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
4231}
4232
4233static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
4234 unsigned int flags)
4235{
4236 bool tgid = flags & TRACE_ITER_RECORD_TGID;
4237 const char *space = " ";
4238 int prec = tgid ? 12 : 2;
4239
4240 print_event_info(buf, m);
4241
4242 seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space);
4243 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
4244 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
4245 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
4246 seq_printf(m, "# %.*s||| / _-=> migrate-disable\n", prec, space);
4247 seq_printf(m, "# %.*s|||| / delay\n", prec, space);
4248 seq_printf(m, "# TASK-PID %.*s CPU# ||||| TIMESTAMP FUNCTION\n", prec, " TGID ");
4249 seq_printf(m, "# | | %.*s | ||||| | |\n", prec, " | ");
4250}
4251
4252void
4253print_trace_header(struct seq_file *m, struct trace_iterator *iter)
4254{
4255 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
4256 struct array_buffer *buf = iter->array_buffer;
4257 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
4258 struct tracer *type = iter->trace;
4259 unsigned long entries;
4260 unsigned long total;
4261 const char *name = "preemption";
4262
4263 name = type->name;
4264
4265 get_total_entries(buf, &total, &entries);
4266
4267 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
4268 name, UTS_RELEASE);
4269 seq_puts(m, "# -----------------------------------"
4270 "---------------------------------\n");
4271 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
4272 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
4273 nsecs_to_usecs(data->saved_latency),
4274 entries,
4275 total,
4276 buf->cpu,
4277#if defined(CONFIG_PREEMPT_NONE)
4278 "server",
4279#elif defined(CONFIG_PREEMPT_VOLUNTARY)
4280 "desktop",
4281#elif defined(CONFIG_PREEMPT)
4282 "preempt",
4283#elif defined(CONFIG_PREEMPT_RT)
4284 "preempt_rt",
4285#else
4286 "unknown",
4287#endif
4288 /* These are reserved for later use */
4289 0, 0, 0, 0);
4290#ifdef CONFIG_SMP
4291 seq_printf(m, " #P:%d)\n", num_online_cpus());
4292#else
4293 seq_puts(m, ")\n");
4294#endif
4295 seq_puts(m, "# -----------------\n");
4296 seq_printf(m, "# | task: %.16s-%d "
4297 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
4298 data->comm, data->pid,
4299 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
4300 data->policy, data->rt_priority);
4301 seq_puts(m, "# -----------------\n");
4302
4303 if (data->critical_start) {
4304 seq_puts(m, "# => started at: ");
4305 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
4306 trace_print_seq(m, &iter->seq);
4307 seq_puts(m, "\n# => ended at: ");
4308 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
4309 trace_print_seq(m, &iter->seq);
4310 seq_puts(m, "\n#\n");
4311 }
4312
4313 seq_puts(m, "#\n");
4314}
4315
4316static void test_cpu_buff_start(struct trace_iterator *iter)
4317{
4318 struct trace_seq *s = &iter->seq;
4319 struct trace_array *tr = iter->tr;
4320
4321 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
4322 return;
4323
4324 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
4325 return;
4326
4327 if (cpumask_available(iter->started) &&
4328 cpumask_test_cpu(iter->cpu, iter->started))
4329 return;
4330
4331 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
4332 return;
4333
4334 if (cpumask_available(iter->started))
4335 cpumask_set_cpu(iter->cpu, iter->started);
4336
4337 /* Don't print started cpu buffer for the first entry of the trace */
4338 if (iter->idx > 1)
4339 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
4340 iter->cpu);
4341}
4342
4343static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
4344{
4345 struct trace_array *tr = iter->tr;
4346 struct trace_seq *s = &iter->seq;
4347 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4348 struct trace_entry *entry;
4349 struct trace_event *event;
4350
4351 entry = iter->ent;
4352
4353 test_cpu_buff_start(iter);
4354
4355 event = ftrace_find_event(entry->type);
4356
4357 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4358 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4359 trace_print_lat_context(iter);
4360 else
4361 trace_print_context(iter);
4362 }
4363
4364 if (trace_seq_has_overflowed(s))
4365 return TRACE_TYPE_PARTIAL_LINE;
4366
4367 if (event)
4368 return event->funcs->trace(iter, sym_flags, event);
4369
4370 trace_seq_printf(s, "Unknown type %d\n", entry->type);
4371
4372 return trace_handle_return(s);
4373}
4374
4375static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
4376{
4377 struct trace_array *tr = iter->tr;
4378 struct trace_seq *s = &iter->seq;
4379 struct trace_entry *entry;
4380 struct trace_event *event;
4381
4382 entry = iter->ent;
4383
4384 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
4385 trace_seq_printf(s, "%d %d %llu ",
4386 entry->pid, iter->cpu, iter->ts);
4387
4388 if (trace_seq_has_overflowed(s))
4389 return TRACE_TYPE_PARTIAL_LINE;
4390
4391 event = ftrace_find_event(entry->type);
4392 if (event)
4393 return event->funcs->raw(iter, 0, event);
4394
4395 trace_seq_printf(s, "%d ?\n", entry->type);
4396
4397 return trace_handle_return(s);
4398}
4399
4400static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
4401{
4402 struct trace_array *tr = iter->tr;
4403 struct trace_seq *s = &iter->seq;
4404 unsigned char newline = '\n';
4405 struct trace_entry *entry;
4406 struct trace_event *event;
4407
4408 entry = iter->ent;
4409
4410 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4411 SEQ_PUT_HEX_FIELD(s, entry->pid);
4412 SEQ_PUT_HEX_FIELD(s, iter->cpu);
4413 SEQ_PUT_HEX_FIELD(s, iter->ts);
4414 if (trace_seq_has_overflowed(s))
4415 return TRACE_TYPE_PARTIAL_LINE;
4416 }
4417
4418 event = ftrace_find_event(entry->type);
4419 if (event) {
4420 enum print_line_t ret = event->funcs->hex(iter, 0, event);
4421 if (ret != TRACE_TYPE_HANDLED)
4422 return ret;
4423 }
4424
4425 SEQ_PUT_FIELD(s, newline);
4426
4427 return trace_handle_return(s);
4428}
4429
4430static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
4431{
4432 struct trace_array *tr = iter->tr;
4433 struct trace_seq *s = &iter->seq;
4434 struct trace_entry *entry;
4435 struct trace_event *event;
4436
4437 entry = iter->ent;
4438
4439 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4440 SEQ_PUT_FIELD(s, entry->pid);
4441 SEQ_PUT_FIELD(s, iter->cpu);
4442 SEQ_PUT_FIELD(s, iter->ts);
4443 if (trace_seq_has_overflowed(s))
4444 return TRACE_TYPE_PARTIAL_LINE;
4445 }
4446
4447 event = ftrace_find_event(entry->type);
4448 return event ? event->funcs->binary(iter, 0, event) :
4449 TRACE_TYPE_HANDLED;
4450}
4451
4452int trace_empty(struct trace_iterator *iter)
4453{
4454 struct ring_buffer_iter *buf_iter;
4455 int cpu;
4456
4457 /* If we are looking at one CPU buffer, only check that one */
4458 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4459 cpu = iter->cpu_file;
4460 buf_iter = trace_buffer_iter(iter, cpu);
4461 if (buf_iter) {
4462 if (!ring_buffer_iter_empty(buf_iter))
4463 return 0;
4464 } else {
4465 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4466 return 0;
4467 }
4468 return 1;
4469 }
4470
4471 for_each_tracing_cpu(cpu) {
4472 buf_iter = trace_buffer_iter(iter, cpu);
4473 if (buf_iter) {
4474 if (!ring_buffer_iter_empty(buf_iter))
4475 return 0;
4476 } else {
4477 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4478 return 0;
4479 }
4480 }
4481
4482 return 1;
4483}
4484
4485/* Called with trace_event_read_lock() held. */
4486enum print_line_t print_trace_line(struct trace_iterator *iter)
4487{
4488 struct trace_array *tr = iter->tr;
4489 unsigned long trace_flags = tr->trace_flags;
4490 enum print_line_t ret;
4491
4492 if (iter->lost_events) {
4493 if (iter->lost_events == (unsigned long)-1)
4494 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
4495 iter->cpu);
4496 else
4497 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
4498 iter->cpu, iter->lost_events);
4499 if (trace_seq_has_overflowed(&iter->seq))
4500 return TRACE_TYPE_PARTIAL_LINE;
4501 }
4502
4503 if (iter->trace && iter->trace->print_line) {
4504 ret = iter->trace->print_line(iter);
4505 if (ret != TRACE_TYPE_UNHANDLED)
4506 return ret;
4507 }
4508
4509 if (iter->ent->type == TRACE_BPUTS &&
4510 trace_flags & TRACE_ITER_PRINTK &&
4511 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4512 return trace_print_bputs_msg_only(iter);
4513
4514 if (iter->ent->type == TRACE_BPRINT &&
4515 trace_flags & TRACE_ITER_PRINTK &&
4516 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4517 return trace_print_bprintk_msg_only(iter);
4518
4519 if (iter->ent->type == TRACE_PRINT &&
4520 trace_flags & TRACE_ITER_PRINTK &&
4521 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4522 return trace_print_printk_msg_only(iter);
4523
4524 if (trace_flags & TRACE_ITER_BIN)
4525 return print_bin_fmt(iter);
4526
4527 if (trace_flags & TRACE_ITER_HEX)
4528 return print_hex_fmt(iter);
4529
4530 if (trace_flags & TRACE_ITER_RAW)
4531 return print_raw_fmt(iter);
4532
4533 return print_trace_fmt(iter);
4534}
4535
4536void trace_latency_header(struct seq_file *m)
4537{
4538 struct trace_iterator *iter = m->private;
4539 struct trace_array *tr = iter->tr;
4540
4541 /* print nothing if the buffers are empty */
4542 if (trace_empty(iter))
4543 return;
4544
4545 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4546 print_trace_header(m, iter);
4547
4548 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4549 print_lat_help_header(m);
4550}
4551
4552void trace_default_header(struct seq_file *m)
4553{
4554 struct trace_iterator *iter = m->private;
4555 struct trace_array *tr = iter->tr;
4556 unsigned long trace_flags = tr->trace_flags;
4557
4558 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4559 return;
4560
4561 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4562 /* print nothing if the buffers are empty */
4563 if (trace_empty(iter))
4564 return;
4565 print_trace_header(m, iter);
4566 if (!(trace_flags & TRACE_ITER_VERBOSE))
4567 print_lat_help_header(m);
4568 } else {
4569 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4570 if (trace_flags & TRACE_ITER_IRQ_INFO)
4571 print_func_help_header_irq(iter->array_buffer,
4572 m, trace_flags);
4573 else
4574 print_func_help_header(iter->array_buffer, m,
4575 trace_flags);
4576 }
4577 }
4578}
4579
4580static void test_ftrace_alive(struct seq_file *m)
4581{
4582 if (!ftrace_is_dead())
4583 return;
4584 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4585 "# MAY BE MISSING FUNCTION EVENTS\n");
4586}
4587
4588#ifdef CONFIG_TRACER_MAX_TRACE
4589static void show_snapshot_main_help(struct seq_file *m)
4590{
4591 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4592 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4593 "# Takes a snapshot of the main buffer.\n"
4594 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4595 "# (Doesn't have to be '2' works with any number that\n"
4596 "# is not a '0' or '1')\n");
4597}
4598
4599static void show_snapshot_percpu_help(struct seq_file *m)
4600{
4601 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4602#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4603 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4604 "# Takes a snapshot of the main buffer for this cpu.\n");
4605#else
4606 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4607 "# Must use main snapshot file to allocate.\n");
4608#endif
4609 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4610 "# (Doesn't have to be '2' works with any number that\n"
4611 "# is not a '0' or '1')\n");
4612}
4613
4614static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
4615{
4616 if (iter->tr->allocated_snapshot)
4617 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4618 else
4619 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4620
4621 seq_puts(m, "# Snapshot commands:\n");
4622 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4623 show_snapshot_main_help(m);
4624 else
4625 show_snapshot_percpu_help(m);
4626}
4627#else
4628/* Should never be called */
4629static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4630#endif
4631
4632static int s_show(struct seq_file *m, void *v)
4633{
4634 struct trace_iterator *iter = v;
4635 int ret;
4636
4637 if (iter->ent == NULL) {
4638 if (iter->tr) {
4639 seq_printf(m, "# tracer: %s\n", iter->trace->name);
4640 seq_puts(m, "#\n");
4641 test_ftrace_alive(m);
4642 }
4643 if (iter->snapshot && trace_empty(iter))
4644 print_snapshot_help(m, iter);
4645 else if (iter->trace && iter->trace->print_header)
4646 iter->trace->print_header(m);
4647 else
4648 trace_default_header(m);
4649
4650 } else if (iter->leftover) {
4651 /*
4652 * If we filled the seq_file buffer earlier, we
4653 * want to just show it now.
4654 */
4655 ret = trace_print_seq(m, &iter->seq);
4656
4657 /* ret should this time be zero, but you never know */
4658 iter->leftover = ret;
4659
4660 } else {
4661 print_trace_line(iter);
4662 ret = trace_print_seq(m, &iter->seq);
4663 /*
4664 * If we overflow the seq_file buffer, then it will
4665 * ask us for this data again at start up.
4666 * Use that instead.
4667 * ret is 0 if seq_file write succeeded.
4668 * -1 otherwise.
4669 */
4670 iter->leftover = ret;
4671 }
4672
4673 return 0;
4674}
4675
4676/*
4677 * Should be used after trace_array_get(), trace_types_lock
4678 * ensures that i_cdev was already initialized.
4679 */
4680static inline int tracing_get_cpu(struct inode *inode)
4681{
4682 if (inode->i_cdev) /* See trace_create_cpu_file() */
4683 return (long)inode->i_cdev - 1;
4684 return RING_BUFFER_ALL_CPUS;
4685}
4686
4687static const struct seq_operations tracer_seq_ops = {
4688 .start = s_start,
4689 .next = s_next,
4690 .stop = s_stop,
4691 .show = s_show,
4692};
4693
4694static struct trace_iterator *
4695__tracing_open(struct inode *inode, struct file *file, bool snapshot)
4696{
4697 struct trace_array *tr = inode->i_private;
4698 struct trace_iterator *iter;
4699 int cpu;
4700
4701 if (tracing_disabled)
4702 return ERR_PTR(-ENODEV);
4703
4704 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4705 if (!iter)
4706 return ERR_PTR(-ENOMEM);
4707
4708 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4709 GFP_KERNEL);
4710 if (!iter->buffer_iter)
4711 goto release;
4712
4713 /*
4714 * trace_find_next_entry() may need to save off iter->ent.
4715 * It will place it into the iter->temp buffer. As most
4716 * events are less than 128, allocate a buffer of that size.
4717 * If one is greater, then trace_find_next_entry() will
4718 * allocate a new buffer to adjust for the bigger iter->ent.
4719 * It's not critical if it fails to get allocated here.
4720 */
4721 iter->temp = kmalloc(128, GFP_KERNEL);
4722 if (iter->temp)
4723 iter->temp_size = 128;
4724
4725 /*
4726 * trace_event_printf() may need to modify given format
4727 * string to replace %p with %px so that it shows real address
4728 * instead of hash value. However, that is only for the event
4729 * tracing, other tracer may not need. Defer the allocation
4730 * until it is needed.
4731 */
4732 iter->fmt = NULL;
4733 iter->fmt_size = 0;
4734
4735 /*
4736 * We make a copy of the current tracer to avoid concurrent
4737 * changes on it while we are reading.
4738 */
4739 mutex_lock(&trace_types_lock);
4740 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4741 if (!iter->trace)
4742 goto fail;
4743
4744 *iter->trace = *tr->current_trace;
4745
4746 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4747 goto fail;
4748
4749 iter->tr = tr;
4750
4751#ifdef CONFIG_TRACER_MAX_TRACE
4752 /* Currently only the top directory has a snapshot */
4753 if (tr->current_trace->print_max || snapshot)
4754 iter->array_buffer = &tr->max_buffer;
4755 else
4756#endif
4757 iter->array_buffer = &tr->array_buffer;
4758 iter->snapshot = snapshot;
4759 iter->pos = -1;
4760 iter->cpu_file = tracing_get_cpu(inode);
4761 mutex_init(&iter->mutex);
4762
4763 /* Notify the tracer early; before we stop tracing. */
4764 if (iter->trace->open)
4765 iter->trace->open(iter);
4766
4767 /* Annotate start of buffers if we had overruns */
4768 if (ring_buffer_overruns(iter->array_buffer->buffer))
4769 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4770
4771 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4772 if (trace_clocks[tr->clock_id].in_ns)
4773 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4774
4775 /*
4776 * If pause-on-trace is enabled, then stop the trace while
4777 * dumping, unless this is the "snapshot" file
4778 */
4779 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4780 tracing_stop_tr(tr);
4781
4782 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4783 for_each_tracing_cpu(cpu) {
4784 iter->buffer_iter[cpu] =
4785 ring_buffer_read_prepare(iter->array_buffer->buffer,
4786 cpu, GFP_KERNEL);
4787 }
4788 ring_buffer_read_prepare_sync();
4789 for_each_tracing_cpu(cpu) {
4790 ring_buffer_read_start(iter->buffer_iter[cpu]);
4791 tracing_iter_reset(iter, cpu);
4792 }
4793 } else {
4794 cpu = iter->cpu_file;
4795 iter->buffer_iter[cpu] =
4796 ring_buffer_read_prepare(iter->array_buffer->buffer,
4797 cpu, GFP_KERNEL);
4798 ring_buffer_read_prepare_sync();
4799 ring_buffer_read_start(iter->buffer_iter[cpu]);
4800 tracing_iter_reset(iter, cpu);
4801 }
4802
4803 mutex_unlock(&trace_types_lock);
4804
4805 return iter;
4806
4807 fail:
4808 mutex_unlock(&trace_types_lock);
4809 kfree(iter->trace);
4810 kfree(iter->temp);
4811 kfree(iter->buffer_iter);
4812release:
4813 seq_release_private(inode, file);
4814 return ERR_PTR(-ENOMEM);
4815}
4816
4817int tracing_open_generic(struct inode *inode, struct file *filp)
4818{
4819 int ret;
4820
4821 ret = tracing_check_open_get_tr(NULL);
4822 if (ret)
4823 return ret;
4824
4825 filp->private_data = inode->i_private;
4826 return 0;
4827}
4828
4829bool tracing_is_disabled(void)
4830{
4831 return (tracing_disabled) ? true: false;
4832}
4833
4834/*
4835 * Open and update trace_array ref count.
4836 * Must have the current trace_array passed to it.
4837 */
4838int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4839{
4840 struct trace_array *tr = inode->i_private;
4841 int ret;
4842
4843 ret = tracing_check_open_get_tr(tr);
4844 if (ret)
4845 return ret;
4846
4847 filp->private_data = inode->i_private;
4848
4849 return 0;
4850}
4851
4852static int tracing_mark_open(struct inode *inode, struct file *filp)
4853{
4854 stream_open(inode, filp);
4855 return tracing_open_generic_tr(inode, filp);
4856}
4857
4858static int tracing_release(struct inode *inode, struct file *file)
4859{
4860 struct trace_array *tr = inode->i_private;
4861 struct seq_file *m = file->private_data;
4862 struct trace_iterator *iter;
4863 int cpu;
4864
4865 if (!(file->f_mode & FMODE_READ)) {
4866 trace_array_put(tr);
4867 return 0;
4868 }
4869
4870 /* Writes do not use seq_file */
4871 iter = m->private;
4872 mutex_lock(&trace_types_lock);
4873
4874 for_each_tracing_cpu(cpu) {
4875 if (iter->buffer_iter[cpu])
4876 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4877 }
4878
4879 if (iter->trace && iter->trace->close)
4880 iter->trace->close(iter);
4881
4882 if (!iter->snapshot && tr->stop_count)
4883 /* reenable tracing if it was previously enabled */
4884 tracing_start_tr(tr);
4885
4886 __trace_array_put(tr);
4887
4888 mutex_unlock(&trace_types_lock);
4889
4890 mutex_destroy(&iter->mutex);
4891 free_cpumask_var(iter->started);
4892 kfree(iter->fmt);
4893 kfree(iter->temp);
4894 kfree(iter->trace);
4895 kfree(iter->buffer_iter);
4896 seq_release_private(inode, file);
4897
4898 return 0;
4899}
4900
4901static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4902{
4903 struct trace_array *tr = inode->i_private;
4904
4905 trace_array_put(tr);
4906 return 0;
4907}
4908
4909static int tracing_single_release_tr(struct inode *inode, struct file *file)
4910{
4911 struct trace_array *tr = inode->i_private;
4912
4913 trace_array_put(tr);
4914
4915 return single_release(inode, file);
4916}
4917
4918static int tracing_open(struct inode *inode, struct file *file)
4919{
4920 struct trace_array *tr = inode->i_private;
4921 struct trace_iterator *iter;
4922 int ret;
4923
4924 ret = tracing_check_open_get_tr(tr);
4925 if (ret)
4926 return ret;
4927
4928 /* If this file was open for write, then erase contents */
4929 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4930 int cpu = tracing_get_cpu(inode);
4931 struct array_buffer *trace_buf = &tr->array_buffer;
4932
4933#ifdef CONFIG_TRACER_MAX_TRACE
4934 if (tr->current_trace->print_max)
4935 trace_buf = &tr->max_buffer;
4936#endif
4937
4938 if (cpu == RING_BUFFER_ALL_CPUS)
4939 tracing_reset_online_cpus(trace_buf);
4940 else
4941 tracing_reset_cpu(trace_buf, cpu);
4942 }
4943
4944 if (file->f_mode & FMODE_READ) {
4945 iter = __tracing_open(inode, file, false);
4946 if (IS_ERR(iter))
4947 ret = PTR_ERR(iter);
4948 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4949 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4950 }
4951
4952 if (ret < 0)
4953 trace_array_put(tr);
4954
4955 return ret;
4956}
4957
4958/*
4959 * Some tracers are not suitable for instance buffers.
4960 * A tracer is always available for the global array (toplevel)
4961 * or if it explicitly states that it is.
4962 */
4963static bool
4964trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4965{
4966 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4967}
4968
4969/* Find the next tracer that this trace array may use */
4970static struct tracer *
4971get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4972{
4973 while (t && !trace_ok_for_array(t, tr))
4974 t = t->next;
4975
4976 return t;
4977}
4978
4979static void *
4980t_next(struct seq_file *m, void *v, loff_t *pos)
4981{
4982 struct trace_array *tr = m->private;
4983 struct tracer *t = v;
4984
4985 (*pos)++;
4986
4987 if (t)
4988 t = get_tracer_for_array(tr, t->next);
4989
4990 return t;
4991}
4992
4993static void *t_start(struct seq_file *m, loff_t *pos)
4994{
4995 struct trace_array *tr = m->private;
4996 struct tracer *t;
4997 loff_t l = 0;
4998
4999 mutex_lock(&trace_types_lock);
5000
5001 t = get_tracer_for_array(tr, trace_types);
5002 for (; t && l < *pos; t = t_next(m, t, &l))
5003 ;
5004
5005 return t;
5006}
5007
5008static void t_stop(struct seq_file *m, void *p)
5009{
5010 mutex_unlock(&trace_types_lock);
5011}
5012
5013static int t_show(struct seq_file *m, void *v)
5014{
5015 struct tracer *t = v;
5016
5017 if (!t)
5018 return 0;
5019
5020 seq_puts(m, t->name);
5021 if (t->next)
5022 seq_putc(m, ' ');
5023 else
5024 seq_putc(m, '\n');
5025
5026 return 0;
5027}
5028
5029static const struct seq_operations show_traces_seq_ops = {
5030 .start = t_start,
5031 .next = t_next,
5032 .stop = t_stop,
5033 .show = t_show,
5034};
5035
5036static int show_traces_open(struct inode *inode, struct file *file)
5037{
5038 struct trace_array *tr = inode->i_private;
5039 struct seq_file *m;
5040 int ret;
5041
5042 ret = tracing_check_open_get_tr(tr);
5043 if (ret)
5044 return ret;
5045
5046 ret = seq_open(file, &show_traces_seq_ops);
5047 if (ret) {
5048 trace_array_put(tr);
5049 return ret;
5050 }
5051
5052 m = file->private_data;
5053 m->private = tr;
5054
5055 return 0;
5056}
5057
5058static int show_traces_release(struct inode *inode, struct file *file)
5059{
5060 struct trace_array *tr = inode->i_private;
5061
5062 trace_array_put(tr);
5063 return seq_release(inode, file);
5064}
5065
5066static ssize_t
5067tracing_write_stub(struct file *filp, const char __user *ubuf,
5068 size_t count, loff_t *ppos)
5069{
5070 return count;
5071}
5072
5073loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
5074{
5075 int ret;
5076
5077 if (file->f_mode & FMODE_READ)
5078 ret = seq_lseek(file, offset, whence);
5079 else
5080 file->f_pos = ret = 0;
5081
5082 return ret;
5083}
5084
5085static const struct file_operations tracing_fops = {
5086 .open = tracing_open,
5087 .read = seq_read,
5088 .write = tracing_write_stub,
5089 .llseek = tracing_lseek,
5090 .release = tracing_release,
5091};
5092
5093static const struct file_operations show_traces_fops = {
5094 .open = show_traces_open,
5095 .read = seq_read,
5096 .llseek = seq_lseek,
5097 .release = show_traces_release,
5098};
5099
5100static ssize_t
5101tracing_cpumask_read(struct file *filp, char __user *ubuf,
5102 size_t count, loff_t *ppos)
5103{
5104 struct trace_array *tr = file_inode(filp)->i_private;
5105 char *mask_str;
5106 int len;
5107
5108 len = snprintf(NULL, 0, "%*pb\n",
5109 cpumask_pr_args(tr->tracing_cpumask)) + 1;
5110 mask_str = kmalloc(len, GFP_KERNEL);
5111 if (!mask_str)
5112 return -ENOMEM;
5113
5114 len = snprintf(mask_str, len, "%*pb\n",
5115 cpumask_pr_args(tr->tracing_cpumask));
5116 if (len >= count) {
5117 count = -EINVAL;
5118 goto out_err;
5119 }
5120 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
5121
5122out_err:
5123 kfree(mask_str);
5124
5125 return count;
5126}
5127
5128int tracing_set_cpumask(struct trace_array *tr,
5129 cpumask_var_t tracing_cpumask_new)
5130{
5131 int cpu;
5132
5133 if (!tr)
5134 return -EINVAL;
5135
5136 local_irq_disable();
5137 arch_spin_lock(&tr->max_lock);
5138 for_each_tracing_cpu(cpu) {
5139 /*
5140 * Increase/decrease the disabled counter if we are
5141 * about to flip a bit in the cpumask:
5142 */
5143 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5144 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5145 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5146 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
5147 }
5148 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5149 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5150 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5151 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
5152 }
5153 }
5154 arch_spin_unlock(&tr->max_lock);
5155 local_irq_enable();
5156
5157 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
5158
5159 return 0;
5160}
5161
5162static ssize_t
5163tracing_cpumask_write(struct file *filp, const char __user *ubuf,
5164 size_t count, loff_t *ppos)
5165{
5166 struct trace_array *tr = file_inode(filp)->i_private;
5167 cpumask_var_t tracing_cpumask_new;
5168 int err;
5169
5170 if (!zalloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
5171 return -ENOMEM;
5172
5173 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
5174 if (err)
5175 goto err_free;
5176
5177 err = tracing_set_cpumask(tr, tracing_cpumask_new);
5178 if (err)
5179 goto err_free;
5180
5181 free_cpumask_var(tracing_cpumask_new);
5182
5183 return count;
5184
5185err_free:
5186 free_cpumask_var(tracing_cpumask_new);
5187
5188 return err;
5189}
5190
5191static const struct file_operations tracing_cpumask_fops = {
5192 .open = tracing_open_generic_tr,
5193 .read = tracing_cpumask_read,
5194 .write = tracing_cpumask_write,
5195 .release = tracing_release_generic_tr,
5196 .llseek = generic_file_llseek,
5197};
5198
5199static int tracing_trace_options_show(struct seq_file *m, void *v)
5200{
5201 struct tracer_opt *trace_opts;
5202 struct trace_array *tr = m->private;
5203 u32 tracer_flags;
5204 int i;
5205
5206 mutex_lock(&trace_types_lock);
5207 tracer_flags = tr->current_trace->flags->val;
5208 trace_opts = tr->current_trace->flags->opts;
5209
5210 for (i = 0; trace_options[i]; i++) {
5211 if (tr->trace_flags & (1 << i))
5212 seq_printf(m, "%s\n", trace_options[i]);
5213 else
5214 seq_printf(m, "no%s\n", trace_options[i]);
5215 }
5216
5217 for (i = 0; trace_opts[i].name; i++) {
5218 if (tracer_flags & trace_opts[i].bit)
5219 seq_printf(m, "%s\n", trace_opts[i].name);
5220 else
5221 seq_printf(m, "no%s\n", trace_opts[i].name);
5222 }
5223 mutex_unlock(&trace_types_lock);
5224
5225 return 0;
5226}
5227
5228static int __set_tracer_option(struct trace_array *tr,
5229 struct tracer_flags *tracer_flags,
5230 struct tracer_opt *opts, int neg)
5231{
5232 struct tracer *trace = tracer_flags->trace;
5233 int ret;
5234
5235 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
5236 if (ret)
5237 return ret;
5238
5239 if (neg)
5240 tracer_flags->val &= ~opts->bit;
5241 else
5242 tracer_flags->val |= opts->bit;
5243 return 0;
5244}
5245
5246/* Try to assign a tracer specific option */
5247static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
5248{
5249 struct tracer *trace = tr->current_trace;
5250 struct tracer_flags *tracer_flags = trace->flags;
5251 struct tracer_opt *opts = NULL;
5252 int i;
5253
5254 for (i = 0; tracer_flags->opts[i].name; i++) {
5255 opts = &tracer_flags->opts[i];
5256
5257 if (strcmp(cmp, opts->name) == 0)
5258 return __set_tracer_option(tr, trace->flags, opts, neg);
5259 }
5260
5261 return -EINVAL;
5262}
5263
5264/* Some tracers require overwrite to stay enabled */
5265int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
5266{
5267 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
5268 return -1;
5269
5270 return 0;
5271}
5272
5273int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
5274{
5275 int *map;
5276
5277 if ((mask == TRACE_ITER_RECORD_TGID) ||
5278 (mask == TRACE_ITER_RECORD_CMD))
5279 lockdep_assert_held(&event_mutex);
5280
5281 /* do nothing if flag is already set */
5282 if (!!(tr->trace_flags & mask) == !!enabled)
5283 return 0;
5284
5285 /* Give the tracer a chance to approve the change */
5286 if (tr->current_trace->flag_changed)
5287 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
5288 return -EINVAL;
5289
5290 if (enabled)
5291 tr->trace_flags |= mask;
5292 else
5293 tr->trace_flags &= ~mask;
5294
5295 if (mask == TRACE_ITER_RECORD_CMD)
5296 trace_event_enable_cmd_record(enabled);
5297
5298 if (mask == TRACE_ITER_RECORD_TGID) {
5299 if (!tgid_map) {
5300 tgid_map_max = pid_max;
5301 map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map),
5302 GFP_KERNEL);
5303
5304 /*
5305 * Pairs with smp_load_acquire() in
5306 * trace_find_tgid_ptr() to ensure that if it observes
5307 * the tgid_map we just allocated then it also observes
5308 * the corresponding tgid_map_max value.
5309 */
5310 smp_store_release(&tgid_map, map);
5311 }
5312 if (!tgid_map) {
5313 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
5314 return -ENOMEM;
5315 }
5316
5317 trace_event_enable_tgid_record(enabled);
5318 }
5319
5320 if (mask == TRACE_ITER_EVENT_FORK)
5321 trace_event_follow_fork(tr, enabled);
5322
5323 if (mask == TRACE_ITER_FUNC_FORK)
5324 ftrace_pid_follow_fork(tr, enabled);
5325
5326 if (mask == TRACE_ITER_OVERWRITE) {
5327 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
5328#ifdef CONFIG_TRACER_MAX_TRACE
5329 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
5330#endif
5331 }
5332
5333 if (mask == TRACE_ITER_PRINTK) {
5334 trace_printk_start_stop_comm(enabled);
5335 trace_printk_control(enabled);
5336 }
5337
5338 return 0;
5339}
5340
5341int trace_set_options(struct trace_array *tr, char *option)
5342{
5343 char *cmp;
5344 int neg = 0;
5345 int ret;
5346 size_t orig_len = strlen(option);
5347 int len;
5348
5349 cmp = strstrip(option);
5350
5351 len = str_has_prefix(cmp, "no");
5352 if (len)
5353 neg = 1;
5354
5355 cmp += len;
5356
5357 mutex_lock(&event_mutex);
5358 mutex_lock(&trace_types_lock);
5359
5360 ret = match_string(trace_options, -1, cmp);
5361 /* If no option could be set, test the specific tracer options */
5362 if (ret < 0)
5363 ret = set_tracer_option(tr, cmp, neg);
5364 else
5365 ret = set_tracer_flag(tr, 1 << ret, !neg);
5366
5367 mutex_unlock(&trace_types_lock);
5368 mutex_unlock(&event_mutex);
5369
5370 /*
5371 * If the first trailing whitespace is replaced with '\0' by strstrip,
5372 * turn it back into a space.
5373 */
5374 if (orig_len > strlen(option))
5375 option[strlen(option)] = ' ';
5376
5377 return ret;
5378}
5379
5380static void __init apply_trace_boot_options(void)
5381{
5382 char *buf = trace_boot_options_buf;
5383 char *option;
5384
5385 while (true) {
5386 option = strsep(&buf, ",");
5387
5388 if (!option)
5389 break;
5390
5391 if (*option)
5392 trace_set_options(&global_trace, option);
5393
5394 /* Put back the comma to allow this to be called again */
5395 if (buf)
5396 *(buf - 1) = ',';
5397 }
5398}
5399
5400static ssize_t
5401tracing_trace_options_write(struct file *filp, const char __user *ubuf,
5402 size_t cnt, loff_t *ppos)
5403{
5404 struct seq_file *m = filp->private_data;
5405 struct trace_array *tr = m->private;
5406 char buf[64];
5407 int ret;
5408
5409 if (cnt >= sizeof(buf))
5410 return -EINVAL;
5411
5412 if (copy_from_user(buf, ubuf, cnt))
5413 return -EFAULT;
5414
5415 buf[cnt] = 0;
5416
5417 ret = trace_set_options(tr, buf);
5418 if (ret < 0)
5419 return ret;
5420
5421 *ppos += cnt;
5422
5423 return cnt;
5424}
5425
5426static int tracing_trace_options_open(struct inode *inode, struct file *file)
5427{
5428 struct trace_array *tr = inode->i_private;
5429 int ret;
5430
5431 ret = tracing_check_open_get_tr(tr);
5432 if (ret)
5433 return ret;
5434
5435 ret = single_open(file, tracing_trace_options_show, inode->i_private);
5436 if (ret < 0)
5437 trace_array_put(tr);
5438
5439 return ret;
5440}
5441
5442static const struct file_operations tracing_iter_fops = {
5443 .open = tracing_trace_options_open,
5444 .read = seq_read,
5445 .llseek = seq_lseek,
5446 .release = tracing_single_release_tr,
5447 .write = tracing_trace_options_write,
5448};
5449
5450static const char readme_msg[] =
5451 "tracing mini-HOWTO:\n\n"
5452 "# echo 0 > tracing_on : quick way to disable tracing\n"
5453 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5454 " Important files:\n"
5455 " trace\t\t\t- The static contents of the buffer\n"
5456 "\t\t\t To clear the buffer write into this file: echo > trace\n"
5457 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5458 " current_tracer\t- function and latency tracers\n"
5459 " available_tracers\t- list of configured tracers for current_tracer\n"
5460 " error_log\t- error log for failed commands (that support it)\n"
5461 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
5462 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
5463 " trace_clock\t\t-change the clock used to order events\n"
5464 " local: Per cpu clock but may not be synced across CPUs\n"
5465 " global: Synced across CPUs but slows tracing down.\n"
5466 " counter: Not a clock, but just an increment\n"
5467 " uptime: Jiffy counter from time of boot\n"
5468 " perf: Same clock that perf events use\n"
5469#ifdef CONFIG_X86_64
5470 " x86-tsc: TSC cycle counter\n"
5471#endif
5472 "\n timestamp_mode\t-view the mode used to timestamp events\n"
5473 " delta: Delta difference against a buffer-wide timestamp\n"
5474 " absolute: Absolute (standalone) timestamp\n"
5475 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5476 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5477 " tracing_cpumask\t- Limit which CPUs to trace\n"
5478 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5479 "\t\t\t Remove sub-buffer with rmdir\n"
5480 " trace_options\t\t- Set format or modify how tracing happens\n"
5481 "\t\t\t Disable an option by prefixing 'no' to the\n"
5482 "\t\t\t option name\n"
5483 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5484#ifdef CONFIG_DYNAMIC_FTRACE
5485 "\n available_filter_functions - list of functions that can be filtered on\n"
5486 " set_ftrace_filter\t- echo function name in here to only trace these\n"
5487 "\t\t\t functions\n"
5488 "\t accepts: func_full_name or glob-matching-pattern\n"
5489 "\t modules: Can select a group via module\n"
5490 "\t Format: :mod:<module-name>\n"
5491 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
5492 "\t triggers: a command to perform when function is hit\n"
5493 "\t Format: <function>:<trigger>[:count]\n"
5494 "\t trigger: traceon, traceoff\n"
5495 "\t\t enable_event:<system>:<event>\n"
5496 "\t\t disable_event:<system>:<event>\n"
5497#ifdef CONFIG_STACKTRACE
5498 "\t\t stacktrace\n"
5499#endif
5500#ifdef CONFIG_TRACER_SNAPSHOT
5501 "\t\t snapshot\n"
5502#endif
5503 "\t\t dump\n"
5504 "\t\t cpudump\n"
5505 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
5506 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
5507 "\t The first one will disable tracing every time do_fault is hit\n"
5508 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
5509 "\t The first time do trap is hit and it disables tracing, the\n"
5510 "\t counter will decrement to 2. If tracing is already disabled,\n"
5511 "\t the counter will not decrement. It only decrements when the\n"
5512 "\t trigger did work\n"
5513 "\t To remove trigger without count:\n"
5514 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
5515 "\t To remove trigger with a count:\n"
5516 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5517 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
5518 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
5519 "\t modules: Can select a group via module command :mod:\n"
5520 "\t Does not accept triggers\n"
5521#endif /* CONFIG_DYNAMIC_FTRACE */
5522#ifdef CONFIG_FUNCTION_TRACER
5523 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5524 "\t\t (function)\n"
5525 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5526 "\t\t (function)\n"
5527#endif
5528#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5529 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5530 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5531 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5532#endif
5533#ifdef CONFIG_TRACER_SNAPSHOT
5534 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5535 "\t\t\t snapshot buffer. Read the contents for more\n"
5536 "\t\t\t information\n"
5537#endif
5538#ifdef CONFIG_STACK_TRACER
5539 " stack_trace\t\t- Shows the max stack trace when active\n"
5540 " stack_max_size\t- Shows current max stack size that was traced\n"
5541 "\t\t\t Write into this file to reset the max size (trigger a\n"
5542 "\t\t\t new trace)\n"
5543#ifdef CONFIG_DYNAMIC_FTRACE
5544 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5545 "\t\t\t traces\n"
5546#endif
5547#endif /* CONFIG_STACK_TRACER */
5548#ifdef CONFIG_DYNAMIC_EVENTS
5549 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5550 "\t\t\t Write into this file to define/undefine new trace events.\n"
5551#endif
5552#ifdef CONFIG_KPROBE_EVENTS
5553 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5554 "\t\t\t Write into this file to define/undefine new trace events.\n"
5555#endif
5556#ifdef CONFIG_UPROBE_EVENTS
5557 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5558 "\t\t\t Write into this file to define/undefine new trace events.\n"
5559#endif
5560#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
5561 "\t accepts: event-definitions (one definition per line)\n"
5562 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
5563 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
5564#ifdef CONFIG_HIST_TRIGGERS
5565 "\t s:[synthetic/]<event> <field> [<field>]\n"
5566#endif
5567 "\t e[:[<group>/]<event>] <attached-group>.<attached-event> [<args>]\n"
5568 "\t -:[<group>/]<event>\n"
5569#ifdef CONFIG_KPROBE_EVENTS
5570 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5571 "place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n"
5572#endif
5573#ifdef CONFIG_UPROBE_EVENTS
5574 " place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
5575#endif
5576 "\t args: <name>=fetcharg[:type]\n"
5577 "\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
5578#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5579 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5580#else
5581 "\t $stack<index>, $stack, $retval, $comm,\n"
5582#endif
5583 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5584 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
5585 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5586 "\t <type>\\[<array-size>\\]\n"
5587#ifdef CONFIG_HIST_TRIGGERS
5588 "\t field: <stype> <name>;\n"
5589 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5590 "\t [unsigned] char/int/long\n"
5591#endif
5592 "\t efield: For event probes ('e' types), the field is on of the fields\n"
5593 "\t of the <attached-group>/<attached-event>.\n"
5594#endif
5595 " events/\t\t- Directory containing all trace event subsystems:\n"
5596 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5597 " events/<system>/\t- Directory containing all trace events for <system>:\n"
5598 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5599 "\t\t\t events\n"
5600 " filter\t\t- If set, only events passing filter are traced\n"
5601 " events/<system>/<event>/\t- Directory containing control files for\n"
5602 "\t\t\t <event>:\n"
5603 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5604 " filter\t\t- If set, only events passing filter are traced\n"
5605 " trigger\t\t- If set, a command to perform when event is hit\n"
5606 "\t Format: <trigger>[:count][if <filter>]\n"
5607 "\t trigger: traceon, traceoff\n"
5608 "\t enable_event:<system>:<event>\n"
5609 "\t disable_event:<system>:<event>\n"
5610#ifdef CONFIG_HIST_TRIGGERS
5611 "\t enable_hist:<system>:<event>\n"
5612 "\t disable_hist:<system>:<event>\n"
5613#endif
5614#ifdef CONFIG_STACKTRACE
5615 "\t\t stacktrace\n"
5616#endif
5617#ifdef CONFIG_TRACER_SNAPSHOT
5618 "\t\t snapshot\n"
5619#endif
5620#ifdef CONFIG_HIST_TRIGGERS
5621 "\t\t hist (see below)\n"
5622#endif
5623 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
5624 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5625 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5626 "\t events/block/block_unplug/trigger\n"
5627 "\t The first disables tracing every time block_unplug is hit.\n"
5628 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
5629 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
5630 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5631 "\t Like function triggers, the counter is only decremented if it\n"
5632 "\t enabled or disabled tracing.\n"
5633 "\t To remove a trigger without a count:\n"
5634 "\t echo '!<trigger> > <system>/<event>/trigger\n"
5635 "\t To remove a trigger with a count:\n"
5636 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5637 "\t Filters can be ignored when removing a trigger.\n"
5638#ifdef CONFIG_HIST_TRIGGERS
5639 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
5640 "\t Format: hist:keys=<field1[,field2,...]>\n"
5641 "\t [:<var1>=<field|var_ref|numeric_literal>[,<var2>=...]]\n"
5642 "\t [:values=<field1[,field2,...]>]\n"
5643 "\t [:sort=<field1[,field2,...]>]\n"
5644 "\t [:size=#entries]\n"
5645 "\t [:pause][:continue][:clear]\n"
5646 "\t [:name=histname1]\n"
5647 "\t [:<handler>.<action>]\n"
5648 "\t [if <filter>]\n\n"
5649 "\t Note, special fields can be used as well:\n"
5650 "\t common_timestamp - to record current timestamp\n"
5651 "\t common_cpu - to record the CPU the event happened on\n"
5652 "\n"
5653 "\t A hist trigger variable can be:\n"
5654 "\t - a reference to a field e.g. x=current_timestamp,\n"
5655 "\t - a reference to another variable e.g. y=$x,\n"
5656 "\t - a numeric literal: e.g. ms_per_sec=1000,\n"
5657 "\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
5658 "\n"
5659 "\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
5660 "\t multiplication(*) and division(/) operators. An operand can be either a\n"
5661 "\t variable reference, field or numeric literal.\n"
5662 "\n"
5663 "\t When a matching event is hit, an entry is added to a hash\n"
5664 "\t table using the key(s) and value(s) named, and the value of a\n"
5665 "\t sum called 'hitcount' is incremented. Keys and values\n"
5666 "\t correspond to fields in the event's format description. Keys\n"
5667 "\t can be any field, or the special string 'stacktrace'.\n"
5668 "\t Compound keys consisting of up to two fields can be specified\n"
5669 "\t by the 'keys' keyword. Values must correspond to numeric\n"
5670 "\t fields. Sort keys consisting of up to two fields can be\n"
5671 "\t specified using the 'sort' keyword. The sort direction can\n"
5672 "\t be modified by appending '.descending' or '.ascending' to a\n"
5673 "\t sort field. The 'size' parameter can be used to specify more\n"
5674 "\t or fewer than the default 2048 entries for the hashtable size.\n"
5675 "\t If a hist trigger is given a name using the 'name' parameter,\n"
5676 "\t its histogram data will be shared with other triggers of the\n"
5677 "\t same name, and trigger hits will update this common data.\n\n"
5678 "\t Reading the 'hist' file for the event will dump the hash\n"
5679 "\t table in its entirety to stdout. If there are multiple hist\n"
5680 "\t triggers attached to an event, there will be a table for each\n"
5681 "\t trigger in the output. The table displayed for a named\n"
5682 "\t trigger will be the same as any other instance having the\n"
5683 "\t same name. The default format used to display a given field\n"
5684 "\t can be modified by appending any of the following modifiers\n"
5685 "\t to the field name, as applicable:\n\n"
5686 "\t .hex display a number as a hex value\n"
5687 "\t .sym display an address as a symbol\n"
5688 "\t .sym-offset display an address as a symbol and offset\n"
5689 "\t .execname display a common_pid as a program name\n"
5690 "\t .syscall display a syscall id as a syscall name\n"
5691 "\t .log2 display log2 value rather than raw number\n"
5692 "\t .buckets=size display values in groups of size rather than raw number\n"
5693 "\t .usecs display a common_timestamp in microseconds\n\n"
5694 "\t The 'pause' parameter can be used to pause an existing hist\n"
5695 "\t trigger or to start a hist trigger but not log any events\n"
5696 "\t until told to do so. 'continue' can be used to start or\n"
5697 "\t restart a paused hist trigger.\n\n"
5698 "\t The 'clear' parameter will clear the contents of a running\n"
5699 "\t hist trigger and leave its current paused/active state\n"
5700 "\t unchanged.\n\n"
5701 "\t The enable_hist and disable_hist triggers can be used to\n"
5702 "\t have one event conditionally start and stop another event's\n"
5703 "\t already-attached hist trigger. The syntax is analogous to\n"
5704 "\t the enable_event and disable_event triggers.\n\n"
5705 "\t Hist trigger handlers and actions are executed whenever a\n"
5706 "\t a histogram entry is added or updated. They take the form:\n\n"
5707 "\t <handler>.<action>\n\n"
5708 "\t The available handlers are:\n\n"
5709 "\t onmatch(matching.event) - invoke on addition or update\n"
5710 "\t onmax(var) - invoke if var exceeds current max\n"
5711 "\t onchange(var) - invoke action if var changes\n\n"
5712 "\t The available actions are:\n\n"
5713 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5714 "\t save(field,...) - save current event fields\n"
5715#ifdef CONFIG_TRACER_SNAPSHOT
5716 "\t snapshot() - snapshot the trace buffer\n\n"
5717#endif
5718#ifdef CONFIG_SYNTH_EVENTS
5719 " events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5720 "\t Write into this file to define/undefine new synthetic events.\n"
5721 "\t example: echo 'myevent u64 lat; char name[]' >> synthetic_events\n"
5722#endif
5723#endif
5724;
5725
5726static ssize_t
5727tracing_readme_read(struct file *filp, char __user *ubuf,
5728 size_t cnt, loff_t *ppos)
5729{
5730 return simple_read_from_buffer(ubuf, cnt, ppos,
5731 readme_msg, strlen(readme_msg));
5732}
5733
5734static const struct file_operations tracing_readme_fops = {
5735 .open = tracing_open_generic,
5736 .read = tracing_readme_read,
5737 .llseek = generic_file_llseek,
5738};
5739
5740static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
5741{
5742 int pid = ++(*pos);
5743
5744 return trace_find_tgid_ptr(pid);
5745}
5746
5747static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5748{
5749 int pid = *pos;
5750
5751 return trace_find_tgid_ptr(pid);
5752}
5753
5754static void saved_tgids_stop(struct seq_file *m, void *v)
5755{
5756}
5757
5758static int saved_tgids_show(struct seq_file *m, void *v)
5759{
5760 int *entry = (int *)v;
5761 int pid = entry - tgid_map;
5762 int tgid = *entry;
5763
5764 if (tgid == 0)
5765 return SEQ_SKIP;
5766
5767 seq_printf(m, "%d %d\n", pid, tgid);
5768 return 0;
5769}
5770
5771static const struct seq_operations tracing_saved_tgids_seq_ops = {
5772 .start = saved_tgids_start,
5773 .stop = saved_tgids_stop,
5774 .next = saved_tgids_next,
5775 .show = saved_tgids_show,
5776};
5777
5778static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5779{
5780 int ret;
5781
5782 ret = tracing_check_open_get_tr(NULL);
5783 if (ret)
5784 return ret;
5785
5786 return seq_open(filp, &tracing_saved_tgids_seq_ops);
5787}
5788
5789
5790static const struct file_operations tracing_saved_tgids_fops = {
5791 .open = tracing_saved_tgids_open,
5792 .read = seq_read,
5793 .llseek = seq_lseek,
5794 .release = seq_release,
5795};
5796
5797static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5798{
5799 unsigned int *ptr = v;
5800
5801 if (*pos || m->count)
5802 ptr++;
5803
5804 (*pos)++;
5805
5806 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5807 ptr++) {
5808 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5809 continue;
5810
5811 return ptr;
5812 }
5813
5814 return NULL;
5815}
5816
5817static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5818{
5819 void *v;
5820 loff_t l = 0;
5821
5822 preempt_disable();
5823 arch_spin_lock(&trace_cmdline_lock);
5824
5825 v = &savedcmd->map_cmdline_to_pid[0];
5826 while (l <= *pos) {
5827 v = saved_cmdlines_next(m, v, &l);
5828 if (!v)
5829 return NULL;
5830 }
5831
5832 return v;
5833}
5834
5835static void saved_cmdlines_stop(struct seq_file *m, void *v)
5836{
5837 arch_spin_unlock(&trace_cmdline_lock);
5838 preempt_enable();
5839}
5840
5841static int saved_cmdlines_show(struct seq_file *m, void *v)
5842{
5843 char buf[TASK_COMM_LEN];
5844 unsigned int *pid = v;
5845
5846 __trace_find_cmdline(*pid, buf);
5847 seq_printf(m, "%d %s\n", *pid, buf);
5848 return 0;
5849}
5850
5851static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5852 .start = saved_cmdlines_start,
5853 .next = saved_cmdlines_next,
5854 .stop = saved_cmdlines_stop,
5855 .show = saved_cmdlines_show,
5856};
5857
5858static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5859{
5860 int ret;
5861
5862 ret = tracing_check_open_get_tr(NULL);
5863 if (ret)
5864 return ret;
5865
5866 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5867}
5868
5869static const struct file_operations tracing_saved_cmdlines_fops = {
5870 .open = tracing_saved_cmdlines_open,
5871 .read = seq_read,
5872 .llseek = seq_lseek,
5873 .release = seq_release,
5874};
5875
5876static ssize_t
5877tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5878 size_t cnt, loff_t *ppos)
5879{
5880 char buf[64];
5881 int r;
5882
5883 arch_spin_lock(&trace_cmdline_lock);
5884 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5885 arch_spin_unlock(&trace_cmdline_lock);
5886
5887 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5888}
5889
5890static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5891{
5892 kfree(s->saved_cmdlines);
5893 kfree(s->map_cmdline_to_pid);
5894 kfree(s);
5895}
5896
5897static int tracing_resize_saved_cmdlines(unsigned int val)
5898{
5899 struct saved_cmdlines_buffer *s, *savedcmd_temp;
5900
5901 s = kmalloc(sizeof(*s), GFP_KERNEL);
5902 if (!s)
5903 return -ENOMEM;
5904
5905 if (allocate_cmdlines_buffer(val, s) < 0) {
5906 kfree(s);
5907 return -ENOMEM;
5908 }
5909
5910 arch_spin_lock(&trace_cmdline_lock);
5911 savedcmd_temp = savedcmd;
5912 savedcmd = s;
5913 arch_spin_unlock(&trace_cmdline_lock);
5914 free_saved_cmdlines_buffer(savedcmd_temp);
5915
5916 return 0;
5917}
5918
5919static ssize_t
5920tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5921 size_t cnt, loff_t *ppos)
5922{
5923 unsigned long val;
5924 int ret;
5925
5926 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5927 if (ret)
5928 return ret;
5929
5930 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5931 if (!val || val > PID_MAX_DEFAULT)
5932 return -EINVAL;
5933
5934 ret = tracing_resize_saved_cmdlines((unsigned int)val);
5935 if (ret < 0)
5936 return ret;
5937
5938 *ppos += cnt;
5939
5940 return cnt;
5941}
5942
5943static const struct file_operations tracing_saved_cmdlines_size_fops = {
5944 .open = tracing_open_generic,
5945 .read = tracing_saved_cmdlines_size_read,
5946 .write = tracing_saved_cmdlines_size_write,
5947};
5948
5949#ifdef CONFIG_TRACE_EVAL_MAP_FILE
5950static union trace_eval_map_item *
5951update_eval_map(union trace_eval_map_item *ptr)
5952{
5953 if (!ptr->map.eval_string) {
5954 if (ptr->tail.next) {
5955 ptr = ptr->tail.next;
5956 /* Set ptr to the next real item (skip head) */
5957 ptr++;
5958 } else
5959 return NULL;
5960 }
5961 return ptr;
5962}
5963
5964static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5965{
5966 union trace_eval_map_item *ptr = v;
5967
5968 /*
5969 * Paranoid! If ptr points to end, we don't want to increment past it.
5970 * This really should never happen.
5971 */
5972 (*pos)++;
5973 ptr = update_eval_map(ptr);
5974 if (WARN_ON_ONCE(!ptr))
5975 return NULL;
5976
5977 ptr++;
5978 ptr = update_eval_map(ptr);
5979
5980 return ptr;
5981}
5982
5983static void *eval_map_start(struct seq_file *m, loff_t *pos)
5984{
5985 union trace_eval_map_item *v;
5986 loff_t l = 0;
5987
5988 mutex_lock(&trace_eval_mutex);
5989
5990 v = trace_eval_maps;
5991 if (v)
5992 v++;
5993
5994 while (v && l < *pos) {
5995 v = eval_map_next(m, v, &l);
5996 }
5997
5998 return v;
5999}
6000
6001static void eval_map_stop(struct seq_file *m, void *v)
6002{
6003 mutex_unlock(&trace_eval_mutex);
6004}
6005
6006static int eval_map_show(struct seq_file *m, void *v)
6007{
6008 union trace_eval_map_item *ptr = v;
6009
6010 seq_printf(m, "%s %ld (%s)\n",
6011 ptr->map.eval_string, ptr->map.eval_value,
6012 ptr->map.system);
6013
6014 return 0;
6015}
6016
6017static const struct seq_operations tracing_eval_map_seq_ops = {
6018 .start = eval_map_start,
6019 .next = eval_map_next,
6020 .stop = eval_map_stop,
6021 .show = eval_map_show,
6022};
6023
6024static int tracing_eval_map_open(struct inode *inode, struct file *filp)
6025{
6026 int ret;
6027
6028 ret = tracing_check_open_get_tr(NULL);
6029 if (ret)
6030 return ret;
6031
6032 return seq_open(filp, &tracing_eval_map_seq_ops);
6033}
6034
6035static const struct file_operations tracing_eval_map_fops = {
6036 .open = tracing_eval_map_open,
6037 .read = seq_read,
6038 .llseek = seq_lseek,
6039 .release = seq_release,
6040};
6041
6042static inline union trace_eval_map_item *
6043trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
6044{
6045 /* Return tail of array given the head */
6046 return ptr + ptr->head.length + 1;
6047}
6048
6049static void
6050trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
6051 int len)
6052{
6053 struct trace_eval_map **stop;
6054 struct trace_eval_map **map;
6055 union trace_eval_map_item *map_array;
6056 union trace_eval_map_item *ptr;
6057
6058 stop = start + len;
6059
6060 /*
6061 * The trace_eval_maps contains the map plus a head and tail item,
6062 * where the head holds the module and length of array, and the
6063 * tail holds a pointer to the next list.
6064 */
6065 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
6066 if (!map_array) {
6067 pr_warn("Unable to allocate trace eval mapping\n");
6068 return;
6069 }
6070
6071 mutex_lock(&trace_eval_mutex);
6072
6073 if (!trace_eval_maps)
6074 trace_eval_maps = map_array;
6075 else {
6076 ptr = trace_eval_maps;
6077 for (;;) {
6078 ptr = trace_eval_jmp_to_tail(ptr);
6079 if (!ptr->tail.next)
6080 break;
6081 ptr = ptr->tail.next;
6082
6083 }
6084 ptr->tail.next = map_array;
6085 }
6086 map_array->head.mod = mod;
6087 map_array->head.length = len;
6088 map_array++;
6089
6090 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
6091 map_array->map = **map;
6092 map_array++;
6093 }
6094 memset(map_array, 0, sizeof(*map_array));
6095
6096 mutex_unlock(&trace_eval_mutex);
6097}
6098
6099static void trace_create_eval_file(struct dentry *d_tracer)
6100{
6101 trace_create_file("eval_map", TRACE_MODE_READ, d_tracer,
6102 NULL, &tracing_eval_map_fops);
6103}
6104
6105#else /* CONFIG_TRACE_EVAL_MAP_FILE */
6106static inline void trace_create_eval_file(struct dentry *d_tracer) { }
6107static inline void trace_insert_eval_map_file(struct module *mod,
6108 struct trace_eval_map **start, int len) { }
6109#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
6110
6111static void trace_insert_eval_map(struct module *mod,
6112 struct trace_eval_map **start, int len)
6113{
6114 struct trace_eval_map **map;
6115
6116 if (len <= 0)
6117 return;
6118
6119 map = start;
6120
6121 trace_event_eval_update(map, len);
6122
6123 trace_insert_eval_map_file(mod, start, len);
6124}
6125
6126static ssize_t
6127tracing_set_trace_read(struct file *filp, char __user *ubuf,
6128 size_t cnt, loff_t *ppos)
6129{
6130 struct trace_array *tr = filp->private_data;
6131 char buf[MAX_TRACER_SIZE+2];
6132 int r;
6133
6134 mutex_lock(&trace_types_lock);
6135 r = sprintf(buf, "%s\n", tr->current_trace->name);
6136 mutex_unlock(&trace_types_lock);
6137
6138 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6139}
6140
6141int tracer_init(struct tracer *t, struct trace_array *tr)
6142{
6143 tracing_reset_online_cpus(&tr->array_buffer);
6144 return t->init(tr);
6145}
6146
6147static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
6148{
6149 int cpu;
6150
6151 for_each_tracing_cpu(cpu)
6152 per_cpu_ptr(buf->data, cpu)->entries = val;
6153}
6154
6155#ifdef CONFIG_TRACER_MAX_TRACE
6156/* resize @tr's buffer to the size of @size_tr's entries */
6157static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
6158 struct array_buffer *size_buf, int cpu_id)
6159{
6160 int cpu, ret = 0;
6161
6162 if (cpu_id == RING_BUFFER_ALL_CPUS) {
6163 for_each_tracing_cpu(cpu) {
6164 ret = ring_buffer_resize(trace_buf->buffer,
6165 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
6166 if (ret < 0)
6167 break;
6168 per_cpu_ptr(trace_buf->data, cpu)->entries =
6169 per_cpu_ptr(size_buf->data, cpu)->entries;
6170 }
6171 } else {
6172 ret = ring_buffer_resize(trace_buf->buffer,
6173 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
6174 if (ret == 0)
6175 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
6176 per_cpu_ptr(size_buf->data, cpu_id)->entries;
6177 }
6178
6179 return ret;
6180}
6181#endif /* CONFIG_TRACER_MAX_TRACE */
6182
6183static int __tracing_resize_ring_buffer(struct trace_array *tr,
6184 unsigned long size, int cpu)
6185{
6186 int ret;
6187
6188 /*
6189 * If kernel or user changes the size of the ring buffer
6190 * we use the size that was given, and we can forget about
6191 * expanding it later.
6192 */
6193 ring_buffer_expanded = true;
6194
6195 /* May be called before buffers are initialized */
6196 if (!tr->array_buffer.buffer)
6197 return 0;
6198
6199 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
6200 if (ret < 0)
6201 return ret;
6202
6203#ifdef CONFIG_TRACER_MAX_TRACE
6204 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
6205 !tr->current_trace->use_max_tr)
6206 goto out;
6207
6208 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
6209 if (ret < 0) {
6210 int r = resize_buffer_duplicate_size(&tr->array_buffer,
6211 &tr->array_buffer, cpu);
6212 if (r < 0) {
6213 /*
6214 * AARGH! We are left with different
6215 * size max buffer!!!!
6216 * The max buffer is our "snapshot" buffer.
6217 * When a tracer needs a snapshot (one of the
6218 * latency tracers), it swaps the max buffer
6219 * with the saved snap shot. We succeeded to
6220 * update the size of the main buffer, but failed to
6221 * update the size of the max buffer. But when we tried
6222 * to reset the main buffer to the original size, we
6223 * failed there too. This is very unlikely to
6224 * happen, but if it does, warn and kill all
6225 * tracing.
6226 */
6227 WARN_ON(1);
6228 tracing_disabled = 1;
6229 }
6230 return ret;
6231 }
6232
6233 if (cpu == RING_BUFFER_ALL_CPUS)
6234 set_buffer_entries(&tr->max_buffer, size);
6235 else
6236 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
6237
6238 out:
6239#endif /* CONFIG_TRACER_MAX_TRACE */
6240
6241 if (cpu == RING_BUFFER_ALL_CPUS)
6242 set_buffer_entries(&tr->array_buffer, size);
6243 else
6244 per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size;
6245
6246 return ret;
6247}
6248
6249ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
6250 unsigned long size, int cpu_id)
6251{
6252 int ret;
6253
6254 mutex_lock(&trace_types_lock);
6255
6256 if (cpu_id != RING_BUFFER_ALL_CPUS) {
6257 /* make sure, this cpu is enabled in the mask */
6258 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
6259 ret = -EINVAL;
6260 goto out;
6261 }
6262 }
6263
6264 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
6265 if (ret < 0)
6266 ret = -ENOMEM;
6267
6268out:
6269 mutex_unlock(&trace_types_lock);
6270
6271 return ret;
6272}
6273
6274
6275/**
6276 * tracing_update_buffers - used by tracing facility to expand ring buffers
6277 *
6278 * To save on memory when the tracing is never used on a system with it
6279 * configured in. The ring buffers are set to a minimum size. But once
6280 * a user starts to use the tracing facility, then they need to grow
6281 * to their default size.
6282 *
6283 * This function is to be called when a tracer is about to be used.
6284 */
6285int tracing_update_buffers(void)
6286{
6287 int ret = 0;
6288
6289 mutex_lock(&trace_types_lock);
6290 if (!ring_buffer_expanded)
6291 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
6292 RING_BUFFER_ALL_CPUS);
6293 mutex_unlock(&trace_types_lock);
6294
6295 return ret;
6296}
6297
6298struct trace_option_dentry;
6299
6300static void
6301create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
6302
6303/*
6304 * Used to clear out the tracer before deletion of an instance.
6305 * Must have trace_types_lock held.
6306 */
6307static void tracing_set_nop(struct trace_array *tr)
6308{
6309 if (tr->current_trace == &nop_trace)
6310 return;
6311
6312 tr->current_trace->enabled--;
6313
6314 if (tr->current_trace->reset)
6315 tr->current_trace->reset(tr);
6316
6317 tr->current_trace = &nop_trace;
6318}
6319
6320static void add_tracer_options(struct trace_array *tr, struct tracer *t)
6321{
6322 /* Only enable if the directory has been created already. */
6323 if (!tr->dir)
6324 return;
6325
6326 create_trace_option_files(tr, t);
6327}
6328
6329int tracing_set_tracer(struct trace_array *tr, const char *buf)
6330{
6331 struct tracer *t;
6332#ifdef CONFIG_TRACER_MAX_TRACE
6333 bool had_max_tr;
6334#endif
6335 int ret = 0;
6336
6337 mutex_lock(&trace_types_lock);
6338
6339 if (!ring_buffer_expanded) {
6340 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
6341 RING_BUFFER_ALL_CPUS);
6342 if (ret < 0)
6343 goto out;
6344 ret = 0;
6345 }
6346
6347 for (t = trace_types; t; t = t->next) {
6348 if (strcmp(t->name, buf) == 0)
6349 break;
6350 }
6351 if (!t) {
6352 ret = -EINVAL;
6353 goto out;
6354 }
6355 if (t == tr->current_trace)
6356 goto out;
6357
6358#ifdef CONFIG_TRACER_SNAPSHOT
6359 if (t->use_max_tr) {
6360 arch_spin_lock(&tr->max_lock);
6361 if (tr->cond_snapshot)
6362 ret = -EBUSY;
6363 arch_spin_unlock(&tr->max_lock);
6364 if (ret)
6365 goto out;
6366 }
6367#endif
6368 /* Some tracers won't work on kernel command line */
6369 if (system_state < SYSTEM_RUNNING && t->noboot) {
6370 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
6371 t->name);
6372 goto out;
6373 }
6374
6375 /* Some tracers are only allowed for the top level buffer */
6376 if (!trace_ok_for_array(t, tr)) {
6377 ret = -EINVAL;
6378 goto out;
6379 }
6380
6381 /* If trace pipe files are being read, we can't change the tracer */
6382 if (tr->trace_ref) {
6383 ret = -EBUSY;
6384 goto out;
6385 }
6386
6387 trace_branch_disable();
6388
6389 tr->current_trace->enabled--;
6390
6391 if (tr->current_trace->reset)
6392 tr->current_trace->reset(tr);
6393
6394 /* Current trace needs to be nop_trace before synchronize_rcu */
6395 tr->current_trace = &nop_trace;
6396
6397#ifdef CONFIG_TRACER_MAX_TRACE
6398 had_max_tr = tr->allocated_snapshot;
6399
6400 if (had_max_tr && !t->use_max_tr) {
6401 /*
6402 * We need to make sure that the update_max_tr sees that
6403 * current_trace changed to nop_trace to keep it from
6404 * swapping the buffers after we resize it.
6405 * The update_max_tr is called from interrupts disabled
6406 * so a synchronized_sched() is sufficient.
6407 */
6408 synchronize_rcu();
6409 free_snapshot(tr);
6410 }
6411#endif
6412
6413#ifdef CONFIG_TRACER_MAX_TRACE
6414 if (t->use_max_tr && !had_max_tr) {
6415 ret = tracing_alloc_snapshot_instance(tr);
6416 if (ret < 0)
6417 goto out;
6418 }
6419#endif
6420
6421 if (t->init) {
6422 ret = tracer_init(t, tr);
6423 if (ret)
6424 goto out;
6425 }
6426
6427 tr->current_trace = t;
6428 tr->current_trace->enabled++;
6429 trace_branch_enable(tr);
6430 out:
6431 mutex_unlock(&trace_types_lock);
6432
6433 return ret;
6434}
6435
6436static ssize_t
6437tracing_set_trace_write(struct file *filp, const char __user *ubuf,
6438 size_t cnt, loff_t *ppos)
6439{
6440 struct trace_array *tr = filp->private_data;
6441 char buf[MAX_TRACER_SIZE+1];
6442 int i;
6443 size_t ret;
6444 int err;
6445
6446 ret = cnt;
6447
6448 if (cnt > MAX_TRACER_SIZE)
6449 cnt = MAX_TRACER_SIZE;
6450
6451 if (copy_from_user(buf, ubuf, cnt))
6452 return -EFAULT;
6453
6454 buf[cnt] = 0;
6455
6456 /* strip ending whitespace. */
6457 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
6458 buf[i] = 0;
6459
6460 err = tracing_set_tracer(tr, buf);
6461 if (err)
6462 return err;
6463
6464 *ppos += ret;
6465
6466 return ret;
6467}
6468
6469static ssize_t
6470tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
6471 size_t cnt, loff_t *ppos)
6472{
6473 char buf[64];
6474 int r;
6475
6476 r = snprintf(buf, sizeof(buf), "%ld\n",
6477 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
6478 if (r > sizeof(buf))
6479 r = sizeof(buf);
6480 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6481}
6482
6483static ssize_t
6484tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
6485 size_t cnt, loff_t *ppos)
6486{
6487 unsigned long val;
6488 int ret;
6489
6490 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6491 if (ret)
6492 return ret;
6493
6494 *ptr = val * 1000;
6495
6496 return cnt;
6497}
6498
6499static ssize_t
6500tracing_thresh_read(struct file *filp, char __user *ubuf,
6501 size_t cnt, loff_t *ppos)
6502{
6503 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
6504}
6505
6506static ssize_t
6507tracing_thresh_write(struct file *filp, const char __user *ubuf,
6508 size_t cnt, loff_t *ppos)
6509{
6510 struct trace_array *tr = filp->private_data;
6511 int ret;
6512
6513 mutex_lock(&trace_types_lock);
6514 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
6515 if (ret < 0)
6516 goto out;
6517
6518 if (tr->current_trace->update_thresh) {
6519 ret = tr->current_trace->update_thresh(tr);
6520 if (ret < 0)
6521 goto out;
6522 }
6523
6524 ret = cnt;
6525out:
6526 mutex_unlock(&trace_types_lock);
6527
6528 return ret;
6529}
6530
6531#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6532
6533static ssize_t
6534tracing_max_lat_read(struct file *filp, char __user *ubuf,
6535 size_t cnt, loff_t *ppos)
6536{
6537 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
6538}
6539
6540static ssize_t
6541tracing_max_lat_write(struct file *filp, const char __user *ubuf,
6542 size_t cnt, loff_t *ppos)
6543{
6544 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
6545}
6546
6547#endif
6548
6549static int tracing_open_pipe(struct inode *inode, struct file *filp)
6550{
6551 struct trace_array *tr = inode->i_private;
6552 struct trace_iterator *iter;
6553 int ret;
6554
6555 ret = tracing_check_open_get_tr(tr);
6556 if (ret)
6557 return ret;
6558
6559 mutex_lock(&trace_types_lock);
6560
6561 /* create a buffer to store the information to pass to userspace */
6562 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6563 if (!iter) {
6564 ret = -ENOMEM;
6565 __trace_array_put(tr);
6566 goto out;
6567 }
6568
6569 trace_seq_init(&iter->seq);
6570 iter->trace = tr->current_trace;
6571
6572 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
6573 ret = -ENOMEM;
6574 goto fail;
6575 }
6576
6577 /* trace pipe does not show start of buffer */
6578 cpumask_setall(iter->started);
6579
6580 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6581 iter->iter_flags |= TRACE_FILE_LAT_FMT;
6582
6583 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6584 if (trace_clocks[tr->clock_id].in_ns)
6585 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6586
6587 iter->tr = tr;
6588 iter->array_buffer = &tr->array_buffer;
6589 iter->cpu_file = tracing_get_cpu(inode);
6590 mutex_init(&iter->mutex);
6591 filp->private_data = iter;
6592
6593 if (iter->trace->pipe_open)
6594 iter->trace->pipe_open(iter);
6595
6596 nonseekable_open(inode, filp);
6597
6598 tr->trace_ref++;
6599out:
6600 mutex_unlock(&trace_types_lock);
6601 return ret;
6602
6603fail:
6604 kfree(iter);
6605 __trace_array_put(tr);
6606 mutex_unlock(&trace_types_lock);
6607 return ret;
6608}
6609
6610static int tracing_release_pipe(struct inode *inode, struct file *file)
6611{
6612 struct trace_iterator *iter = file->private_data;
6613 struct trace_array *tr = inode->i_private;
6614
6615 mutex_lock(&trace_types_lock);
6616
6617 tr->trace_ref--;
6618
6619 if (iter->trace->pipe_close)
6620 iter->trace->pipe_close(iter);
6621
6622 mutex_unlock(&trace_types_lock);
6623
6624 free_cpumask_var(iter->started);
6625 mutex_destroy(&iter->mutex);
6626 kfree(iter);
6627
6628 trace_array_put(tr);
6629
6630 return 0;
6631}
6632
6633static __poll_t
6634trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
6635{
6636 struct trace_array *tr = iter->tr;
6637
6638 /* Iterators are static, they should be filled or empty */
6639 if (trace_buffer_iter(iter, iter->cpu_file))
6640 return EPOLLIN | EPOLLRDNORM;
6641
6642 if (tr->trace_flags & TRACE_ITER_BLOCK)
6643 /*
6644 * Always select as readable when in blocking mode
6645 */
6646 return EPOLLIN | EPOLLRDNORM;
6647 else
6648 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
6649 filp, poll_table);
6650}
6651
6652static __poll_t
6653tracing_poll_pipe(struct file *filp, poll_table *poll_table)
6654{
6655 struct trace_iterator *iter = filp->private_data;
6656
6657 return trace_poll(iter, filp, poll_table);
6658}
6659
6660/* Must be called with iter->mutex held. */
6661static int tracing_wait_pipe(struct file *filp)
6662{
6663 struct trace_iterator *iter = filp->private_data;
6664 int ret;
6665
6666 while (trace_empty(iter)) {
6667
6668 if ((filp->f_flags & O_NONBLOCK)) {
6669 return -EAGAIN;
6670 }
6671
6672 /*
6673 * We block until we read something and tracing is disabled.
6674 * We still block if tracing is disabled, but we have never
6675 * read anything. This allows a user to cat this file, and
6676 * then enable tracing. But after we have read something,
6677 * we give an EOF when tracing is again disabled.
6678 *
6679 * iter->pos will be 0 if we haven't read anything.
6680 */
6681 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6682 break;
6683
6684 mutex_unlock(&iter->mutex);
6685
6686 ret = wait_on_pipe(iter, 0);
6687
6688 mutex_lock(&iter->mutex);
6689
6690 if (ret)
6691 return ret;
6692 }
6693
6694 return 1;
6695}
6696
6697/*
6698 * Consumer reader.
6699 */
6700static ssize_t
6701tracing_read_pipe(struct file *filp, char __user *ubuf,
6702 size_t cnt, loff_t *ppos)
6703{
6704 struct trace_iterator *iter = filp->private_data;
6705 ssize_t sret;
6706
6707 /*
6708 * Avoid more than one consumer on a single file descriptor
6709 * This is just a matter of traces coherency, the ring buffer itself
6710 * is protected.
6711 */
6712 mutex_lock(&iter->mutex);
6713
6714 /* return any leftover data */
6715 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6716 if (sret != -EBUSY)
6717 goto out;
6718
6719 trace_seq_init(&iter->seq);
6720
6721 if (iter->trace->read) {
6722 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6723 if (sret)
6724 goto out;
6725 }
6726
6727waitagain:
6728 sret = tracing_wait_pipe(filp);
6729 if (sret <= 0)
6730 goto out;
6731
6732 /* stop when tracing is finished */
6733 if (trace_empty(iter)) {
6734 sret = 0;
6735 goto out;
6736 }
6737
6738 if (cnt >= PAGE_SIZE)
6739 cnt = PAGE_SIZE - 1;
6740
6741 /* reset all but tr, trace, and overruns */
6742 trace_iterator_reset(iter);
6743 cpumask_clear(iter->started);
6744 trace_seq_init(&iter->seq);
6745
6746 trace_event_read_lock();
6747 trace_access_lock(iter->cpu_file);
6748 while (trace_find_next_entry_inc(iter) != NULL) {
6749 enum print_line_t ret;
6750 int save_len = iter->seq.seq.len;
6751
6752 ret = print_trace_line(iter);
6753 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6754 /* don't print partial lines */
6755 iter->seq.seq.len = save_len;
6756 break;
6757 }
6758 if (ret != TRACE_TYPE_NO_CONSUME)
6759 trace_consume(iter);
6760
6761 if (trace_seq_used(&iter->seq) >= cnt)
6762 break;
6763
6764 /*
6765 * Setting the full flag means we reached the trace_seq buffer
6766 * size and we should leave by partial output condition above.
6767 * One of the trace_seq_* functions is not used properly.
6768 */
6769 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6770 iter->ent->type);
6771 }
6772 trace_access_unlock(iter->cpu_file);
6773 trace_event_read_unlock();
6774
6775 /* Now copy what we have to the user */
6776 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6777 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6778 trace_seq_init(&iter->seq);
6779
6780 /*
6781 * If there was nothing to send to user, in spite of consuming trace
6782 * entries, go back to wait for more entries.
6783 */
6784 if (sret == -EBUSY)
6785 goto waitagain;
6786
6787out:
6788 mutex_unlock(&iter->mutex);
6789
6790 return sret;
6791}
6792
6793static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6794 unsigned int idx)
6795{
6796 __free_page(spd->pages[idx]);
6797}
6798
6799static size_t
6800tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6801{
6802 size_t count;
6803 int save_len;
6804 int ret;
6805
6806 /* Seq buffer is page-sized, exactly what we need. */
6807 for (;;) {
6808 save_len = iter->seq.seq.len;
6809 ret = print_trace_line(iter);
6810
6811 if (trace_seq_has_overflowed(&iter->seq)) {
6812 iter->seq.seq.len = save_len;
6813 break;
6814 }
6815
6816 /*
6817 * This should not be hit, because it should only
6818 * be set if the iter->seq overflowed. But check it
6819 * anyway to be safe.
6820 */
6821 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6822 iter->seq.seq.len = save_len;
6823 break;
6824 }
6825
6826 count = trace_seq_used(&iter->seq) - save_len;
6827 if (rem < count) {
6828 rem = 0;
6829 iter->seq.seq.len = save_len;
6830 break;
6831 }
6832
6833 if (ret != TRACE_TYPE_NO_CONSUME)
6834 trace_consume(iter);
6835 rem -= count;
6836 if (!trace_find_next_entry_inc(iter)) {
6837 rem = 0;
6838 iter->ent = NULL;
6839 break;
6840 }
6841 }
6842
6843 return rem;
6844}
6845
6846static ssize_t tracing_splice_read_pipe(struct file *filp,
6847 loff_t *ppos,
6848 struct pipe_inode_info *pipe,
6849 size_t len,
6850 unsigned int flags)
6851{
6852 struct page *pages_def[PIPE_DEF_BUFFERS];
6853 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6854 struct trace_iterator *iter = filp->private_data;
6855 struct splice_pipe_desc spd = {
6856 .pages = pages_def,
6857 .partial = partial_def,
6858 .nr_pages = 0, /* This gets updated below. */
6859 .nr_pages_max = PIPE_DEF_BUFFERS,
6860 .ops = &default_pipe_buf_ops,
6861 .spd_release = tracing_spd_release_pipe,
6862 };
6863 ssize_t ret;
6864 size_t rem;
6865 unsigned int i;
6866
6867 if (splice_grow_spd(pipe, &spd))
6868 return -ENOMEM;
6869
6870 mutex_lock(&iter->mutex);
6871
6872 if (iter->trace->splice_read) {
6873 ret = iter->trace->splice_read(iter, filp,
6874 ppos, pipe, len, flags);
6875 if (ret)
6876 goto out_err;
6877 }
6878
6879 ret = tracing_wait_pipe(filp);
6880 if (ret <= 0)
6881 goto out_err;
6882
6883 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6884 ret = -EFAULT;
6885 goto out_err;
6886 }
6887
6888 trace_event_read_lock();
6889 trace_access_lock(iter->cpu_file);
6890
6891 /* Fill as many pages as possible. */
6892 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6893 spd.pages[i] = alloc_page(GFP_KERNEL);
6894 if (!spd.pages[i])
6895 break;
6896
6897 rem = tracing_fill_pipe_page(rem, iter);
6898
6899 /* Copy the data into the page, so we can start over. */
6900 ret = trace_seq_to_buffer(&iter->seq,
6901 page_address(spd.pages[i]),
6902 trace_seq_used(&iter->seq));
6903 if (ret < 0) {
6904 __free_page(spd.pages[i]);
6905 break;
6906 }
6907 spd.partial[i].offset = 0;
6908 spd.partial[i].len = trace_seq_used(&iter->seq);
6909
6910 trace_seq_init(&iter->seq);
6911 }
6912
6913 trace_access_unlock(iter->cpu_file);
6914 trace_event_read_unlock();
6915 mutex_unlock(&iter->mutex);
6916
6917 spd.nr_pages = i;
6918
6919 if (i)
6920 ret = splice_to_pipe(pipe, &spd);
6921 else
6922 ret = 0;
6923out:
6924 splice_shrink_spd(&spd);
6925 return ret;
6926
6927out_err:
6928 mutex_unlock(&iter->mutex);
6929 goto out;
6930}
6931
6932static ssize_t
6933tracing_entries_read(struct file *filp, char __user *ubuf,
6934 size_t cnt, loff_t *ppos)
6935{
6936 struct inode *inode = file_inode(filp);
6937 struct trace_array *tr = inode->i_private;
6938 int cpu = tracing_get_cpu(inode);
6939 char buf[64];
6940 int r = 0;
6941 ssize_t ret;
6942
6943 mutex_lock(&trace_types_lock);
6944
6945 if (cpu == RING_BUFFER_ALL_CPUS) {
6946 int cpu, buf_size_same;
6947 unsigned long size;
6948
6949 size = 0;
6950 buf_size_same = 1;
6951 /* check if all cpu sizes are same */
6952 for_each_tracing_cpu(cpu) {
6953 /* fill in the size from first enabled cpu */
6954 if (size == 0)
6955 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
6956 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
6957 buf_size_same = 0;
6958 break;
6959 }
6960 }
6961
6962 if (buf_size_same) {
6963 if (!ring_buffer_expanded)
6964 r = sprintf(buf, "%lu (expanded: %lu)\n",
6965 size >> 10,
6966 trace_buf_size >> 10);
6967 else
6968 r = sprintf(buf, "%lu\n", size >> 10);
6969 } else
6970 r = sprintf(buf, "X\n");
6971 } else
6972 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
6973
6974 mutex_unlock(&trace_types_lock);
6975
6976 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6977 return ret;
6978}
6979
6980static ssize_t
6981tracing_entries_write(struct file *filp, const char __user *ubuf,
6982 size_t cnt, loff_t *ppos)
6983{
6984 struct inode *inode = file_inode(filp);
6985 struct trace_array *tr = inode->i_private;
6986 unsigned long val;
6987 int ret;
6988
6989 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6990 if (ret)
6991 return ret;
6992
6993 /* must have at least 1 entry */
6994 if (!val)
6995 return -EINVAL;
6996
6997 /* value is in KB */
6998 val <<= 10;
6999 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
7000 if (ret < 0)
7001 return ret;
7002
7003 *ppos += cnt;
7004
7005 return cnt;
7006}
7007
7008static ssize_t
7009tracing_total_entries_read(struct file *filp, char __user *ubuf,
7010 size_t cnt, loff_t *ppos)
7011{
7012 struct trace_array *tr = filp->private_data;
7013 char buf[64];
7014 int r, cpu;
7015 unsigned long size = 0, expanded_size = 0;
7016
7017 mutex_lock(&trace_types_lock);
7018 for_each_tracing_cpu(cpu) {
7019 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
7020 if (!ring_buffer_expanded)
7021 expanded_size += trace_buf_size >> 10;
7022 }
7023 if (ring_buffer_expanded)
7024 r = sprintf(buf, "%lu\n", size);
7025 else
7026 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
7027 mutex_unlock(&trace_types_lock);
7028
7029 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7030}
7031
7032static ssize_t
7033tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
7034 size_t cnt, loff_t *ppos)
7035{
7036 /*
7037 * There is no need to read what the user has written, this function
7038 * is just to make sure that there is no error when "echo" is used
7039 */
7040
7041 *ppos += cnt;
7042
7043 return cnt;
7044}
7045
7046static int
7047tracing_free_buffer_release(struct inode *inode, struct file *filp)
7048{
7049 struct trace_array *tr = inode->i_private;
7050
7051 /* disable tracing ? */
7052 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
7053 tracer_tracing_off(tr);
7054 /* resize the ring buffer to 0 */
7055 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
7056
7057 trace_array_put(tr);
7058
7059 return 0;
7060}
7061
7062static ssize_t
7063tracing_mark_write(struct file *filp, const char __user *ubuf,
7064 size_t cnt, loff_t *fpos)
7065{
7066 struct trace_array *tr = filp->private_data;
7067 struct ring_buffer_event *event;
7068 enum event_trigger_type tt = ETT_NONE;
7069 struct trace_buffer *buffer;
7070 struct print_entry *entry;
7071 ssize_t written;
7072 int size;
7073 int len;
7074
7075/* Used in tracing_mark_raw_write() as well */
7076#define FAULTED_STR "<faulted>"
7077#define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
7078
7079 if (tracing_disabled)
7080 return -EINVAL;
7081
7082 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7083 return -EINVAL;
7084
7085 if (cnt > TRACE_BUF_SIZE)
7086 cnt = TRACE_BUF_SIZE;
7087
7088 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7089
7090 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
7091
7092 /* If less than "<faulted>", then make sure we can still add that */
7093 if (cnt < FAULTED_SIZE)
7094 size += FAULTED_SIZE - cnt;
7095
7096 buffer = tr->array_buffer.buffer;
7097 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
7098 tracing_gen_ctx());
7099 if (unlikely(!event))
7100 /* Ring buffer disabled, return as if not open for write */
7101 return -EBADF;
7102
7103 entry = ring_buffer_event_data(event);
7104 entry->ip = _THIS_IP_;
7105
7106 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
7107 if (len) {
7108 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7109 cnt = FAULTED_SIZE;
7110 written = -EFAULT;
7111 } else
7112 written = cnt;
7113
7114 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
7115 /* do not add \n before testing triggers, but add \0 */
7116 entry->buf[cnt] = '\0';
7117 tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event);
7118 }
7119
7120 if (entry->buf[cnt - 1] != '\n') {
7121 entry->buf[cnt] = '\n';
7122 entry->buf[cnt + 1] = '\0';
7123 } else
7124 entry->buf[cnt] = '\0';
7125
7126 if (static_branch_unlikely(&trace_marker_exports_enabled))
7127 ftrace_exports(event, TRACE_EXPORT_MARKER);
7128 __buffer_unlock_commit(buffer, event);
7129
7130 if (tt)
7131 event_triggers_post_call(tr->trace_marker_file, tt);
7132
7133 return written;
7134}
7135
7136/* Limit it for now to 3K (including tag) */
7137#define RAW_DATA_MAX_SIZE (1024*3)
7138
7139static ssize_t
7140tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
7141 size_t cnt, loff_t *fpos)
7142{
7143 struct trace_array *tr = filp->private_data;
7144 struct ring_buffer_event *event;
7145 struct trace_buffer *buffer;
7146 struct raw_data_entry *entry;
7147 ssize_t written;
7148 int size;
7149 int len;
7150
7151#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
7152
7153 if (tracing_disabled)
7154 return -EINVAL;
7155
7156 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7157 return -EINVAL;
7158
7159 /* The marker must at least have a tag id */
7160 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
7161 return -EINVAL;
7162
7163 if (cnt > TRACE_BUF_SIZE)
7164 cnt = TRACE_BUF_SIZE;
7165
7166 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7167
7168 size = sizeof(*entry) + cnt;
7169 if (cnt < FAULT_SIZE_ID)
7170 size += FAULT_SIZE_ID - cnt;
7171
7172 buffer = tr->array_buffer.buffer;
7173 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
7174 tracing_gen_ctx());
7175 if (!event)
7176 /* Ring buffer disabled, return as if not open for write */
7177 return -EBADF;
7178
7179 entry = ring_buffer_event_data(event);
7180
7181 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
7182 if (len) {
7183 entry->id = -1;
7184 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7185 written = -EFAULT;
7186 } else
7187 written = cnt;
7188
7189 __buffer_unlock_commit(buffer, event);
7190
7191 return written;
7192}
7193
7194static int tracing_clock_show(struct seq_file *m, void *v)
7195{
7196 struct trace_array *tr = m->private;
7197 int i;
7198
7199 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
7200 seq_printf(m,
7201 "%s%s%s%s", i ? " " : "",
7202 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
7203 i == tr->clock_id ? "]" : "");
7204 seq_putc(m, '\n');
7205
7206 return 0;
7207}
7208
7209int tracing_set_clock(struct trace_array *tr, const char *clockstr)
7210{
7211 int i;
7212
7213 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
7214 if (strcmp(trace_clocks[i].name, clockstr) == 0)
7215 break;
7216 }
7217 if (i == ARRAY_SIZE(trace_clocks))
7218 return -EINVAL;
7219
7220 mutex_lock(&trace_types_lock);
7221
7222 tr->clock_id = i;
7223
7224 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
7225
7226 /*
7227 * New clock may not be consistent with the previous clock.
7228 * Reset the buffer so that it doesn't have incomparable timestamps.
7229 */
7230 tracing_reset_online_cpus(&tr->array_buffer);
7231
7232#ifdef CONFIG_TRACER_MAX_TRACE
7233 if (tr->max_buffer.buffer)
7234 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
7235 tracing_reset_online_cpus(&tr->max_buffer);
7236#endif
7237
7238 mutex_unlock(&trace_types_lock);
7239
7240 return 0;
7241}
7242
7243static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
7244 size_t cnt, loff_t *fpos)
7245{
7246 struct seq_file *m = filp->private_data;
7247 struct trace_array *tr = m->private;
7248 char buf[64];
7249 const char *clockstr;
7250 int ret;
7251
7252 if (cnt >= sizeof(buf))
7253 return -EINVAL;
7254
7255 if (copy_from_user(buf, ubuf, cnt))
7256 return -EFAULT;
7257
7258 buf[cnt] = 0;
7259
7260 clockstr = strstrip(buf);
7261
7262 ret = tracing_set_clock(tr, clockstr);
7263 if (ret)
7264 return ret;
7265
7266 *fpos += cnt;
7267
7268 return cnt;
7269}
7270
7271static int tracing_clock_open(struct inode *inode, struct file *file)
7272{
7273 struct trace_array *tr = inode->i_private;
7274 int ret;
7275
7276 ret = tracing_check_open_get_tr(tr);
7277 if (ret)
7278 return ret;
7279
7280 ret = single_open(file, tracing_clock_show, inode->i_private);
7281 if (ret < 0)
7282 trace_array_put(tr);
7283
7284 return ret;
7285}
7286
7287static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
7288{
7289 struct trace_array *tr = m->private;
7290
7291 mutex_lock(&trace_types_lock);
7292
7293 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
7294 seq_puts(m, "delta [absolute]\n");
7295 else
7296 seq_puts(m, "[delta] absolute\n");
7297
7298 mutex_unlock(&trace_types_lock);
7299
7300 return 0;
7301}
7302
7303static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
7304{
7305 struct trace_array *tr = inode->i_private;
7306 int ret;
7307
7308 ret = tracing_check_open_get_tr(tr);
7309 if (ret)
7310 return ret;
7311
7312 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
7313 if (ret < 0)
7314 trace_array_put(tr);
7315
7316 return ret;
7317}
7318
7319u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_event *rbe)
7320{
7321 if (rbe == this_cpu_read(trace_buffered_event))
7322 return ring_buffer_time_stamp(buffer);
7323
7324 return ring_buffer_event_time_stamp(buffer, rbe);
7325}
7326
7327/*
7328 * Set or disable using the per CPU trace_buffer_event when possible.
7329 */
7330int tracing_set_filter_buffering(struct trace_array *tr, bool set)
7331{
7332 int ret = 0;
7333
7334 mutex_lock(&trace_types_lock);
7335
7336 if (set && tr->no_filter_buffering_ref++)
7337 goto out;
7338
7339 if (!set) {
7340 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) {
7341 ret = -EINVAL;
7342 goto out;
7343 }
7344
7345 --tr->no_filter_buffering_ref;
7346 }
7347 out:
7348 mutex_unlock(&trace_types_lock);
7349
7350 return ret;
7351}
7352
7353struct ftrace_buffer_info {
7354 struct trace_iterator iter;
7355 void *spare;
7356 unsigned int spare_cpu;
7357 unsigned int read;
7358};
7359
7360#ifdef CONFIG_TRACER_SNAPSHOT
7361static int tracing_snapshot_open(struct inode *inode, struct file *file)
7362{
7363 struct trace_array *tr = inode->i_private;
7364 struct trace_iterator *iter;
7365 struct seq_file *m;
7366 int ret;
7367
7368 ret = tracing_check_open_get_tr(tr);
7369 if (ret)
7370 return ret;
7371
7372 if (file->f_mode & FMODE_READ) {
7373 iter = __tracing_open(inode, file, true);
7374 if (IS_ERR(iter))
7375 ret = PTR_ERR(iter);
7376 } else {
7377 /* Writes still need the seq_file to hold the private data */
7378 ret = -ENOMEM;
7379 m = kzalloc(sizeof(*m), GFP_KERNEL);
7380 if (!m)
7381 goto out;
7382 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
7383 if (!iter) {
7384 kfree(m);
7385 goto out;
7386 }
7387 ret = 0;
7388
7389 iter->tr = tr;
7390 iter->array_buffer = &tr->max_buffer;
7391 iter->cpu_file = tracing_get_cpu(inode);
7392 m->private = iter;
7393 file->private_data = m;
7394 }
7395out:
7396 if (ret < 0)
7397 trace_array_put(tr);
7398
7399 return ret;
7400}
7401
7402static ssize_t
7403tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
7404 loff_t *ppos)
7405{
7406 struct seq_file *m = filp->private_data;
7407 struct trace_iterator *iter = m->private;
7408 struct trace_array *tr = iter->tr;
7409 unsigned long val;
7410 int ret;
7411
7412 ret = tracing_update_buffers();
7413 if (ret < 0)
7414 return ret;
7415
7416 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7417 if (ret)
7418 return ret;
7419
7420 mutex_lock(&trace_types_lock);
7421
7422 if (tr->current_trace->use_max_tr) {
7423 ret = -EBUSY;
7424 goto out;
7425 }
7426
7427 arch_spin_lock(&tr->max_lock);
7428 if (tr->cond_snapshot)
7429 ret = -EBUSY;
7430 arch_spin_unlock(&tr->max_lock);
7431 if (ret)
7432 goto out;
7433
7434 switch (val) {
7435 case 0:
7436 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7437 ret = -EINVAL;
7438 break;
7439 }
7440 if (tr->allocated_snapshot)
7441 free_snapshot(tr);
7442 break;
7443 case 1:
7444/* Only allow per-cpu swap if the ring buffer supports it */
7445#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
7446 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7447 ret = -EINVAL;
7448 break;
7449 }
7450#endif
7451 if (tr->allocated_snapshot)
7452 ret = resize_buffer_duplicate_size(&tr->max_buffer,
7453 &tr->array_buffer, iter->cpu_file);
7454 else
7455 ret = tracing_alloc_snapshot_instance(tr);
7456 if (ret < 0)
7457 break;
7458 local_irq_disable();
7459 /* Now, we're going to swap */
7460 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7461 update_max_tr(tr, current, smp_processor_id(), NULL);
7462 else
7463 update_max_tr_single(tr, current, iter->cpu_file);
7464 local_irq_enable();
7465 break;
7466 default:
7467 if (tr->allocated_snapshot) {
7468 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7469 tracing_reset_online_cpus(&tr->max_buffer);
7470 else
7471 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
7472 }
7473 break;
7474 }
7475
7476 if (ret >= 0) {
7477 *ppos += cnt;
7478 ret = cnt;
7479 }
7480out:
7481 mutex_unlock(&trace_types_lock);
7482 return ret;
7483}
7484
7485static int tracing_snapshot_release(struct inode *inode, struct file *file)
7486{
7487 struct seq_file *m = file->private_data;
7488 int ret;
7489
7490 ret = tracing_release(inode, file);
7491
7492 if (file->f_mode & FMODE_READ)
7493 return ret;
7494
7495 /* If write only, the seq_file is just a stub */
7496 if (m)
7497 kfree(m->private);
7498 kfree(m);
7499
7500 return 0;
7501}
7502
7503static int tracing_buffers_open(struct inode *inode, struct file *filp);
7504static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7505 size_t count, loff_t *ppos);
7506static int tracing_buffers_release(struct inode *inode, struct file *file);
7507static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7508 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7509
7510static int snapshot_raw_open(struct inode *inode, struct file *filp)
7511{
7512 struct ftrace_buffer_info *info;
7513 int ret;
7514
7515 /* The following checks for tracefs lockdown */
7516 ret = tracing_buffers_open(inode, filp);
7517 if (ret < 0)
7518 return ret;
7519
7520 info = filp->private_data;
7521
7522 if (info->iter.trace->use_max_tr) {
7523 tracing_buffers_release(inode, filp);
7524 return -EBUSY;
7525 }
7526
7527 info->iter.snapshot = true;
7528 info->iter.array_buffer = &info->iter.tr->max_buffer;
7529
7530 return ret;
7531}
7532
7533#endif /* CONFIG_TRACER_SNAPSHOT */
7534
7535
7536static const struct file_operations tracing_thresh_fops = {
7537 .open = tracing_open_generic,
7538 .read = tracing_thresh_read,
7539 .write = tracing_thresh_write,
7540 .llseek = generic_file_llseek,
7541};
7542
7543#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7544static const struct file_operations tracing_max_lat_fops = {
7545 .open = tracing_open_generic,
7546 .read = tracing_max_lat_read,
7547 .write = tracing_max_lat_write,
7548 .llseek = generic_file_llseek,
7549};
7550#endif
7551
7552static const struct file_operations set_tracer_fops = {
7553 .open = tracing_open_generic,
7554 .read = tracing_set_trace_read,
7555 .write = tracing_set_trace_write,
7556 .llseek = generic_file_llseek,
7557};
7558
7559static const struct file_operations tracing_pipe_fops = {
7560 .open = tracing_open_pipe,
7561 .poll = tracing_poll_pipe,
7562 .read = tracing_read_pipe,
7563 .splice_read = tracing_splice_read_pipe,
7564 .release = tracing_release_pipe,
7565 .llseek = no_llseek,
7566};
7567
7568static const struct file_operations tracing_entries_fops = {
7569 .open = tracing_open_generic_tr,
7570 .read = tracing_entries_read,
7571 .write = tracing_entries_write,
7572 .llseek = generic_file_llseek,
7573 .release = tracing_release_generic_tr,
7574};
7575
7576static const struct file_operations tracing_total_entries_fops = {
7577 .open = tracing_open_generic_tr,
7578 .read = tracing_total_entries_read,
7579 .llseek = generic_file_llseek,
7580 .release = tracing_release_generic_tr,
7581};
7582
7583static const struct file_operations tracing_free_buffer_fops = {
7584 .open = tracing_open_generic_tr,
7585 .write = tracing_free_buffer_write,
7586 .release = tracing_free_buffer_release,
7587};
7588
7589static const struct file_operations tracing_mark_fops = {
7590 .open = tracing_mark_open,
7591 .write = tracing_mark_write,
7592 .release = tracing_release_generic_tr,
7593};
7594
7595static const struct file_operations tracing_mark_raw_fops = {
7596 .open = tracing_mark_open,
7597 .write = tracing_mark_raw_write,
7598 .release = tracing_release_generic_tr,
7599};
7600
7601static const struct file_operations trace_clock_fops = {
7602 .open = tracing_clock_open,
7603 .read = seq_read,
7604 .llseek = seq_lseek,
7605 .release = tracing_single_release_tr,
7606 .write = tracing_clock_write,
7607};
7608
7609static const struct file_operations trace_time_stamp_mode_fops = {
7610 .open = tracing_time_stamp_mode_open,
7611 .read = seq_read,
7612 .llseek = seq_lseek,
7613 .release = tracing_single_release_tr,
7614};
7615
7616#ifdef CONFIG_TRACER_SNAPSHOT
7617static const struct file_operations snapshot_fops = {
7618 .open = tracing_snapshot_open,
7619 .read = seq_read,
7620 .write = tracing_snapshot_write,
7621 .llseek = tracing_lseek,
7622 .release = tracing_snapshot_release,
7623};
7624
7625static const struct file_operations snapshot_raw_fops = {
7626 .open = snapshot_raw_open,
7627 .read = tracing_buffers_read,
7628 .release = tracing_buffers_release,
7629 .splice_read = tracing_buffers_splice_read,
7630 .llseek = no_llseek,
7631};
7632
7633#endif /* CONFIG_TRACER_SNAPSHOT */
7634
7635/*
7636 * trace_min_max_write - Write a u64 value to a trace_min_max_param struct
7637 * @filp: The active open file structure
7638 * @ubuf: The userspace provided buffer to read value into
7639 * @cnt: The maximum number of bytes to read
7640 * @ppos: The current "file" position
7641 *
7642 * This function implements the write interface for a struct trace_min_max_param.
7643 * The filp->private_data must point to a trace_min_max_param structure that
7644 * defines where to write the value, the min and the max acceptable values,
7645 * and a lock to protect the write.
7646 */
7647static ssize_t
7648trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
7649{
7650 struct trace_min_max_param *param = filp->private_data;
7651 u64 val;
7652 int err;
7653
7654 if (!param)
7655 return -EFAULT;
7656
7657 err = kstrtoull_from_user(ubuf, cnt, 10, &val);
7658 if (err)
7659 return err;
7660
7661 if (param->lock)
7662 mutex_lock(param->lock);
7663
7664 if (param->min && val < *param->min)
7665 err = -EINVAL;
7666
7667 if (param->max && val > *param->max)
7668 err = -EINVAL;
7669
7670 if (!err)
7671 *param->val = val;
7672
7673 if (param->lock)
7674 mutex_unlock(param->lock);
7675
7676 if (err)
7677 return err;
7678
7679 return cnt;
7680}
7681
7682/*
7683 * trace_min_max_read - Read a u64 value from a trace_min_max_param struct
7684 * @filp: The active open file structure
7685 * @ubuf: The userspace provided buffer to read value into
7686 * @cnt: The maximum number of bytes to read
7687 * @ppos: The current "file" position
7688 *
7689 * This function implements the read interface for a struct trace_min_max_param.
7690 * The filp->private_data must point to a trace_min_max_param struct with valid
7691 * data.
7692 */
7693static ssize_t
7694trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
7695{
7696 struct trace_min_max_param *param = filp->private_data;
7697 char buf[U64_STR_SIZE];
7698 int len;
7699 u64 val;
7700
7701 if (!param)
7702 return -EFAULT;
7703
7704 val = *param->val;
7705
7706 if (cnt > sizeof(buf))
7707 cnt = sizeof(buf);
7708
7709 len = snprintf(buf, sizeof(buf), "%llu\n", val);
7710
7711 return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
7712}
7713
7714const struct file_operations trace_min_max_fops = {
7715 .open = tracing_open_generic,
7716 .read = trace_min_max_read,
7717 .write = trace_min_max_write,
7718};
7719
7720#define TRACING_LOG_ERRS_MAX 8
7721#define TRACING_LOG_LOC_MAX 128
7722
7723#define CMD_PREFIX " Command: "
7724
7725struct err_info {
7726 const char **errs; /* ptr to loc-specific array of err strings */
7727 u8 type; /* index into errs -> specific err string */
7728 u8 pos; /* MAX_FILTER_STR_VAL = 256 */
7729 u64 ts;
7730};
7731
7732struct tracing_log_err {
7733 struct list_head list;
7734 struct err_info info;
7735 char loc[TRACING_LOG_LOC_MAX]; /* err location */
7736 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
7737};
7738
7739static DEFINE_MUTEX(tracing_err_log_lock);
7740
7741static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
7742{
7743 struct tracing_log_err *err;
7744
7745 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7746 err = kzalloc(sizeof(*err), GFP_KERNEL);
7747 if (!err)
7748 err = ERR_PTR(-ENOMEM);
7749 else
7750 tr->n_err_log_entries++;
7751
7752 return err;
7753 }
7754
7755 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7756 list_del(&err->list);
7757
7758 return err;
7759}
7760
7761/**
7762 * err_pos - find the position of a string within a command for error careting
7763 * @cmd: The tracing command that caused the error
7764 * @str: The string to position the caret at within @cmd
7765 *
7766 * Finds the position of the first occurrence of @str within @cmd. The
7767 * return value can be passed to tracing_log_err() for caret placement
7768 * within @cmd.
7769 *
7770 * Returns the index within @cmd of the first occurrence of @str or 0
7771 * if @str was not found.
7772 */
7773unsigned int err_pos(char *cmd, const char *str)
7774{
7775 char *found;
7776
7777 if (WARN_ON(!strlen(cmd)))
7778 return 0;
7779
7780 found = strstr(cmd, str);
7781 if (found)
7782 return found - cmd;
7783
7784 return 0;
7785}
7786
7787/**
7788 * tracing_log_err - write an error to the tracing error log
7789 * @tr: The associated trace array for the error (NULL for top level array)
7790 * @loc: A string describing where the error occurred
7791 * @cmd: The tracing command that caused the error
7792 * @errs: The array of loc-specific static error strings
7793 * @type: The index into errs[], which produces the specific static err string
7794 * @pos: The position the caret should be placed in the cmd
7795 *
7796 * Writes an error into tracing/error_log of the form:
7797 *
7798 * <loc>: error: <text>
7799 * Command: <cmd>
7800 * ^
7801 *
7802 * tracing/error_log is a small log file containing the last
7803 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7804 * unless there has been a tracing error, and the error log can be
7805 * cleared and have its memory freed by writing the empty string in
7806 * truncation mode to it i.e. echo > tracing/error_log.
7807 *
7808 * NOTE: the @errs array along with the @type param are used to
7809 * produce a static error string - this string is not copied and saved
7810 * when the error is logged - only a pointer to it is saved. See
7811 * existing callers for examples of how static strings are typically
7812 * defined for use with tracing_log_err().
7813 */
7814void tracing_log_err(struct trace_array *tr,
7815 const char *loc, const char *cmd,
7816 const char **errs, u8 type, u8 pos)
7817{
7818 struct tracing_log_err *err;
7819
7820 if (!tr)
7821 tr = &global_trace;
7822
7823 mutex_lock(&tracing_err_log_lock);
7824 err = get_tracing_log_err(tr);
7825 if (PTR_ERR(err) == -ENOMEM) {
7826 mutex_unlock(&tracing_err_log_lock);
7827 return;
7828 }
7829
7830 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7831 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
7832
7833 err->info.errs = errs;
7834 err->info.type = type;
7835 err->info.pos = pos;
7836 err->info.ts = local_clock();
7837
7838 list_add_tail(&err->list, &tr->err_log);
7839 mutex_unlock(&tracing_err_log_lock);
7840}
7841
7842static void clear_tracing_err_log(struct trace_array *tr)
7843{
7844 struct tracing_log_err *err, *next;
7845
7846 mutex_lock(&tracing_err_log_lock);
7847 list_for_each_entry_safe(err, next, &tr->err_log, list) {
7848 list_del(&err->list);
7849 kfree(err);
7850 }
7851
7852 tr->n_err_log_entries = 0;
7853 mutex_unlock(&tracing_err_log_lock);
7854}
7855
7856static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7857{
7858 struct trace_array *tr = m->private;
7859
7860 mutex_lock(&tracing_err_log_lock);
7861
7862 return seq_list_start(&tr->err_log, *pos);
7863}
7864
7865static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7866{
7867 struct trace_array *tr = m->private;
7868
7869 return seq_list_next(v, &tr->err_log, pos);
7870}
7871
7872static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7873{
7874 mutex_unlock(&tracing_err_log_lock);
7875}
7876
7877static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
7878{
7879 u8 i;
7880
7881 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7882 seq_putc(m, ' ');
7883 for (i = 0; i < pos; i++)
7884 seq_putc(m, ' ');
7885 seq_puts(m, "^\n");
7886}
7887
7888static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7889{
7890 struct tracing_log_err *err = v;
7891
7892 if (err) {
7893 const char *err_text = err->info.errs[err->info.type];
7894 u64 sec = err->info.ts;
7895 u32 nsec;
7896
7897 nsec = do_div(sec, NSEC_PER_SEC);
7898 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7899 err->loc, err_text);
7900 seq_printf(m, "%s", err->cmd);
7901 tracing_err_log_show_pos(m, err->info.pos);
7902 }
7903
7904 return 0;
7905}
7906
7907static const struct seq_operations tracing_err_log_seq_ops = {
7908 .start = tracing_err_log_seq_start,
7909 .next = tracing_err_log_seq_next,
7910 .stop = tracing_err_log_seq_stop,
7911 .show = tracing_err_log_seq_show
7912};
7913
7914static int tracing_err_log_open(struct inode *inode, struct file *file)
7915{
7916 struct trace_array *tr = inode->i_private;
7917 int ret = 0;
7918
7919 ret = tracing_check_open_get_tr(tr);
7920 if (ret)
7921 return ret;
7922
7923 /* If this file was opened for write, then erase contents */
7924 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7925 clear_tracing_err_log(tr);
7926
7927 if (file->f_mode & FMODE_READ) {
7928 ret = seq_open(file, &tracing_err_log_seq_ops);
7929 if (!ret) {
7930 struct seq_file *m = file->private_data;
7931 m->private = tr;
7932 } else {
7933 trace_array_put(tr);
7934 }
7935 }
7936 return ret;
7937}
7938
7939static ssize_t tracing_err_log_write(struct file *file,
7940 const char __user *buffer,
7941 size_t count, loff_t *ppos)
7942{
7943 return count;
7944}
7945
7946static int tracing_err_log_release(struct inode *inode, struct file *file)
7947{
7948 struct trace_array *tr = inode->i_private;
7949
7950 trace_array_put(tr);
7951
7952 if (file->f_mode & FMODE_READ)
7953 seq_release(inode, file);
7954
7955 return 0;
7956}
7957
7958static const struct file_operations tracing_err_log_fops = {
7959 .open = tracing_err_log_open,
7960 .write = tracing_err_log_write,
7961 .read = seq_read,
7962 .llseek = seq_lseek,
7963 .release = tracing_err_log_release,
7964};
7965
7966static int tracing_buffers_open(struct inode *inode, struct file *filp)
7967{
7968 struct trace_array *tr = inode->i_private;
7969 struct ftrace_buffer_info *info;
7970 int ret;
7971
7972 ret = tracing_check_open_get_tr(tr);
7973 if (ret)
7974 return ret;
7975
7976 info = kvzalloc(sizeof(*info), GFP_KERNEL);
7977 if (!info) {
7978 trace_array_put(tr);
7979 return -ENOMEM;
7980 }
7981
7982 mutex_lock(&trace_types_lock);
7983
7984 info->iter.tr = tr;
7985 info->iter.cpu_file = tracing_get_cpu(inode);
7986 info->iter.trace = tr->current_trace;
7987 info->iter.array_buffer = &tr->array_buffer;
7988 info->spare = NULL;
7989 /* Force reading ring buffer for first read */
7990 info->read = (unsigned int)-1;
7991
7992 filp->private_data = info;
7993
7994 tr->trace_ref++;
7995
7996 mutex_unlock(&trace_types_lock);
7997
7998 ret = nonseekable_open(inode, filp);
7999 if (ret < 0)
8000 trace_array_put(tr);
8001
8002 return ret;
8003}
8004
8005static __poll_t
8006tracing_buffers_poll(struct file *filp, poll_table *poll_table)
8007{
8008 struct ftrace_buffer_info *info = filp->private_data;
8009 struct trace_iterator *iter = &info->iter;
8010
8011 return trace_poll(iter, filp, poll_table);
8012}
8013
8014static ssize_t
8015tracing_buffers_read(struct file *filp, char __user *ubuf,
8016 size_t count, loff_t *ppos)
8017{
8018 struct ftrace_buffer_info *info = filp->private_data;
8019 struct trace_iterator *iter = &info->iter;
8020 ssize_t ret = 0;
8021 ssize_t size;
8022
8023 if (!count)
8024 return 0;
8025
8026#ifdef CONFIG_TRACER_MAX_TRACE
8027 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8028 return -EBUSY;
8029#endif
8030
8031 if (!info->spare) {
8032 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
8033 iter->cpu_file);
8034 if (IS_ERR(info->spare)) {
8035 ret = PTR_ERR(info->spare);
8036 info->spare = NULL;
8037 } else {
8038 info->spare_cpu = iter->cpu_file;
8039 }
8040 }
8041 if (!info->spare)
8042 return ret;
8043
8044 /* Do we have previous read data to read? */
8045 if (info->read < PAGE_SIZE)
8046 goto read;
8047
8048 again:
8049 trace_access_lock(iter->cpu_file);
8050 ret = ring_buffer_read_page(iter->array_buffer->buffer,
8051 &info->spare,
8052 count,
8053 iter->cpu_file, 0);
8054 trace_access_unlock(iter->cpu_file);
8055
8056 if (ret < 0) {
8057 if (trace_empty(iter)) {
8058 if ((filp->f_flags & O_NONBLOCK))
8059 return -EAGAIN;
8060
8061 ret = wait_on_pipe(iter, 0);
8062 if (ret)
8063 return ret;
8064
8065 goto again;
8066 }
8067 return 0;
8068 }
8069
8070 info->read = 0;
8071 read:
8072 size = PAGE_SIZE - info->read;
8073 if (size > count)
8074 size = count;
8075
8076 ret = copy_to_user(ubuf, info->spare + info->read, size);
8077 if (ret == size)
8078 return -EFAULT;
8079
8080 size -= ret;
8081
8082 *ppos += size;
8083 info->read += size;
8084
8085 return size;
8086}
8087
8088static int tracing_buffers_release(struct inode *inode, struct file *file)
8089{
8090 struct ftrace_buffer_info *info = file->private_data;
8091 struct trace_iterator *iter = &info->iter;
8092
8093 mutex_lock(&trace_types_lock);
8094
8095 iter->tr->trace_ref--;
8096
8097 __trace_array_put(iter->tr);
8098
8099 if (info->spare)
8100 ring_buffer_free_read_page(iter->array_buffer->buffer,
8101 info->spare_cpu, info->spare);
8102 kvfree(info);
8103
8104 mutex_unlock(&trace_types_lock);
8105
8106 return 0;
8107}
8108
8109struct buffer_ref {
8110 struct trace_buffer *buffer;
8111 void *page;
8112 int cpu;
8113 refcount_t refcount;
8114};
8115
8116static void buffer_ref_release(struct buffer_ref *ref)
8117{
8118 if (!refcount_dec_and_test(&ref->refcount))
8119 return;
8120 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
8121 kfree(ref);
8122}
8123
8124static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
8125 struct pipe_buffer *buf)
8126{
8127 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
8128
8129 buffer_ref_release(ref);
8130 buf->private = 0;
8131}
8132
8133static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
8134 struct pipe_buffer *buf)
8135{
8136 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
8137
8138 if (refcount_read(&ref->refcount) > INT_MAX/2)
8139 return false;
8140
8141 refcount_inc(&ref->refcount);
8142 return true;
8143}
8144
8145/* Pipe buffer operations for a buffer. */
8146static const struct pipe_buf_operations buffer_pipe_buf_ops = {
8147 .release = buffer_pipe_buf_release,
8148 .get = buffer_pipe_buf_get,
8149};
8150
8151/*
8152 * Callback from splice_to_pipe(), if we need to release some pages
8153 * at the end of the spd in case we error'ed out in filling the pipe.
8154 */
8155static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
8156{
8157 struct buffer_ref *ref =
8158 (struct buffer_ref *)spd->partial[i].private;
8159
8160 buffer_ref_release(ref);
8161 spd->partial[i].private = 0;
8162}
8163
8164static ssize_t
8165tracing_buffers_splice_read(struct file *file, loff_t *ppos,
8166 struct pipe_inode_info *pipe, size_t len,
8167 unsigned int flags)
8168{
8169 struct ftrace_buffer_info *info = file->private_data;
8170 struct trace_iterator *iter = &info->iter;
8171 struct partial_page partial_def[PIPE_DEF_BUFFERS];
8172 struct page *pages_def[PIPE_DEF_BUFFERS];
8173 struct splice_pipe_desc spd = {
8174 .pages = pages_def,
8175 .partial = partial_def,
8176 .nr_pages_max = PIPE_DEF_BUFFERS,
8177 .ops = &buffer_pipe_buf_ops,
8178 .spd_release = buffer_spd_release,
8179 };
8180 struct buffer_ref *ref;
8181 int entries, i;
8182 ssize_t ret = 0;
8183
8184#ifdef CONFIG_TRACER_MAX_TRACE
8185 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8186 return -EBUSY;
8187#endif
8188
8189 if (*ppos & (PAGE_SIZE - 1))
8190 return -EINVAL;
8191
8192 if (len & (PAGE_SIZE - 1)) {
8193 if (len < PAGE_SIZE)
8194 return -EINVAL;
8195 len &= PAGE_MASK;
8196 }
8197
8198 if (splice_grow_spd(pipe, &spd))
8199 return -ENOMEM;
8200
8201 again:
8202 trace_access_lock(iter->cpu_file);
8203 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8204
8205 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
8206 struct page *page;
8207 int r;
8208
8209 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
8210 if (!ref) {
8211 ret = -ENOMEM;
8212 break;
8213 }
8214
8215 refcount_set(&ref->refcount, 1);
8216 ref->buffer = iter->array_buffer->buffer;
8217 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
8218 if (IS_ERR(ref->page)) {
8219 ret = PTR_ERR(ref->page);
8220 ref->page = NULL;
8221 kfree(ref);
8222 break;
8223 }
8224 ref->cpu = iter->cpu_file;
8225
8226 r = ring_buffer_read_page(ref->buffer, &ref->page,
8227 len, iter->cpu_file, 1);
8228 if (r < 0) {
8229 ring_buffer_free_read_page(ref->buffer, ref->cpu,
8230 ref->page);
8231 kfree(ref);
8232 break;
8233 }
8234
8235 page = virt_to_page(ref->page);
8236
8237 spd.pages[i] = page;
8238 spd.partial[i].len = PAGE_SIZE;
8239 spd.partial[i].offset = 0;
8240 spd.partial[i].private = (unsigned long)ref;
8241 spd.nr_pages++;
8242 *ppos += PAGE_SIZE;
8243
8244 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8245 }
8246
8247 trace_access_unlock(iter->cpu_file);
8248 spd.nr_pages = i;
8249
8250 /* did we read anything? */
8251 if (!spd.nr_pages) {
8252 if (ret)
8253 goto out;
8254
8255 ret = -EAGAIN;
8256 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
8257 goto out;
8258
8259 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
8260 if (ret)
8261 goto out;
8262
8263 goto again;
8264 }
8265
8266 ret = splice_to_pipe(pipe, &spd);
8267out:
8268 splice_shrink_spd(&spd);
8269
8270 return ret;
8271}
8272
8273static const struct file_operations tracing_buffers_fops = {
8274 .open = tracing_buffers_open,
8275 .read = tracing_buffers_read,
8276 .poll = tracing_buffers_poll,
8277 .release = tracing_buffers_release,
8278 .splice_read = tracing_buffers_splice_read,
8279 .llseek = no_llseek,
8280};
8281
8282static ssize_t
8283tracing_stats_read(struct file *filp, char __user *ubuf,
8284 size_t count, loff_t *ppos)
8285{
8286 struct inode *inode = file_inode(filp);
8287 struct trace_array *tr = inode->i_private;
8288 struct array_buffer *trace_buf = &tr->array_buffer;
8289 int cpu = tracing_get_cpu(inode);
8290 struct trace_seq *s;
8291 unsigned long cnt;
8292 unsigned long long t;
8293 unsigned long usec_rem;
8294
8295 s = kmalloc(sizeof(*s), GFP_KERNEL);
8296 if (!s)
8297 return -ENOMEM;
8298
8299 trace_seq_init(s);
8300
8301 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
8302 trace_seq_printf(s, "entries: %ld\n", cnt);
8303
8304 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
8305 trace_seq_printf(s, "overrun: %ld\n", cnt);
8306
8307 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
8308 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
8309
8310 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
8311 trace_seq_printf(s, "bytes: %ld\n", cnt);
8312
8313 if (trace_clocks[tr->clock_id].in_ns) {
8314 /* local or global for trace_clock */
8315 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8316 usec_rem = do_div(t, USEC_PER_SEC);
8317 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
8318 t, usec_rem);
8319
8320 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer));
8321 usec_rem = do_div(t, USEC_PER_SEC);
8322 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
8323 } else {
8324 /* counter or tsc mode for trace_clock */
8325 trace_seq_printf(s, "oldest event ts: %llu\n",
8326 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8327
8328 trace_seq_printf(s, "now ts: %llu\n",
8329 ring_buffer_time_stamp(trace_buf->buffer));
8330 }
8331
8332 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
8333 trace_seq_printf(s, "dropped events: %ld\n", cnt);
8334
8335 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
8336 trace_seq_printf(s, "read events: %ld\n", cnt);
8337
8338 count = simple_read_from_buffer(ubuf, count, ppos,
8339 s->buffer, trace_seq_used(s));
8340
8341 kfree(s);
8342
8343 return count;
8344}
8345
8346static const struct file_operations tracing_stats_fops = {
8347 .open = tracing_open_generic_tr,
8348 .read = tracing_stats_read,
8349 .llseek = generic_file_llseek,
8350 .release = tracing_release_generic_tr,
8351};
8352
8353#ifdef CONFIG_DYNAMIC_FTRACE
8354
8355static ssize_t
8356tracing_read_dyn_info(struct file *filp, char __user *ubuf,
8357 size_t cnt, loff_t *ppos)
8358{
8359 ssize_t ret;
8360 char *buf;
8361 int r;
8362
8363 /* 256 should be plenty to hold the amount needed */
8364 buf = kmalloc(256, GFP_KERNEL);
8365 if (!buf)
8366 return -ENOMEM;
8367
8368 r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
8369 ftrace_update_tot_cnt,
8370 ftrace_number_of_pages,
8371 ftrace_number_of_groups);
8372
8373 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8374 kfree(buf);
8375 return ret;
8376}
8377
8378static const struct file_operations tracing_dyn_info_fops = {
8379 .open = tracing_open_generic,
8380 .read = tracing_read_dyn_info,
8381 .llseek = generic_file_llseek,
8382};
8383#endif /* CONFIG_DYNAMIC_FTRACE */
8384
8385#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
8386static void
8387ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
8388 struct trace_array *tr, struct ftrace_probe_ops *ops,
8389 void *data)
8390{
8391 tracing_snapshot_instance(tr);
8392}
8393
8394static void
8395ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
8396 struct trace_array *tr, struct ftrace_probe_ops *ops,
8397 void *data)
8398{
8399 struct ftrace_func_mapper *mapper = data;
8400 long *count = NULL;
8401
8402 if (mapper)
8403 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8404
8405 if (count) {
8406
8407 if (*count <= 0)
8408 return;
8409
8410 (*count)--;
8411 }
8412
8413 tracing_snapshot_instance(tr);
8414}
8415
8416static int
8417ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
8418 struct ftrace_probe_ops *ops, void *data)
8419{
8420 struct ftrace_func_mapper *mapper = data;
8421 long *count = NULL;
8422
8423 seq_printf(m, "%ps:", (void *)ip);
8424
8425 seq_puts(m, "snapshot");
8426
8427 if (mapper)
8428 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8429
8430 if (count)
8431 seq_printf(m, ":count=%ld\n", *count);
8432 else
8433 seq_puts(m, ":unlimited\n");
8434
8435 return 0;
8436}
8437
8438static int
8439ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
8440 unsigned long ip, void *init_data, void **data)
8441{
8442 struct ftrace_func_mapper *mapper = *data;
8443
8444 if (!mapper) {
8445 mapper = allocate_ftrace_func_mapper();
8446 if (!mapper)
8447 return -ENOMEM;
8448 *data = mapper;
8449 }
8450
8451 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
8452}
8453
8454static void
8455ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
8456 unsigned long ip, void *data)
8457{
8458 struct ftrace_func_mapper *mapper = data;
8459
8460 if (!ip) {
8461 if (!mapper)
8462 return;
8463 free_ftrace_func_mapper(mapper, NULL);
8464 return;
8465 }
8466
8467 ftrace_func_mapper_remove_ip(mapper, ip);
8468}
8469
8470static struct ftrace_probe_ops snapshot_probe_ops = {
8471 .func = ftrace_snapshot,
8472 .print = ftrace_snapshot_print,
8473};
8474
8475static struct ftrace_probe_ops snapshot_count_probe_ops = {
8476 .func = ftrace_count_snapshot,
8477 .print = ftrace_snapshot_print,
8478 .init = ftrace_snapshot_init,
8479 .free = ftrace_snapshot_free,
8480};
8481
8482static int
8483ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
8484 char *glob, char *cmd, char *param, int enable)
8485{
8486 struct ftrace_probe_ops *ops;
8487 void *count = (void *)-1;
8488 char *number;
8489 int ret;
8490
8491 if (!tr)
8492 return -ENODEV;
8493
8494 /* hash funcs only work with set_ftrace_filter */
8495 if (!enable)
8496 return -EINVAL;
8497
8498 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
8499
8500 if (glob[0] == '!')
8501 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
8502
8503 if (!param)
8504 goto out_reg;
8505
8506 number = strsep(&param, ":");
8507
8508 if (!strlen(number))
8509 goto out_reg;
8510
8511 /*
8512 * We use the callback data field (which is a pointer)
8513 * as our counter.
8514 */
8515 ret = kstrtoul(number, 0, (unsigned long *)&count);
8516 if (ret)
8517 return ret;
8518
8519 out_reg:
8520 ret = tracing_alloc_snapshot_instance(tr);
8521 if (ret < 0)
8522 goto out;
8523
8524 ret = register_ftrace_function_probe(glob, tr, ops, count);
8525
8526 out:
8527 return ret < 0 ? ret : 0;
8528}
8529
8530static struct ftrace_func_command ftrace_snapshot_cmd = {
8531 .name = "snapshot",
8532 .func = ftrace_trace_snapshot_callback,
8533};
8534
8535static __init int register_snapshot_cmd(void)
8536{
8537 return register_ftrace_command(&ftrace_snapshot_cmd);
8538}
8539#else
8540static inline __init int register_snapshot_cmd(void) { return 0; }
8541#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
8542
8543static struct dentry *tracing_get_dentry(struct trace_array *tr)
8544{
8545 if (WARN_ON(!tr->dir))
8546 return ERR_PTR(-ENODEV);
8547
8548 /* Top directory uses NULL as the parent */
8549 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
8550 return NULL;
8551
8552 /* All sub buffers have a descriptor */
8553 return tr->dir;
8554}
8555
8556static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
8557{
8558 struct dentry *d_tracer;
8559
8560 if (tr->percpu_dir)
8561 return tr->percpu_dir;
8562
8563 d_tracer = tracing_get_dentry(tr);
8564 if (IS_ERR(d_tracer))
8565 return NULL;
8566
8567 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
8568
8569 MEM_FAIL(!tr->percpu_dir,
8570 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8571
8572 return tr->percpu_dir;
8573}
8574
8575static struct dentry *
8576trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
8577 void *data, long cpu, const struct file_operations *fops)
8578{
8579 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8580
8581 if (ret) /* See tracing_get_cpu() */
8582 d_inode(ret)->i_cdev = (void *)(cpu + 1);
8583 return ret;
8584}
8585
8586static void
8587tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8588{
8589 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8590 struct dentry *d_cpu;
8591 char cpu_dir[30]; /* 30 characters should be more than enough */
8592
8593 if (!d_percpu)
8594 return;
8595
8596 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8597 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8598 if (!d_cpu) {
8599 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8600 return;
8601 }
8602
8603 /* per cpu trace_pipe */
8604 trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu,
8605 tr, cpu, &tracing_pipe_fops);
8606
8607 /* per cpu trace */
8608 trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu,
8609 tr, cpu, &tracing_fops);
8610
8611 trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu,
8612 tr, cpu, &tracing_buffers_fops);
8613
8614 trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu,
8615 tr, cpu, &tracing_stats_fops);
8616
8617 trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu,
8618 tr, cpu, &tracing_entries_fops);
8619
8620#ifdef CONFIG_TRACER_SNAPSHOT
8621 trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu,
8622 tr, cpu, &snapshot_fops);
8623
8624 trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu,
8625 tr, cpu, &snapshot_raw_fops);
8626#endif
8627}
8628
8629#ifdef CONFIG_FTRACE_SELFTEST
8630/* Let selftest have access to static functions in this file */
8631#include "trace_selftest.c"
8632#endif
8633
8634static ssize_t
8635trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
8636 loff_t *ppos)
8637{
8638 struct trace_option_dentry *topt = filp->private_data;
8639 char *buf;
8640
8641 if (topt->flags->val & topt->opt->bit)
8642 buf = "1\n";
8643 else
8644 buf = "0\n";
8645
8646 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8647}
8648
8649static ssize_t
8650trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
8651 loff_t *ppos)
8652{
8653 struct trace_option_dentry *topt = filp->private_data;
8654 unsigned long val;
8655 int ret;
8656
8657 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8658 if (ret)
8659 return ret;
8660
8661 if (val != 0 && val != 1)
8662 return -EINVAL;
8663
8664 if (!!(topt->flags->val & topt->opt->bit) != val) {
8665 mutex_lock(&trace_types_lock);
8666 ret = __set_tracer_option(topt->tr, topt->flags,
8667 topt->opt, !val);
8668 mutex_unlock(&trace_types_lock);
8669 if (ret)
8670 return ret;
8671 }
8672
8673 *ppos += cnt;
8674
8675 return cnt;
8676}
8677
8678
8679static const struct file_operations trace_options_fops = {
8680 .open = tracing_open_generic,
8681 .read = trace_options_read,
8682 .write = trace_options_write,
8683 .llseek = generic_file_llseek,
8684};
8685
8686/*
8687 * In order to pass in both the trace_array descriptor as well as the index
8688 * to the flag that the trace option file represents, the trace_array
8689 * has a character array of trace_flags_index[], which holds the index
8690 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8691 * The address of this character array is passed to the flag option file
8692 * read/write callbacks.
8693 *
8694 * In order to extract both the index and the trace_array descriptor,
8695 * get_tr_index() uses the following algorithm.
8696 *
8697 * idx = *ptr;
8698 *
8699 * As the pointer itself contains the address of the index (remember
8700 * index[1] == 1).
8701 *
8702 * Then to get the trace_array descriptor, by subtracting that index
8703 * from the ptr, we get to the start of the index itself.
8704 *
8705 * ptr - idx == &index[0]
8706 *
8707 * Then a simple container_of() from that pointer gets us to the
8708 * trace_array descriptor.
8709 */
8710static void get_tr_index(void *data, struct trace_array **ptr,
8711 unsigned int *pindex)
8712{
8713 *pindex = *(unsigned char *)data;
8714
8715 *ptr = container_of(data - *pindex, struct trace_array,
8716 trace_flags_index);
8717}
8718
8719static ssize_t
8720trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
8721 loff_t *ppos)
8722{
8723 void *tr_index = filp->private_data;
8724 struct trace_array *tr;
8725 unsigned int index;
8726 char *buf;
8727
8728 get_tr_index(tr_index, &tr, &index);
8729
8730 if (tr->trace_flags & (1 << index))
8731 buf = "1\n";
8732 else
8733 buf = "0\n";
8734
8735 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8736}
8737
8738static ssize_t
8739trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
8740 loff_t *ppos)
8741{
8742 void *tr_index = filp->private_data;
8743 struct trace_array *tr;
8744 unsigned int index;
8745 unsigned long val;
8746 int ret;
8747
8748 get_tr_index(tr_index, &tr, &index);
8749
8750 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8751 if (ret)
8752 return ret;
8753
8754 if (val != 0 && val != 1)
8755 return -EINVAL;
8756
8757 mutex_lock(&event_mutex);
8758 mutex_lock(&trace_types_lock);
8759 ret = set_tracer_flag(tr, 1 << index, val);
8760 mutex_unlock(&trace_types_lock);
8761 mutex_unlock(&event_mutex);
8762
8763 if (ret < 0)
8764 return ret;
8765
8766 *ppos += cnt;
8767
8768 return cnt;
8769}
8770
8771static const struct file_operations trace_options_core_fops = {
8772 .open = tracing_open_generic,
8773 .read = trace_options_core_read,
8774 .write = trace_options_core_write,
8775 .llseek = generic_file_llseek,
8776};
8777
8778struct dentry *trace_create_file(const char *name,
8779 umode_t mode,
8780 struct dentry *parent,
8781 void *data,
8782 const struct file_operations *fops)
8783{
8784 struct dentry *ret;
8785
8786 ret = tracefs_create_file(name, mode, parent, data, fops);
8787 if (!ret)
8788 pr_warn("Could not create tracefs '%s' entry\n", name);
8789
8790 return ret;
8791}
8792
8793
8794static struct dentry *trace_options_init_dentry(struct trace_array *tr)
8795{
8796 struct dentry *d_tracer;
8797
8798 if (tr->options)
8799 return tr->options;
8800
8801 d_tracer = tracing_get_dentry(tr);
8802 if (IS_ERR(d_tracer))
8803 return NULL;
8804
8805 tr->options = tracefs_create_dir("options", d_tracer);
8806 if (!tr->options) {
8807 pr_warn("Could not create tracefs directory 'options'\n");
8808 return NULL;
8809 }
8810
8811 return tr->options;
8812}
8813
8814static void
8815create_trace_option_file(struct trace_array *tr,
8816 struct trace_option_dentry *topt,
8817 struct tracer_flags *flags,
8818 struct tracer_opt *opt)
8819{
8820 struct dentry *t_options;
8821
8822 t_options = trace_options_init_dentry(tr);
8823 if (!t_options)
8824 return;
8825
8826 topt->flags = flags;
8827 topt->opt = opt;
8828 topt->tr = tr;
8829
8830 topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,
8831 t_options, topt, &trace_options_fops);
8832
8833}
8834
8835static void
8836create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
8837{
8838 struct trace_option_dentry *topts;
8839 struct trace_options *tr_topts;
8840 struct tracer_flags *flags;
8841 struct tracer_opt *opts;
8842 int cnt;
8843 int i;
8844
8845 if (!tracer)
8846 return;
8847
8848 flags = tracer->flags;
8849
8850 if (!flags || !flags->opts)
8851 return;
8852
8853 /*
8854 * If this is an instance, only create flags for tracers
8855 * the instance may have.
8856 */
8857 if (!trace_ok_for_array(tracer, tr))
8858 return;
8859
8860 for (i = 0; i < tr->nr_topts; i++) {
8861 /* Make sure there's no duplicate flags. */
8862 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8863 return;
8864 }
8865
8866 opts = flags->opts;
8867
8868 for (cnt = 0; opts[cnt].name; cnt++)
8869 ;
8870
8871 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
8872 if (!topts)
8873 return;
8874
8875 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
8876 GFP_KERNEL);
8877 if (!tr_topts) {
8878 kfree(topts);
8879 return;
8880 }
8881
8882 tr->topts = tr_topts;
8883 tr->topts[tr->nr_topts].tracer = tracer;
8884 tr->topts[tr->nr_topts].topts = topts;
8885 tr->nr_topts++;
8886
8887 for (cnt = 0; opts[cnt].name; cnt++) {
8888 create_trace_option_file(tr, &topts[cnt], flags,
8889 &opts[cnt]);
8890 MEM_FAIL(topts[cnt].entry == NULL,
8891 "Failed to create trace option: %s",
8892 opts[cnt].name);
8893 }
8894}
8895
8896static struct dentry *
8897create_trace_option_core_file(struct trace_array *tr,
8898 const char *option, long index)
8899{
8900 struct dentry *t_options;
8901
8902 t_options = trace_options_init_dentry(tr);
8903 if (!t_options)
8904 return NULL;
8905
8906 return trace_create_file(option, TRACE_MODE_WRITE, t_options,
8907 (void *)&tr->trace_flags_index[index],
8908 &trace_options_core_fops);
8909}
8910
8911static void create_trace_options_dir(struct trace_array *tr)
8912{
8913 struct dentry *t_options;
8914 bool top_level = tr == &global_trace;
8915 int i;
8916
8917 t_options = trace_options_init_dentry(tr);
8918 if (!t_options)
8919 return;
8920
8921 for (i = 0; trace_options[i]; i++) {
8922 if (top_level ||
8923 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8924 create_trace_option_core_file(tr, trace_options[i], i);
8925 }
8926}
8927
8928static ssize_t
8929rb_simple_read(struct file *filp, char __user *ubuf,
8930 size_t cnt, loff_t *ppos)
8931{
8932 struct trace_array *tr = filp->private_data;
8933 char buf[64];
8934 int r;
8935
8936 r = tracer_tracing_is_on(tr);
8937 r = sprintf(buf, "%d\n", r);
8938
8939 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8940}
8941
8942static ssize_t
8943rb_simple_write(struct file *filp, const char __user *ubuf,
8944 size_t cnt, loff_t *ppos)
8945{
8946 struct trace_array *tr = filp->private_data;
8947 struct trace_buffer *buffer = tr->array_buffer.buffer;
8948 unsigned long val;
8949 int ret;
8950
8951 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8952 if (ret)
8953 return ret;
8954
8955 if (buffer) {
8956 mutex_lock(&trace_types_lock);
8957 if (!!val == tracer_tracing_is_on(tr)) {
8958 val = 0; /* do nothing */
8959 } else if (val) {
8960 tracer_tracing_on(tr);
8961 if (tr->current_trace->start)
8962 tr->current_trace->start(tr);
8963 } else {
8964 tracer_tracing_off(tr);
8965 if (tr->current_trace->stop)
8966 tr->current_trace->stop(tr);
8967 }
8968 mutex_unlock(&trace_types_lock);
8969 }
8970
8971 (*ppos)++;
8972
8973 return cnt;
8974}
8975
8976static const struct file_operations rb_simple_fops = {
8977 .open = tracing_open_generic_tr,
8978 .read = rb_simple_read,
8979 .write = rb_simple_write,
8980 .release = tracing_release_generic_tr,
8981 .llseek = default_llseek,
8982};
8983
8984static ssize_t
8985buffer_percent_read(struct file *filp, char __user *ubuf,
8986 size_t cnt, loff_t *ppos)
8987{
8988 struct trace_array *tr = filp->private_data;
8989 char buf[64];
8990 int r;
8991
8992 r = tr->buffer_percent;
8993 r = sprintf(buf, "%d\n", r);
8994
8995 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8996}
8997
8998static ssize_t
8999buffer_percent_write(struct file *filp, const char __user *ubuf,
9000 size_t cnt, loff_t *ppos)
9001{
9002 struct trace_array *tr = filp->private_data;
9003 unsigned long val;
9004 int ret;
9005
9006 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9007 if (ret)
9008 return ret;
9009
9010 if (val > 100)
9011 return -EINVAL;
9012
9013 if (!val)
9014 val = 1;
9015
9016 tr->buffer_percent = val;
9017
9018 (*ppos)++;
9019
9020 return cnt;
9021}
9022
9023static const struct file_operations buffer_percent_fops = {
9024 .open = tracing_open_generic_tr,
9025 .read = buffer_percent_read,
9026 .write = buffer_percent_write,
9027 .release = tracing_release_generic_tr,
9028 .llseek = default_llseek,
9029};
9030
9031static struct dentry *trace_instance_dir;
9032
9033static void
9034init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
9035
9036static int
9037allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
9038{
9039 enum ring_buffer_flags rb_flags;
9040
9041 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
9042
9043 buf->tr = tr;
9044
9045 buf->buffer = ring_buffer_alloc(size, rb_flags);
9046 if (!buf->buffer)
9047 return -ENOMEM;
9048
9049 buf->data = alloc_percpu(struct trace_array_cpu);
9050 if (!buf->data) {
9051 ring_buffer_free(buf->buffer);
9052 buf->buffer = NULL;
9053 return -ENOMEM;
9054 }
9055
9056 /* Allocate the first page for all buffers */
9057 set_buffer_entries(&tr->array_buffer,
9058 ring_buffer_size(tr->array_buffer.buffer, 0));
9059
9060 return 0;
9061}
9062
9063static int allocate_trace_buffers(struct trace_array *tr, int size)
9064{
9065 int ret;
9066
9067 ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
9068 if (ret)
9069 return ret;
9070
9071#ifdef CONFIG_TRACER_MAX_TRACE
9072 ret = allocate_trace_buffer(tr, &tr->max_buffer,
9073 allocate_snapshot ? size : 1);
9074 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
9075 ring_buffer_free(tr->array_buffer.buffer);
9076 tr->array_buffer.buffer = NULL;
9077 free_percpu(tr->array_buffer.data);
9078 tr->array_buffer.data = NULL;
9079 return -ENOMEM;
9080 }
9081 tr->allocated_snapshot = allocate_snapshot;
9082
9083 /*
9084 * Only the top level trace array gets its snapshot allocated
9085 * from the kernel command line.
9086 */
9087 allocate_snapshot = false;
9088#endif
9089
9090 return 0;
9091}
9092
9093static void free_trace_buffer(struct array_buffer *buf)
9094{
9095 if (buf->buffer) {
9096 ring_buffer_free(buf->buffer);
9097 buf->buffer = NULL;
9098 free_percpu(buf->data);
9099 buf->data = NULL;
9100 }
9101}
9102
9103static void free_trace_buffers(struct trace_array *tr)
9104{
9105 if (!tr)
9106 return;
9107
9108 free_trace_buffer(&tr->array_buffer);
9109
9110#ifdef CONFIG_TRACER_MAX_TRACE
9111 free_trace_buffer(&tr->max_buffer);
9112#endif
9113}
9114
9115static void init_trace_flags_index(struct trace_array *tr)
9116{
9117 int i;
9118
9119 /* Used by the trace options files */
9120 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
9121 tr->trace_flags_index[i] = i;
9122}
9123
9124static void __update_tracer_options(struct trace_array *tr)
9125{
9126 struct tracer *t;
9127
9128 for (t = trace_types; t; t = t->next)
9129 add_tracer_options(tr, t);
9130}
9131
9132static void update_tracer_options(struct trace_array *tr)
9133{
9134 mutex_lock(&trace_types_lock);
9135 __update_tracer_options(tr);
9136 mutex_unlock(&trace_types_lock);
9137}
9138
9139/* Must have trace_types_lock held */
9140struct trace_array *trace_array_find(const char *instance)
9141{
9142 struct trace_array *tr, *found = NULL;
9143
9144 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9145 if (tr->name && strcmp(tr->name, instance) == 0) {
9146 found = tr;
9147 break;
9148 }
9149 }
9150
9151 return found;
9152}
9153
9154struct trace_array *trace_array_find_get(const char *instance)
9155{
9156 struct trace_array *tr;
9157
9158 mutex_lock(&trace_types_lock);
9159 tr = trace_array_find(instance);
9160 if (tr)
9161 tr->ref++;
9162 mutex_unlock(&trace_types_lock);
9163
9164 return tr;
9165}
9166
9167static int trace_array_create_dir(struct trace_array *tr)
9168{
9169 int ret;
9170
9171 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
9172 if (!tr->dir)
9173 return -EINVAL;
9174
9175 ret = event_trace_add_tracer(tr->dir, tr);
9176 if (ret) {
9177 tracefs_remove(tr->dir);
9178 return ret;
9179 }
9180
9181 init_tracer_tracefs(tr, tr->dir);
9182 __update_tracer_options(tr);
9183
9184 return ret;
9185}
9186
9187static struct trace_array *trace_array_create(const char *name)
9188{
9189 struct trace_array *tr;
9190 int ret;
9191
9192 ret = -ENOMEM;
9193 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
9194 if (!tr)
9195 return ERR_PTR(ret);
9196
9197 tr->name = kstrdup(name, GFP_KERNEL);
9198 if (!tr->name)
9199 goto out_free_tr;
9200
9201 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
9202 goto out_free_tr;
9203
9204 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
9205
9206 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
9207
9208 raw_spin_lock_init(&tr->start_lock);
9209
9210 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9211
9212 tr->current_trace = &nop_trace;
9213
9214 INIT_LIST_HEAD(&tr->systems);
9215 INIT_LIST_HEAD(&tr->events);
9216 INIT_LIST_HEAD(&tr->hist_vars);
9217 INIT_LIST_HEAD(&tr->err_log);
9218
9219 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
9220 goto out_free_tr;
9221
9222 if (ftrace_allocate_ftrace_ops(tr) < 0)
9223 goto out_free_tr;
9224
9225 ftrace_init_trace_array(tr);
9226
9227 init_trace_flags_index(tr);
9228
9229 if (trace_instance_dir) {
9230 ret = trace_array_create_dir(tr);
9231 if (ret)
9232 goto out_free_tr;
9233 } else
9234 __trace_early_add_events(tr);
9235
9236 list_add(&tr->list, &ftrace_trace_arrays);
9237
9238 tr->ref++;
9239
9240 return tr;
9241
9242 out_free_tr:
9243 ftrace_free_ftrace_ops(tr);
9244 free_trace_buffers(tr);
9245 free_cpumask_var(tr->tracing_cpumask);
9246 kfree(tr->name);
9247 kfree(tr);
9248
9249 return ERR_PTR(ret);
9250}
9251
9252static int instance_mkdir(const char *name)
9253{
9254 struct trace_array *tr;
9255 int ret;
9256
9257 mutex_lock(&event_mutex);
9258 mutex_lock(&trace_types_lock);
9259
9260 ret = -EEXIST;
9261 if (trace_array_find(name))
9262 goto out_unlock;
9263
9264 tr = trace_array_create(name);
9265
9266 ret = PTR_ERR_OR_ZERO(tr);
9267
9268out_unlock:
9269 mutex_unlock(&trace_types_lock);
9270 mutex_unlock(&event_mutex);
9271 return ret;
9272}
9273
9274/**
9275 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
9276 * @name: The name of the trace array to be looked up/created.
9277 *
9278 * Returns pointer to trace array with given name.
9279 * NULL, if it cannot be created.
9280 *
9281 * NOTE: This function increments the reference counter associated with the
9282 * trace array returned. This makes sure it cannot be freed while in use.
9283 * Use trace_array_put() once the trace array is no longer needed.
9284 * If the trace_array is to be freed, trace_array_destroy() needs to
9285 * be called after the trace_array_put(), or simply let user space delete
9286 * it from the tracefs instances directory. But until the
9287 * trace_array_put() is called, user space can not delete it.
9288 *
9289 */
9290struct trace_array *trace_array_get_by_name(const char *name)
9291{
9292 struct trace_array *tr;
9293
9294 mutex_lock(&event_mutex);
9295 mutex_lock(&trace_types_lock);
9296
9297 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9298 if (tr->name && strcmp(tr->name, name) == 0)
9299 goto out_unlock;
9300 }
9301
9302 tr = trace_array_create(name);
9303
9304 if (IS_ERR(tr))
9305 tr = NULL;
9306out_unlock:
9307 if (tr)
9308 tr->ref++;
9309
9310 mutex_unlock(&trace_types_lock);
9311 mutex_unlock(&event_mutex);
9312 return tr;
9313}
9314EXPORT_SYMBOL_GPL(trace_array_get_by_name);
9315
9316static int __remove_instance(struct trace_array *tr)
9317{
9318 int i;
9319
9320 /* Reference counter for a newly created trace array = 1. */
9321 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))
9322 return -EBUSY;
9323
9324 list_del(&tr->list);
9325
9326 /* Disable all the flags that were enabled coming in */
9327 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
9328 if ((1 << i) & ZEROED_TRACE_FLAGS)
9329 set_tracer_flag(tr, 1 << i, 0);
9330 }
9331
9332 tracing_set_nop(tr);
9333 clear_ftrace_function_probes(tr);
9334 event_trace_del_tracer(tr);
9335 ftrace_clear_pids(tr);
9336 ftrace_destroy_function_files(tr);
9337 tracefs_remove(tr->dir);
9338 free_percpu(tr->last_func_repeats);
9339 free_trace_buffers(tr);
9340
9341 for (i = 0; i < tr->nr_topts; i++) {
9342 kfree(tr->topts[i].topts);
9343 }
9344 kfree(tr->topts);
9345
9346 free_cpumask_var(tr->tracing_cpumask);
9347 kfree(tr->name);
9348 kfree(tr);
9349
9350 return 0;
9351}
9352
9353int trace_array_destroy(struct trace_array *this_tr)
9354{
9355 struct trace_array *tr;
9356 int ret;
9357
9358 if (!this_tr)
9359 return -EINVAL;
9360
9361 mutex_lock(&event_mutex);
9362 mutex_lock(&trace_types_lock);
9363
9364 ret = -ENODEV;
9365
9366 /* Making sure trace array exists before destroying it. */
9367 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9368 if (tr == this_tr) {
9369 ret = __remove_instance(tr);
9370 break;
9371 }
9372 }
9373
9374 mutex_unlock(&trace_types_lock);
9375 mutex_unlock(&event_mutex);
9376
9377 return ret;
9378}
9379EXPORT_SYMBOL_GPL(trace_array_destroy);
9380
9381static int instance_rmdir(const char *name)
9382{
9383 struct trace_array *tr;
9384 int ret;
9385
9386 mutex_lock(&event_mutex);
9387 mutex_lock(&trace_types_lock);
9388
9389 ret = -ENODEV;
9390 tr = trace_array_find(name);
9391 if (tr)
9392 ret = __remove_instance(tr);
9393
9394 mutex_unlock(&trace_types_lock);
9395 mutex_unlock(&event_mutex);
9396
9397 return ret;
9398}
9399
9400static __init void create_trace_instances(struct dentry *d_tracer)
9401{
9402 struct trace_array *tr;
9403
9404 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
9405 instance_mkdir,
9406 instance_rmdir);
9407 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
9408 return;
9409
9410 mutex_lock(&event_mutex);
9411 mutex_lock(&trace_types_lock);
9412
9413 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9414 if (!tr->name)
9415 continue;
9416 if (MEM_FAIL(trace_array_create_dir(tr) < 0,
9417 "Failed to create instance directory\n"))
9418 break;
9419 }
9420
9421 mutex_unlock(&trace_types_lock);
9422 mutex_unlock(&event_mutex);
9423}
9424
9425static void
9426init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
9427{
9428 struct trace_event_file *file;
9429 int cpu;
9430
9431 trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer,
9432 tr, &show_traces_fops);
9433
9434 trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer,
9435 tr, &set_tracer_fops);
9436
9437 trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer,
9438 tr, &tracing_cpumask_fops);
9439
9440 trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer,
9441 tr, &tracing_iter_fops);
9442
9443 trace_create_file("trace", TRACE_MODE_WRITE, d_tracer,
9444 tr, &tracing_fops);
9445
9446 trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer,
9447 tr, &tracing_pipe_fops);
9448
9449 trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer,
9450 tr, &tracing_entries_fops);
9451
9452 trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer,
9453 tr, &tracing_total_entries_fops);
9454
9455 trace_create_file("free_buffer", 0200, d_tracer,
9456 tr, &tracing_free_buffer_fops);
9457
9458 trace_create_file("trace_marker", 0220, d_tracer,
9459 tr, &tracing_mark_fops);
9460
9461 file = __find_event_file(tr, "ftrace", "print");
9462 if (file && file->dir)
9463 trace_create_file("trigger", TRACE_MODE_WRITE, file->dir,
9464 file, &event_trigger_fops);
9465 tr->trace_marker_file = file;
9466
9467 trace_create_file("trace_marker_raw", 0220, d_tracer,
9468 tr, &tracing_mark_raw_fops);
9469
9470 trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr,
9471 &trace_clock_fops);
9472
9473 trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer,
9474 tr, &rb_simple_fops);
9475
9476 trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr,
9477 &trace_time_stamp_mode_fops);
9478
9479 tr->buffer_percent = 50;
9480
9481 trace_create_file("buffer_percent", TRACE_MODE_READ, d_tracer,
9482 tr, &buffer_percent_fops);
9483
9484 create_trace_options_dir(tr);
9485
9486 trace_create_maxlat_file(tr, d_tracer);
9487
9488 if (ftrace_create_function_files(tr, d_tracer))
9489 MEM_FAIL(1, "Could not allocate function filter files");
9490
9491#ifdef CONFIG_TRACER_SNAPSHOT
9492 trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer,
9493 tr, &snapshot_fops);
9494#endif
9495
9496 trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer,
9497 tr, &tracing_err_log_fops);
9498
9499 for_each_tracing_cpu(cpu)
9500 tracing_init_tracefs_percpu(tr, cpu);
9501
9502 ftrace_init_tracefs(tr, d_tracer);
9503}
9504
9505static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
9506{
9507 struct vfsmount *mnt;
9508 struct file_system_type *type;
9509
9510 /*
9511 * To maintain backward compatibility for tools that mount
9512 * debugfs to get to the tracing facility, tracefs is automatically
9513 * mounted to the debugfs/tracing directory.
9514 */
9515 type = get_fs_type("tracefs");
9516 if (!type)
9517 return NULL;
9518 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
9519 put_filesystem(type);
9520 if (IS_ERR(mnt))
9521 return NULL;
9522 mntget(mnt);
9523
9524 return mnt;
9525}
9526
9527/**
9528 * tracing_init_dentry - initialize top level trace array
9529 *
9530 * This is called when creating files or directories in the tracing
9531 * directory. It is called via fs_initcall() by any of the boot up code
9532 * and expects to return the dentry of the top level tracing directory.
9533 */
9534int tracing_init_dentry(void)
9535{
9536 struct trace_array *tr = &global_trace;
9537
9538 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9539 pr_warn("Tracing disabled due to lockdown\n");
9540 return -EPERM;
9541 }
9542
9543 /* The top level trace array uses NULL as parent */
9544 if (tr->dir)
9545 return 0;
9546
9547 if (WARN_ON(!tracefs_initialized()))
9548 return -ENODEV;
9549
9550 /*
9551 * As there may still be users that expect the tracing
9552 * files to exist in debugfs/tracing, we must automount
9553 * the tracefs file system there, so older tools still
9554 * work with the newer kernel.
9555 */
9556 tr->dir = debugfs_create_automount("tracing", NULL,
9557 trace_automount, NULL);
9558
9559 return 0;
9560}
9561
9562extern struct trace_eval_map *__start_ftrace_eval_maps[];
9563extern struct trace_eval_map *__stop_ftrace_eval_maps[];
9564
9565static struct workqueue_struct *eval_map_wq __initdata;
9566static struct work_struct eval_map_work __initdata;
9567
9568static void __init eval_map_work_func(struct work_struct *work)
9569{
9570 int len;
9571
9572 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
9573 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
9574}
9575
9576static int __init trace_eval_init(void)
9577{
9578 INIT_WORK(&eval_map_work, eval_map_work_func);
9579
9580 eval_map_wq = alloc_workqueue("eval_map_wq", WQ_UNBOUND, 0);
9581 if (!eval_map_wq) {
9582 pr_err("Unable to allocate eval_map_wq\n");
9583 /* Do work here */
9584 eval_map_work_func(&eval_map_work);
9585 return -ENOMEM;
9586 }
9587
9588 queue_work(eval_map_wq, &eval_map_work);
9589 return 0;
9590}
9591
9592static int __init trace_eval_sync(void)
9593{
9594 /* Make sure the eval map updates are finished */
9595 if (eval_map_wq)
9596 destroy_workqueue(eval_map_wq);
9597 return 0;
9598}
9599
9600late_initcall_sync(trace_eval_sync);
9601
9602
9603#ifdef CONFIG_MODULES
9604static void trace_module_add_evals(struct module *mod)
9605{
9606 if (!mod->num_trace_evals)
9607 return;
9608
9609 /*
9610 * Modules with bad taint do not have events created, do
9611 * not bother with enums either.
9612 */
9613 if (trace_module_has_bad_taint(mod))
9614 return;
9615
9616 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
9617}
9618
9619#ifdef CONFIG_TRACE_EVAL_MAP_FILE
9620static void trace_module_remove_evals(struct module *mod)
9621{
9622 union trace_eval_map_item *map;
9623 union trace_eval_map_item **last = &trace_eval_maps;
9624
9625 if (!mod->num_trace_evals)
9626 return;
9627
9628 mutex_lock(&trace_eval_mutex);
9629
9630 map = trace_eval_maps;
9631
9632 while (map) {
9633 if (map->head.mod == mod)
9634 break;
9635 map = trace_eval_jmp_to_tail(map);
9636 last = &map->tail.next;
9637 map = map->tail.next;
9638 }
9639 if (!map)
9640 goto out;
9641
9642 *last = trace_eval_jmp_to_tail(map)->tail.next;
9643 kfree(map);
9644 out:
9645 mutex_unlock(&trace_eval_mutex);
9646}
9647#else
9648static inline void trace_module_remove_evals(struct module *mod) { }
9649#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9650
9651static int trace_module_notify(struct notifier_block *self,
9652 unsigned long val, void *data)
9653{
9654 struct module *mod = data;
9655
9656 switch (val) {
9657 case MODULE_STATE_COMING:
9658 trace_module_add_evals(mod);
9659 break;
9660 case MODULE_STATE_GOING:
9661 trace_module_remove_evals(mod);
9662 break;
9663 }
9664
9665 return NOTIFY_OK;
9666}
9667
9668static struct notifier_block trace_module_nb = {
9669 .notifier_call = trace_module_notify,
9670 .priority = 0,
9671};
9672#endif /* CONFIG_MODULES */
9673
9674static __init int tracer_init_tracefs(void)
9675{
9676 int ret;
9677
9678 trace_access_lock_init();
9679
9680 ret = tracing_init_dentry();
9681 if (ret)
9682 return 0;
9683
9684 event_trace_init();
9685
9686 init_tracer_tracefs(&global_trace, NULL);
9687 ftrace_init_tracefs_toplevel(&global_trace, NULL);
9688
9689 trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL,
9690 &global_trace, &tracing_thresh_fops);
9691
9692 trace_create_file("README", TRACE_MODE_READ, NULL,
9693 NULL, &tracing_readme_fops);
9694
9695 trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL,
9696 NULL, &tracing_saved_cmdlines_fops);
9697
9698 trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
9699 NULL, &tracing_saved_cmdlines_size_fops);
9700
9701 trace_create_file("saved_tgids", TRACE_MODE_READ, NULL,
9702 NULL, &tracing_saved_tgids_fops);
9703
9704 trace_eval_init();
9705
9706 trace_create_eval_file(NULL);
9707
9708#ifdef CONFIG_MODULES
9709 register_module_notifier(&trace_module_nb);
9710#endif
9711
9712#ifdef CONFIG_DYNAMIC_FTRACE
9713 trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
9714 NULL, &tracing_dyn_info_fops);
9715#endif
9716
9717 create_trace_instances(NULL);
9718
9719 update_tracer_options(&global_trace);
9720
9721 return 0;
9722}
9723
9724fs_initcall(tracer_init_tracefs);
9725
9726static int trace_panic_handler(struct notifier_block *this,
9727 unsigned long event, void *unused)
9728{
9729 if (ftrace_dump_on_oops)
9730 ftrace_dump(ftrace_dump_on_oops);
9731 return NOTIFY_OK;
9732}
9733
9734static struct notifier_block trace_panic_notifier = {
9735 .notifier_call = trace_panic_handler,
9736 .next = NULL,
9737 .priority = 150 /* priority: INT_MAX >= x >= 0 */
9738};
9739
9740static int trace_die_handler(struct notifier_block *self,
9741 unsigned long val,
9742 void *data)
9743{
9744 switch (val) {
9745 case DIE_OOPS:
9746 if (ftrace_dump_on_oops)
9747 ftrace_dump(ftrace_dump_on_oops);
9748 break;
9749 default:
9750 break;
9751 }
9752 return NOTIFY_OK;
9753}
9754
9755static struct notifier_block trace_die_notifier = {
9756 .notifier_call = trace_die_handler,
9757 .priority = 200
9758};
9759
9760/*
9761 * printk is set to max of 1024, we really don't need it that big.
9762 * Nothing should be printing 1000 characters anyway.
9763 */
9764#define TRACE_MAX_PRINT 1000
9765
9766/*
9767 * Define here KERN_TRACE so that we have one place to modify
9768 * it if we decide to change what log level the ftrace dump
9769 * should be at.
9770 */
9771#define KERN_TRACE KERN_EMERG
9772
9773void
9774trace_printk_seq(struct trace_seq *s)
9775{
9776 /* Probably should print a warning here. */
9777 if (s->seq.len >= TRACE_MAX_PRINT)
9778 s->seq.len = TRACE_MAX_PRINT;
9779
9780 /*
9781 * More paranoid code. Although the buffer size is set to
9782 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9783 * an extra layer of protection.
9784 */
9785 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9786 s->seq.len = s->seq.size - 1;
9787
9788 /* should be zero ended, but we are paranoid. */
9789 s->buffer[s->seq.len] = 0;
9790
9791 printk(KERN_TRACE "%s", s->buffer);
9792
9793 trace_seq_init(s);
9794}
9795
9796void trace_init_global_iter(struct trace_iterator *iter)
9797{
9798 iter->tr = &global_trace;
9799 iter->trace = iter->tr->current_trace;
9800 iter->cpu_file = RING_BUFFER_ALL_CPUS;
9801 iter->array_buffer = &global_trace.array_buffer;
9802
9803 if (iter->trace && iter->trace->open)
9804 iter->trace->open(iter);
9805
9806 /* Annotate start of buffers if we had overruns */
9807 if (ring_buffer_overruns(iter->array_buffer->buffer))
9808 iter->iter_flags |= TRACE_FILE_ANNOTATE;
9809
9810 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
9811 if (trace_clocks[iter->tr->clock_id].in_ns)
9812 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9813}
9814
9815void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
9816{
9817 /* use static because iter can be a bit big for the stack */
9818 static struct trace_iterator iter;
9819 static atomic_t dump_running;
9820 struct trace_array *tr = &global_trace;
9821 unsigned int old_userobj;
9822 unsigned long flags;
9823 int cnt = 0, cpu;
9824
9825 /* Only allow one dump user at a time. */
9826 if (atomic_inc_return(&dump_running) != 1) {
9827 atomic_dec(&dump_running);
9828 return;
9829 }
9830
9831 /*
9832 * Always turn off tracing when we dump.
9833 * We don't need to show trace output of what happens
9834 * between multiple crashes.
9835 *
9836 * If the user does a sysrq-z, then they can re-enable
9837 * tracing with echo 1 > tracing_on.
9838 */
9839 tracing_off();
9840
9841 local_irq_save(flags);
9842
9843 /* Simulate the iterator */
9844 trace_init_global_iter(&iter);
9845 /* Can not use kmalloc for iter.temp and iter.fmt */
9846 iter.temp = static_temp_buf;
9847 iter.temp_size = STATIC_TEMP_BUF_SIZE;
9848 iter.fmt = static_fmt_buf;
9849 iter.fmt_size = STATIC_FMT_BUF_SIZE;
9850
9851 for_each_tracing_cpu(cpu) {
9852 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9853 }
9854
9855 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
9856
9857 /* don't look at user memory in panic mode */
9858 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
9859
9860 switch (oops_dump_mode) {
9861 case DUMP_ALL:
9862 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9863 break;
9864 case DUMP_ORIG:
9865 iter.cpu_file = raw_smp_processor_id();
9866 break;
9867 case DUMP_NONE:
9868 goto out_enable;
9869 default:
9870 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
9871 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9872 }
9873
9874 printk(KERN_TRACE "Dumping ftrace buffer:\n");
9875
9876 /* Did function tracer already get disabled? */
9877 if (ftrace_is_dead()) {
9878 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9879 printk("# MAY BE MISSING FUNCTION EVENTS\n");
9880 }
9881
9882 /*
9883 * We need to stop all tracing on all CPUS to read
9884 * the next buffer. This is a bit expensive, but is
9885 * not done often. We fill all what we can read,
9886 * and then release the locks again.
9887 */
9888
9889 while (!trace_empty(&iter)) {
9890
9891 if (!cnt)
9892 printk(KERN_TRACE "---------------------------------\n");
9893
9894 cnt++;
9895
9896 trace_iterator_reset(&iter);
9897 iter.iter_flags |= TRACE_FILE_LAT_FMT;
9898
9899 if (trace_find_next_entry_inc(&iter) != NULL) {
9900 int ret;
9901
9902 ret = print_trace_line(&iter);
9903 if (ret != TRACE_TYPE_NO_CONSUME)
9904 trace_consume(&iter);
9905 }
9906 touch_nmi_watchdog();
9907
9908 trace_printk_seq(&iter.seq);
9909 }
9910
9911 if (!cnt)
9912 printk(KERN_TRACE " (ftrace buffer empty)\n");
9913 else
9914 printk(KERN_TRACE "---------------------------------\n");
9915
9916 out_enable:
9917 tr->trace_flags |= old_userobj;
9918
9919 for_each_tracing_cpu(cpu) {
9920 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9921 }
9922 atomic_dec(&dump_running);
9923 local_irq_restore(flags);
9924}
9925EXPORT_SYMBOL_GPL(ftrace_dump);
9926
9927#define WRITE_BUFSIZE 4096
9928
9929ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9930 size_t count, loff_t *ppos,
9931 int (*createfn)(const char *))
9932{
9933 char *kbuf, *buf, *tmp;
9934 int ret = 0;
9935 size_t done = 0;
9936 size_t size;
9937
9938 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9939 if (!kbuf)
9940 return -ENOMEM;
9941
9942 while (done < count) {
9943 size = count - done;
9944
9945 if (size >= WRITE_BUFSIZE)
9946 size = WRITE_BUFSIZE - 1;
9947
9948 if (copy_from_user(kbuf, buffer + done, size)) {
9949 ret = -EFAULT;
9950 goto out;
9951 }
9952 kbuf[size] = '\0';
9953 buf = kbuf;
9954 do {
9955 tmp = strchr(buf, '\n');
9956 if (tmp) {
9957 *tmp = '\0';
9958 size = tmp - buf + 1;
9959 } else {
9960 size = strlen(buf);
9961 if (done + size < count) {
9962 if (buf != kbuf)
9963 break;
9964 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9965 pr_warn("Line length is too long: Should be less than %d\n",
9966 WRITE_BUFSIZE - 2);
9967 ret = -EINVAL;
9968 goto out;
9969 }
9970 }
9971 done += size;
9972
9973 /* Remove comments */
9974 tmp = strchr(buf, '#');
9975
9976 if (tmp)
9977 *tmp = '\0';
9978
9979 ret = createfn(buf);
9980 if (ret)
9981 goto out;
9982 buf += size;
9983
9984 } while (done < count);
9985 }
9986 ret = done;
9987
9988out:
9989 kfree(kbuf);
9990
9991 return ret;
9992}
9993
9994__init static int tracer_alloc_buffers(void)
9995{
9996 int ring_buf_size;
9997 int ret = -ENOMEM;
9998
9999
10000 if (security_locked_down(LOCKDOWN_TRACEFS)) {
10001 pr_warn("Tracing disabled due to lockdown\n");
10002 return -EPERM;
10003 }
10004
10005 /*
10006 * Make sure we don't accidentally add more trace options
10007 * than we have bits for.
10008 */
10009 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
10010
10011 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
10012 goto out;
10013
10014 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
10015 goto out_free_buffer_mask;
10016
10017 /* Only allocate trace_printk buffers if a trace_printk exists */
10018 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
10019 /* Must be called before global_trace.buffer is allocated */
10020 trace_printk_init_buffers();
10021
10022 /* To save memory, keep the ring buffer size to its minimum */
10023 if (ring_buffer_expanded)
10024 ring_buf_size = trace_buf_size;
10025 else
10026 ring_buf_size = 1;
10027
10028 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
10029 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
10030
10031 raw_spin_lock_init(&global_trace.start_lock);
10032
10033 /*
10034 * The prepare callbacks allocates some memory for the ring buffer. We
10035 * don't free the buffer if the CPU goes down. If we were to free
10036 * the buffer, then the user would lose any trace that was in the
10037 * buffer. The memory will be removed once the "instance" is removed.
10038 */
10039 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
10040 "trace/RB:preapre", trace_rb_cpu_prepare,
10041 NULL);
10042 if (ret < 0)
10043 goto out_free_cpumask;
10044 /* Used for event triggers */
10045 ret = -ENOMEM;
10046 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
10047 if (!temp_buffer)
10048 goto out_rm_hp_state;
10049
10050 if (trace_create_savedcmd() < 0)
10051 goto out_free_temp_buffer;
10052
10053 /* TODO: make the number of buffers hot pluggable with CPUS */
10054 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
10055 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
10056 goto out_free_savedcmd;
10057 }
10058
10059 if (global_trace.buffer_disabled)
10060 tracing_off();
10061
10062 if (trace_boot_clock) {
10063 ret = tracing_set_clock(&global_trace, trace_boot_clock);
10064 if (ret < 0)
10065 pr_warn("Trace clock %s not defined, going back to default\n",
10066 trace_boot_clock);
10067 }
10068
10069 /*
10070 * register_tracer() might reference current_trace, so it
10071 * needs to be set before we register anything. This is
10072 * just a bootstrap of current_trace anyway.
10073 */
10074 global_trace.current_trace = &nop_trace;
10075
10076 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
10077
10078 ftrace_init_global_array_ops(&global_trace);
10079
10080 init_trace_flags_index(&global_trace);
10081
10082 register_tracer(&nop_trace);
10083
10084 /* Function tracing may start here (via kernel command line) */
10085 init_function_trace();
10086
10087 /* All seems OK, enable tracing */
10088 tracing_disabled = 0;
10089
10090 atomic_notifier_chain_register(&panic_notifier_list,
10091 &trace_panic_notifier);
10092
10093 register_die_notifier(&trace_die_notifier);
10094
10095 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
10096
10097 INIT_LIST_HEAD(&global_trace.systems);
10098 INIT_LIST_HEAD(&global_trace.events);
10099 INIT_LIST_HEAD(&global_trace.hist_vars);
10100 INIT_LIST_HEAD(&global_trace.err_log);
10101 list_add(&global_trace.list, &ftrace_trace_arrays);
10102
10103 apply_trace_boot_options();
10104
10105 register_snapshot_cmd();
10106
10107 test_can_verify();
10108
10109 return 0;
10110
10111out_free_savedcmd:
10112 free_saved_cmdlines_buffer(savedcmd);
10113out_free_temp_buffer:
10114 ring_buffer_free(temp_buffer);
10115out_rm_hp_state:
10116 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
10117out_free_cpumask:
10118 free_cpumask_var(global_trace.tracing_cpumask);
10119out_free_buffer_mask:
10120 free_cpumask_var(tracing_buffer_mask);
10121out:
10122 return ret;
10123}
10124
10125void __init early_trace_init(void)
10126{
10127 if (tracepoint_printk) {
10128 tracepoint_print_iter =
10129 kzalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
10130 if (MEM_FAIL(!tracepoint_print_iter,
10131 "Failed to allocate trace iterator\n"))
10132 tracepoint_printk = 0;
10133 else
10134 static_key_enable(&tracepoint_printk_key.key);
10135 }
10136 tracer_alloc_buffers();
10137}
10138
10139void __init trace_init(void)
10140{
10141 trace_event_init();
10142}
10143
10144__init static void clear_boot_tracer(void)
10145{
10146 /*
10147 * The default tracer at boot buffer is an init section.
10148 * This function is called in lateinit. If we did not
10149 * find the boot tracer, then clear it out, to prevent
10150 * later registration from accessing the buffer that is
10151 * about to be freed.
10152 */
10153 if (!default_bootup_tracer)
10154 return;
10155
10156 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
10157 default_bootup_tracer);
10158 default_bootup_tracer = NULL;
10159}
10160
10161#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
10162__init static void tracing_set_default_clock(void)
10163{
10164 /* sched_clock_stable() is determined in late_initcall */
10165 if (!trace_boot_clock && !sched_clock_stable()) {
10166 if (security_locked_down(LOCKDOWN_TRACEFS)) {
10167 pr_warn("Can not set tracing clock due to lockdown\n");
10168 return;
10169 }
10170
10171 printk(KERN_WARNING
10172 "Unstable clock detected, switching default tracing clock to \"global\"\n"
10173 "If you want to keep using the local clock, then add:\n"
10174 " \"trace_clock=local\"\n"
10175 "on the kernel command line\n");
10176 tracing_set_clock(&global_trace, "global");
10177 }
10178}
10179#else
10180static inline void tracing_set_default_clock(void) { }
10181#endif
10182
10183__init static int late_trace_init(void)
10184{
10185 if (tracepoint_printk && tracepoint_printk_stop_on_boot) {
10186 static_key_disable(&tracepoint_printk_key.key);
10187 tracepoint_printk = 0;
10188 }
10189
10190 tracing_set_default_clock();
10191 clear_boot_tracer();
10192 return 0;
10193}
10194
10195late_initcall_sync(late_trace_init);