projects/linux·blob v5.12 · refs

9757 lines · 235.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * ring buffer based function tracer
4 *
5 * Copyright (C) 2007-2012 Steven Rostedt <[email protected]>
6 * Copyright (C) 2008 Ingo Molnar <[email protected]>
7 *
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <[email protected]>
10 *
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
14 */
15#include <linux/ring_buffer.h>
16#include <generated/utsrelease.h>
17#include <linux/stacktrace.h>
18#include <linux/writeback.h>
19#include <linux/kallsyms.h>
20#include <linux/security.h>
21#include <linux/seq_file.h>
22#include <linux/notifier.h>
23#include <linux/irqflags.h>
24#include <linux/debugfs.h>
25#include <linux/tracefs.h>
26#include <linux/pagemap.h>
27#include <linux/hardirq.h>
28#include <linux/linkage.h>
29#include <linux/uaccess.h>
30#include <linux/vmalloc.h>
31#include <linux/ftrace.h>
32#include <linux/module.h>
33#include <linux/percpu.h>
34#include <linux/splice.h>
35#include <linux/kdebug.h>
36#include <linux/string.h>
37#include <linux/mount.h>
38#include <linux/rwsem.h>
39#include <linux/slab.h>
40#include <linux/ctype.h>
41#include <linux/init.h>
42#include <linux/poll.h>
43#include <linux/nmi.h>
44#include <linux/fs.h>
45#include <linux/trace.h>
46#include <linux/sched/clock.h>
47#include <linux/sched/rt.h>
48#include <linux/fsnotify.h>
49#include <linux/irq_work.h>
50#include <linux/workqueue.h>
51
52#include "trace.h"
53#include "trace_output.h"
54
55/*
56 * On boot up, the ring buffer is set to the minimum size, so that
57 * we do not waste memory on systems that are not using tracing.
58 */
59bool ring_buffer_expanded;
60
61/*
62 * We need to change this state when a selftest is running.
63 * A selftest will lurk into the ring-buffer to count the
64 * entries inserted during the selftest although some concurrent
65 * insertions into the ring-buffer such as trace_printk could occurred
66 * at the same time, giving false positive or negative results.
67 */
68static bool __read_mostly tracing_selftest_running;
69
70/*
71 * If boot-time tracing including tracers/events via kernel cmdline
72 * is running, we do not want to run SELFTEST.
73 */
74bool __read_mostly tracing_selftest_disabled;
75
76#ifdef CONFIG_FTRACE_STARTUP_TEST
77void __init disable_tracing_selftest(const char *reason)
78{
79 if (!tracing_selftest_disabled) {
80 tracing_selftest_disabled = true;
81 pr_info("Ftrace startup test is disabled due to %s\n", reason);
82 }
83}
84#endif
85
86/* Pipe tracepoints to printk */
87struct trace_iterator *tracepoint_print_iter;
88int tracepoint_printk;
89static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
90
91/* For tracers that don't implement custom flags */
92static struct tracer_opt dummy_tracer_opt[] = {
93 { }
94};
95
96static int
97dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
98{
99 return 0;
100}
101
102/*
103 * To prevent the comm cache from being overwritten when no
104 * tracing is active, only save the comm when a trace event
105 * occurred.
106 */
107static DEFINE_PER_CPU(bool, trace_taskinfo_save);
108
109/*
110 * Kill all tracing for good (never come back).
111 * It is initialized to 1 but will turn to zero if the initialization
112 * of the tracer is successful. But that is the only place that sets
113 * this back to zero.
114 */
115static int tracing_disabled = 1;
116
117cpumask_var_t __read_mostly tracing_buffer_mask;
118
119/*
120 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
121 *
122 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
123 * is set, then ftrace_dump is called. This will output the contents
124 * of the ftrace buffers to the console. This is very useful for
125 * capturing traces that lead to crashes and outputing it to a
126 * serial console.
127 *
128 * It is default off, but you can enable it with either specifying
129 * "ftrace_dump_on_oops" in the kernel command line, or setting
130 * /proc/sys/kernel/ftrace_dump_on_oops
131 * Set 1 if you want to dump buffers of all CPUs
132 * Set 2 if you want to dump the buffer of the CPU that triggered oops
133 */
134
135enum ftrace_dump_mode ftrace_dump_on_oops;
136
137/* When set, tracing will stop when a WARN*() is hit */
138int __disable_trace_on_warning;
139
140#ifdef CONFIG_TRACE_EVAL_MAP_FILE
141/* Map of enums to their values, for "eval_map" file */
142struct trace_eval_map_head {
143 struct module *mod;
144 unsigned long length;
145};
146
147union trace_eval_map_item;
148
149struct trace_eval_map_tail {
150 /*
151 * "end" is first and points to NULL as it must be different
152 * than "mod" or "eval_string"
153 */
154 union trace_eval_map_item *next;
155 const char *end; /* points to NULL */
156};
157
158static DEFINE_MUTEX(trace_eval_mutex);
159
160/*
161 * The trace_eval_maps are saved in an array with two extra elements,
162 * one at the beginning, and one at the end. The beginning item contains
163 * the count of the saved maps (head.length), and the module they
164 * belong to if not built in (head.mod). The ending item contains a
165 * pointer to the next array of saved eval_map items.
166 */
167union trace_eval_map_item {
168 struct trace_eval_map map;
169 struct trace_eval_map_head head;
170 struct trace_eval_map_tail tail;
171};
172
173static union trace_eval_map_item *trace_eval_maps;
174#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
175
176int tracing_set_tracer(struct trace_array *tr, const char *buf);
177static void ftrace_trace_userstack(struct trace_array *tr,
178 struct trace_buffer *buffer,
179 unsigned int trace_ctx);
180
181#define MAX_TRACER_SIZE 100
182static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
183static char *default_bootup_tracer;
184
185static bool allocate_snapshot;
186
187static int __init set_cmdline_ftrace(char *str)
188{
189 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
190 default_bootup_tracer = bootup_tracer_buf;
191 /* We are using ftrace early, expand it */
192 ring_buffer_expanded = true;
193 return 1;
194}
195__setup("ftrace=", set_cmdline_ftrace);
196
197static int __init set_ftrace_dump_on_oops(char *str)
198{
199 if (*str++ != '=' || !*str) {
200 ftrace_dump_on_oops = DUMP_ALL;
201 return 1;
202 }
203
204 if (!strcmp("orig_cpu", str)) {
205 ftrace_dump_on_oops = DUMP_ORIG;
206 return 1;
207 }
208
209 return 0;
210}
211__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
212
213static int __init stop_trace_on_warning(char *str)
214{
215 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
216 __disable_trace_on_warning = 1;
217 return 1;
218}
219__setup("traceoff_on_warning", stop_trace_on_warning);
220
221static int __init boot_alloc_snapshot(char *str)
222{
223 allocate_snapshot = true;
224 /* We also need the main ring buffer expanded */
225 ring_buffer_expanded = true;
226 return 1;
227}
228__setup("alloc_snapshot", boot_alloc_snapshot);
229
230
231static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
232
233static int __init set_trace_boot_options(char *str)
234{
235 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
236 return 0;
237}
238__setup("trace_options=", set_trace_boot_options);
239
240static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
241static char *trace_boot_clock __initdata;
242
243static int __init set_trace_boot_clock(char *str)
244{
245 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
246 trace_boot_clock = trace_boot_clock_buf;
247 return 0;
248}
249__setup("trace_clock=", set_trace_boot_clock);
250
251static int __init set_tracepoint_printk(char *str)
252{
253 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
254 tracepoint_printk = 1;
255 return 1;
256}
257__setup("tp_printk", set_tracepoint_printk);
258
259unsigned long long ns2usecs(u64 nsec)
260{
261 nsec += 500;
262 do_div(nsec, 1000);
263 return nsec;
264}
265
266static void
267trace_process_export(struct trace_export *export,
268 struct ring_buffer_event *event, int flag)
269{
270 struct trace_entry *entry;
271 unsigned int size = 0;
272
273 if (export->flags & flag) {
274 entry = ring_buffer_event_data(event);
275 size = ring_buffer_event_length(event);
276 export->write(export, entry, size);
277 }
278}
279
280static DEFINE_MUTEX(ftrace_export_lock);
281
282static struct trace_export __rcu *ftrace_exports_list __read_mostly;
283
284static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
285static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
286static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
287
288static inline void ftrace_exports_enable(struct trace_export *export)
289{
290 if (export->flags & TRACE_EXPORT_FUNCTION)
291 static_branch_inc(&trace_function_exports_enabled);
292
293 if (export->flags & TRACE_EXPORT_EVENT)
294 static_branch_inc(&trace_event_exports_enabled);
295
296 if (export->flags & TRACE_EXPORT_MARKER)
297 static_branch_inc(&trace_marker_exports_enabled);
298}
299
300static inline void ftrace_exports_disable(struct trace_export *export)
301{
302 if (export->flags & TRACE_EXPORT_FUNCTION)
303 static_branch_dec(&trace_function_exports_enabled);
304
305 if (export->flags & TRACE_EXPORT_EVENT)
306 static_branch_dec(&trace_event_exports_enabled);
307
308 if (export->flags & TRACE_EXPORT_MARKER)
309 static_branch_dec(&trace_marker_exports_enabled);
310}
311
312static void ftrace_exports(struct ring_buffer_event *event, int flag)
313{
314 struct trace_export *export;
315
316 preempt_disable_notrace();
317
318 export = rcu_dereference_raw_check(ftrace_exports_list);
319 while (export) {
320 trace_process_export(export, event, flag);
321 export = rcu_dereference_raw_check(export->next);
322 }
323
324 preempt_enable_notrace();
325}
326
327static inline void
328add_trace_export(struct trace_export **list, struct trace_export *export)
329{
330 rcu_assign_pointer(export->next, *list);
331 /*
332 * We are entering export into the list but another
333 * CPU might be walking that list. We need to make sure
334 * the export->next pointer is valid before another CPU sees
335 * the export pointer included into the list.
336 */
337 rcu_assign_pointer(*list, export);
338}
339
340static inline int
341rm_trace_export(struct trace_export **list, struct trace_export *export)
342{
343 struct trace_export **p;
344
345 for (p = list; *p != NULL; p = &(*p)->next)
346 if (*p == export)
347 break;
348
349 if (*p != export)
350 return -1;
351
352 rcu_assign_pointer(*p, (*p)->next);
353
354 return 0;
355}
356
357static inline void
358add_ftrace_export(struct trace_export **list, struct trace_export *export)
359{
360 ftrace_exports_enable(export);
361
362 add_trace_export(list, export);
363}
364
365static inline int
366rm_ftrace_export(struct trace_export **list, struct trace_export *export)
367{
368 int ret;
369
370 ret = rm_trace_export(list, export);
371 ftrace_exports_disable(export);
372
373 return ret;
374}
375
376int register_ftrace_export(struct trace_export *export)
377{
378 if (WARN_ON_ONCE(!export->write))
379 return -1;
380
381 mutex_lock(&ftrace_export_lock);
382
383 add_ftrace_export(&ftrace_exports_list, export);
384
385 mutex_unlock(&ftrace_export_lock);
386
387 return 0;
388}
389EXPORT_SYMBOL_GPL(register_ftrace_export);
390
391int unregister_ftrace_export(struct trace_export *export)
392{
393 int ret;
394
395 mutex_lock(&ftrace_export_lock);
396
397 ret = rm_ftrace_export(&ftrace_exports_list, export);
398
399 mutex_unlock(&ftrace_export_lock);
400
401 return ret;
402}
403EXPORT_SYMBOL_GPL(unregister_ftrace_export);
404
405/* trace_flags holds trace_options default values */
406#define TRACE_DEFAULT_FLAGS \
407 (FUNCTION_DEFAULT_FLAGS | \
408 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
409 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
410 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
411 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | \
412 TRACE_ITER_HASH_PTR)
413
414/* trace_options that are only supported by global_trace */
415#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
416 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
417
418/* trace_flags that are default zero for instances */
419#define ZEROED_TRACE_FLAGS \
420 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
421
422/*
423 * The global_trace is the descriptor that holds the top-level tracing
424 * buffers for the live tracing.
425 */
426static struct trace_array global_trace = {
427 .trace_flags = TRACE_DEFAULT_FLAGS,
428};
429
430LIST_HEAD(ftrace_trace_arrays);
431
432int trace_array_get(struct trace_array *this_tr)
433{
434 struct trace_array *tr;
435 int ret = -ENODEV;
436
437 mutex_lock(&trace_types_lock);
438 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
439 if (tr == this_tr) {
440 tr->ref++;
441 ret = 0;
442 break;
443 }
444 }
445 mutex_unlock(&trace_types_lock);
446
447 return ret;
448}
449
450static void __trace_array_put(struct trace_array *this_tr)
451{
452 WARN_ON(!this_tr->ref);
453 this_tr->ref--;
454}
455
456/**
457 * trace_array_put - Decrement the reference counter for this trace array.
458 * @this_tr : pointer to the trace array
459 *
460 * NOTE: Use this when we no longer need the trace array returned by
461 * trace_array_get_by_name(). This ensures the trace array can be later
462 * destroyed.
463 *
464 */
465void trace_array_put(struct trace_array *this_tr)
466{
467 if (!this_tr)
468 return;
469
470 mutex_lock(&trace_types_lock);
471 __trace_array_put(this_tr);
472 mutex_unlock(&trace_types_lock);
473}
474EXPORT_SYMBOL_GPL(trace_array_put);
475
476int tracing_check_open_get_tr(struct trace_array *tr)
477{
478 int ret;
479
480 ret = security_locked_down(LOCKDOWN_TRACEFS);
481 if (ret)
482 return ret;
483
484 if (tracing_disabled)
485 return -ENODEV;
486
487 if (tr && trace_array_get(tr) < 0)
488 return -ENODEV;
489
490 return 0;
491}
492
493int call_filter_check_discard(struct trace_event_call *call, void *rec,
494 struct trace_buffer *buffer,
495 struct ring_buffer_event *event)
496{
497 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
498 !filter_match_preds(call->filter, rec)) {
499 __trace_event_discard_commit(buffer, event);
500 return 1;
501 }
502
503 return 0;
504}
505
506void trace_free_pid_list(struct trace_pid_list *pid_list)
507{
508 vfree(pid_list->pids);
509 kfree(pid_list);
510}
511
512/**
513 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
514 * @filtered_pids: The list of pids to check
515 * @search_pid: The PID to find in @filtered_pids
516 *
517 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
518 */
519bool
520trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
521{
522 /*
523 * If pid_max changed after filtered_pids was created, we
524 * by default ignore all pids greater than the previous pid_max.
525 */
526 if (search_pid >= filtered_pids->pid_max)
527 return false;
528
529 return test_bit(search_pid, filtered_pids->pids);
530}
531
532/**
533 * trace_ignore_this_task - should a task be ignored for tracing
534 * @filtered_pids: The list of pids to check
535 * @filtered_no_pids: The list of pids not to be traced
536 * @task: The task that should be ignored if not filtered
537 *
538 * Checks if @task should be traced or not from @filtered_pids.
539 * Returns true if @task should *NOT* be traced.
540 * Returns false if @task should be traced.
541 */
542bool
543trace_ignore_this_task(struct trace_pid_list *filtered_pids,
544 struct trace_pid_list *filtered_no_pids,
545 struct task_struct *task)
546{
547 /*
548 * If filterd_no_pids is not empty, and the task's pid is listed
549 * in filtered_no_pids, then return true.
550 * Otherwise, if filtered_pids is empty, that means we can
551 * trace all tasks. If it has content, then only trace pids
552 * within filtered_pids.
553 */
554
555 return (filtered_pids &&
556 !trace_find_filtered_pid(filtered_pids, task->pid)) ||
557 (filtered_no_pids &&
558 trace_find_filtered_pid(filtered_no_pids, task->pid));
559}
560
561/**
562 * trace_filter_add_remove_task - Add or remove a task from a pid_list
563 * @pid_list: The list to modify
564 * @self: The current task for fork or NULL for exit
565 * @task: The task to add or remove
566 *
567 * If adding a task, if @self is defined, the task is only added if @self
568 * is also included in @pid_list. This happens on fork and tasks should
569 * only be added when the parent is listed. If @self is NULL, then the
570 * @task pid will be removed from the list, which would happen on exit
571 * of a task.
572 */
573void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
574 struct task_struct *self,
575 struct task_struct *task)
576{
577 if (!pid_list)
578 return;
579
580 /* For forks, we only add if the forking task is listed */
581 if (self) {
582 if (!trace_find_filtered_pid(pid_list, self->pid))
583 return;
584 }
585
586 /* Sorry, but we don't support pid_max changing after setting */
587 if (task->pid >= pid_list->pid_max)
588 return;
589
590 /* "self" is set for forks, and NULL for exits */
591 if (self)
592 set_bit(task->pid, pid_list->pids);
593 else
594 clear_bit(task->pid, pid_list->pids);
595}
596
597/**
598 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
599 * @pid_list: The pid list to show
600 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
601 * @pos: The position of the file
602 *
603 * This is used by the seq_file "next" operation to iterate the pids
604 * listed in a trace_pid_list structure.
605 *
606 * Returns the pid+1 as we want to display pid of zero, but NULL would
607 * stop the iteration.
608 */
609void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
610{
611 unsigned long pid = (unsigned long)v;
612
613 (*pos)++;
614
615 /* pid already is +1 of the actual prevous bit */
616 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
617
618 /* Return pid + 1 to allow zero to be represented */
619 if (pid < pid_list->pid_max)
620 return (void *)(pid + 1);
621
622 return NULL;
623}
624
625/**
626 * trace_pid_start - Used for seq_file to start reading pid lists
627 * @pid_list: The pid list to show
628 * @pos: The position of the file
629 *
630 * This is used by seq_file "start" operation to start the iteration
631 * of listing pids.
632 *
633 * Returns the pid+1 as we want to display pid of zero, but NULL would
634 * stop the iteration.
635 */
636void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
637{
638 unsigned long pid;
639 loff_t l = 0;
640
641 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
642 if (pid >= pid_list->pid_max)
643 return NULL;
644
645 /* Return pid + 1 so that zero can be the exit value */
646 for (pid++; pid && l < *pos;
647 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
648 ;
649 return (void *)pid;
650}
651
652/**
653 * trace_pid_show - show the current pid in seq_file processing
654 * @m: The seq_file structure to write into
655 * @v: A void pointer of the pid (+1) value to display
656 *
657 * Can be directly used by seq_file operations to display the current
658 * pid value.
659 */
660int trace_pid_show(struct seq_file *m, void *v)
661{
662 unsigned long pid = (unsigned long)v - 1;
663
664 seq_printf(m, "%lu\n", pid);
665 return 0;
666}
667
668/* 128 should be much more than enough */
669#define PID_BUF_SIZE 127
670
671int trace_pid_write(struct trace_pid_list *filtered_pids,
672 struct trace_pid_list **new_pid_list,
673 const char __user *ubuf, size_t cnt)
674{
675 struct trace_pid_list *pid_list;
676 struct trace_parser parser;
677 unsigned long val;
678 int nr_pids = 0;
679 ssize_t read = 0;
680 ssize_t ret = 0;
681 loff_t pos;
682 pid_t pid;
683
684 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
685 return -ENOMEM;
686
687 /*
688 * Always recreate a new array. The write is an all or nothing
689 * operation. Always create a new array when adding new pids by
690 * the user. If the operation fails, then the current list is
691 * not modified.
692 */
693 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
694 if (!pid_list) {
695 trace_parser_put(&parser);
696 return -ENOMEM;
697 }
698
699 pid_list->pid_max = READ_ONCE(pid_max);
700
701 /* Only truncating will shrink pid_max */
702 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
703 pid_list->pid_max = filtered_pids->pid_max;
704
705 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
706 if (!pid_list->pids) {
707 trace_parser_put(&parser);
708 kfree(pid_list);
709 return -ENOMEM;
710 }
711
712 if (filtered_pids) {
713 /* copy the current bits to the new max */
714 for_each_set_bit(pid, filtered_pids->pids,
715 filtered_pids->pid_max) {
716 set_bit(pid, pid_list->pids);
717 nr_pids++;
718 }
719 }
720
721 while (cnt > 0) {
722
723 pos = 0;
724
725 ret = trace_get_user(&parser, ubuf, cnt, &pos);
726 if (ret < 0 || !trace_parser_loaded(&parser))
727 break;
728
729 read += ret;
730 ubuf += ret;
731 cnt -= ret;
732
733 ret = -EINVAL;
734 if (kstrtoul(parser.buffer, 0, &val))
735 break;
736 if (val >= pid_list->pid_max)
737 break;
738
739 pid = (pid_t)val;
740
741 set_bit(pid, pid_list->pids);
742 nr_pids++;
743
744 trace_parser_clear(&parser);
745 ret = 0;
746 }
747 trace_parser_put(&parser);
748
749 if (ret < 0) {
750 trace_free_pid_list(pid_list);
751 return ret;
752 }
753
754 if (!nr_pids) {
755 /* Cleared the list of pids */
756 trace_free_pid_list(pid_list);
757 read = ret;
758 pid_list = NULL;
759 }
760
761 *new_pid_list = pid_list;
762
763 return read;
764}
765
766static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
767{
768 u64 ts;
769
770 /* Early boot up does not have a buffer yet */
771 if (!buf->buffer)
772 return trace_clock_local();
773
774 ts = ring_buffer_time_stamp(buf->buffer, cpu);
775 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
776
777 return ts;
778}
779
780u64 ftrace_now(int cpu)
781{
782 return buffer_ftrace_now(&global_trace.array_buffer, cpu);
783}
784
785/**
786 * tracing_is_enabled - Show if global_trace has been enabled
787 *
788 * Shows if the global trace has been enabled or not. It uses the
789 * mirror flag "buffer_disabled" to be used in fast paths such as for
790 * the irqsoff tracer. But it may be inaccurate due to races. If you
791 * need to know the accurate state, use tracing_is_on() which is a little
792 * slower, but accurate.
793 */
794int tracing_is_enabled(void)
795{
796 /*
797 * For quick access (irqsoff uses this in fast path), just
798 * return the mirror variable of the state of the ring buffer.
799 * It's a little racy, but we don't really care.
800 */
801 smp_rmb();
802 return !global_trace.buffer_disabled;
803}
804
805/*
806 * trace_buf_size is the size in bytes that is allocated
807 * for a buffer. Note, the number of bytes is always rounded
808 * to page size.
809 *
810 * This number is purposely set to a low number of 16384.
811 * If the dump on oops happens, it will be much appreciated
812 * to not have to wait for all that output. Anyway this can be
813 * boot time and run time configurable.
814 */
815#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
816
817static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
818
819/* trace_types holds a link list of available tracers. */
820static struct tracer *trace_types __read_mostly;
821
822/*
823 * trace_types_lock is used to protect the trace_types list.
824 */
825DEFINE_MUTEX(trace_types_lock);
826
827/*
828 * serialize the access of the ring buffer
829 *
830 * ring buffer serializes readers, but it is low level protection.
831 * The validity of the events (which returns by ring_buffer_peek() ..etc)
832 * are not protected by ring buffer.
833 *
834 * The content of events may become garbage if we allow other process consumes
835 * these events concurrently:
836 * A) the page of the consumed events may become a normal page
837 * (not reader page) in ring buffer, and this page will be rewrited
838 * by events producer.
839 * B) The page of the consumed events may become a page for splice_read,
840 * and this page will be returned to system.
841 *
842 * These primitives allow multi process access to different cpu ring buffer
843 * concurrently.
844 *
845 * These primitives don't distinguish read-only and read-consume access.
846 * Multi read-only access are also serialized.
847 */
848
849#ifdef CONFIG_SMP
850static DECLARE_RWSEM(all_cpu_access_lock);
851static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
852
853static inline void trace_access_lock(int cpu)
854{
855 if (cpu == RING_BUFFER_ALL_CPUS) {
856 /* gain it for accessing the whole ring buffer. */
857 down_write(&all_cpu_access_lock);
858 } else {
859 /* gain it for accessing a cpu ring buffer. */
860
861 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
862 down_read(&all_cpu_access_lock);
863
864 /* Secondly block other access to this @cpu ring buffer. */
865 mutex_lock(&per_cpu(cpu_access_lock, cpu));
866 }
867}
868
869static inline void trace_access_unlock(int cpu)
870{
871 if (cpu == RING_BUFFER_ALL_CPUS) {
872 up_write(&all_cpu_access_lock);
873 } else {
874 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
875 up_read(&all_cpu_access_lock);
876 }
877}
878
879static inline void trace_access_lock_init(void)
880{
881 int cpu;
882
883 for_each_possible_cpu(cpu)
884 mutex_init(&per_cpu(cpu_access_lock, cpu));
885}
886
887#else
888
889static DEFINE_MUTEX(access_lock);
890
891static inline void trace_access_lock(int cpu)
892{
893 (void)cpu;
894 mutex_lock(&access_lock);
895}
896
897static inline void trace_access_unlock(int cpu)
898{
899 (void)cpu;
900 mutex_unlock(&access_lock);
901}
902
903static inline void trace_access_lock_init(void)
904{
905}
906
907#endif
908
909#ifdef CONFIG_STACKTRACE
910static void __ftrace_trace_stack(struct trace_buffer *buffer,
911 unsigned int trace_ctx,
912 int skip, struct pt_regs *regs);
913static inline void ftrace_trace_stack(struct trace_array *tr,
914 struct trace_buffer *buffer,
915 unsigned int trace_ctx,
916 int skip, struct pt_regs *regs);
917
918#else
919static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
920 unsigned int trace_ctx,
921 int skip, struct pt_regs *regs)
922{
923}
924static inline void ftrace_trace_stack(struct trace_array *tr,
925 struct trace_buffer *buffer,
926 unsigned long trace_ctx,
927 int skip, struct pt_regs *regs)
928{
929}
930
931#endif
932
933static __always_inline void
934trace_event_setup(struct ring_buffer_event *event,
935 int type, unsigned int trace_ctx)
936{
937 struct trace_entry *ent = ring_buffer_event_data(event);
938
939 tracing_generic_entry_update(ent, type, trace_ctx);
940}
941
942static __always_inline struct ring_buffer_event *
943__trace_buffer_lock_reserve(struct trace_buffer *buffer,
944 int type,
945 unsigned long len,
946 unsigned int trace_ctx)
947{
948 struct ring_buffer_event *event;
949
950 event = ring_buffer_lock_reserve(buffer, len);
951 if (event != NULL)
952 trace_event_setup(event, type, trace_ctx);
953
954 return event;
955}
956
957void tracer_tracing_on(struct trace_array *tr)
958{
959 if (tr->array_buffer.buffer)
960 ring_buffer_record_on(tr->array_buffer.buffer);
961 /*
962 * This flag is looked at when buffers haven't been allocated
963 * yet, or by some tracers (like irqsoff), that just want to
964 * know if the ring buffer has been disabled, but it can handle
965 * races of where it gets disabled but we still do a record.
966 * As the check is in the fast path of the tracers, it is more
967 * important to be fast than accurate.
968 */
969 tr->buffer_disabled = 0;
970 /* Make the flag seen by readers */
971 smp_wmb();
972}
973
974/**
975 * tracing_on - enable tracing buffers
976 *
977 * This function enables tracing buffers that may have been
978 * disabled with tracing_off.
979 */
980void tracing_on(void)
981{
982 tracer_tracing_on(&global_trace);
983}
984EXPORT_SYMBOL_GPL(tracing_on);
985
986
987static __always_inline void
988__buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)
989{
990 __this_cpu_write(trace_taskinfo_save, true);
991
992 /* If this is the temp buffer, we need to commit fully */
993 if (this_cpu_read(trace_buffered_event) == event) {
994 /* Length is in event->array[0] */
995 ring_buffer_write(buffer, event->array[0], &event->array[1]);
996 /* Release the temp buffer */
997 this_cpu_dec(trace_buffered_event_cnt);
998 } else
999 ring_buffer_unlock_commit(buffer, event);
1000}
1001
1002/**
1003 * __trace_puts - write a constant string into the trace buffer.
1004 * @ip: The address of the caller
1005 * @str: The constant string to write
1006 * @size: The size of the string.
1007 */
1008int __trace_puts(unsigned long ip, const char *str, int size)
1009{
1010 struct ring_buffer_event *event;
1011 struct trace_buffer *buffer;
1012 struct print_entry *entry;
1013 unsigned int trace_ctx;
1014 int alloc;
1015
1016 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
1017 return 0;
1018
1019 if (unlikely(tracing_selftest_running || tracing_disabled))
1020 return 0;
1021
1022 alloc = sizeof(*entry) + size + 2; /* possible \n added */
1023
1024 trace_ctx = tracing_gen_ctx();
1025 buffer = global_trace.array_buffer.buffer;
1026 ring_buffer_nest_start(buffer);
1027 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
1028 trace_ctx);
1029 if (!event) {
1030 size = 0;
1031 goto out;
1032 }
1033
1034 entry = ring_buffer_event_data(event);
1035 entry->ip = ip;
1036
1037 memcpy(&entry->buf, str, size);
1038
1039 /* Add a newline if necessary */
1040 if (entry->buf[size - 1] != '\n') {
1041 entry->buf[size] = '\n';
1042 entry->buf[size + 1] = '\0';
1043 } else
1044 entry->buf[size] = '\0';
1045
1046 __buffer_unlock_commit(buffer, event);
1047 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
1048 out:
1049 ring_buffer_nest_end(buffer);
1050 return size;
1051}
1052EXPORT_SYMBOL_GPL(__trace_puts);
1053
1054/**
1055 * __trace_bputs - write the pointer to a constant string into trace buffer
1056 * @ip: The address of the caller
1057 * @str: The constant string to write to the buffer to
1058 */
1059int __trace_bputs(unsigned long ip, const char *str)
1060{
1061 struct ring_buffer_event *event;
1062 struct trace_buffer *buffer;
1063 struct bputs_entry *entry;
1064 unsigned int trace_ctx;
1065 int size = sizeof(struct bputs_entry);
1066 int ret = 0;
1067
1068 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
1069 return 0;
1070
1071 if (unlikely(tracing_selftest_running || tracing_disabled))
1072 return 0;
1073
1074 trace_ctx = tracing_gen_ctx();
1075 buffer = global_trace.array_buffer.buffer;
1076
1077 ring_buffer_nest_start(buffer);
1078 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
1079 trace_ctx);
1080 if (!event)
1081 goto out;
1082
1083 entry = ring_buffer_event_data(event);
1084 entry->ip = ip;
1085 entry->str = str;
1086
1087 __buffer_unlock_commit(buffer, event);
1088 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
1089
1090 ret = 1;
1091 out:
1092 ring_buffer_nest_end(buffer);
1093 return ret;
1094}
1095EXPORT_SYMBOL_GPL(__trace_bputs);
1096
1097#ifdef CONFIG_TRACER_SNAPSHOT
1098static void tracing_snapshot_instance_cond(struct trace_array *tr,
1099 void *cond_data)
1100{
1101 struct tracer *tracer = tr->current_trace;
1102 unsigned long flags;
1103
1104 if (in_nmi()) {
1105 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
1106 internal_trace_puts("*** snapshot is being ignored ***\n");
1107 return;
1108 }
1109
1110 if (!tr->allocated_snapshot) {
1111 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
1112 internal_trace_puts("*** stopping trace here! ***\n");
1113 tracing_off();
1114 return;
1115 }
1116
1117 /* Note, snapshot can not be used when the tracer uses it */
1118 if (tracer->use_max_tr) {
1119 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
1120 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
1121 return;
1122 }
1123
1124 local_irq_save(flags);
1125 update_max_tr(tr, current, smp_processor_id(), cond_data);
1126 local_irq_restore(flags);
1127}
1128
1129void tracing_snapshot_instance(struct trace_array *tr)
1130{
1131 tracing_snapshot_instance_cond(tr, NULL);
1132}
1133
1134/**
1135 * tracing_snapshot - take a snapshot of the current buffer.
1136 *
1137 * This causes a swap between the snapshot buffer and the current live
1138 * tracing buffer. You can use this to take snapshots of the live
1139 * trace when some condition is triggered, but continue to trace.
1140 *
1141 * Note, make sure to allocate the snapshot with either
1142 * a tracing_snapshot_alloc(), or by doing it manually
1143 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
1144 *
1145 * If the snapshot buffer is not allocated, it will stop tracing.
1146 * Basically making a permanent snapshot.
1147 */
1148void tracing_snapshot(void)
1149{
1150 struct trace_array *tr = &global_trace;
1151
1152 tracing_snapshot_instance(tr);
1153}
1154EXPORT_SYMBOL_GPL(tracing_snapshot);
1155
1156/**
1157 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1158 * @tr: The tracing instance to snapshot
1159 * @cond_data: The data to be tested conditionally, and possibly saved
1160 *
1161 * This is the same as tracing_snapshot() except that the snapshot is
1162 * conditional - the snapshot will only happen if the
1163 * cond_snapshot.update() implementation receiving the cond_data
1164 * returns true, which means that the trace array's cond_snapshot
1165 * update() operation used the cond_data to determine whether the
1166 * snapshot should be taken, and if it was, presumably saved it along
1167 * with the snapshot.
1168 */
1169void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1170{
1171 tracing_snapshot_instance_cond(tr, cond_data);
1172}
1173EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1174
1175/**
1176 * tracing_snapshot_cond_data - get the user data associated with a snapshot
1177 * @tr: The tracing instance
1178 *
1179 * When the user enables a conditional snapshot using
1180 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1181 * with the snapshot. This accessor is used to retrieve it.
1182 *
1183 * Should not be called from cond_snapshot.update(), since it takes
1184 * the tr->max_lock lock, which the code calling
1185 * cond_snapshot.update() has already done.
1186 *
1187 * Returns the cond_data associated with the trace array's snapshot.
1188 */
1189void *tracing_cond_snapshot_data(struct trace_array *tr)
1190{
1191 void *cond_data = NULL;
1192
1193 arch_spin_lock(&tr->max_lock);
1194
1195 if (tr->cond_snapshot)
1196 cond_data = tr->cond_snapshot->cond_data;
1197
1198 arch_spin_unlock(&tr->max_lock);
1199
1200 return cond_data;
1201}
1202EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1203
1204static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1205 struct array_buffer *size_buf, int cpu_id);
1206static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1207
1208int tracing_alloc_snapshot_instance(struct trace_array *tr)
1209{
1210 int ret;
1211
1212 if (!tr->allocated_snapshot) {
1213
1214 /* allocate spare buffer */
1215 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1216 &tr->array_buffer, RING_BUFFER_ALL_CPUS);
1217 if (ret < 0)
1218 return ret;
1219
1220 tr->allocated_snapshot = true;
1221 }
1222
1223 return 0;
1224}
1225
1226static void free_snapshot(struct trace_array *tr)
1227{
1228 /*
1229 * We don't free the ring buffer. instead, resize it because
1230 * The max_tr ring buffer has some state (e.g. ring->clock) and
1231 * we want preserve it.
1232 */
1233 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1234 set_buffer_entries(&tr->max_buffer, 1);
1235 tracing_reset_online_cpus(&tr->max_buffer);
1236 tr->allocated_snapshot = false;
1237}
1238
1239/**
1240 * tracing_alloc_snapshot - allocate snapshot buffer.
1241 *
1242 * This only allocates the snapshot buffer if it isn't already
1243 * allocated - it doesn't also take a snapshot.
1244 *
1245 * This is meant to be used in cases where the snapshot buffer needs
1246 * to be set up for events that can't sleep but need to be able to
1247 * trigger a snapshot.
1248 */
1249int tracing_alloc_snapshot(void)
1250{
1251 struct trace_array *tr = &global_trace;
1252 int ret;
1253
1254 ret = tracing_alloc_snapshot_instance(tr);
1255 WARN_ON(ret < 0);
1256
1257 return ret;
1258}
1259EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1260
1261/**
1262 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1263 *
1264 * This is similar to tracing_snapshot(), but it will allocate the
1265 * snapshot buffer if it isn't already allocated. Use this only
1266 * where it is safe to sleep, as the allocation may sleep.
1267 *
1268 * This causes a swap between the snapshot buffer and the current live
1269 * tracing buffer. You can use this to take snapshots of the live
1270 * trace when some condition is triggered, but continue to trace.
1271 */
1272void tracing_snapshot_alloc(void)
1273{
1274 int ret;
1275
1276 ret = tracing_alloc_snapshot();
1277 if (ret < 0)
1278 return;
1279
1280 tracing_snapshot();
1281}
1282EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1283
1284/**
1285 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1286 * @tr: The tracing instance
1287 * @cond_data: User data to associate with the snapshot
1288 * @update: Implementation of the cond_snapshot update function
1289 *
1290 * Check whether the conditional snapshot for the given instance has
1291 * already been enabled, or if the current tracer is already using a
1292 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1293 * save the cond_data and update function inside.
1294 *
1295 * Returns 0 if successful, error otherwise.
1296 */
1297int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1298 cond_update_fn_t update)
1299{
1300 struct cond_snapshot *cond_snapshot;
1301 int ret = 0;
1302
1303 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1304 if (!cond_snapshot)
1305 return -ENOMEM;
1306
1307 cond_snapshot->cond_data = cond_data;
1308 cond_snapshot->update = update;
1309
1310 mutex_lock(&trace_types_lock);
1311
1312 ret = tracing_alloc_snapshot_instance(tr);
1313 if (ret)
1314 goto fail_unlock;
1315
1316 if (tr->current_trace->use_max_tr) {
1317 ret = -EBUSY;
1318 goto fail_unlock;
1319 }
1320
1321 /*
1322 * The cond_snapshot can only change to NULL without the
1323 * trace_types_lock. We don't care if we race with it going
1324 * to NULL, but we want to make sure that it's not set to
1325 * something other than NULL when we get here, which we can
1326 * do safely with only holding the trace_types_lock and not
1327 * having to take the max_lock.
1328 */
1329 if (tr->cond_snapshot) {
1330 ret = -EBUSY;
1331 goto fail_unlock;
1332 }
1333
1334 arch_spin_lock(&tr->max_lock);
1335 tr->cond_snapshot = cond_snapshot;
1336 arch_spin_unlock(&tr->max_lock);
1337
1338 mutex_unlock(&trace_types_lock);
1339
1340 return ret;
1341
1342 fail_unlock:
1343 mutex_unlock(&trace_types_lock);
1344 kfree(cond_snapshot);
1345 return ret;
1346}
1347EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1348
1349/**
1350 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1351 * @tr: The tracing instance
1352 *
1353 * Check whether the conditional snapshot for the given instance is
1354 * enabled; if so, free the cond_snapshot associated with it,
1355 * otherwise return -EINVAL.
1356 *
1357 * Returns 0 if successful, error otherwise.
1358 */
1359int tracing_snapshot_cond_disable(struct trace_array *tr)
1360{
1361 int ret = 0;
1362
1363 arch_spin_lock(&tr->max_lock);
1364
1365 if (!tr->cond_snapshot)
1366 ret = -EINVAL;
1367 else {
1368 kfree(tr->cond_snapshot);
1369 tr->cond_snapshot = NULL;
1370 }
1371
1372 arch_spin_unlock(&tr->max_lock);
1373
1374 return ret;
1375}
1376EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1377#else
1378void tracing_snapshot(void)
1379{
1380 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1381}
1382EXPORT_SYMBOL_GPL(tracing_snapshot);
1383void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1384{
1385 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1386}
1387EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1388int tracing_alloc_snapshot(void)
1389{
1390 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1391 return -ENODEV;
1392}
1393EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1394void tracing_snapshot_alloc(void)
1395{
1396 /* Give warning */
1397 tracing_snapshot();
1398}
1399EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1400void *tracing_cond_snapshot_data(struct trace_array *tr)
1401{
1402 return NULL;
1403}
1404EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1405int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1406{
1407 return -ENODEV;
1408}
1409EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1410int tracing_snapshot_cond_disable(struct trace_array *tr)
1411{
1412 return false;
1413}
1414EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1415#endif /* CONFIG_TRACER_SNAPSHOT */
1416
1417void tracer_tracing_off(struct trace_array *tr)
1418{
1419 if (tr->array_buffer.buffer)
1420 ring_buffer_record_off(tr->array_buffer.buffer);
1421 /*
1422 * This flag is looked at when buffers haven't been allocated
1423 * yet, or by some tracers (like irqsoff), that just want to
1424 * know if the ring buffer has been disabled, but it can handle
1425 * races of where it gets disabled but we still do a record.
1426 * As the check is in the fast path of the tracers, it is more
1427 * important to be fast than accurate.
1428 */
1429 tr->buffer_disabled = 1;
1430 /* Make the flag seen by readers */
1431 smp_wmb();
1432}
1433
1434/**
1435 * tracing_off - turn off tracing buffers
1436 *
1437 * This function stops the tracing buffers from recording data.
1438 * It does not disable any overhead the tracers themselves may
1439 * be causing. This function simply causes all recording to
1440 * the ring buffers to fail.
1441 */
1442void tracing_off(void)
1443{
1444 tracer_tracing_off(&global_trace);
1445}
1446EXPORT_SYMBOL_GPL(tracing_off);
1447
1448void disable_trace_on_warning(void)
1449{
1450 if (__disable_trace_on_warning) {
1451 trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
1452 "Disabling tracing due to warning\n");
1453 tracing_off();
1454 }
1455}
1456
1457/**
1458 * tracer_tracing_is_on - show real state of ring buffer enabled
1459 * @tr : the trace array to know if ring buffer is enabled
1460 *
1461 * Shows real state of the ring buffer if it is enabled or not.
1462 */
1463bool tracer_tracing_is_on(struct trace_array *tr)
1464{
1465 if (tr->array_buffer.buffer)
1466 return ring_buffer_record_is_on(tr->array_buffer.buffer);
1467 return !tr->buffer_disabled;
1468}
1469
1470/**
1471 * tracing_is_on - show state of ring buffers enabled
1472 */
1473int tracing_is_on(void)
1474{
1475 return tracer_tracing_is_on(&global_trace);
1476}
1477EXPORT_SYMBOL_GPL(tracing_is_on);
1478
1479static int __init set_buf_size(char *str)
1480{
1481 unsigned long buf_size;
1482
1483 if (!str)
1484 return 0;
1485 buf_size = memparse(str, &str);
1486 /* nr_entries can not be zero */
1487 if (buf_size == 0)
1488 return 0;
1489 trace_buf_size = buf_size;
1490 return 1;
1491}
1492__setup("trace_buf_size=", set_buf_size);
1493
1494static int __init set_tracing_thresh(char *str)
1495{
1496 unsigned long threshold;
1497 int ret;
1498
1499 if (!str)
1500 return 0;
1501 ret = kstrtoul(str, 0, &threshold);
1502 if (ret < 0)
1503 return 0;
1504 tracing_thresh = threshold * 1000;
1505 return 1;
1506}
1507__setup("tracing_thresh=", set_tracing_thresh);
1508
1509unsigned long nsecs_to_usecs(unsigned long nsecs)
1510{
1511 return nsecs / 1000;
1512}
1513
1514/*
1515 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1516 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1517 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1518 * of strings in the order that the evals (enum) were defined.
1519 */
1520#undef C
1521#define C(a, b) b
1522
1523/* These must match the bit postions in trace_iterator_flags */
1524static const char *trace_options[] = {
1525 TRACE_FLAGS
1526 NULL
1527};
1528
1529static struct {
1530 u64 (*func)(void);
1531 const char *name;
1532 int in_ns; /* is this clock in nanoseconds? */
1533} trace_clocks[] = {
1534 { trace_clock_local, "local", 1 },
1535 { trace_clock_global, "global", 1 },
1536 { trace_clock_counter, "counter", 0 },
1537 { trace_clock_jiffies, "uptime", 0 },
1538 { trace_clock, "perf", 1 },
1539 { ktime_get_mono_fast_ns, "mono", 1 },
1540 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1541 { ktime_get_boot_fast_ns, "boot", 1 },
1542 ARCH_TRACE_CLOCKS
1543};
1544
1545bool trace_clock_in_ns(struct trace_array *tr)
1546{
1547 if (trace_clocks[tr->clock_id].in_ns)
1548 return true;
1549
1550 return false;
1551}
1552
1553/*
1554 * trace_parser_get_init - gets the buffer for trace parser
1555 */
1556int trace_parser_get_init(struct trace_parser *parser, int size)
1557{
1558 memset(parser, 0, sizeof(*parser));
1559
1560 parser->buffer = kmalloc(size, GFP_KERNEL);
1561 if (!parser->buffer)
1562 return 1;
1563
1564 parser->size = size;
1565 return 0;
1566}
1567
1568/*
1569 * trace_parser_put - frees the buffer for trace parser
1570 */
1571void trace_parser_put(struct trace_parser *parser)
1572{
1573 kfree(parser->buffer);
1574 parser->buffer = NULL;
1575}
1576
1577/*
1578 * trace_get_user - reads the user input string separated by space
1579 * (matched by isspace(ch))
1580 *
1581 * For each string found the 'struct trace_parser' is updated,
1582 * and the function returns.
1583 *
1584 * Returns number of bytes read.
1585 *
1586 * See kernel/trace/trace.h for 'struct trace_parser' details.
1587 */
1588int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1589 size_t cnt, loff_t *ppos)
1590{
1591 char ch;
1592 size_t read = 0;
1593 ssize_t ret;
1594
1595 if (!*ppos)
1596 trace_parser_clear(parser);
1597
1598 ret = get_user(ch, ubuf++);
1599 if (ret)
1600 goto out;
1601
1602 read++;
1603 cnt--;
1604
1605 /*
1606 * The parser is not finished with the last write,
1607 * continue reading the user input without skipping spaces.
1608 */
1609 if (!parser->cont) {
1610 /* skip white space */
1611 while (cnt && isspace(ch)) {
1612 ret = get_user(ch, ubuf++);
1613 if (ret)
1614 goto out;
1615 read++;
1616 cnt--;
1617 }
1618
1619 parser->idx = 0;
1620
1621 /* only spaces were written */
1622 if (isspace(ch) || !ch) {
1623 *ppos += read;
1624 ret = read;
1625 goto out;
1626 }
1627 }
1628
1629 /* read the non-space input */
1630 while (cnt && !isspace(ch) && ch) {
1631 if (parser->idx < parser->size - 1)
1632 parser->buffer[parser->idx++] = ch;
1633 else {
1634 ret = -EINVAL;
1635 goto out;
1636 }
1637 ret = get_user(ch, ubuf++);
1638 if (ret)
1639 goto out;
1640 read++;
1641 cnt--;
1642 }
1643
1644 /* We either got finished input or we have to wait for another call. */
1645 if (isspace(ch) || !ch) {
1646 parser->buffer[parser->idx] = 0;
1647 parser->cont = false;
1648 } else if (parser->idx < parser->size - 1) {
1649 parser->cont = true;
1650 parser->buffer[parser->idx++] = ch;
1651 /* Make sure the parsed string always terminates with '\0'. */
1652 parser->buffer[parser->idx] = 0;
1653 } else {
1654 ret = -EINVAL;
1655 goto out;
1656 }
1657
1658 *ppos += read;
1659 ret = read;
1660
1661out:
1662 return ret;
1663}
1664
1665/* TODO add a seq_buf_to_buffer() */
1666static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1667{
1668 int len;
1669
1670 if (trace_seq_used(s) <= s->seq.readpos)
1671 return -EBUSY;
1672
1673 len = trace_seq_used(s) - s->seq.readpos;
1674 if (cnt > len)
1675 cnt = len;
1676 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1677
1678 s->seq.readpos += cnt;
1679 return cnt;
1680}
1681
1682unsigned long __read_mostly tracing_thresh;
1683static const struct file_operations tracing_max_lat_fops;
1684
1685#if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1686 defined(CONFIG_FSNOTIFY)
1687
1688static struct workqueue_struct *fsnotify_wq;
1689
1690static void latency_fsnotify_workfn(struct work_struct *work)
1691{
1692 struct trace_array *tr = container_of(work, struct trace_array,
1693 fsnotify_work);
1694 fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);
1695}
1696
1697static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1698{
1699 struct trace_array *tr = container_of(iwork, struct trace_array,
1700 fsnotify_irqwork);
1701 queue_work(fsnotify_wq, &tr->fsnotify_work);
1702}
1703
1704static void trace_create_maxlat_file(struct trace_array *tr,
1705 struct dentry *d_tracer)
1706{
1707 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1708 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1709 tr->d_max_latency = trace_create_file("tracing_max_latency", 0644,
1710 d_tracer, &tr->max_latency,
1711 &tracing_max_lat_fops);
1712}
1713
1714__init static int latency_fsnotify_init(void)
1715{
1716 fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1717 WQ_UNBOUND | WQ_HIGHPRI, 0);
1718 if (!fsnotify_wq) {
1719 pr_err("Unable to allocate tr_max_lat_wq\n");
1720 return -ENOMEM;
1721 }
1722 return 0;
1723}
1724
1725late_initcall_sync(latency_fsnotify_init);
1726
1727void latency_fsnotify(struct trace_array *tr)
1728{
1729 if (!fsnotify_wq)
1730 return;
1731 /*
1732 * We cannot call queue_work(&tr->fsnotify_work) from here because it's
1733 * possible that we are called from __schedule() or do_idle(), which
1734 * could cause a deadlock.
1735 */
1736 irq_work_queue(&tr->fsnotify_irqwork);
1737}
1738
1739/*
1740 * (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1741 * defined(CONFIG_FSNOTIFY)
1742 */
1743#else
1744
1745#define trace_create_maxlat_file(tr, d_tracer) \
1746 trace_create_file("tracing_max_latency", 0644, d_tracer, \
1747 &tr->max_latency, &tracing_max_lat_fops)
1748
1749#endif
1750
1751#ifdef CONFIG_TRACER_MAX_TRACE
1752/*
1753 * Copy the new maximum trace into the separate maximum-trace
1754 * structure. (this way the maximum trace is permanently saved,
1755 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1756 */
1757static void
1758__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1759{
1760 struct array_buffer *trace_buf = &tr->array_buffer;
1761 struct array_buffer *max_buf = &tr->max_buffer;
1762 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1763 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1764
1765 max_buf->cpu = cpu;
1766 max_buf->time_start = data->preempt_timestamp;
1767
1768 max_data->saved_latency = tr->max_latency;
1769 max_data->critical_start = data->critical_start;
1770 max_data->critical_end = data->critical_end;
1771
1772 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1773 max_data->pid = tsk->pid;
1774 /*
1775 * If tsk == current, then use current_uid(), as that does not use
1776 * RCU. The irq tracer can be called out of RCU scope.
1777 */
1778 if (tsk == current)
1779 max_data->uid = current_uid();
1780 else
1781 max_data->uid = task_uid(tsk);
1782
1783 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1784 max_data->policy = tsk->policy;
1785 max_data->rt_priority = tsk->rt_priority;
1786
1787 /* record this tasks comm */
1788 tracing_record_cmdline(tsk);
1789 latency_fsnotify(tr);
1790}
1791
1792/**
1793 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1794 * @tr: tracer
1795 * @tsk: the task with the latency
1796 * @cpu: The cpu that initiated the trace.
1797 * @cond_data: User data associated with a conditional snapshot
1798 *
1799 * Flip the buffers between the @tr and the max_tr and record information
1800 * about which task was the cause of this latency.
1801 */
1802void
1803update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1804 void *cond_data)
1805{
1806 if (tr->stop_count)
1807 return;
1808
1809 WARN_ON_ONCE(!irqs_disabled());
1810
1811 if (!tr->allocated_snapshot) {
1812 /* Only the nop tracer should hit this when disabling */
1813 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1814 return;
1815 }
1816
1817 arch_spin_lock(&tr->max_lock);
1818
1819 /* Inherit the recordable setting from array_buffer */
1820 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1821 ring_buffer_record_on(tr->max_buffer.buffer);
1822 else
1823 ring_buffer_record_off(tr->max_buffer.buffer);
1824
1825#ifdef CONFIG_TRACER_SNAPSHOT
1826 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1827 goto out_unlock;
1828#endif
1829 swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1830
1831 __update_max_tr(tr, tsk, cpu);
1832
1833 out_unlock:
1834 arch_spin_unlock(&tr->max_lock);
1835}
1836
1837/**
1838 * update_max_tr_single - only copy one trace over, and reset the rest
1839 * @tr: tracer
1840 * @tsk: task with the latency
1841 * @cpu: the cpu of the buffer to copy.
1842 *
1843 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1844 */
1845void
1846update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1847{
1848 int ret;
1849
1850 if (tr->stop_count)
1851 return;
1852
1853 WARN_ON_ONCE(!irqs_disabled());
1854 if (!tr->allocated_snapshot) {
1855 /* Only the nop tracer should hit this when disabling */
1856 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1857 return;
1858 }
1859
1860 arch_spin_lock(&tr->max_lock);
1861
1862 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
1863
1864 if (ret == -EBUSY) {
1865 /*
1866 * We failed to swap the buffer due to a commit taking
1867 * place on this CPU. We fail to record, but we reset
1868 * the max trace buffer (no one writes directly to it)
1869 * and flag that it failed.
1870 */
1871 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1872 "Failed to swap buffers due to commit in progress\n");
1873 }
1874
1875 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1876
1877 __update_max_tr(tr, tsk, cpu);
1878 arch_spin_unlock(&tr->max_lock);
1879}
1880#endif /* CONFIG_TRACER_MAX_TRACE */
1881
1882static int wait_on_pipe(struct trace_iterator *iter, int full)
1883{
1884 /* Iterators are static, they should be filled or empty */
1885 if (trace_buffer_iter(iter, iter->cpu_file))
1886 return 0;
1887
1888 return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file,
1889 full);
1890}
1891
1892#ifdef CONFIG_FTRACE_STARTUP_TEST
1893static bool selftests_can_run;
1894
1895struct trace_selftests {
1896 struct list_head list;
1897 struct tracer *type;
1898};
1899
1900static LIST_HEAD(postponed_selftests);
1901
1902static int save_selftest(struct tracer *type)
1903{
1904 struct trace_selftests *selftest;
1905
1906 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1907 if (!selftest)
1908 return -ENOMEM;
1909
1910 selftest->type = type;
1911 list_add(&selftest->list, &postponed_selftests);
1912 return 0;
1913}
1914
1915static int run_tracer_selftest(struct tracer *type)
1916{
1917 struct trace_array *tr = &global_trace;
1918 struct tracer *saved_tracer = tr->current_trace;
1919 int ret;
1920
1921 if (!type->selftest || tracing_selftest_disabled)
1922 return 0;
1923
1924 /*
1925 * If a tracer registers early in boot up (before scheduling is
1926 * initialized and such), then do not run its selftests yet.
1927 * Instead, run it a little later in the boot process.
1928 */
1929 if (!selftests_can_run)
1930 return save_selftest(type);
1931
1932 if (!tracing_is_on()) {
1933 pr_warn("Selftest for tracer %s skipped due to tracing disabled\n",
1934 type->name);
1935 return 0;
1936 }
1937
1938 /*
1939 * Run a selftest on this tracer.
1940 * Here we reset the trace buffer, and set the current
1941 * tracer to be this tracer. The tracer can then run some
1942 * internal tracing to verify that everything is in order.
1943 * If we fail, we do not register this tracer.
1944 */
1945 tracing_reset_online_cpus(&tr->array_buffer);
1946
1947 tr->current_trace = type;
1948
1949#ifdef CONFIG_TRACER_MAX_TRACE
1950 if (type->use_max_tr) {
1951 /* If we expanded the buffers, make sure the max is expanded too */
1952 if (ring_buffer_expanded)
1953 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1954 RING_BUFFER_ALL_CPUS);
1955 tr->allocated_snapshot = true;
1956 }
1957#endif
1958
1959 /* the test is responsible for initializing and enabling */
1960 pr_info("Testing tracer %s: ", type->name);
1961 ret = type->selftest(type, tr);
1962 /* the test is responsible for resetting too */
1963 tr->current_trace = saved_tracer;
1964 if (ret) {
1965 printk(KERN_CONT "FAILED!\n");
1966 /* Add the warning after printing 'FAILED' */
1967 WARN_ON(1);
1968 return -1;
1969 }
1970 /* Only reset on passing, to avoid touching corrupted buffers */
1971 tracing_reset_online_cpus(&tr->array_buffer);
1972
1973#ifdef CONFIG_TRACER_MAX_TRACE
1974 if (type->use_max_tr) {
1975 tr->allocated_snapshot = false;
1976
1977 /* Shrink the max buffer again */
1978 if (ring_buffer_expanded)
1979 ring_buffer_resize(tr->max_buffer.buffer, 1,
1980 RING_BUFFER_ALL_CPUS);
1981 }
1982#endif
1983
1984 printk(KERN_CONT "PASSED\n");
1985 return 0;
1986}
1987
1988static __init int init_trace_selftests(void)
1989{
1990 struct trace_selftests *p, *n;
1991 struct tracer *t, **last;
1992 int ret;
1993
1994 selftests_can_run = true;
1995
1996 mutex_lock(&trace_types_lock);
1997
1998 if (list_empty(&postponed_selftests))
1999 goto out;
2000
2001 pr_info("Running postponed tracer tests:\n");
2002
2003 tracing_selftest_running = true;
2004 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
2005 /* This loop can take minutes when sanitizers are enabled, so
2006 * lets make sure we allow RCU processing.
2007 */
2008 cond_resched();
2009 ret = run_tracer_selftest(p->type);
2010 /* If the test fails, then warn and remove from available_tracers */
2011 if (ret < 0) {
2012 WARN(1, "tracer: %s failed selftest, disabling\n",
2013 p->type->name);
2014 last = &trace_types;
2015 for (t = trace_types; t; t = t->next) {
2016 if (t == p->type) {
2017 *last = t->next;
2018 break;
2019 }
2020 last = &t->next;
2021 }
2022 }
2023 list_del(&p->list);
2024 kfree(p);
2025 }
2026 tracing_selftest_running = false;
2027
2028 out:
2029 mutex_unlock(&trace_types_lock);
2030
2031 return 0;
2032}
2033core_initcall(init_trace_selftests);
2034#else
2035static inline int run_tracer_selftest(struct tracer *type)
2036{
2037 return 0;
2038}
2039#endif /* CONFIG_FTRACE_STARTUP_TEST */
2040
2041static void add_tracer_options(struct trace_array *tr, struct tracer *t);
2042
2043static void __init apply_trace_boot_options(void);
2044
2045/**
2046 * register_tracer - register a tracer with the ftrace system.
2047 * @type: the plugin for the tracer
2048 *
2049 * Register a new plugin tracer.
2050 */
2051int __init register_tracer(struct tracer *type)
2052{
2053 struct tracer *t;
2054 int ret = 0;
2055
2056 if (!type->name) {
2057 pr_info("Tracer must have a name\n");
2058 return -1;
2059 }
2060
2061 if (strlen(type->name) >= MAX_TRACER_SIZE) {
2062 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
2063 return -1;
2064 }
2065
2066 if (security_locked_down(LOCKDOWN_TRACEFS)) {
2067 pr_warn("Can not register tracer %s due to lockdown\n",
2068 type->name);
2069 return -EPERM;
2070 }
2071
2072 mutex_lock(&trace_types_lock);
2073
2074 tracing_selftest_running = true;
2075
2076 for (t = trace_types; t; t = t->next) {
2077 if (strcmp(type->name, t->name) == 0) {
2078 /* already found */
2079 pr_info("Tracer %s already registered\n",
2080 type->name);
2081 ret = -1;
2082 goto out;
2083 }
2084 }
2085
2086 if (!type->set_flag)
2087 type->set_flag = &dummy_set_flag;
2088 if (!type->flags) {
2089 /*allocate a dummy tracer_flags*/
2090 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
2091 if (!type->flags) {
2092 ret = -ENOMEM;
2093 goto out;
2094 }
2095 type->flags->val = 0;
2096 type->flags->opts = dummy_tracer_opt;
2097 } else
2098 if (!type->flags->opts)
2099 type->flags->opts = dummy_tracer_opt;
2100
2101 /* store the tracer for __set_tracer_option */
2102 type->flags->trace = type;
2103
2104 ret = run_tracer_selftest(type);
2105 if (ret < 0)
2106 goto out;
2107
2108 type->next = trace_types;
2109 trace_types = type;
2110 add_tracer_options(&global_trace, type);
2111
2112 out:
2113 tracing_selftest_running = false;
2114 mutex_unlock(&trace_types_lock);
2115
2116 if (ret || !default_bootup_tracer)
2117 goto out_unlock;
2118
2119 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
2120 goto out_unlock;
2121
2122 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
2123 /* Do we want this tracer to start on bootup? */
2124 tracing_set_tracer(&global_trace, type->name);
2125 default_bootup_tracer = NULL;
2126
2127 apply_trace_boot_options();
2128
2129 /* disable other selftests, since this will break it. */
2130 disable_tracing_selftest("running a tracer");
2131
2132 out_unlock:
2133 return ret;
2134}
2135
2136static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
2137{
2138 struct trace_buffer *buffer = buf->buffer;
2139
2140 if (!buffer)
2141 return;
2142
2143 ring_buffer_record_disable(buffer);
2144
2145 /* Make sure all commits have finished */
2146 synchronize_rcu();
2147 ring_buffer_reset_cpu(buffer, cpu);
2148
2149 ring_buffer_record_enable(buffer);
2150}
2151
2152void tracing_reset_online_cpus(struct array_buffer *buf)
2153{
2154 struct trace_buffer *buffer = buf->buffer;
2155
2156 if (!buffer)
2157 return;
2158
2159 ring_buffer_record_disable(buffer);
2160
2161 /* Make sure all commits have finished */
2162 synchronize_rcu();
2163
2164 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2165
2166 ring_buffer_reset_online_cpus(buffer);
2167
2168 ring_buffer_record_enable(buffer);
2169}
2170
2171/* Must have trace_types_lock held */
2172void tracing_reset_all_online_cpus(void)
2173{
2174 struct trace_array *tr;
2175
2176 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2177 if (!tr->clear_trace)
2178 continue;
2179 tr->clear_trace = false;
2180 tracing_reset_online_cpus(&tr->array_buffer);
2181#ifdef CONFIG_TRACER_MAX_TRACE
2182 tracing_reset_online_cpus(&tr->max_buffer);
2183#endif
2184 }
2185}
2186
2187static int *tgid_map;
2188
2189#define SAVED_CMDLINES_DEFAULT 128
2190#define NO_CMDLINE_MAP UINT_MAX
2191static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2192struct saved_cmdlines_buffer {
2193 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
2194 unsigned *map_cmdline_to_pid;
2195 unsigned cmdline_num;
2196 int cmdline_idx;
2197 char *saved_cmdlines;
2198};
2199static struct saved_cmdlines_buffer *savedcmd;
2200
2201/* temporary disable recording */
2202static atomic_t trace_record_taskinfo_disabled __read_mostly;
2203
2204static inline char *get_saved_cmdlines(int idx)
2205{
2206 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
2207}
2208
2209static inline void set_cmdline(int idx, const char *cmdline)
2210{
2211 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
2212}
2213
2214static int allocate_cmdlines_buffer(unsigned int val,
2215 struct saved_cmdlines_buffer *s)
2216{
2217 s->map_cmdline_to_pid = kmalloc_array(val,
2218 sizeof(*s->map_cmdline_to_pid),
2219 GFP_KERNEL);
2220 if (!s->map_cmdline_to_pid)
2221 return -ENOMEM;
2222
2223 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
2224 if (!s->saved_cmdlines) {
2225 kfree(s->map_cmdline_to_pid);
2226 return -ENOMEM;
2227 }
2228
2229 s->cmdline_idx = 0;
2230 s->cmdline_num = val;
2231 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
2232 sizeof(s->map_pid_to_cmdline));
2233 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
2234 val * sizeof(*s->map_cmdline_to_pid));
2235
2236 return 0;
2237}
2238
2239static int trace_create_savedcmd(void)
2240{
2241 int ret;
2242
2243 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
2244 if (!savedcmd)
2245 return -ENOMEM;
2246
2247 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
2248 if (ret < 0) {
2249 kfree(savedcmd);
2250 savedcmd = NULL;
2251 return -ENOMEM;
2252 }
2253
2254 return 0;
2255}
2256
2257int is_tracing_stopped(void)
2258{
2259 return global_trace.stop_count;
2260}
2261
2262/**
2263 * tracing_start - quick start of the tracer
2264 *
2265 * If tracing is enabled but was stopped by tracing_stop,
2266 * this will start the tracer back up.
2267 */
2268void tracing_start(void)
2269{
2270 struct trace_buffer *buffer;
2271 unsigned long flags;
2272
2273 if (tracing_disabled)
2274 return;
2275
2276 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2277 if (--global_trace.stop_count) {
2278 if (global_trace.stop_count < 0) {
2279 /* Someone screwed up their debugging */
2280 WARN_ON_ONCE(1);
2281 global_trace.stop_count = 0;
2282 }
2283 goto out;
2284 }
2285
2286 /* Prevent the buffers from switching */
2287 arch_spin_lock(&global_trace.max_lock);
2288
2289 buffer = global_trace.array_buffer.buffer;
2290 if (buffer)
2291 ring_buffer_record_enable(buffer);
2292
2293#ifdef CONFIG_TRACER_MAX_TRACE
2294 buffer = global_trace.max_buffer.buffer;
2295 if (buffer)
2296 ring_buffer_record_enable(buffer);
2297#endif
2298
2299 arch_spin_unlock(&global_trace.max_lock);
2300
2301 out:
2302 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2303}
2304
2305static void tracing_start_tr(struct trace_array *tr)
2306{
2307 struct trace_buffer *buffer;
2308 unsigned long flags;
2309
2310 if (tracing_disabled)
2311 return;
2312
2313 /* If global, we need to also start the max tracer */
2314 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2315 return tracing_start();
2316
2317 raw_spin_lock_irqsave(&tr->start_lock, flags);
2318
2319 if (--tr->stop_count) {
2320 if (tr->stop_count < 0) {
2321 /* Someone screwed up their debugging */
2322 WARN_ON_ONCE(1);
2323 tr->stop_count = 0;
2324 }
2325 goto out;
2326 }
2327
2328 buffer = tr->array_buffer.buffer;
2329 if (buffer)
2330 ring_buffer_record_enable(buffer);
2331
2332 out:
2333 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2334}
2335
2336/**
2337 * tracing_stop - quick stop of the tracer
2338 *
2339 * Light weight way to stop tracing. Use in conjunction with
2340 * tracing_start.
2341 */
2342void tracing_stop(void)
2343{
2344 struct trace_buffer *buffer;
2345 unsigned long flags;
2346
2347 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2348 if (global_trace.stop_count++)
2349 goto out;
2350
2351 /* Prevent the buffers from switching */
2352 arch_spin_lock(&global_trace.max_lock);
2353
2354 buffer = global_trace.array_buffer.buffer;
2355 if (buffer)
2356 ring_buffer_record_disable(buffer);
2357
2358#ifdef CONFIG_TRACER_MAX_TRACE
2359 buffer = global_trace.max_buffer.buffer;
2360 if (buffer)
2361 ring_buffer_record_disable(buffer);
2362#endif
2363
2364 arch_spin_unlock(&global_trace.max_lock);
2365
2366 out:
2367 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2368}
2369
2370static void tracing_stop_tr(struct trace_array *tr)
2371{
2372 struct trace_buffer *buffer;
2373 unsigned long flags;
2374
2375 /* If global, we need to also stop the max tracer */
2376 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2377 return tracing_stop();
2378
2379 raw_spin_lock_irqsave(&tr->start_lock, flags);
2380 if (tr->stop_count++)
2381 goto out;
2382
2383 buffer = tr->array_buffer.buffer;
2384 if (buffer)
2385 ring_buffer_record_disable(buffer);
2386
2387 out:
2388 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2389}
2390
2391static int trace_save_cmdline(struct task_struct *tsk)
2392{
2393 unsigned pid, idx;
2394
2395 /* treat recording of idle task as a success */
2396 if (!tsk->pid)
2397 return 1;
2398
2399 if (unlikely(tsk->pid > PID_MAX_DEFAULT))
2400 return 0;
2401
2402 /*
2403 * It's not the end of the world if we don't get
2404 * the lock, but we also don't want to spin
2405 * nor do we want to disable interrupts,
2406 * so if we miss here, then better luck next time.
2407 */
2408 if (!arch_spin_trylock(&trace_cmdline_lock))
2409 return 0;
2410
2411 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2412 if (idx == NO_CMDLINE_MAP) {
2413 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2414
2415 /*
2416 * Check whether the cmdline buffer at idx has a pid
2417 * mapped. We are going to overwrite that entry so we
2418 * need to clear the map_pid_to_cmdline. Otherwise we
2419 * would read the new comm for the old pid.
2420 */
2421 pid = savedcmd->map_cmdline_to_pid[idx];
2422 if (pid != NO_CMDLINE_MAP)
2423 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
2424
2425 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2426 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
2427
2428 savedcmd->cmdline_idx = idx;
2429 }
2430
2431 set_cmdline(idx, tsk->comm);
2432
2433 arch_spin_unlock(&trace_cmdline_lock);
2434
2435 return 1;
2436}
2437
2438static void __trace_find_cmdline(int pid, char comm[])
2439{
2440 unsigned map;
2441
2442 if (!pid) {
2443 strcpy(comm, "<idle>");
2444 return;
2445 }
2446
2447 if (WARN_ON_ONCE(pid < 0)) {
2448 strcpy(comm, "<XXX>");
2449 return;
2450 }
2451
2452 if (pid > PID_MAX_DEFAULT) {
2453 strcpy(comm, "<...>");
2454 return;
2455 }
2456
2457 map = savedcmd->map_pid_to_cmdline[pid];
2458 if (map != NO_CMDLINE_MAP)
2459 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2460 else
2461 strcpy(comm, "<...>");
2462}
2463
2464void trace_find_cmdline(int pid, char comm[])
2465{
2466 preempt_disable();
2467 arch_spin_lock(&trace_cmdline_lock);
2468
2469 __trace_find_cmdline(pid, comm);
2470
2471 arch_spin_unlock(&trace_cmdline_lock);
2472 preempt_enable();
2473}
2474
2475int trace_find_tgid(int pid)
2476{
2477 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2478 return 0;
2479
2480 return tgid_map[pid];
2481}
2482
2483static int trace_save_tgid(struct task_struct *tsk)
2484{
2485 /* treat recording of idle task as a success */
2486 if (!tsk->pid)
2487 return 1;
2488
2489 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
2490 return 0;
2491
2492 tgid_map[tsk->pid] = tsk->tgid;
2493 return 1;
2494}
2495
2496static bool tracing_record_taskinfo_skip(int flags)
2497{
2498 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2499 return true;
2500 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2501 return true;
2502 if (!__this_cpu_read(trace_taskinfo_save))
2503 return true;
2504 return false;
2505}
2506
2507/**
2508 * tracing_record_taskinfo - record the task info of a task
2509 *
2510 * @task: task to record
2511 * @flags: TRACE_RECORD_CMDLINE for recording comm
2512 * TRACE_RECORD_TGID for recording tgid
2513 */
2514void tracing_record_taskinfo(struct task_struct *task, int flags)
2515{
2516 bool done;
2517
2518 if (tracing_record_taskinfo_skip(flags))
2519 return;
2520
2521 /*
2522 * Record as much task information as possible. If some fail, continue
2523 * to try to record the others.
2524 */
2525 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2526 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2527
2528 /* If recording any information failed, retry again soon. */
2529 if (!done)
2530 return;
2531
2532 __this_cpu_write(trace_taskinfo_save, false);
2533}
2534
2535/**
2536 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2537 *
2538 * @prev: previous task during sched_switch
2539 * @next: next task during sched_switch
2540 * @flags: TRACE_RECORD_CMDLINE for recording comm
2541 * TRACE_RECORD_TGID for recording tgid
2542 */
2543void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2544 struct task_struct *next, int flags)
2545{
2546 bool done;
2547
2548 if (tracing_record_taskinfo_skip(flags))
2549 return;
2550
2551 /*
2552 * Record as much task information as possible. If some fail, continue
2553 * to try to record the others.
2554 */
2555 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2556 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2557 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2558 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2559
2560 /* If recording any information failed, retry again soon. */
2561 if (!done)
2562 return;
2563
2564 __this_cpu_write(trace_taskinfo_save, false);
2565}
2566
2567/* Helpers to record a specific task information */
2568void tracing_record_cmdline(struct task_struct *task)
2569{
2570 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2571}
2572
2573void tracing_record_tgid(struct task_struct *task)
2574{
2575 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2576}
2577
2578/*
2579 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2580 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2581 * simplifies those functions and keeps them in sync.
2582 */
2583enum print_line_t trace_handle_return(struct trace_seq *s)
2584{
2585 return trace_seq_has_overflowed(s) ?
2586 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2587}
2588EXPORT_SYMBOL_GPL(trace_handle_return);
2589
2590unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
2591{
2592 unsigned int trace_flags = irqs_status;
2593 unsigned int pc;
2594
2595 pc = preempt_count();
2596
2597 if (pc & NMI_MASK)
2598 trace_flags |= TRACE_FLAG_NMI;
2599 if (pc & HARDIRQ_MASK)
2600 trace_flags |= TRACE_FLAG_HARDIRQ;
2601 if (in_serving_softirq())
2602 trace_flags |= TRACE_FLAG_SOFTIRQ;
2603
2604 if (tif_need_resched())
2605 trace_flags |= TRACE_FLAG_NEED_RESCHED;
2606 if (test_preempt_need_resched())
2607 trace_flags |= TRACE_FLAG_PREEMPT_RESCHED;
2608 return (trace_flags << 16) | (pc & 0xff);
2609}
2610
2611struct ring_buffer_event *
2612trace_buffer_lock_reserve(struct trace_buffer *buffer,
2613 int type,
2614 unsigned long len,
2615 unsigned int trace_ctx)
2616{
2617 return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
2618}
2619
2620DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2621DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2622static int trace_buffered_event_ref;
2623
2624/**
2625 * trace_buffered_event_enable - enable buffering events
2626 *
2627 * When events are being filtered, it is quicker to use a temporary
2628 * buffer to write the event data into if there's a likely chance
2629 * that it will not be committed. The discard of the ring buffer
2630 * is not as fast as committing, and is much slower than copying
2631 * a commit.
2632 *
2633 * When an event is to be filtered, allocate per cpu buffers to
2634 * write the event data into, and if the event is filtered and discarded
2635 * it is simply dropped, otherwise, the entire data is to be committed
2636 * in one shot.
2637 */
2638void trace_buffered_event_enable(void)
2639{
2640 struct ring_buffer_event *event;
2641 struct page *page;
2642 int cpu;
2643
2644 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2645
2646 if (trace_buffered_event_ref++)
2647 return;
2648
2649 for_each_tracing_cpu(cpu) {
2650 page = alloc_pages_node(cpu_to_node(cpu),
2651 GFP_KERNEL | __GFP_NORETRY, 0);
2652 if (!page)
2653 goto failed;
2654
2655 event = page_address(page);
2656 memset(event, 0, sizeof(*event));
2657
2658 per_cpu(trace_buffered_event, cpu) = event;
2659
2660 preempt_disable();
2661 if (cpu == smp_processor_id() &&
2662 __this_cpu_read(trace_buffered_event) !=
2663 per_cpu(trace_buffered_event, cpu))
2664 WARN_ON_ONCE(1);
2665 preempt_enable();
2666 }
2667
2668 return;
2669 failed:
2670 trace_buffered_event_disable();
2671}
2672
2673static void enable_trace_buffered_event(void *data)
2674{
2675 /* Probably not needed, but do it anyway */
2676 smp_rmb();
2677 this_cpu_dec(trace_buffered_event_cnt);
2678}
2679
2680static void disable_trace_buffered_event(void *data)
2681{
2682 this_cpu_inc(trace_buffered_event_cnt);
2683}
2684
2685/**
2686 * trace_buffered_event_disable - disable buffering events
2687 *
2688 * When a filter is removed, it is faster to not use the buffered
2689 * events, and to commit directly into the ring buffer. Free up
2690 * the temp buffers when there are no more users. This requires
2691 * special synchronization with current events.
2692 */
2693void trace_buffered_event_disable(void)
2694{
2695 int cpu;
2696
2697 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2698
2699 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2700 return;
2701
2702 if (--trace_buffered_event_ref)
2703 return;
2704
2705 preempt_disable();
2706 /* For each CPU, set the buffer as used. */
2707 smp_call_function_many(tracing_buffer_mask,
2708 disable_trace_buffered_event, NULL, 1);
2709 preempt_enable();
2710
2711 /* Wait for all current users to finish */
2712 synchronize_rcu();
2713
2714 for_each_tracing_cpu(cpu) {
2715 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2716 per_cpu(trace_buffered_event, cpu) = NULL;
2717 }
2718 /*
2719 * Make sure trace_buffered_event is NULL before clearing
2720 * trace_buffered_event_cnt.
2721 */
2722 smp_wmb();
2723
2724 preempt_disable();
2725 /* Do the work on each cpu */
2726 smp_call_function_many(tracing_buffer_mask,
2727 enable_trace_buffered_event, NULL, 1);
2728 preempt_enable();
2729}
2730
2731static struct trace_buffer *temp_buffer;
2732
2733struct ring_buffer_event *
2734trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2735 struct trace_event_file *trace_file,
2736 int type, unsigned long len,
2737 unsigned int trace_ctx)
2738{
2739 struct ring_buffer_event *entry;
2740 int val;
2741
2742 *current_rb = trace_file->tr->array_buffer.buffer;
2743
2744 if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
2745 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2746 (entry = this_cpu_read(trace_buffered_event))) {
2747 /* Try to use the per cpu buffer first */
2748 val = this_cpu_inc_return(trace_buffered_event_cnt);
2749 if ((len < (PAGE_SIZE - sizeof(*entry))) && val == 1) {
2750 trace_event_setup(entry, type, trace_ctx);
2751 entry->array[0] = len;
2752 return entry;
2753 }
2754 this_cpu_dec(trace_buffered_event_cnt);
2755 }
2756
2757 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2758 trace_ctx);
2759 /*
2760 * If tracing is off, but we have triggers enabled
2761 * we still need to look at the event data. Use the temp_buffer
2762 * to store the trace event for the trigger to use. It's recursive
2763 * safe and will not be recorded anywhere.
2764 */
2765 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2766 *current_rb = temp_buffer;
2767 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2768 trace_ctx);
2769 }
2770 return entry;
2771}
2772EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2773
2774static DEFINE_SPINLOCK(tracepoint_iter_lock);
2775static DEFINE_MUTEX(tracepoint_printk_mutex);
2776
2777static void output_printk(struct trace_event_buffer *fbuffer)
2778{
2779 struct trace_event_call *event_call;
2780 struct trace_event_file *file;
2781 struct trace_event *event;
2782 unsigned long flags;
2783 struct trace_iterator *iter = tracepoint_print_iter;
2784
2785 /* We should never get here if iter is NULL */
2786 if (WARN_ON_ONCE(!iter))
2787 return;
2788
2789 event_call = fbuffer->trace_file->event_call;
2790 if (!event_call || !event_call->event.funcs ||
2791 !event_call->event.funcs->trace)
2792 return;
2793
2794 file = fbuffer->trace_file;
2795 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
2796 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2797 !filter_match_preds(file->filter, fbuffer->entry)))
2798 return;
2799
2800 event = &fbuffer->trace_file->event_call->event;
2801
2802 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2803 trace_seq_init(&iter->seq);
2804 iter->ent = fbuffer->entry;
2805 event_call->event.funcs->trace(iter, 0, event);
2806 trace_seq_putc(&iter->seq, 0);
2807 printk("%s", iter->seq.buffer);
2808
2809 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2810}
2811
2812int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2813 void *buffer, size_t *lenp,
2814 loff_t *ppos)
2815{
2816 int save_tracepoint_printk;
2817 int ret;
2818
2819 mutex_lock(&tracepoint_printk_mutex);
2820 save_tracepoint_printk = tracepoint_printk;
2821
2822 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2823
2824 /*
2825 * This will force exiting early, as tracepoint_printk
2826 * is always zero when tracepoint_printk_iter is not allocated
2827 */
2828 if (!tracepoint_print_iter)
2829 tracepoint_printk = 0;
2830
2831 if (save_tracepoint_printk == tracepoint_printk)
2832 goto out;
2833
2834 if (tracepoint_printk)
2835 static_key_enable(&tracepoint_printk_key.key);
2836 else
2837 static_key_disable(&tracepoint_printk_key.key);
2838
2839 out:
2840 mutex_unlock(&tracepoint_printk_mutex);
2841
2842 return ret;
2843}
2844
2845void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2846{
2847 if (static_key_false(&tracepoint_printk_key.key))
2848 output_printk(fbuffer);
2849
2850 if (static_branch_unlikely(&trace_event_exports_enabled))
2851 ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
2852 event_trigger_unlock_commit_regs(fbuffer->trace_file, fbuffer->buffer,
2853 fbuffer->event, fbuffer->entry,
2854 fbuffer->trace_ctx, fbuffer->regs);
2855}
2856EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2857
2858/*
2859 * Skip 3:
2860 *
2861 * trace_buffer_unlock_commit_regs()
2862 * trace_event_buffer_commit()
2863 * trace_event_raw_event_xxx()
2864 */
2865# define STACK_SKIP 3
2866
2867void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2868 struct trace_buffer *buffer,
2869 struct ring_buffer_event *event,
2870 unsigned int trace_ctx,
2871 struct pt_regs *regs)
2872{
2873 __buffer_unlock_commit(buffer, event);
2874
2875 /*
2876 * If regs is not set, then skip the necessary functions.
2877 * Note, we can still get here via blktrace, wakeup tracer
2878 * and mmiotrace, but that's ok if they lose a function or
2879 * two. They are not that meaningful.
2880 */
2881 ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs);
2882 ftrace_trace_userstack(tr, buffer, trace_ctx);
2883}
2884
2885/*
2886 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2887 */
2888void
2889trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2890 struct ring_buffer_event *event)
2891{
2892 __buffer_unlock_commit(buffer, event);
2893}
2894
2895void
2896trace_function(struct trace_array *tr, unsigned long ip, unsigned long
2897 parent_ip, unsigned int trace_ctx)
2898{
2899 struct trace_event_call *call = &event_function;
2900 struct trace_buffer *buffer = tr->array_buffer.buffer;
2901 struct ring_buffer_event *event;
2902 struct ftrace_entry *entry;
2903
2904 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2905 trace_ctx);
2906 if (!event)
2907 return;
2908 entry = ring_buffer_event_data(event);
2909 entry->ip = ip;
2910 entry->parent_ip = parent_ip;
2911
2912 if (!call_filter_check_discard(call, entry, buffer, event)) {
2913 if (static_branch_unlikely(&trace_function_exports_enabled))
2914 ftrace_exports(event, TRACE_EXPORT_FUNCTION);
2915 __buffer_unlock_commit(buffer, event);
2916 }
2917}
2918
2919#ifdef CONFIG_STACKTRACE
2920
2921/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2922#define FTRACE_KSTACK_NESTING 4
2923
2924#define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2925
2926struct ftrace_stack {
2927 unsigned long calls[FTRACE_KSTACK_ENTRIES];
2928};
2929
2930
2931struct ftrace_stacks {
2932 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2933};
2934
2935static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2936static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2937
2938static void __ftrace_trace_stack(struct trace_buffer *buffer,
2939 unsigned int trace_ctx,
2940 int skip, struct pt_regs *regs)
2941{
2942 struct trace_event_call *call = &event_kernel_stack;
2943 struct ring_buffer_event *event;
2944 unsigned int size, nr_entries;
2945 struct ftrace_stack *fstack;
2946 struct stack_entry *entry;
2947 int stackidx;
2948
2949 /*
2950 * Add one, for this function and the call to save_stack_trace()
2951 * If regs is set, then these functions will not be in the way.
2952 */
2953#ifndef CONFIG_UNWINDER_ORC
2954 if (!regs)
2955 skip++;
2956#endif
2957
2958 preempt_disable_notrace();
2959
2960 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2961
2962 /* This should never happen. If it does, yell once and skip */
2963 if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
2964 goto out;
2965
2966 /*
2967 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2968 * interrupt will either see the value pre increment or post
2969 * increment. If the interrupt happens pre increment it will have
2970 * restored the counter when it returns. We just need a barrier to
2971 * keep gcc from moving things around.
2972 */
2973 barrier();
2974
2975 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2976 size = ARRAY_SIZE(fstack->calls);
2977
2978 if (regs) {
2979 nr_entries = stack_trace_save_regs(regs, fstack->calls,
2980 size, skip);
2981 } else {
2982 nr_entries = stack_trace_save(fstack->calls, size, skip);
2983 }
2984
2985 size = nr_entries * sizeof(unsigned long);
2986 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2987 (sizeof(*entry) - sizeof(entry->caller)) + size,
2988 trace_ctx);
2989 if (!event)
2990 goto out;
2991 entry = ring_buffer_event_data(event);
2992
2993 memcpy(&entry->caller, fstack->calls, size);
2994 entry->size = nr_entries;
2995
2996 if (!call_filter_check_discard(call, entry, buffer, event))
2997 __buffer_unlock_commit(buffer, event);
2998
2999 out:
3000 /* Again, don't let gcc optimize things here */
3001 barrier();
3002 __this_cpu_dec(ftrace_stack_reserve);
3003 preempt_enable_notrace();
3004
3005}
3006
3007static inline void ftrace_trace_stack(struct trace_array *tr,
3008 struct trace_buffer *buffer,
3009 unsigned int trace_ctx,
3010 int skip, struct pt_regs *regs)
3011{
3012 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
3013 return;
3014
3015 __ftrace_trace_stack(buffer, trace_ctx, skip, regs);
3016}
3017
3018void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
3019 int skip)
3020{
3021 struct trace_buffer *buffer = tr->array_buffer.buffer;
3022
3023 if (rcu_is_watching()) {
3024 __ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3025 return;
3026 }
3027
3028 /*
3029 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
3030 * but if the above rcu_is_watching() failed, then the NMI
3031 * triggered someplace critical, and rcu_irq_enter() should
3032 * not be called from NMI.
3033 */
3034 if (unlikely(in_nmi()))
3035 return;
3036
3037 rcu_irq_enter_irqson();
3038 __ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3039 rcu_irq_exit_irqson();
3040}
3041
3042/**
3043 * trace_dump_stack - record a stack back trace in the trace buffer
3044 * @skip: Number of functions to skip (helper handlers)
3045 */
3046void trace_dump_stack(int skip)
3047{
3048 if (tracing_disabled || tracing_selftest_running)
3049 return;
3050
3051#ifndef CONFIG_UNWINDER_ORC
3052 /* Skip 1 to skip this function. */
3053 skip++;
3054#endif
3055 __ftrace_trace_stack(global_trace.array_buffer.buffer,
3056 tracing_gen_ctx(), skip, NULL);
3057}
3058EXPORT_SYMBOL_GPL(trace_dump_stack);
3059
3060#ifdef CONFIG_USER_STACKTRACE_SUPPORT
3061static DEFINE_PER_CPU(int, user_stack_count);
3062
3063static void
3064ftrace_trace_userstack(struct trace_array *tr,
3065 struct trace_buffer *buffer, unsigned int trace_ctx)
3066{
3067 struct trace_event_call *call = &event_user_stack;
3068 struct ring_buffer_event *event;
3069 struct userstack_entry *entry;
3070
3071 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
3072 return;
3073
3074 /*
3075 * NMIs can not handle page faults, even with fix ups.
3076 * The save user stack can (and often does) fault.
3077 */
3078 if (unlikely(in_nmi()))
3079 return;
3080
3081 /*
3082 * prevent recursion, since the user stack tracing may
3083 * trigger other kernel events.
3084 */
3085 preempt_disable();
3086 if (__this_cpu_read(user_stack_count))
3087 goto out;
3088
3089 __this_cpu_inc(user_stack_count);
3090
3091 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3092 sizeof(*entry), trace_ctx);
3093 if (!event)
3094 goto out_drop_count;
3095 entry = ring_buffer_event_data(event);
3096
3097 entry->tgid = current->tgid;
3098 memset(&entry->caller, 0, sizeof(entry->caller));
3099
3100 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3101 if (!call_filter_check_discard(call, entry, buffer, event))
3102 __buffer_unlock_commit(buffer, event);
3103
3104 out_drop_count:
3105 __this_cpu_dec(user_stack_count);
3106 out:
3107 preempt_enable();
3108}
3109#else /* CONFIG_USER_STACKTRACE_SUPPORT */
3110static void ftrace_trace_userstack(struct trace_array *tr,
3111 struct trace_buffer *buffer,
3112 unsigned int trace_ctx)
3113{
3114}
3115#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3116
3117#endif /* CONFIG_STACKTRACE */
3118
3119/* created for use with alloc_percpu */
3120struct trace_buffer_struct {
3121 int nesting;
3122 char buffer[4][TRACE_BUF_SIZE];
3123};
3124
3125static struct trace_buffer_struct *trace_percpu_buffer;
3126
3127/*
3128 * This allows for lockless recording. If we're nested too deeply, then
3129 * this returns NULL.
3130 */
3131static char *get_trace_buf(void)
3132{
3133 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3134
3135 if (!buffer || buffer->nesting >= 4)
3136 return NULL;
3137
3138 buffer->nesting++;
3139
3140 /* Interrupts must see nesting incremented before we use the buffer */
3141 barrier();
3142 return &buffer->buffer[buffer->nesting - 1][0];
3143}
3144
3145static void put_trace_buf(void)
3146{
3147 /* Don't let the decrement of nesting leak before this */
3148 barrier();
3149 this_cpu_dec(trace_percpu_buffer->nesting);
3150}
3151
3152static int alloc_percpu_trace_buffer(void)
3153{
3154 struct trace_buffer_struct *buffers;
3155
3156 if (trace_percpu_buffer)
3157 return 0;
3158
3159 buffers = alloc_percpu(struct trace_buffer_struct);
3160 if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3161 return -ENOMEM;
3162
3163 trace_percpu_buffer = buffers;
3164 return 0;
3165}
3166
3167static int buffers_allocated;
3168
3169void trace_printk_init_buffers(void)
3170{
3171 if (buffers_allocated)
3172 return;
3173
3174 if (alloc_percpu_trace_buffer())
3175 return;
3176
3177 /* trace_printk() is for debug use only. Don't use it in production. */
3178
3179 pr_warn("\n");
3180 pr_warn("**********************************************************\n");
3181 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3182 pr_warn("** **\n");
3183 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3184 pr_warn("** **\n");
3185 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3186 pr_warn("** unsafe for production use. **\n");
3187 pr_warn("** **\n");
3188 pr_warn("** If you see this message and you are not debugging **\n");
3189 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3190 pr_warn("** **\n");
3191 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3192 pr_warn("**********************************************************\n");
3193
3194 /* Expand the buffers to set size */
3195 tracing_update_buffers();
3196
3197 buffers_allocated = 1;
3198
3199 /*
3200 * trace_printk_init_buffers() can be called by modules.
3201 * If that happens, then we need to start cmdline recording
3202 * directly here. If the global_trace.buffer is already
3203 * allocated here, then this was called by module code.
3204 */
3205 if (global_trace.array_buffer.buffer)
3206 tracing_start_cmdline_record();
3207}
3208EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3209
3210void trace_printk_start_comm(void)
3211{
3212 /* Start tracing comms if trace printk is set */
3213 if (!buffers_allocated)
3214 return;
3215 tracing_start_cmdline_record();
3216}
3217
3218static void trace_printk_start_stop_comm(int enabled)
3219{
3220 if (!buffers_allocated)
3221 return;
3222
3223 if (enabled)
3224 tracing_start_cmdline_record();
3225 else
3226 tracing_stop_cmdline_record();
3227}
3228
3229/**
3230 * trace_vbprintk - write binary msg to tracing buffer
3231 * @ip: The address of the caller
3232 * @fmt: The string format to write to the buffer
3233 * @args: Arguments for @fmt
3234 */
3235int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3236{
3237 struct trace_event_call *call = &event_bprint;
3238 struct ring_buffer_event *event;
3239 struct trace_buffer *buffer;
3240 struct trace_array *tr = &global_trace;
3241 struct bprint_entry *entry;
3242 unsigned int trace_ctx;
3243 char *tbuffer;
3244 int len = 0, size;
3245
3246 if (unlikely(tracing_selftest_running || tracing_disabled))
3247 return 0;
3248
3249 /* Don't pollute graph traces with trace_vprintk internals */
3250 pause_graph_tracing();
3251
3252 trace_ctx = tracing_gen_ctx();
3253 preempt_disable_notrace();
3254
3255 tbuffer = get_trace_buf();
3256 if (!tbuffer) {
3257 len = 0;
3258 goto out_nobuffer;
3259 }
3260
3261 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3262
3263 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3264 goto out_put;
3265
3266 size = sizeof(*entry) + sizeof(u32) * len;
3267 buffer = tr->array_buffer.buffer;
3268 ring_buffer_nest_start(buffer);
3269 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3270 trace_ctx);
3271 if (!event)
3272 goto out;
3273 entry = ring_buffer_event_data(event);
3274 entry->ip = ip;
3275 entry->fmt = fmt;
3276
3277 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3278 if (!call_filter_check_discard(call, entry, buffer, event)) {
3279 __buffer_unlock_commit(buffer, event);
3280 ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL);
3281 }
3282
3283out:
3284 ring_buffer_nest_end(buffer);
3285out_put:
3286 put_trace_buf();
3287
3288out_nobuffer:
3289 preempt_enable_notrace();
3290 unpause_graph_tracing();
3291
3292 return len;
3293}
3294EXPORT_SYMBOL_GPL(trace_vbprintk);
3295
3296__printf(3, 0)
3297static int
3298__trace_array_vprintk(struct trace_buffer *buffer,
3299 unsigned long ip, const char *fmt, va_list args)
3300{
3301 struct trace_event_call *call = &event_print;
3302 struct ring_buffer_event *event;
3303 int len = 0, size;
3304 struct print_entry *entry;
3305 unsigned int trace_ctx;
3306 char *tbuffer;
3307
3308 if (tracing_disabled || tracing_selftest_running)
3309 return 0;
3310
3311 /* Don't pollute graph traces with trace_vprintk internals */
3312 pause_graph_tracing();
3313
3314 trace_ctx = tracing_gen_ctx();
3315 preempt_disable_notrace();
3316
3317
3318 tbuffer = get_trace_buf();
3319 if (!tbuffer) {
3320 len = 0;
3321 goto out_nobuffer;
3322 }
3323
3324 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3325
3326 size = sizeof(*entry) + len + 1;
3327 ring_buffer_nest_start(buffer);
3328 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3329 trace_ctx);
3330 if (!event)
3331 goto out;
3332 entry = ring_buffer_event_data(event);
3333 entry->ip = ip;
3334
3335 memcpy(&entry->buf, tbuffer, len + 1);
3336 if (!call_filter_check_discard(call, entry, buffer, event)) {
3337 __buffer_unlock_commit(buffer, event);
3338 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 6, NULL);
3339 }
3340
3341out:
3342 ring_buffer_nest_end(buffer);
3343 put_trace_buf();
3344
3345out_nobuffer:
3346 preempt_enable_notrace();
3347 unpause_graph_tracing();
3348
3349 return len;
3350}
3351
3352__printf(3, 0)
3353int trace_array_vprintk(struct trace_array *tr,
3354 unsigned long ip, const char *fmt, va_list args)
3355{
3356 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3357}
3358
3359/**
3360 * trace_array_printk - Print a message to a specific instance
3361 * @tr: The instance trace_array descriptor
3362 * @ip: The instruction pointer that this is called from.
3363 * @fmt: The format to print (printf format)
3364 *
3365 * If a subsystem sets up its own instance, they have the right to
3366 * printk strings into their tracing instance buffer using this
3367 * function. Note, this function will not write into the top level
3368 * buffer (use trace_printk() for that), as writing into the top level
3369 * buffer should only have events that can be individually disabled.
3370 * trace_printk() is only used for debugging a kernel, and should not
3371 * be ever encorporated in normal use.
3372 *
3373 * trace_array_printk() can be used, as it will not add noise to the
3374 * top level tracing buffer.
3375 *
3376 * Note, trace_array_init_printk() must be called on @tr before this
3377 * can be used.
3378 */
3379__printf(3, 0)
3380int trace_array_printk(struct trace_array *tr,
3381 unsigned long ip, const char *fmt, ...)
3382{
3383 int ret;
3384 va_list ap;
3385
3386 if (!tr)
3387 return -ENOENT;
3388
3389 /* This is only allowed for created instances */
3390 if (tr == &global_trace)
3391 return 0;
3392
3393 if (!(tr->trace_flags & TRACE_ITER_PRINTK))
3394 return 0;
3395
3396 va_start(ap, fmt);
3397 ret = trace_array_vprintk(tr, ip, fmt, ap);
3398 va_end(ap);
3399 return ret;
3400}
3401EXPORT_SYMBOL_GPL(trace_array_printk);
3402
3403/**
3404 * trace_array_init_printk - Initialize buffers for trace_array_printk()
3405 * @tr: The trace array to initialize the buffers for
3406 *
3407 * As trace_array_printk() only writes into instances, they are OK to
3408 * have in the kernel (unlike trace_printk()). This needs to be called
3409 * before trace_array_printk() can be used on a trace_array.
3410 */
3411int trace_array_init_printk(struct trace_array *tr)
3412{
3413 if (!tr)
3414 return -ENOENT;
3415
3416 /* This is only allowed for created instances */
3417 if (tr == &global_trace)
3418 return -EINVAL;
3419
3420 return alloc_percpu_trace_buffer();
3421}
3422EXPORT_SYMBOL_GPL(trace_array_init_printk);
3423
3424__printf(3, 4)
3425int trace_array_printk_buf(struct trace_buffer *buffer,
3426 unsigned long ip, const char *fmt, ...)
3427{
3428 int ret;
3429 va_list ap;
3430
3431 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3432 return 0;
3433
3434 va_start(ap, fmt);
3435 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3436 va_end(ap);
3437 return ret;
3438}
3439
3440__printf(2, 0)
3441int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3442{
3443 return trace_array_vprintk(&global_trace, ip, fmt, args);
3444}
3445EXPORT_SYMBOL_GPL(trace_vprintk);
3446
3447static void trace_iterator_increment(struct trace_iterator *iter)
3448{
3449 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3450
3451 iter->idx++;
3452 if (buf_iter)
3453 ring_buffer_iter_advance(buf_iter);
3454}
3455
3456static struct trace_entry *
3457peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3458 unsigned long *lost_events)
3459{
3460 struct ring_buffer_event *event;
3461 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3462
3463 if (buf_iter) {
3464 event = ring_buffer_iter_peek(buf_iter, ts);
3465 if (lost_events)
3466 *lost_events = ring_buffer_iter_dropped(buf_iter) ?
3467 (unsigned long)-1 : 0;
3468 } else {
3469 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3470 lost_events);
3471 }
3472
3473 if (event) {
3474 iter->ent_size = ring_buffer_event_length(event);
3475 return ring_buffer_event_data(event);
3476 }
3477 iter->ent_size = 0;
3478 return NULL;
3479}
3480
3481static struct trace_entry *
3482__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3483 unsigned long *missing_events, u64 *ent_ts)
3484{
3485 struct trace_buffer *buffer = iter->array_buffer->buffer;
3486 struct trace_entry *ent, *next = NULL;
3487 unsigned long lost_events = 0, next_lost = 0;
3488 int cpu_file = iter->cpu_file;
3489 u64 next_ts = 0, ts;
3490 int next_cpu = -1;
3491 int next_size = 0;
3492 int cpu;
3493
3494 /*
3495 * If we are in a per_cpu trace file, don't bother by iterating over
3496 * all cpu and peek directly.
3497 */
3498 if (cpu_file > RING_BUFFER_ALL_CPUS) {
3499 if (ring_buffer_empty_cpu(buffer, cpu_file))
3500 return NULL;
3501 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3502 if (ent_cpu)
3503 *ent_cpu = cpu_file;
3504
3505 return ent;
3506 }
3507
3508 for_each_tracing_cpu(cpu) {
3509
3510 if (ring_buffer_empty_cpu(buffer, cpu))
3511 continue;
3512
3513 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3514
3515 /*
3516 * Pick the entry with the smallest timestamp:
3517 */
3518 if (ent && (!next || ts < next_ts)) {
3519 next = ent;
3520 next_cpu = cpu;
3521 next_ts = ts;
3522 next_lost = lost_events;
3523 next_size = iter->ent_size;
3524 }
3525 }
3526
3527 iter->ent_size = next_size;
3528
3529 if (ent_cpu)
3530 *ent_cpu = next_cpu;
3531
3532 if (ent_ts)
3533 *ent_ts = next_ts;
3534
3535 if (missing_events)
3536 *missing_events = next_lost;
3537
3538 return next;
3539}
3540
3541#define STATIC_FMT_BUF_SIZE 128
3542static char static_fmt_buf[STATIC_FMT_BUF_SIZE];
3543
3544static char *trace_iter_expand_format(struct trace_iterator *iter)
3545{
3546 char *tmp;
3547
3548 /*
3549 * iter->tr is NULL when used with tp_printk, which makes
3550 * this get called where it is not safe to call krealloc().
3551 */
3552 if (!iter->tr || iter->fmt == static_fmt_buf)
3553 return NULL;
3554
3555 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE,
3556 GFP_KERNEL);
3557 if (tmp) {
3558 iter->fmt_size += STATIC_FMT_BUF_SIZE;
3559 iter->fmt = tmp;
3560 }
3561
3562 return tmp;
3563}
3564
3565const char *trace_event_format(struct trace_iterator *iter, const char *fmt)
3566{
3567 const char *p, *new_fmt;
3568 char *q;
3569
3570 if (WARN_ON_ONCE(!fmt))
3571 return fmt;
3572
3573 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR)
3574 return fmt;
3575
3576 p = fmt;
3577 new_fmt = q = iter->fmt;
3578 while (*p) {
3579 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {
3580 if (!trace_iter_expand_format(iter))
3581 return fmt;
3582
3583 q += iter->fmt - new_fmt;
3584 new_fmt = iter->fmt;
3585 }
3586
3587 *q++ = *p++;
3588
3589 /* Replace %p with %px */
3590 if (p[-1] == '%') {
3591 if (p[0] == '%') {
3592 *q++ = *p++;
3593 } else if (p[0] == 'p' && !isalnum(p[1])) {
3594 *q++ = *p++;
3595 *q++ = 'x';
3596 }
3597 }
3598 }
3599 *q = '\0';
3600
3601 return new_fmt;
3602}
3603
3604#define STATIC_TEMP_BUF_SIZE 128
3605static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
3606
3607/* Find the next real entry, without updating the iterator itself */
3608struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3609 int *ent_cpu, u64 *ent_ts)
3610{
3611 /* __find_next_entry will reset ent_size */
3612 int ent_size = iter->ent_size;
3613 struct trace_entry *entry;
3614
3615 /*
3616 * If called from ftrace_dump(), then the iter->temp buffer
3617 * will be the static_temp_buf and not created from kmalloc.
3618 * If the entry size is greater than the buffer, we can
3619 * not save it. Just return NULL in that case. This is only
3620 * used to add markers when two consecutive events' time
3621 * stamps have a large delta. See trace_print_lat_context()
3622 */
3623 if (iter->temp == static_temp_buf &&
3624 STATIC_TEMP_BUF_SIZE < ent_size)
3625 return NULL;
3626
3627 /*
3628 * The __find_next_entry() may call peek_next_entry(), which may
3629 * call ring_buffer_peek() that may make the contents of iter->ent
3630 * undefined. Need to copy iter->ent now.
3631 */
3632 if (iter->ent && iter->ent != iter->temp) {
3633 if ((!iter->temp || iter->temp_size < iter->ent_size) &&
3634 !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
3635 void *temp;
3636 temp = kmalloc(iter->ent_size, GFP_KERNEL);
3637 if (!temp)
3638 return NULL;
3639 kfree(iter->temp);
3640 iter->temp = temp;
3641 iter->temp_size = iter->ent_size;
3642 }
3643 memcpy(iter->temp, iter->ent, iter->ent_size);
3644 iter->ent = iter->temp;
3645 }
3646 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3647 /* Put back the original ent_size */
3648 iter->ent_size = ent_size;
3649
3650 return entry;
3651}
3652
3653/* Find the next real entry, and increment the iterator to the next entry */
3654void *trace_find_next_entry_inc(struct trace_iterator *iter)
3655{
3656 iter->ent = __find_next_entry(iter, &iter->cpu,
3657 &iter->lost_events, &iter->ts);
3658
3659 if (iter->ent)
3660 trace_iterator_increment(iter);
3661
3662 return iter->ent ? iter : NULL;
3663}
3664
3665static void trace_consume(struct trace_iterator *iter)
3666{
3667 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
3668 &iter->lost_events);
3669}
3670
3671static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3672{
3673 struct trace_iterator *iter = m->private;
3674 int i = (int)*pos;
3675 void *ent;
3676
3677 WARN_ON_ONCE(iter->leftover);
3678
3679 (*pos)++;
3680
3681 /* can't go backwards */
3682 if (iter->idx > i)
3683 return NULL;
3684
3685 if (iter->idx < 0)
3686 ent = trace_find_next_entry_inc(iter);
3687 else
3688 ent = iter;
3689
3690 while (ent && iter->idx < i)
3691 ent = trace_find_next_entry_inc(iter);
3692
3693 iter->pos = *pos;
3694
3695 return ent;
3696}
3697
3698void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3699{
3700 struct ring_buffer_iter *buf_iter;
3701 unsigned long entries = 0;
3702 u64 ts;
3703
3704 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
3705
3706 buf_iter = trace_buffer_iter(iter, cpu);
3707 if (!buf_iter)
3708 return;
3709
3710 ring_buffer_iter_reset(buf_iter);
3711
3712 /*
3713 * We could have the case with the max latency tracers
3714 * that a reset never took place on a cpu. This is evident
3715 * by the timestamp being before the start of the buffer.
3716 */
3717 while (ring_buffer_iter_peek(buf_iter, &ts)) {
3718 if (ts >= iter->array_buffer->time_start)
3719 break;
3720 entries++;
3721 ring_buffer_iter_advance(buf_iter);
3722 }
3723
3724 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
3725}
3726
3727/*
3728 * The current tracer is copied to avoid a global locking
3729 * all around.
3730 */
3731static void *s_start(struct seq_file *m, loff_t *pos)
3732{
3733 struct trace_iterator *iter = m->private;
3734 struct trace_array *tr = iter->tr;
3735 int cpu_file = iter->cpu_file;
3736 void *p = NULL;
3737 loff_t l = 0;
3738 int cpu;
3739
3740 /*
3741 * copy the tracer to avoid using a global lock all around.
3742 * iter->trace is a copy of current_trace, the pointer to the
3743 * name may be used instead of a strcmp(), as iter->trace->name
3744 * will point to the same string as current_trace->name.
3745 */
3746 mutex_lock(&trace_types_lock);
3747 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3748 *iter->trace = *tr->current_trace;
3749 mutex_unlock(&trace_types_lock);
3750
3751#ifdef CONFIG_TRACER_MAX_TRACE
3752 if (iter->snapshot && iter->trace->use_max_tr)
3753 return ERR_PTR(-EBUSY);
3754#endif
3755
3756 if (!iter->snapshot)
3757 atomic_inc(&trace_record_taskinfo_disabled);
3758
3759 if (*pos != iter->pos) {
3760 iter->ent = NULL;
3761 iter->cpu = 0;
3762 iter->idx = -1;
3763
3764 if (cpu_file == RING_BUFFER_ALL_CPUS) {
3765 for_each_tracing_cpu(cpu)
3766 tracing_iter_reset(iter, cpu);
3767 } else
3768 tracing_iter_reset(iter, cpu_file);
3769
3770 iter->leftover = 0;
3771 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3772 ;
3773
3774 } else {
3775 /*
3776 * If we overflowed the seq_file before, then we want
3777 * to just reuse the trace_seq buffer again.
3778 */
3779 if (iter->leftover)
3780 p = iter;
3781 else {
3782 l = *pos - 1;
3783 p = s_next(m, p, &l);
3784 }
3785 }
3786
3787 trace_event_read_lock();
3788 trace_access_lock(cpu_file);
3789 return p;
3790}
3791
3792static void s_stop(struct seq_file *m, void *p)
3793{
3794 struct trace_iterator *iter = m->private;
3795
3796#ifdef CONFIG_TRACER_MAX_TRACE
3797 if (iter->snapshot && iter->trace->use_max_tr)
3798 return;
3799#endif
3800
3801 if (!iter->snapshot)
3802 atomic_dec(&trace_record_taskinfo_disabled);
3803
3804 trace_access_unlock(iter->cpu_file);
3805 trace_event_read_unlock();
3806}
3807
3808static void
3809get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
3810 unsigned long *entries, int cpu)
3811{
3812 unsigned long count;
3813
3814 count = ring_buffer_entries_cpu(buf->buffer, cpu);
3815 /*
3816 * If this buffer has skipped entries, then we hold all
3817 * entries for the trace and we need to ignore the
3818 * ones before the time stamp.
3819 */
3820 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3821 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3822 /* total is the same as the entries */
3823 *total = count;
3824 } else
3825 *total = count +
3826 ring_buffer_overrun_cpu(buf->buffer, cpu);
3827 *entries = count;
3828}
3829
3830static void
3831get_total_entries(struct array_buffer *buf,
3832 unsigned long *total, unsigned long *entries)
3833{
3834 unsigned long t, e;
3835 int cpu;
3836
3837 *total = 0;
3838 *entries = 0;
3839
3840 for_each_tracing_cpu(cpu) {
3841 get_total_entries_cpu(buf, &t, &e, cpu);
3842 *total += t;
3843 *entries += e;
3844 }
3845}
3846
3847unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
3848{
3849 unsigned long total, entries;
3850
3851 if (!tr)
3852 tr = &global_trace;
3853
3854 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
3855
3856 return entries;
3857}
3858
3859unsigned long trace_total_entries(struct trace_array *tr)
3860{
3861 unsigned long total, entries;
3862
3863 if (!tr)
3864 tr = &global_trace;
3865
3866 get_total_entries(&tr->array_buffer, &total, &entries);
3867
3868 return entries;
3869}
3870
3871static void print_lat_help_header(struct seq_file *m)
3872{
3873 seq_puts(m, "# _------=> CPU# \n"
3874 "# / _-----=> irqs-off \n"
3875 "# | / _----=> need-resched \n"
3876 "# || / _---=> hardirq/softirq \n"
3877 "# ||| / _--=> preempt-depth \n"
3878 "# |||| / delay \n"
3879 "# cmd pid ||||| time | caller \n"
3880 "# \\ / ||||| \\ | / \n");
3881}
3882
3883static void print_event_info(struct array_buffer *buf, struct seq_file *m)
3884{
3885 unsigned long total;
3886 unsigned long entries;
3887
3888 get_total_entries(buf, &total, &entries);
3889 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3890 entries, total, num_online_cpus());
3891 seq_puts(m, "#\n");
3892}
3893
3894static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
3895 unsigned int flags)
3896{
3897 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3898
3899 print_event_info(buf, m);
3900
3901 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
3902 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
3903}
3904
3905static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
3906 unsigned int flags)
3907{
3908 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3909 const char *space = " ";
3910 int prec = tgid ? 12 : 2;
3911
3912 print_event_info(buf, m);
3913
3914 seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space);
3915 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
3916 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
3917 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
3918 seq_printf(m, "# %.*s||| / delay\n", prec, space);
3919 seq_printf(m, "# TASK-PID %.*s CPU# |||| TIMESTAMP FUNCTION\n", prec, " TGID ");
3920 seq_printf(m, "# | | %.*s | |||| | |\n", prec, " | ");
3921}
3922
3923void
3924print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3925{
3926 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3927 struct array_buffer *buf = iter->array_buffer;
3928 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3929 struct tracer *type = iter->trace;
3930 unsigned long entries;
3931 unsigned long total;
3932 const char *name = "preemption";
3933
3934 name = type->name;
3935
3936 get_total_entries(buf, &total, &entries);
3937
3938 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
3939 name, UTS_RELEASE);
3940 seq_puts(m, "# -----------------------------------"
3941 "---------------------------------\n");
3942 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3943 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3944 nsecs_to_usecs(data->saved_latency),
3945 entries,
3946 total,
3947 buf->cpu,
3948#if defined(CONFIG_PREEMPT_NONE)
3949 "server",
3950#elif defined(CONFIG_PREEMPT_VOLUNTARY)
3951 "desktop",
3952#elif defined(CONFIG_PREEMPT)
3953 "preempt",
3954#elif defined(CONFIG_PREEMPT_RT)
3955 "preempt_rt",
3956#else
3957 "unknown",
3958#endif
3959 /* These are reserved for later use */
3960 0, 0, 0, 0);
3961#ifdef CONFIG_SMP
3962 seq_printf(m, " #P:%d)\n", num_online_cpus());
3963#else
3964 seq_puts(m, ")\n");
3965#endif
3966 seq_puts(m, "# -----------------\n");
3967 seq_printf(m, "# | task: %.16s-%d "
3968 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3969 data->comm, data->pid,
3970 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3971 data->policy, data->rt_priority);
3972 seq_puts(m, "# -----------------\n");
3973
3974 if (data->critical_start) {
3975 seq_puts(m, "# => started at: ");
3976 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3977 trace_print_seq(m, &iter->seq);
3978 seq_puts(m, "\n# => ended at: ");
3979 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3980 trace_print_seq(m, &iter->seq);
3981 seq_puts(m, "\n#\n");
3982 }
3983
3984 seq_puts(m, "#\n");
3985}
3986
3987static void test_cpu_buff_start(struct trace_iterator *iter)
3988{
3989 struct trace_seq *s = &iter->seq;
3990 struct trace_array *tr = iter->tr;
3991
3992 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3993 return;
3994
3995 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3996 return;
3997
3998 if (cpumask_available(iter->started) &&
3999 cpumask_test_cpu(iter->cpu, iter->started))
4000 return;
4001
4002 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
4003 return;
4004
4005 if (cpumask_available(iter->started))
4006 cpumask_set_cpu(iter->cpu, iter->started);
4007
4008 /* Don't print started cpu buffer for the first entry of the trace */
4009 if (iter->idx > 1)
4010 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
4011 iter->cpu);
4012}
4013
4014static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
4015{
4016 struct trace_array *tr = iter->tr;
4017 struct trace_seq *s = &iter->seq;
4018 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4019 struct trace_entry *entry;
4020 struct trace_event *event;
4021
4022 entry = iter->ent;
4023
4024 test_cpu_buff_start(iter);
4025
4026 event = ftrace_find_event(entry->type);
4027
4028 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4029 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4030 trace_print_lat_context(iter);
4031 else
4032 trace_print_context(iter);
4033 }
4034
4035 if (trace_seq_has_overflowed(s))
4036 return TRACE_TYPE_PARTIAL_LINE;
4037
4038 if (event)
4039 return event->funcs->trace(iter, sym_flags, event);
4040
4041 trace_seq_printf(s, "Unknown type %d\n", entry->type);
4042
4043 return trace_handle_return(s);
4044}
4045
4046static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
4047{
4048 struct trace_array *tr = iter->tr;
4049 struct trace_seq *s = &iter->seq;
4050 struct trace_entry *entry;
4051 struct trace_event *event;
4052
4053 entry = iter->ent;
4054
4055 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
4056 trace_seq_printf(s, "%d %d %llu ",
4057 entry->pid, iter->cpu, iter->ts);
4058
4059 if (trace_seq_has_overflowed(s))
4060 return TRACE_TYPE_PARTIAL_LINE;
4061
4062 event = ftrace_find_event(entry->type);
4063 if (event)
4064 return event->funcs->raw(iter, 0, event);
4065
4066 trace_seq_printf(s, "%d ?\n", entry->type);
4067
4068 return trace_handle_return(s);
4069}
4070
4071static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
4072{
4073 struct trace_array *tr = iter->tr;
4074 struct trace_seq *s = &iter->seq;
4075 unsigned char newline = '\n';
4076 struct trace_entry *entry;
4077 struct trace_event *event;
4078
4079 entry = iter->ent;
4080
4081 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4082 SEQ_PUT_HEX_FIELD(s, entry->pid);
4083 SEQ_PUT_HEX_FIELD(s, iter->cpu);
4084 SEQ_PUT_HEX_FIELD(s, iter->ts);
4085 if (trace_seq_has_overflowed(s))
4086 return TRACE_TYPE_PARTIAL_LINE;
4087 }
4088
4089 event = ftrace_find_event(entry->type);
4090 if (event) {
4091 enum print_line_t ret = event->funcs->hex(iter, 0, event);
4092 if (ret != TRACE_TYPE_HANDLED)
4093 return ret;
4094 }
4095
4096 SEQ_PUT_FIELD(s, newline);
4097
4098 return trace_handle_return(s);
4099}
4100
4101static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
4102{
4103 struct trace_array *tr = iter->tr;
4104 struct trace_seq *s = &iter->seq;
4105 struct trace_entry *entry;
4106 struct trace_event *event;
4107
4108 entry = iter->ent;
4109
4110 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4111 SEQ_PUT_FIELD(s, entry->pid);
4112 SEQ_PUT_FIELD(s, iter->cpu);
4113 SEQ_PUT_FIELD(s, iter->ts);
4114 if (trace_seq_has_overflowed(s))
4115 return TRACE_TYPE_PARTIAL_LINE;
4116 }
4117
4118 event = ftrace_find_event(entry->type);
4119 return event ? event->funcs->binary(iter, 0, event) :
4120 TRACE_TYPE_HANDLED;
4121}
4122
4123int trace_empty(struct trace_iterator *iter)
4124{
4125 struct ring_buffer_iter *buf_iter;
4126 int cpu;
4127
4128 /* If we are looking at one CPU buffer, only check that one */
4129 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4130 cpu = iter->cpu_file;
4131 buf_iter = trace_buffer_iter(iter, cpu);
4132 if (buf_iter) {
4133 if (!ring_buffer_iter_empty(buf_iter))
4134 return 0;
4135 } else {
4136 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4137 return 0;
4138 }
4139 return 1;
4140 }
4141
4142 for_each_tracing_cpu(cpu) {
4143 buf_iter = trace_buffer_iter(iter, cpu);
4144 if (buf_iter) {
4145 if (!ring_buffer_iter_empty(buf_iter))
4146 return 0;
4147 } else {
4148 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4149 return 0;
4150 }
4151 }
4152
4153 return 1;
4154}
4155
4156/* Called with trace_event_read_lock() held. */
4157enum print_line_t print_trace_line(struct trace_iterator *iter)
4158{
4159 struct trace_array *tr = iter->tr;
4160 unsigned long trace_flags = tr->trace_flags;
4161 enum print_line_t ret;
4162
4163 if (iter->lost_events) {
4164 if (iter->lost_events == (unsigned long)-1)
4165 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
4166 iter->cpu);
4167 else
4168 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
4169 iter->cpu, iter->lost_events);
4170 if (trace_seq_has_overflowed(&iter->seq))
4171 return TRACE_TYPE_PARTIAL_LINE;
4172 }
4173
4174 if (iter->trace && iter->trace->print_line) {
4175 ret = iter->trace->print_line(iter);
4176 if (ret != TRACE_TYPE_UNHANDLED)
4177 return ret;
4178 }
4179
4180 if (iter->ent->type == TRACE_BPUTS &&
4181 trace_flags & TRACE_ITER_PRINTK &&
4182 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4183 return trace_print_bputs_msg_only(iter);
4184
4185 if (iter->ent->type == TRACE_BPRINT &&
4186 trace_flags & TRACE_ITER_PRINTK &&
4187 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4188 return trace_print_bprintk_msg_only(iter);
4189
4190 if (iter->ent->type == TRACE_PRINT &&
4191 trace_flags & TRACE_ITER_PRINTK &&
4192 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4193 return trace_print_printk_msg_only(iter);
4194
4195 if (trace_flags & TRACE_ITER_BIN)
4196 return print_bin_fmt(iter);
4197
4198 if (trace_flags & TRACE_ITER_HEX)
4199 return print_hex_fmt(iter);
4200
4201 if (trace_flags & TRACE_ITER_RAW)
4202 return print_raw_fmt(iter);
4203
4204 return print_trace_fmt(iter);
4205}
4206
4207void trace_latency_header(struct seq_file *m)
4208{
4209 struct trace_iterator *iter = m->private;
4210 struct trace_array *tr = iter->tr;
4211
4212 /* print nothing if the buffers are empty */
4213 if (trace_empty(iter))
4214 return;
4215
4216 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4217 print_trace_header(m, iter);
4218
4219 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4220 print_lat_help_header(m);
4221}
4222
4223void trace_default_header(struct seq_file *m)
4224{
4225 struct trace_iterator *iter = m->private;
4226 struct trace_array *tr = iter->tr;
4227 unsigned long trace_flags = tr->trace_flags;
4228
4229 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4230 return;
4231
4232 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4233 /* print nothing if the buffers are empty */
4234 if (trace_empty(iter))
4235 return;
4236 print_trace_header(m, iter);
4237 if (!(trace_flags & TRACE_ITER_VERBOSE))
4238 print_lat_help_header(m);
4239 } else {
4240 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4241 if (trace_flags & TRACE_ITER_IRQ_INFO)
4242 print_func_help_header_irq(iter->array_buffer,
4243 m, trace_flags);
4244 else
4245 print_func_help_header(iter->array_buffer, m,
4246 trace_flags);
4247 }
4248 }
4249}
4250
4251static void test_ftrace_alive(struct seq_file *m)
4252{
4253 if (!ftrace_is_dead())
4254 return;
4255 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4256 "# MAY BE MISSING FUNCTION EVENTS\n");
4257}
4258
4259#ifdef CONFIG_TRACER_MAX_TRACE
4260static void show_snapshot_main_help(struct seq_file *m)
4261{
4262 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4263 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4264 "# Takes a snapshot of the main buffer.\n"
4265 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4266 "# (Doesn't have to be '2' works with any number that\n"
4267 "# is not a '0' or '1')\n");
4268}
4269
4270static void show_snapshot_percpu_help(struct seq_file *m)
4271{
4272 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4273#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4274 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4275 "# Takes a snapshot of the main buffer for this cpu.\n");
4276#else
4277 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4278 "# Must use main snapshot file to allocate.\n");
4279#endif
4280 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4281 "# (Doesn't have to be '2' works with any number that\n"
4282 "# is not a '0' or '1')\n");
4283}
4284
4285static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
4286{
4287 if (iter->tr->allocated_snapshot)
4288 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4289 else
4290 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4291
4292 seq_puts(m, "# Snapshot commands:\n");
4293 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4294 show_snapshot_main_help(m);
4295 else
4296 show_snapshot_percpu_help(m);
4297}
4298#else
4299/* Should never be called */
4300static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4301#endif
4302
4303static int s_show(struct seq_file *m, void *v)
4304{
4305 struct trace_iterator *iter = v;
4306 int ret;
4307
4308 if (iter->ent == NULL) {
4309 if (iter->tr) {
4310 seq_printf(m, "# tracer: %s\n", iter->trace->name);
4311 seq_puts(m, "#\n");
4312 test_ftrace_alive(m);
4313 }
4314 if (iter->snapshot && trace_empty(iter))
4315 print_snapshot_help(m, iter);
4316 else if (iter->trace && iter->trace->print_header)
4317 iter->trace->print_header(m);
4318 else
4319 trace_default_header(m);
4320
4321 } else if (iter->leftover) {
4322 /*
4323 * If we filled the seq_file buffer earlier, we
4324 * want to just show it now.
4325 */
4326 ret = trace_print_seq(m, &iter->seq);
4327
4328 /* ret should this time be zero, but you never know */
4329 iter->leftover = ret;
4330
4331 } else {
4332 print_trace_line(iter);
4333 ret = trace_print_seq(m, &iter->seq);
4334 /*
4335 * If we overflow the seq_file buffer, then it will
4336 * ask us for this data again at start up.
4337 * Use that instead.
4338 * ret is 0 if seq_file write succeeded.
4339 * -1 otherwise.
4340 */
4341 iter->leftover = ret;
4342 }
4343
4344 return 0;
4345}
4346
4347/*
4348 * Should be used after trace_array_get(), trace_types_lock
4349 * ensures that i_cdev was already initialized.
4350 */
4351static inline int tracing_get_cpu(struct inode *inode)
4352{
4353 if (inode->i_cdev) /* See trace_create_cpu_file() */
4354 return (long)inode->i_cdev - 1;
4355 return RING_BUFFER_ALL_CPUS;
4356}
4357
4358static const struct seq_operations tracer_seq_ops = {
4359 .start = s_start,
4360 .next = s_next,
4361 .stop = s_stop,
4362 .show = s_show,
4363};
4364
4365static struct trace_iterator *
4366__tracing_open(struct inode *inode, struct file *file, bool snapshot)
4367{
4368 struct trace_array *tr = inode->i_private;
4369 struct trace_iterator *iter;
4370 int cpu;
4371
4372 if (tracing_disabled)
4373 return ERR_PTR(-ENODEV);
4374
4375 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4376 if (!iter)
4377 return ERR_PTR(-ENOMEM);
4378
4379 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4380 GFP_KERNEL);
4381 if (!iter->buffer_iter)
4382 goto release;
4383
4384 /*
4385 * trace_find_next_entry() may need to save off iter->ent.
4386 * It will place it into the iter->temp buffer. As most
4387 * events are less than 128, allocate a buffer of that size.
4388 * If one is greater, then trace_find_next_entry() will
4389 * allocate a new buffer to adjust for the bigger iter->ent.
4390 * It's not critical if it fails to get allocated here.
4391 */
4392 iter->temp = kmalloc(128, GFP_KERNEL);
4393 if (iter->temp)
4394 iter->temp_size = 128;
4395
4396 /*
4397 * trace_event_printf() may need to modify given format
4398 * string to replace %p with %px so that it shows real address
4399 * instead of hash value. However, that is only for the event
4400 * tracing, other tracer may not need. Defer the allocation
4401 * until it is needed.
4402 */
4403 iter->fmt = NULL;
4404 iter->fmt_size = 0;
4405
4406 /*
4407 * We make a copy of the current tracer to avoid concurrent
4408 * changes on it while we are reading.
4409 */
4410 mutex_lock(&trace_types_lock);
4411 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4412 if (!iter->trace)
4413 goto fail;
4414
4415 *iter->trace = *tr->current_trace;
4416
4417 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4418 goto fail;
4419
4420 iter->tr = tr;
4421
4422#ifdef CONFIG_TRACER_MAX_TRACE
4423 /* Currently only the top directory has a snapshot */
4424 if (tr->current_trace->print_max || snapshot)
4425 iter->array_buffer = &tr->max_buffer;
4426 else
4427#endif
4428 iter->array_buffer = &tr->array_buffer;
4429 iter->snapshot = snapshot;
4430 iter->pos = -1;
4431 iter->cpu_file = tracing_get_cpu(inode);
4432 mutex_init(&iter->mutex);
4433
4434 /* Notify the tracer early; before we stop tracing. */
4435 if (iter->trace->open)
4436 iter->trace->open(iter);
4437
4438 /* Annotate start of buffers if we had overruns */
4439 if (ring_buffer_overruns(iter->array_buffer->buffer))
4440 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4441
4442 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4443 if (trace_clocks[tr->clock_id].in_ns)
4444 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4445
4446 /*
4447 * If pause-on-trace is enabled, then stop the trace while
4448 * dumping, unless this is the "snapshot" file
4449 */
4450 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4451 tracing_stop_tr(tr);
4452
4453 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4454 for_each_tracing_cpu(cpu) {
4455 iter->buffer_iter[cpu] =
4456 ring_buffer_read_prepare(iter->array_buffer->buffer,
4457 cpu, GFP_KERNEL);
4458 }
4459 ring_buffer_read_prepare_sync();
4460 for_each_tracing_cpu(cpu) {
4461 ring_buffer_read_start(iter->buffer_iter[cpu]);
4462 tracing_iter_reset(iter, cpu);
4463 }
4464 } else {
4465 cpu = iter->cpu_file;
4466 iter->buffer_iter[cpu] =
4467 ring_buffer_read_prepare(iter->array_buffer->buffer,
4468 cpu, GFP_KERNEL);
4469 ring_buffer_read_prepare_sync();
4470 ring_buffer_read_start(iter->buffer_iter[cpu]);
4471 tracing_iter_reset(iter, cpu);
4472 }
4473
4474 mutex_unlock(&trace_types_lock);
4475
4476 return iter;
4477
4478 fail:
4479 mutex_unlock(&trace_types_lock);
4480 kfree(iter->trace);
4481 kfree(iter->temp);
4482 kfree(iter->buffer_iter);
4483release:
4484 seq_release_private(inode, file);
4485 return ERR_PTR(-ENOMEM);
4486}
4487
4488int tracing_open_generic(struct inode *inode, struct file *filp)
4489{
4490 int ret;
4491
4492 ret = tracing_check_open_get_tr(NULL);
4493 if (ret)
4494 return ret;
4495
4496 filp->private_data = inode->i_private;
4497 return 0;
4498}
4499
4500bool tracing_is_disabled(void)
4501{
4502 return (tracing_disabled) ? true: false;
4503}
4504
4505/*
4506 * Open and update trace_array ref count.
4507 * Must have the current trace_array passed to it.
4508 */
4509int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4510{
4511 struct trace_array *tr = inode->i_private;
4512 int ret;
4513
4514 ret = tracing_check_open_get_tr(tr);
4515 if (ret)
4516 return ret;
4517
4518 filp->private_data = inode->i_private;
4519
4520 return 0;
4521}
4522
4523static int tracing_release(struct inode *inode, struct file *file)
4524{
4525 struct trace_array *tr = inode->i_private;
4526 struct seq_file *m = file->private_data;
4527 struct trace_iterator *iter;
4528 int cpu;
4529
4530 if (!(file->f_mode & FMODE_READ)) {
4531 trace_array_put(tr);
4532 return 0;
4533 }
4534
4535 /* Writes do not use seq_file */
4536 iter = m->private;
4537 mutex_lock(&trace_types_lock);
4538
4539 for_each_tracing_cpu(cpu) {
4540 if (iter->buffer_iter[cpu])
4541 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4542 }
4543
4544 if (iter->trace && iter->trace->close)
4545 iter->trace->close(iter);
4546
4547 if (!iter->snapshot && tr->stop_count)
4548 /* reenable tracing if it was previously enabled */
4549 tracing_start_tr(tr);
4550
4551 __trace_array_put(tr);
4552
4553 mutex_unlock(&trace_types_lock);
4554
4555 mutex_destroy(&iter->mutex);
4556 free_cpumask_var(iter->started);
4557 kfree(iter->fmt);
4558 kfree(iter->temp);
4559 kfree(iter->trace);
4560 kfree(iter->buffer_iter);
4561 seq_release_private(inode, file);
4562
4563 return 0;
4564}
4565
4566static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4567{
4568 struct trace_array *tr = inode->i_private;
4569
4570 trace_array_put(tr);
4571 return 0;
4572}
4573
4574static int tracing_single_release_tr(struct inode *inode, struct file *file)
4575{
4576 struct trace_array *tr = inode->i_private;
4577
4578 trace_array_put(tr);
4579
4580 return single_release(inode, file);
4581}
4582
4583static int tracing_open(struct inode *inode, struct file *file)
4584{
4585 struct trace_array *tr = inode->i_private;
4586 struct trace_iterator *iter;
4587 int ret;
4588
4589 ret = tracing_check_open_get_tr(tr);
4590 if (ret)
4591 return ret;
4592
4593 /* If this file was open for write, then erase contents */
4594 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4595 int cpu = tracing_get_cpu(inode);
4596 struct array_buffer *trace_buf = &tr->array_buffer;
4597
4598#ifdef CONFIG_TRACER_MAX_TRACE
4599 if (tr->current_trace->print_max)
4600 trace_buf = &tr->max_buffer;
4601#endif
4602
4603 if (cpu == RING_BUFFER_ALL_CPUS)
4604 tracing_reset_online_cpus(trace_buf);
4605 else
4606 tracing_reset_cpu(trace_buf, cpu);
4607 }
4608
4609 if (file->f_mode & FMODE_READ) {
4610 iter = __tracing_open(inode, file, false);
4611 if (IS_ERR(iter))
4612 ret = PTR_ERR(iter);
4613 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4614 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4615 }
4616
4617 if (ret < 0)
4618 trace_array_put(tr);
4619
4620 return ret;
4621}
4622
4623/*
4624 * Some tracers are not suitable for instance buffers.
4625 * A tracer is always available for the global array (toplevel)
4626 * or if it explicitly states that it is.
4627 */
4628static bool
4629trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4630{
4631 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4632}
4633
4634/* Find the next tracer that this trace array may use */
4635static struct tracer *
4636get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4637{
4638 while (t && !trace_ok_for_array(t, tr))
4639 t = t->next;
4640
4641 return t;
4642}
4643
4644static void *
4645t_next(struct seq_file *m, void *v, loff_t *pos)
4646{
4647 struct trace_array *tr = m->private;
4648 struct tracer *t = v;
4649
4650 (*pos)++;
4651
4652 if (t)
4653 t = get_tracer_for_array(tr, t->next);
4654
4655 return t;
4656}
4657
4658static void *t_start(struct seq_file *m, loff_t *pos)
4659{
4660 struct trace_array *tr = m->private;
4661 struct tracer *t;
4662 loff_t l = 0;
4663
4664 mutex_lock(&trace_types_lock);
4665
4666 t = get_tracer_for_array(tr, trace_types);
4667 for (; t && l < *pos; t = t_next(m, t, &l))
4668 ;
4669
4670 return t;
4671}
4672
4673static void t_stop(struct seq_file *m, void *p)
4674{
4675 mutex_unlock(&trace_types_lock);
4676}
4677
4678static int t_show(struct seq_file *m, void *v)
4679{
4680 struct tracer *t = v;
4681
4682 if (!t)
4683 return 0;
4684
4685 seq_puts(m, t->name);
4686 if (t->next)
4687 seq_putc(m, ' ');
4688 else
4689 seq_putc(m, '\n');
4690
4691 return 0;
4692}
4693
4694static const struct seq_operations show_traces_seq_ops = {
4695 .start = t_start,
4696 .next = t_next,
4697 .stop = t_stop,
4698 .show = t_show,
4699};
4700
4701static int show_traces_open(struct inode *inode, struct file *file)
4702{
4703 struct trace_array *tr = inode->i_private;
4704 struct seq_file *m;
4705 int ret;
4706
4707 ret = tracing_check_open_get_tr(tr);
4708 if (ret)
4709 return ret;
4710
4711 ret = seq_open(file, &show_traces_seq_ops);
4712 if (ret) {
4713 trace_array_put(tr);
4714 return ret;
4715 }
4716
4717 m = file->private_data;
4718 m->private = tr;
4719
4720 return 0;
4721}
4722
4723static int show_traces_release(struct inode *inode, struct file *file)
4724{
4725 struct trace_array *tr = inode->i_private;
4726
4727 trace_array_put(tr);
4728 return seq_release(inode, file);
4729}
4730
4731static ssize_t
4732tracing_write_stub(struct file *filp, const char __user *ubuf,
4733 size_t count, loff_t *ppos)
4734{
4735 return count;
4736}
4737
4738loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4739{
4740 int ret;
4741
4742 if (file->f_mode & FMODE_READ)
4743 ret = seq_lseek(file, offset, whence);
4744 else
4745 file->f_pos = ret = 0;
4746
4747 return ret;
4748}
4749
4750static const struct file_operations tracing_fops = {
4751 .open = tracing_open,
4752 .read = seq_read,
4753 .write = tracing_write_stub,
4754 .llseek = tracing_lseek,
4755 .release = tracing_release,
4756};
4757
4758static const struct file_operations show_traces_fops = {
4759 .open = show_traces_open,
4760 .read = seq_read,
4761 .llseek = seq_lseek,
4762 .release = show_traces_release,
4763};
4764
4765static ssize_t
4766tracing_cpumask_read(struct file *filp, char __user *ubuf,
4767 size_t count, loff_t *ppos)
4768{
4769 struct trace_array *tr = file_inode(filp)->i_private;
4770 char *mask_str;
4771 int len;
4772
4773 len = snprintf(NULL, 0, "%*pb\n",
4774 cpumask_pr_args(tr->tracing_cpumask)) + 1;
4775 mask_str = kmalloc(len, GFP_KERNEL);
4776 if (!mask_str)
4777 return -ENOMEM;
4778
4779 len = snprintf(mask_str, len, "%*pb\n",
4780 cpumask_pr_args(tr->tracing_cpumask));
4781 if (len >= count) {
4782 count = -EINVAL;
4783 goto out_err;
4784 }
4785 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
4786
4787out_err:
4788 kfree(mask_str);
4789
4790 return count;
4791}
4792
4793int tracing_set_cpumask(struct trace_array *tr,
4794 cpumask_var_t tracing_cpumask_new)
4795{
4796 int cpu;
4797
4798 if (!tr)
4799 return -EINVAL;
4800
4801 local_irq_disable();
4802 arch_spin_lock(&tr->max_lock);
4803 for_each_tracing_cpu(cpu) {
4804 /*
4805 * Increase/decrease the disabled counter if we are
4806 * about to flip a bit in the cpumask:
4807 */
4808 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4809 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4810 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
4811 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
4812 }
4813 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4814 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4815 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
4816 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
4817 }
4818 }
4819 arch_spin_unlock(&tr->max_lock);
4820 local_irq_enable();
4821
4822 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
4823
4824 return 0;
4825}
4826
4827static ssize_t
4828tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4829 size_t count, loff_t *ppos)
4830{
4831 struct trace_array *tr = file_inode(filp)->i_private;
4832 cpumask_var_t tracing_cpumask_new;
4833 int err;
4834
4835 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4836 return -ENOMEM;
4837
4838 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4839 if (err)
4840 goto err_free;
4841
4842 err = tracing_set_cpumask(tr, tracing_cpumask_new);
4843 if (err)
4844 goto err_free;
4845
4846 free_cpumask_var(tracing_cpumask_new);
4847
4848 return count;
4849
4850err_free:
4851 free_cpumask_var(tracing_cpumask_new);
4852
4853 return err;
4854}
4855
4856static const struct file_operations tracing_cpumask_fops = {
4857 .open = tracing_open_generic_tr,
4858 .read = tracing_cpumask_read,
4859 .write = tracing_cpumask_write,
4860 .release = tracing_release_generic_tr,
4861 .llseek = generic_file_llseek,
4862};
4863
4864static int tracing_trace_options_show(struct seq_file *m, void *v)
4865{
4866 struct tracer_opt *trace_opts;
4867 struct trace_array *tr = m->private;
4868 u32 tracer_flags;
4869 int i;
4870
4871 mutex_lock(&trace_types_lock);
4872 tracer_flags = tr->current_trace->flags->val;
4873 trace_opts = tr->current_trace->flags->opts;
4874
4875 for (i = 0; trace_options[i]; i++) {
4876 if (tr->trace_flags & (1 << i))
4877 seq_printf(m, "%s\n", trace_options[i]);
4878 else
4879 seq_printf(m, "no%s\n", trace_options[i]);
4880 }
4881
4882 for (i = 0; trace_opts[i].name; i++) {
4883 if (tracer_flags & trace_opts[i].bit)
4884 seq_printf(m, "%s\n", trace_opts[i].name);
4885 else
4886 seq_printf(m, "no%s\n", trace_opts[i].name);
4887 }
4888 mutex_unlock(&trace_types_lock);
4889
4890 return 0;
4891}
4892
4893static int __set_tracer_option(struct trace_array *tr,
4894 struct tracer_flags *tracer_flags,
4895 struct tracer_opt *opts, int neg)
4896{
4897 struct tracer *trace = tracer_flags->trace;
4898 int ret;
4899
4900 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
4901 if (ret)
4902 return ret;
4903
4904 if (neg)
4905 tracer_flags->val &= ~opts->bit;
4906 else
4907 tracer_flags->val |= opts->bit;
4908 return 0;
4909}
4910
4911/* Try to assign a tracer specific option */
4912static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
4913{
4914 struct tracer *trace = tr->current_trace;
4915 struct tracer_flags *tracer_flags = trace->flags;
4916 struct tracer_opt *opts = NULL;
4917 int i;
4918
4919 for (i = 0; tracer_flags->opts[i].name; i++) {
4920 opts = &tracer_flags->opts[i];
4921
4922 if (strcmp(cmp, opts->name) == 0)
4923 return __set_tracer_option(tr, trace->flags, opts, neg);
4924 }
4925
4926 return -EINVAL;
4927}
4928
4929/* Some tracers require overwrite to stay enabled */
4930int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4931{
4932 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4933 return -1;
4934
4935 return 0;
4936}
4937
4938int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
4939{
4940 if ((mask == TRACE_ITER_RECORD_TGID) ||
4941 (mask == TRACE_ITER_RECORD_CMD))
4942 lockdep_assert_held(&event_mutex);
4943
4944 /* do nothing if flag is already set */
4945 if (!!(tr->trace_flags & mask) == !!enabled)
4946 return 0;
4947
4948 /* Give the tracer a chance to approve the change */
4949 if (tr->current_trace->flag_changed)
4950 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4951 return -EINVAL;
4952
4953 if (enabled)
4954 tr->trace_flags |= mask;
4955 else
4956 tr->trace_flags &= ~mask;
4957
4958 if (mask == TRACE_ITER_RECORD_CMD)
4959 trace_event_enable_cmd_record(enabled);
4960
4961 if (mask == TRACE_ITER_RECORD_TGID) {
4962 if (!tgid_map)
4963 tgid_map = kvcalloc(PID_MAX_DEFAULT + 1,
4964 sizeof(*tgid_map),
4965 GFP_KERNEL);
4966 if (!tgid_map) {
4967 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4968 return -ENOMEM;
4969 }
4970
4971 trace_event_enable_tgid_record(enabled);
4972 }
4973
4974 if (mask == TRACE_ITER_EVENT_FORK)
4975 trace_event_follow_fork(tr, enabled);
4976
4977 if (mask == TRACE_ITER_FUNC_FORK)
4978 ftrace_pid_follow_fork(tr, enabled);
4979
4980 if (mask == TRACE_ITER_OVERWRITE) {
4981 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
4982#ifdef CONFIG_TRACER_MAX_TRACE
4983 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4984#endif
4985 }
4986
4987 if (mask == TRACE_ITER_PRINTK) {
4988 trace_printk_start_stop_comm(enabled);
4989 trace_printk_control(enabled);
4990 }
4991
4992 return 0;
4993}
4994
4995int trace_set_options(struct trace_array *tr, char *option)
4996{
4997 char *cmp;
4998 int neg = 0;
4999 int ret;
5000 size_t orig_len = strlen(option);
5001 int len;
5002
5003 cmp = strstrip(option);
5004
5005 len = str_has_prefix(cmp, "no");
5006 if (len)
5007 neg = 1;
5008
5009 cmp += len;
5010
5011 mutex_lock(&event_mutex);
5012 mutex_lock(&trace_types_lock);
5013
5014 ret = match_string(trace_options, -1, cmp);
5015 /* If no option could be set, test the specific tracer options */
5016 if (ret < 0)
5017 ret = set_tracer_option(tr, cmp, neg);
5018 else
5019 ret = set_tracer_flag(tr, 1 << ret, !neg);
5020
5021 mutex_unlock(&trace_types_lock);
5022 mutex_unlock(&event_mutex);
5023
5024 /*
5025 * If the first trailing whitespace is replaced with '\0' by strstrip,
5026 * turn it back into a space.
5027 */
5028 if (orig_len > strlen(option))
5029 option[strlen(option)] = ' ';
5030
5031 return ret;
5032}
5033
5034static void __init apply_trace_boot_options(void)
5035{
5036 char *buf = trace_boot_options_buf;
5037 char *option;
5038
5039 while (true) {
5040 option = strsep(&buf, ",");
5041
5042 if (!option)
5043 break;
5044
5045 if (*option)
5046 trace_set_options(&global_trace, option);
5047
5048 /* Put back the comma to allow this to be called again */
5049 if (buf)
5050 *(buf - 1) = ',';
5051 }
5052}
5053
5054static ssize_t
5055tracing_trace_options_write(struct file *filp, const char __user *ubuf,
5056 size_t cnt, loff_t *ppos)
5057{
5058 struct seq_file *m = filp->private_data;
5059 struct trace_array *tr = m->private;
5060 char buf[64];
5061 int ret;
5062
5063 if (cnt >= sizeof(buf))
5064 return -EINVAL;
5065
5066 if (copy_from_user(buf, ubuf, cnt))
5067 return -EFAULT;
5068
5069 buf[cnt] = 0;
5070
5071 ret = trace_set_options(tr, buf);
5072 if (ret < 0)
5073 return ret;
5074
5075 *ppos += cnt;
5076
5077 return cnt;
5078}
5079
5080static int tracing_trace_options_open(struct inode *inode, struct file *file)
5081{
5082 struct trace_array *tr = inode->i_private;
5083 int ret;
5084
5085 ret = tracing_check_open_get_tr(tr);
5086 if (ret)
5087 return ret;
5088
5089 ret = single_open(file, tracing_trace_options_show, inode->i_private);
5090 if (ret < 0)
5091 trace_array_put(tr);
5092
5093 return ret;
5094}
5095
5096static const struct file_operations tracing_iter_fops = {
5097 .open = tracing_trace_options_open,
5098 .read = seq_read,
5099 .llseek = seq_lseek,
5100 .release = tracing_single_release_tr,
5101 .write = tracing_trace_options_write,
5102};
5103
5104static const char readme_msg[] =
5105 "tracing mini-HOWTO:\n\n"
5106 "# echo 0 > tracing_on : quick way to disable tracing\n"
5107 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5108 " Important files:\n"
5109 " trace\t\t\t- The static contents of the buffer\n"
5110 "\t\t\t To clear the buffer write into this file: echo > trace\n"
5111 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5112 " current_tracer\t- function and latency tracers\n"
5113 " available_tracers\t- list of configured tracers for current_tracer\n"
5114 " error_log\t- error log for failed commands (that support it)\n"
5115 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
5116 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
5117 " trace_clock\t\t-change the clock used to order events\n"
5118 " local: Per cpu clock but may not be synced across CPUs\n"
5119 " global: Synced across CPUs but slows tracing down.\n"
5120 " counter: Not a clock, but just an increment\n"
5121 " uptime: Jiffy counter from time of boot\n"
5122 " perf: Same clock that perf events use\n"
5123#ifdef CONFIG_X86_64
5124 " x86-tsc: TSC cycle counter\n"
5125#endif
5126 "\n timestamp_mode\t-view the mode used to timestamp events\n"
5127 " delta: Delta difference against a buffer-wide timestamp\n"
5128 " absolute: Absolute (standalone) timestamp\n"
5129 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5130 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5131 " tracing_cpumask\t- Limit which CPUs to trace\n"
5132 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5133 "\t\t\t Remove sub-buffer with rmdir\n"
5134 " trace_options\t\t- Set format or modify how tracing happens\n"
5135 "\t\t\t Disable an option by prefixing 'no' to the\n"
5136 "\t\t\t option name\n"
5137 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5138#ifdef CONFIG_DYNAMIC_FTRACE
5139 "\n available_filter_functions - list of functions that can be filtered on\n"
5140 " set_ftrace_filter\t- echo function name in here to only trace these\n"
5141 "\t\t\t functions\n"
5142 "\t accepts: func_full_name or glob-matching-pattern\n"
5143 "\t modules: Can select a group via module\n"
5144 "\t Format: :mod:<module-name>\n"
5145 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
5146 "\t triggers: a command to perform when function is hit\n"
5147 "\t Format: <function>:<trigger>[:count]\n"
5148 "\t trigger: traceon, traceoff\n"
5149 "\t\t enable_event:<system>:<event>\n"
5150 "\t\t disable_event:<system>:<event>\n"
5151#ifdef CONFIG_STACKTRACE
5152 "\t\t stacktrace\n"
5153#endif
5154#ifdef CONFIG_TRACER_SNAPSHOT
5155 "\t\t snapshot\n"
5156#endif
5157 "\t\t dump\n"
5158 "\t\t cpudump\n"
5159 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
5160 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
5161 "\t The first one will disable tracing every time do_fault is hit\n"
5162 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
5163 "\t The first time do trap is hit and it disables tracing, the\n"
5164 "\t counter will decrement to 2. If tracing is already disabled,\n"
5165 "\t the counter will not decrement. It only decrements when the\n"
5166 "\t trigger did work\n"
5167 "\t To remove trigger without count:\n"
5168 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
5169 "\t To remove trigger with a count:\n"
5170 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5171 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
5172 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
5173 "\t modules: Can select a group via module command :mod:\n"
5174 "\t Does not accept triggers\n"
5175#endif /* CONFIG_DYNAMIC_FTRACE */
5176#ifdef CONFIG_FUNCTION_TRACER
5177 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5178 "\t\t (function)\n"
5179 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5180 "\t\t (function)\n"
5181#endif
5182#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5183 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5184 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5185 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5186#endif
5187#ifdef CONFIG_TRACER_SNAPSHOT
5188 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5189 "\t\t\t snapshot buffer. Read the contents for more\n"
5190 "\t\t\t information\n"
5191#endif
5192#ifdef CONFIG_STACK_TRACER
5193 " stack_trace\t\t- Shows the max stack trace when active\n"
5194 " stack_max_size\t- Shows current max stack size that was traced\n"
5195 "\t\t\t Write into this file to reset the max size (trigger a\n"
5196 "\t\t\t new trace)\n"
5197#ifdef CONFIG_DYNAMIC_FTRACE
5198 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5199 "\t\t\t traces\n"
5200#endif
5201#endif /* CONFIG_STACK_TRACER */
5202#ifdef CONFIG_DYNAMIC_EVENTS
5203 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5204 "\t\t\t Write into this file to define/undefine new trace events.\n"
5205#endif
5206#ifdef CONFIG_KPROBE_EVENTS
5207 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5208 "\t\t\t Write into this file to define/undefine new trace events.\n"
5209#endif
5210#ifdef CONFIG_UPROBE_EVENTS
5211 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5212 "\t\t\t Write into this file to define/undefine new trace events.\n"
5213#endif
5214#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
5215 "\t accepts: event-definitions (one definition per line)\n"
5216 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
5217 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
5218#ifdef CONFIG_HIST_TRIGGERS
5219 "\t s:[synthetic/]<event> <field> [<field>]\n"
5220#endif
5221 "\t -:[<group>/]<event>\n"
5222#ifdef CONFIG_KPROBE_EVENTS
5223 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5224 "place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n"
5225#endif
5226#ifdef CONFIG_UPROBE_EVENTS
5227 " place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
5228#endif
5229 "\t args: <name>=fetcharg[:type]\n"
5230 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
5231#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5232 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5233#else
5234 "\t $stack<index>, $stack, $retval, $comm,\n"
5235#endif
5236 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5237 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
5238 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5239 "\t <type>\\[<array-size>\\]\n"
5240#ifdef CONFIG_HIST_TRIGGERS
5241 "\t field: <stype> <name>;\n"
5242 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5243 "\t [unsigned] char/int/long\n"
5244#endif
5245#endif
5246 " events/\t\t- Directory containing all trace event subsystems:\n"
5247 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5248 " events/<system>/\t- Directory containing all trace events for <system>:\n"
5249 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5250 "\t\t\t events\n"
5251 " filter\t\t- If set, only events passing filter are traced\n"
5252 " events/<system>/<event>/\t- Directory containing control files for\n"
5253 "\t\t\t <event>:\n"
5254 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5255 " filter\t\t- If set, only events passing filter are traced\n"
5256 " trigger\t\t- If set, a command to perform when event is hit\n"
5257 "\t Format: <trigger>[:count][if <filter>]\n"
5258 "\t trigger: traceon, traceoff\n"
5259 "\t enable_event:<system>:<event>\n"
5260 "\t disable_event:<system>:<event>\n"
5261#ifdef CONFIG_HIST_TRIGGERS
5262 "\t enable_hist:<system>:<event>\n"
5263 "\t disable_hist:<system>:<event>\n"
5264#endif
5265#ifdef CONFIG_STACKTRACE
5266 "\t\t stacktrace\n"
5267#endif
5268#ifdef CONFIG_TRACER_SNAPSHOT
5269 "\t\t snapshot\n"
5270#endif
5271#ifdef CONFIG_HIST_TRIGGERS
5272 "\t\t hist (see below)\n"
5273#endif
5274 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
5275 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5276 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5277 "\t events/block/block_unplug/trigger\n"
5278 "\t The first disables tracing every time block_unplug is hit.\n"
5279 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
5280 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
5281 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5282 "\t Like function triggers, the counter is only decremented if it\n"
5283 "\t enabled or disabled tracing.\n"
5284 "\t To remove a trigger without a count:\n"
5285 "\t echo '!<trigger> > <system>/<event>/trigger\n"
5286 "\t To remove a trigger with a count:\n"
5287 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5288 "\t Filters can be ignored when removing a trigger.\n"
5289#ifdef CONFIG_HIST_TRIGGERS
5290 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
5291 "\t Format: hist:keys=<field1[,field2,...]>\n"
5292 "\t [:values=<field1[,field2,...]>]\n"
5293 "\t [:sort=<field1[,field2,...]>]\n"
5294 "\t [:size=#entries]\n"
5295 "\t [:pause][:continue][:clear]\n"
5296 "\t [:name=histname1]\n"
5297 "\t [:<handler>.<action>]\n"
5298 "\t [if <filter>]\n\n"
5299 "\t When a matching event is hit, an entry is added to a hash\n"
5300 "\t table using the key(s) and value(s) named, and the value of a\n"
5301 "\t sum called 'hitcount' is incremented. Keys and values\n"
5302 "\t correspond to fields in the event's format description. Keys\n"
5303 "\t can be any field, or the special string 'stacktrace'.\n"
5304 "\t Compound keys consisting of up to two fields can be specified\n"
5305 "\t by the 'keys' keyword. Values must correspond to numeric\n"
5306 "\t fields. Sort keys consisting of up to two fields can be\n"
5307 "\t specified using the 'sort' keyword. The sort direction can\n"
5308 "\t be modified by appending '.descending' or '.ascending' to a\n"
5309 "\t sort field. The 'size' parameter can be used to specify more\n"
5310 "\t or fewer than the default 2048 entries for the hashtable size.\n"
5311 "\t If a hist trigger is given a name using the 'name' parameter,\n"
5312 "\t its histogram data will be shared with other triggers of the\n"
5313 "\t same name, and trigger hits will update this common data.\n\n"
5314 "\t Reading the 'hist' file for the event will dump the hash\n"
5315 "\t table in its entirety to stdout. If there are multiple hist\n"
5316 "\t triggers attached to an event, there will be a table for each\n"
5317 "\t trigger in the output. The table displayed for a named\n"
5318 "\t trigger will be the same as any other instance having the\n"
5319 "\t same name. The default format used to display a given field\n"
5320 "\t can be modified by appending any of the following modifiers\n"
5321 "\t to the field name, as applicable:\n\n"
5322 "\t .hex display a number as a hex value\n"
5323 "\t .sym display an address as a symbol\n"
5324 "\t .sym-offset display an address as a symbol and offset\n"
5325 "\t .execname display a common_pid as a program name\n"
5326 "\t .syscall display a syscall id as a syscall name\n"
5327 "\t .log2 display log2 value rather than raw number\n"
5328 "\t .usecs display a common_timestamp in microseconds\n\n"
5329 "\t The 'pause' parameter can be used to pause an existing hist\n"
5330 "\t trigger or to start a hist trigger but not log any events\n"
5331 "\t until told to do so. 'continue' can be used to start or\n"
5332 "\t restart a paused hist trigger.\n\n"
5333 "\t The 'clear' parameter will clear the contents of a running\n"
5334 "\t hist trigger and leave its current paused/active state\n"
5335 "\t unchanged.\n\n"
5336 "\t The enable_hist and disable_hist triggers can be used to\n"
5337 "\t have one event conditionally start and stop another event's\n"
5338 "\t already-attached hist trigger. The syntax is analogous to\n"
5339 "\t the enable_event and disable_event triggers.\n\n"
5340 "\t Hist trigger handlers and actions are executed whenever a\n"
5341 "\t a histogram entry is added or updated. They take the form:\n\n"
5342 "\t <handler>.<action>\n\n"
5343 "\t The available handlers are:\n\n"
5344 "\t onmatch(matching.event) - invoke on addition or update\n"
5345 "\t onmax(var) - invoke if var exceeds current max\n"
5346 "\t onchange(var) - invoke action if var changes\n\n"
5347 "\t The available actions are:\n\n"
5348 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5349 "\t save(field,...) - save current event fields\n"
5350#ifdef CONFIG_TRACER_SNAPSHOT
5351 "\t snapshot() - snapshot the trace buffer\n\n"
5352#endif
5353#ifdef CONFIG_SYNTH_EVENTS
5354 " events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5355 "\t Write into this file to define/undefine new synthetic events.\n"
5356 "\t example: echo 'myevent u64 lat; char name[]' >> synthetic_events\n"
5357#endif
5358#endif
5359;
5360
5361static ssize_t
5362tracing_readme_read(struct file *filp, char __user *ubuf,
5363 size_t cnt, loff_t *ppos)
5364{
5365 return simple_read_from_buffer(ubuf, cnt, ppos,
5366 readme_msg, strlen(readme_msg));
5367}
5368
5369static const struct file_operations tracing_readme_fops = {
5370 .open = tracing_open_generic,
5371 .read = tracing_readme_read,
5372 .llseek = generic_file_llseek,
5373};
5374
5375static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
5376{
5377 int *ptr = v;
5378
5379 if (*pos || m->count)
5380 ptr++;
5381
5382 (*pos)++;
5383
5384 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
5385 if (trace_find_tgid(*ptr))
5386 return ptr;
5387 }
5388
5389 return NULL;
5390}
5391
5392static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5393{
5394 void *v;
5395 loff_t l = 0;
5396
5397 if (!tgid_map)
5398 return NULL;
5399
5400 v = &tgid_map[0];
5401 while (l <= *pos) {
5402 v = saved_tgids_next(m, v, &l);
5403 if (!v)
5404 return NULL;
5405 }
5406
5407 return v;
5408}
5409
5410static void saved_tgids_stop(struct seq_file *m, void *v)
5411{
5412}
5413
5414static int saved_tgids_show(struct seq_file *m, void *v)
5415{
5416 int pid = (int *)v - tgid_map;
5417
5418 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
5419 return 0;
5420}
5421
5422static const struct seq_operations tracing_saved_tgids_seq_ops = {
5423 .start = saved_tgids_start,
5424 .stop = saved_tgids_stop,
5425 .next = saved_tgids_next,
5426 .show = saved_tgids_show,
5427};
5428
5429static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5430{
5431 int ret;
5432
5433 ret = tracing_check_open_get_tr(NULL);
5434 if (ret)
5435 return ret;
5436
5437 return seq_open(filp, &tracing_saved_tgids_seq_ops);
5438}
5439
5440
5441static const struct file_operations tracing_saved_tgids_fops = {
5442 .open = tracing_saved_tgids_open,
5443 .read = seq_read,
5444 .llseek = seq_lseek,
5445 .release = seq_release,
5446};
5447
5448static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5449{
5450 unsigned int *ptr = v;
5451
5452 if (*pos || m->count)
5453 ptr++;
5454
5455 (*pos)++;
5456
5457 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5458 ptr++) {
5459 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5460 continue;
5461
5462 return ptr;
5463 }
5464
5465 return NULL;
5466}
5467
5468static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5469{
5470 void *v;
5471 loff_t l = 0;
5472
5473 preempt_disable();
5474 arch_spin_lock(&trace_cmdline_lock);
5475
5476 v = &savedcmd->map_cmdline_to_pid[0];
5477 while (l <= *pos) {
5478 v = saved_cmdlines_next(m, v, &l);
5479 if (!v)
5480 return NULL;
5481 }
5482
5483 return v;
5484}
5485
5486static void saved_cmdlines_stop(struct seq_file *m, void *v)
5487{
5488 arch_spin_unlock(&trace_cmdline_lock);
5489 preempt_enable();
5490}
5491
5492static int saved_cmdlines_show(struct seq_file *m, void *v)
5493{
5494 char buf[TASK_COMM_LEN];
5495 unsigned int *pid = v;
5496
5497 __trace_find_cmdline(*pid, buf);
5498 seq_printf(m, "%d %s\n", *pid, buf);
5499 return 0;
5500}
5501
5502static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5503 .start = saved_cmdlines_start,
5504 .next = saved_cmdlines_next,
5505 .stop = saved_cmdlines_stop,
5506 .show = saved_cmdlines_show,
5507};
5508
5509static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5510{
5511 int ret;
5512
5513 ret = tracing_check_open_get_tr(NULL);
5514 if (ret)
5515 return ret;
5516
5517 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5518}
5519
5520static const struct file_operations tracing_saved_cmdlines_fops = {
5521 .open = tracing_saved_cmdlines_open,
5522 .read = seq_read,
5523 .llseek = seq_lseek,
5524 .release = seq_release,
5525};
5526
5527static ssize_t
5528tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5529 size_t cnt, loff_t *ppos)
5530{
5531 char buf[64];
5532 int r;
5533
5534 arch_spin_lock(&trace_cmdline_lock);
5535 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5536 arch_spin_unlock(&trace_cmdline_lock);
5537
5538 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5539}
5540
5541static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5542{
5543 kfree(s->saved_cmdlines);
5544 kfree(s->map_cmdline_to_pid);
5545 kfree(s);
5546}
5547
5548static int tracing_resize_saved_cmdlines(unsigned int val)
5549{
5550 struct saved_cmdlines_buffer *s, *savedcmd_temp;
5551
5552 s = kmalloc(sizeof(*s), GFP_KERNEL);
5553 if (!s)
5554 return -ENOMEM;
5555
5556 if (allocate_cmdlines_buffer(val, s) < 0) {
5557 kfree(s);
5558 return -ENOMEM;
5559 }
5560
5561 arch_spin_lock(&trace_cmdline_lock);
5562 savedcmd_temp = savedcmd;
5563 savedcmd = s;
5564 arch_spin_unlock(&trace_cmdline_lock);
5565 free_saved_cmdlines_buffer(savedcmd_temp);
5566
5567 return 0;
5568}
5569
5570static ssize_t
5571tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5572 size_t cnt, loff_t *ppos)
5573{
5574 unsigned long val;
5575 int ret;
5576
5577 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5578 if (ret)
5579 return ret;
5580
5581 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5582 if (!val || val > PID_MAX_DEFAULT)
5583 return -EINVAL;
5584
5585 ret = tracing_resize_saved_cmdlines((unsigned int)val);
5586 if (ret < 0)
5587 return ret;
5588
5589 *ppos += cnt;
5590
5591 return cnt;
5592}
5593
5594static const struct file_operations tracing_saved_cmdlines_size_fops = {
5595 .open = tracing_open_generic,
5596 .read = tracing_saved_cmdlines_size_read,
5597 .write = tracing_saved_cmdlines_size_write,
5598};
5599
5600#ifdef CONFIG_TRACE_EVAL_MAP_FILE
5601static union trace_eval_map_item *
5602update_eval_map(union trace_eval_map_item *ptr)
5603{
5604 if (!ptr->map.eval_string) {
5605 if (ptr->tail.next) {
5606 ptr = ptr->tail.next;
5607 /* Set ptr to the next real item (skip head) */
5608 ptr++;
5609 } else
5610 return NULL;
5611 }
5612 return ptr;
5613}
5614
5615static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5616{
5617 union trace_eval_map_item *ptr = v;
5618
5619 /*
5620 * Paranoid! If ptr points to end, we don't want to increment past it.
5621 * This really should never happen.
5622 */
5623 (*pos)++;
5624 ptr = update_eval_map(ptr);
5625 if (WARN_ON_ONCE(!ptr))
5626 return NULL;
5627
5628 ptr++;
5629 ptr = update_eval_map(ptr);
5630
5631 return ptr;
5632}
5633
5634static void *eval_map_start(struct seq_file *m, loff_t *pos)
5635{
5636 union trace_eval_map_item *v;
5637 loff_t l = 0;
5638
5639 mutex_lock(&trace_eval_mutex);
5640
5641 v = trace_eval_maps;
5642 if (v)
5643 v++;
5644
5645 while (v && l < *pos) {
5646 v = eval_map_next(m, v, &l);
5647 }
5648
5649 return v;
5650}
5651
5652static void eval_map_stop(struct seq_file *m, void *v)
5653{
5654 mutex_unlock(&trace_eval_mutex);
5655}
5656
5657static int eval_map_show(struct seq_file *m, void *v)
5658{
5659 union trace_eval_map_item *ptr = v;
5660
5661 seq_printf(m, "%s %ld (%s)\n",
5662 ptr->map.eval_string, ptr->map.eval_value,
5663 ptr->map.system);
5664
5665 return 0;
5666}
5667
5668static const struct seq_operations tracing_eval_map_seq_ops = {
5669 .start = eval_map_start,
5670 .next = eval_map_next,
5671 .stop = eval_map_stop,
5672 .show = eval_map_show,
5673};
5674
5675static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5676{
5677 int ret;
5678
5679 ret = tracing_check_open_get_tr(NULL);
5680 if (ret)
5681 return ret;
5682
5683 return seq_open(filp, &tracing_eval_map_seq_ops);
5684}
5685
5686static const struct file_operations tracing_eval_map_fops = {
5687 .open = tracing_eval_map_open,
5688 .read = seq_read,
5689 .llseek = seq_lseek,
5690 .release = seq_release,
5691};
5692
5693static inline union trace_eval_map_item *
5694trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5695{
5696 /* Return tail of array given the head */
5697 return ptr + ptr->head.length + 1;
5698}
5699
5700static void
5701trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5702 int len)
5703{
5704 struct trace_eval_map **stop;
5705 struct trace_eval_map **map;
5706 union trace_eval_map_item *map_array;
5707 union trace_eval_map_item *ptr;
5708
5709 stop = start + len;
5710
5711 /*
5712 * The trace_eval_maps contains the map plus a head and tail item,
5713 * where the head holds the module and length of array, and the
5714 * tail holds a pointer to the next list.
5715 */
5716 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
5717 if (!map_array) {
5718 pr_warn("Unable to allocate trace eval mapping\n");
5719 return;
5720 }
5721
5722 mutex_lock(&trace_eval_mutex);
5723
5724 if (!trace_eval_maps)
5725 trace_eval_maps = map_array;
5726 else {
5727 ptr = trace_eval_maps;
5728 for (;;) {
5729 ptr = trace_eval_jmp_to_tail(ptr);
5730 if (!ptr->tail.next)
5731 break;
5732 ptr = ptr->tail.next;
5733
5734 }
5735 ptr->tail.next = map_array;
5736 }
5737 map_array->head.mod = mod;
5738 map_array->head.length = len;
5739 map_array++;
5740
5741 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5742 map_array->map = **map;
5743 map_array++;
5744 }
5745 memset(map_array, 0, sizeof(*map_array));
5746
5747 mutex_unlock(&trace_eval_mutex);
5748}
5749
5750static void trace_create_eval_file(struct dentry *d_tracer)
5751{
5752 trace_create_file("eval_map", 0444, d_tracer,
5753 NULL, &tracing_eval_map_fops);
5754}
5755
5756#else /* CONFIG_TRACE_EVAL_MAP_FILE */
5757static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5758static inline void trace_insert_eval_map_file(struct module *mod,
5759 struct trace_eval_map **start, int len) { }
5760#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5761
5762static void trace_insert_eval_map(struct module *mod,
5763 struct trace_eval_map **start, int len)
5764{
5765 struct trace_eval_map **map;
5766
5767 if (len <= 0)
5768 return;
5769
5770 map = start;
5771
5772 trace_event_eval_update(map, len);
5773
5774 trace_insert_eval_map_file(mod, start, len);
5775}
5776
5777static ssize_t
5778tracing_set_trace_read(struct file *filp, char __user *ubuf,
5779 size_t cnt, loff_t *ppos)
5780{
5781 struct trace_array *tr = filp->private_data;
5782 char buf[MAX_TRACER_SIZE+2];
5783 int r;
5784
5785 mutex_lock(&trace_types_lock);
5786 r = sprintf(buf, "%s\n", tr->current_trace->name);
5787 mutex_unlock(&trace_types_lock);
5788
5789 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5790}
5791
5792int tracer_init(struct tracer *t, struct trace_array *tr)
5793{
5794 tracing_reset_online_cpus(&tr->array_buffer);
5795 return t->init(tr);
5796}
5797
5798static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
5799{
5800 int cpu;
5801
5802 for_each_tracing_cpu(cpu)
5803 per_cpu_ptr(buf->data, cpu)->entries = val;
5804}
5805
5806#ifdef CONFIG_TRACER_MAX_TRACE
5807/* resize @tr's buffer to the size of @size_tr's entries */
5808static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
5809 struct array_buffer *size_buf, int cpu_id)
5810{
5811 int cpu, ret = 0;
5812
5813 if (cpu_id == RING_BUFFER_ALL_CPUS) {
5814 for_each_tracing_cpu(cpu) {
5815 ret = ring_buffer_resize(trace_buf->buffer,
5816 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5817 if (ret < 0)
5818 break;
5819 per_cpu_ptr(trace_buf->data, cpu)->entries =
5820 per_cpu_ptr(size_buf->data, cpu)->entries;
5821 }
5822 } else {
5823 ret = ring_buffer_resize(trace_buf->buffer,
5824 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5825 if (ret == 0)
5826 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5827 per_cpu_ptr(size_buf->data, cpu_id)->entries;
5828 }
5829
5830 return ret;
5831}
5832#endif /* CONFIG_TRACER_MAX_TRACE */
5833
5834static int __tracing_resize_ring_buffer(struct trace_array *tr,
5835 unsigned long size, int cpu)
5836{
5837 int ret;
5838
5839 /*
5840 * If kernel or user changes the size of the ring buffer
5841 * we use the size that was given, and we can forget about
5842 * expanding it later.
5843 */
5844 ring_buffer_expanded = true;
5845
5846 /* May be called before buffers are initialized */
5847 if (!tr->array_buffer.buffer)
5848 return 0;
5849
5850 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
5851 if (ret < 0)
5852 return ret;
5853
5854#ifdef CONFIG_TRACER_MAX_TRACE
5855 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5856 !tr->current_trace->use_max_tr)
5857 goto out;
5858
5859 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5860 if (ret < 0) {
5861 int r = resize_buffer_duplicate_size(&tr->array_buffer,
5862 &tr->array_buffer, cpu);
5863 if (r < 0) {
5864 /*
5865 * AARGH! We are left with different
5866 * size max buffer!!!!
5867 * The max buffer is our "snapshot" buffer.
5868 * When a tracer needs a snapshot (one of the
5869 * latency tracers), it swaps the max buffer
5870 * with the saved snap shot. We succeeded to
5871 * update the size of the main buffer, but failed to
5872 * update the size of the max buffer. But when we tried
5873 * to reset the main buffer to the original size, we
5874 * failed there too. This is very unlikely to
5875 * happen, but if it does, warn and kill all
5876 * tracing.
5877 */
5878 WARN_ON(1);
5879 tracing_disabled = 1;
5880 }
5881 return ret;
5882 }
5883
5884 if (cpu == RING_BUFFER_ALL_CPUS)
5885 set_buffer_entries(&tr->max_buffer, size);
5886 else
5887 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
5888
5889 out:
5890#endif /* CONFIG_TRACER_MAX_TRACE */
5891
5892 if (cpu == RING_BUFFER_ALL_CPUS)
5893 set_buffer_entries(&tr->array_buffer, size);
5894 else
5895 per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size;
5896
5897 return ret;
5898}
5899
5900ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5901 unsigned long size, int cpu_id)
5902{
5903 int ret = size;
5904
5905 mutex_lock(&trace_types_lock);
5906
5907 if (cpu_id != RING_BUFFER_ALL_CPUS) {
5908 /* make sure, this cpu is enabled in the mask */
5909 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5910 ret = -EINVAL;
5911 goto out;
5912 }
5913 }
5914
5915 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
5916 if (ret < 0)
5917 ret = -ENOMEM;
5918
5919out:
5920 mutex_unlock(&trace_types_lock);
5921
5922 return ret;
5923}
5924
5925
5926/**
5927 * tracing_update_buffers - used by tracing facility to expand ring buffers
5928 *
5929 * To save on memory when the tracing is never used on a system with it
5930 * configured in. The ring buffers are set to a minimum size. But once
5931 * a user starts to use the tracing facility, then they need to grow
5932 * to their default size.
5933 *
5934 * This function is to be called when a tracer is about to be used.
5935 */
5936int tracing_update_buffers(void)
5937{
5938 int ret = 0;
5939
5940 mutex_lock(&trace_types_lock);
5941 if (!ring_buffer_expanded)
5942 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
5943 RING_BUFFER_ALL_CPUS);
5944 mutex_unlock(&trace_types_lock);
5945
5946 return ret;
5947}
5948
5949struct trace_option_dentry;
5950
5951static void
5952create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
5953
5954/*
5955 * Used to clear out the tracer before deletion of an instance.
5956 * Must have trace_types_lock held.
5957 */
5958static void tracing_set_nop(struct trace_array *tr)
5959{
5960 if (tr->current_trace == &nop_trace)
5961 return;
5962
5963 tr->current_trace->enabled--;
5964
5965 if (tr->current_trace->reset)
5966 tr->current_trace->reset(tr);
5967
5968 tr->current_trace = &nop_trace;
5969}
5970
5971static void add_tracer_options(struct trace_array *tr, struct tracer *t)
5972{
5973 /* Only enable if the directory has been created already. */
5974 if (!tr->dir)
5975 return;
5976
5977 create_trace_option_files(tr, t);
5978}
5979
5980int tracing_set_tracer(struct trace_array *tr, const char *buf)
5981{
5982 struct tracer *t;
5983#ifdef CONFIG_TRACER_MAX_TRACE
5984 bool had_max_tr;
5985#endif
5986 int ret = 0;
5987
5988 mutex_lock(&trace_types_lock);
5989
5990 if (!ring_buffer_expanded) {
5991 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5992 RING_BUFFER_ALL_CPUS);
5993 if (ret < 0)
5994 goto out;
5995 ret = 0;
5996 }
5997
5998 for (t = trace_types; t; t = t->next) {
5999 if (strcmp(t->name, buf) == 0)
6000 break;
6001 }
6002 if (!t) {
6003 ret = -EINVAL;
6004 goto out;
6005 }
6006 if (t == tr->current_trace)
6007 goto out;
6008
6009#ifdef CONFIG_TRACER_SNAPSHOT
6010 if (t->use_max_tr) {
6011 arch_spin_lock(&tr->max_lock);
6012 if (tr->cond_snapshot)
6013 ret = -EBUSY;
6014 arch_spin_unlock(&tr->max_lock);
6015 if (ret)
6016 goto out;
6017 }
6018#endif
6019 /* Some tracers won't work on kernel command line */
6020 if (system_state < SYSTEM_RUNNING && t->noboot) {
6021 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
6022 t->name);
6023 goto out;
6024 }
6025
6026 /* Some tracers are only allowed for the top level buffer */
6027 if (!trace_ok_for_array(t, tr)) {
6028 ret = -EINVAL;
6029 goto out;
6030 }
6031
6032 /* If trace pipe files are being read, we can't change the tracer */
6033 if (tr->trace_ref) {
6034 ret = -EBUSY;
6035 goto out;
6036 }
6037
6038 trace_branch_disable();
6039
6040 tr->current_trace->enabled--;
6041
6042 if (tr->current_trace->reset)
6043 tr->current_trace->reset(tr);
6044
6045 /* Current trace needs to be nop_trace before synchronize_rcu */
6046 tr->current_trace = &nop_trace;
6047
6048#ifdef CONFIG_TRACER_MAX_TRACE
6049 had_max_tr = tr->allocated_snapshot;
6050
6051 if (had_max_tr && !t->use_max_tr) {
6052 /*
6053 * We need to make sure that the update_max_tr sees that
6054 * current_trace changed to nop_trace to keep it from
6055 * swapping the buffers after we resize it.
6056 * The update_max_tr is called from interrupts disabled
6057 * so a synchronized_sched() is sufficient.
6058 */
6059 synchronize_rcu();
6060 free_snapshot(tr);
6061 }
6062#endif
6063
6064#ifdef CONFIG_TRACER_MAX_TRACE
6065 if (t->use_max_tr && !had_max_tr) {
6066 ret = tracing_alloc_snapshot_instance(tr);
6067 if (ret < 0)
6068 goto out;
6069 }
6070#endif
6071
6072 if (t->init) {
6073 ret = tracer_init(t, tr);
6074 if (ret)
6075 goto out;
6076 }
6077
6078 tr->current_trace = t;
6079 tr->current_trace->enabled++;
6080 trace_branch_enable(tr);
6081 out:
6082 mutex_unlock(&trace_types_lock);
6083
6084 return ret;
6085}
6086
6087static ssize_t
6088tracing_set_trace_write(struct file *filp, const char __user *ubuf,
6089 size_t cnt, loff_t *ppos)
6090{
6091 struct trace_array *tr = filp->private_data;
6092 char buf[MAX_TRACER_SIZE+1];
6093 int i;
6094 size_t ret;
6095 int err;
6096
6097 ret = cnt;
6098
6099 if (cnt > MAX_TRACER_SIZE)
6100 cnt = MAX_TRACER_SIZE;
6101
6102 if (copy_from_user(buf, ubuf, cnt))
6103 return -EFAULT;
6104
6105 buf[cnt] = 0;
6106
6107 /* strip ending whitespace. */
6108 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
6109 buf[i] = 0;
6110
6111 err = tracing_set_tracer(tr, buf);
6112 if (err)
6113 return err;
6114
6115 *ppos += ret;
6116
6117 return ret;
6118}
6119
6120static ssize_t
6121tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
6122 size_t cnt, loff_t *ppos)
6123{
6124 char buf[64];
6125 int r;
6126
6127 r = snprintf(buf, sizeof(buf), "%ld\n",
6128 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
6129 if (r > sizeof(buf))
6130 r = sizeof(buf);
6131 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6132}
6133
6134static ssize_t
6135tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
6136 size_t cnt, loff_t *ppos)
6137{
6138 unsigned long val;
6139 int ret;
6140
6141 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6142 if (ret)
6143 return ret;
6144
6145 *ptr = val * 1000;
6146
6147 return cnt;
6148}
6149
6150static ssize_t
6151tracing_thresh_read(struct file *filp, char __user *ubuf,
6152 size_t cnt, loff_t *ppos)
6153{
6154 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
6155}
6156
6157static ssize_t
6158tracing_thresh_write(struct file *filp, const char __user *ubuf,
6159 size_t cnt, loff_t *ppos)
6160{
6161 struct trace_array *tr = filp->private_data;
6162 int ret;
6163
6164 mutex_lock(&trace_types_lock);
6165 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
6166 if (ret < 0)
6167 goto out;
6168
6169 if (tr->current_trace->update_thresh) {
6170 ret = tr->current_trace->update_thresh(tr);
6171 if (ret < 0)
6172 goto out;
6173 }
6174
6175 ret = cnt;
6176out:
6177 mutex_unlock(&trace_types_lock);
6178
6179 return ret;
6180}
6181
6182#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6183
6184static ssize_t
6185tracing_max_lat_read(struct file *filp, char __user *ubuf,
6186 size_t cnt, loff_t *ppos)
6187{
6188 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
6189}
6190
6191static ssize_t
6192tracing_max_lat_write(struct file *filp, const char __user *ubuf,
6193 size_t cnt, loff_t *ppos)
6194{
6195 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
6196}
6197
6198#endif
6199
6200static int tracing_open_pipe(struct inode *inode, struct file *filp)
6201{
6202 struct trace_array *tr = inode->i_private;
6203 struct trace_iterator *iter;
6204 int ret;
6205
6206 ret = tracing_check_open_get_tr(tr);
6207 if (ret)
6208 return ret;
6209
6210 mutex_lock(&trace_types_lock);
6211
6212 /* create a buffer to store the information to pass to userspace */
6213 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6214 if (!iter) {
6215 ret = -ENOMEM;
6216 __trace_array_put(tr);
6217 goto out;
6218 }
6219
6220 trace_seq_init(&iter->seq);
6221 iter->trace = tr->current_trace;
6222
6223 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
6224 ret = -ENOMEM;
6225 goto fail;
6226 }
6227
6228 /* trace pipe does not show start of buffer */
6229 cpumask_setall(iter->started);
6230
6231 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6232 iter->iter_flags |= TRACE_FILE_LAT_FMT;
6233
6234 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6235 if (trace_clocks[tr->clock_id].in_ns)
6236 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6237
6238 iter->tr = tr;
6239 iter->array_buffer = &tr->array_buffer;
6240 iter->cpu_file = tracing_get_cpu(inode);
6241 mutex_init(&iter->mutex);
6242 filp->private_data = iter;
6243
6244 if (iter->trace->pipe_open)
6245 iter->trace->pipe_open(iter);
6246
6247 nonseekable_open(inode, filp);
6248
6249 tr->trace_ref++;
6250out:
6251 mutex_unlock(&trace_types_lock);
6252 return ret;
6253
6254fail:
6255 kfree(iter);
6256 __trace_array_put(tr);
6257 mutex_unlock(&trace_types_lock);
6258 return ret;
6259}
6260
6261static int tracing_release_pipe(struct inode *inode, struct file *file)
6262{
6263 struct trace_iterator *iter = file->private_data;
6264 struct trace_array *tr = inode->i_private;
6265
6266 mutex_lock(&trace_types_lock);
6267
6268 tr->trace_ref--;
6269
6270 if (iter->trace->pipe_close)
6271 iter->trace->pipe_close(iter);
6272
6273 mutex_unlock(&trace_types_lock);
6274
6275 free_cpumask_var(iter->started);
6276 mutex_destroy(&iter->mutex);
6277 kfree(iter);
6278
6279 trace_array_put(tr);
6280
6281 return 0;
6282}
6283
6284static __poll_t
6285trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
6286{
6287 struct trace_array *tr = iter->tr;
6288
6289 /* Iterators are static, they should be filled or empty */
6290 if (trace_buffer_iter(iter, iter->cpu_file))
6291 return EPOLLIN | EPOLLRDNORM;
6292
6293 if (tr->trace_flags & TRACE_ITER_BLOCK)
6294 /*
6295 * Always select as readable when in blocking mode
6296 */
6297 return EPOLLIN | EPOLLRDNORM;
6298 else
6299 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
6300 filp, poll_table);
6301}
6302
6303static __poll_t
6304tracing_poll_pipe(struct file *filp, poll_table *poll_table)
6305{
6306 struct trace_iterator *iter = filp->private_data;
6307
6308 return trace_poll(iter, filp, poll_table);
6309}
6310
6311/* Must be called with iter->mutex held. */
6312static int tracing_wait_pipe(struct file *filp)
6313{
6314 struct trace_iterator *iter = filp->private_data;
6315 int ret;
6316
6317 while (trace_empty(iter)) {
6318
6319 if ((filp->f_flags & O_NONBLOCK)) {
6320 return -EAGAIN;
6321 }
6322
6323 /*
6324 * We block until we read something and tracing is disabled.
6325 * We still block if tracing is disabled, but we have never
6326 * read anything. This allows a user to cat this file, and
6327 * then enable tracing. But after we have read something,
6328 * we give an EOF when tracing is again disabled.
6329 *
6330 * iter->pos will be 0 if we haven't read anything.
6331 */
6332 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6333 break;
6334
6335 mutex_unlock(&iter->mutex);
6336
6337 ret = wait_on_pipe(iter, 0);
6338
6339 mutex_lock(&iter->mutex);
6340
6341 if (ret)
6342 return ret;
6343 }
6344
6345 return 1;
6346}
6347
6348/*
6349 * Consumer reader.
6350 */
6351static ssize_t
6352tracing_read_pipe(struct file *filp, char __user *ubuf,
6353 size_t cnt, loff_t *ppos)
6354{
6355 struct trace_iterator *iter = filp->private_data;
6356 ssize_t sret;
6357
6358 /*
6359 * Avoid more than one consumer on a single file descriptor
6360 * This is just a matter of traces coherency, the ring buffer itself
6361 * is protected.
6362 */
6363 mutex_lock(&iter->mutex);
6364
6365 /* return any leftover data */
6366 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6367 if (sret != -EBUSY)
6368 goto out;
6369
6370 trace_seq_init(&iter->seq);
6371
6372 if (iter->trace->read) {
6373 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6374 if (sret)
6375 goto out;
6376 }
6377
6378waitagain:
6379 sret = tracing_wait_pipe(filp);
6380 if (sret <= 0)
6381 goto out;
6382
6383 /* stop when tracing is finished */
6384 if (trace_empty(iter)) {
6385 sret = 0;
6386 goto out;
6387 }
6388
6389 if (cnt >= PAGE_SIZE)
6390 cnt = PAGE_SIZE - 1;
6391
6392 /* reset all but tr, trace, and overruns */
6393 memset(&iter->seq, 0,
6394 sizeof(struct trace_iterator) -
6395 offsetof(struct trace_iterator, seq));
6396 cpumask_clear(iter->started);
6397 trace_seq_init(&iter->seq);
6398 iter->pos = -1;
6399
6400 trace_event_read_lock();
6401 trace_access_lock(iter->cpu_file);
6402 while (trace_find_next_entry_inc(iter) != NULL) {
6403 enum print_line_t ret;
6404 int save_len = iter->seq.seq.len;
6405
6406 ret = print_trace_line(iter);
6407 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6408 /* don't print partial lines */
6409 iter->seq.seq.len = save_len;
6410 break;
6411 }
6412 if (ret != TRACE_TYPE_NO_CONSUME)
6413 trace_consume(iter);
6414
6415 if (trace_seq_used(&iter->seq) >= cnt)
6416 break;
6417
6418 /*
6419 * Setting the full flag means we reached the trace_seq buffer
6420 * size and we should leave by partial output condition above.
6421 * One of the trace_seq_* functions is not used properly.
6422 */
6423 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6424 iter->ent->type);
6425 }
6426 trace_access_unlock(iter->cpu_file);
6427 trace_event_read_unlock();
6428
6429 /* Now copy what we have to the user */
6430 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6431 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6432 trace_seq_init(&iter->seq);
6433
6434 /*
6435 * If there was nothing to send to user, in spite of consuming trace
6436 * entries, go back to wait for more entries.
6437 */
6438 if (sret == -EBUSY)
6439 goto waitagain;
6440
6441out:
6442 mutex_unlock(&iter->mutex);
6443
6444 return sret;
6445}
6446
6447static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6448 unsigned int idx)
6449{
6450 __free_page(spd->pages[idx]);
6451}
6452
6453static size_t
6454tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6455{
6456 size_t count;
6457 int save_len;
6458 int ret;
6459
6460 /* Seq buffer is page-sized, exactly what we need. */
6461 for (;;) {
6462 save_len = iter->seq.seq.len;
6463 ret = print_trace_line(iter);
6464
6465 if (trace_seq_has_overflowed(&iter->seq)) {
6466 iter->seq.seq.len = save_len;
6467 break;
6468 }
6469
6470 /*
6471 * This should not be hit, because it should only
6472 * be set if the iter->seq overflowed. But check it
6473 * anyway to be safe.
6474 */
6475 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6476 iter->seq.seq.len = save_len;
6477 break;
6478 }
6479
6480 count = trace_seq_used(&iter->seq) - save_len;
6481 if (rem < count) {
6482 rem = 0;
6483 iter->seq.seq.len = save_len;
6484 break;
6485 }
6486
6487 if (ret != TRACE_TYPE_NO_CONSUME)
6488 trace_consume(iter);
6489 rem -= count;
6490 if (!trace_find_next_entry_inc(iter)) {
6491 rem = 0;
6492 iter->ent = NULL;
6493 break;
6494 }
6495 }
6496
6497 return rem;
6498}
6499
6500static ssize_t tracing_splice_read_pipe(struct file *filp,
6501 loff_t *ppos,
6502 struct pipe_inode_info *pipe,
6503 size_t len,
6504 unsigned int flags)
6505{
6506 struct page *pages_def[PIPE_DEF_BUFFERS];
6507 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6508 struct trace_iterator *iter = filp->private_data;
6509 struct splice_pipe_desc spd = {
6510 .pages = pages_def,
6511 .partial = partial_def,
6512 .nr_pages = 0, /* This gets updated below. */
6513 .nr_pages_max = PIPE_DEF_BUFFERS,
6514 .ops = &default_pipe_buf_ops,
6515 .spd_release = tracing_spd_release_pipe,
6516 };
6517 ssize_t ret;
6518 size_t rem;
6519 unsigned int i;
6520
6521 if (splice_grow_spd(pipe, &spd))
6522 return -ENOMEM;
6523
6524 mutex_lock(&iter->mutex);
6525
6526 if (iter->trace->splice_read) {
6527 ret = iter->trace->splice_read(iter, filp,
6528 ppos, pipe, len, flags);
6529 if (ret)
6530 goto out_err;
6531 }
6532
6533 ret = tracing_wait_pipe(filp);
6534 if (ret <= 0)
6535 goto out_err;
6536
6537 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6538 ret = -EFAULT;
6539 goto out_err;
6540 }
6541
6542 trace_event_read_lock();
6543 trace_access_lock(iter->cpu_file);
6544
6545 /* Fill as many pages as possible. */
6546 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6547 spd.pages[i] = alloc_page(GFP_KERNEL);
6548 if (!spd.pages[i])
6549 break;
6550
6551 rem = tracing_fill_pipe_page(rem, iter);
6552
6553 /* Copy the data into the page, so we can start over. */
6554 ret = trace_seq_to_buffer(&iter->seq,
6555 page_address(spd.pages[i]),
6556 trace_seq_used(&iter->seq));
6557 if (ret < 0) {
6558 __free_page(spd.pages[i]);
6559 break;
6560 }
6561 spd.partial[i].offset = 0;
6562 spd.partial[i].len = trace_seq_used(&iter->seq);
6563
6564 trace_seq_init(&iter->seq);
6565 }
6566
6567 trace_access_unlock(iter->cpu_file);
6568 trace_event_read_unlock();
6569 mutex_unlock(&iter->mutex);
6570
6571 spd.nr_pages = i;
6572
6573 if (i)
6574 ret = splice_to_pipe(pipe, &spd);
6575 else
6576 ret = 0;
6577out:
6578 splice_shrink_spd(&spd);
6579 return ret;
6580
6581out_err:
6582 mutex_unlock(&iter->mutex);
6583 goto out;
6584}
6585
6586static ssize_t
6587tracing_entries_read(struct file *filp, char __user *ubuf,
6588 size_t cnt, loff_t *ppos)
6589{
6590 struct inode *inode = file_inode(filp);
6591 struct trace_array *tr = inode->i_private;
6592 int cpu = tracing_get_cpu(inode);
6593 char buf[64];
6594 int r = 0;
6595 ssize_t ret;
6596
6597 mutex_lock(&trace_types_lock);
6598
6599 if (cpu == RING_BUFFER_ALL_CPUS) {
6600 int cpu, buf_size_same;
6601 unsigned long size;
6602
6603 size = 0;
6604 buf_size_same = 1;
6605 /* check if all cpu sizes are same */
6606 for_each_tracing_cpu(cpu) {
6607 /* fill in the size from first enabled cpu */
6608 if (size == 0)
6609 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
6610 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
6611 buf_size_same = 0;
6612 break;
6613 }
6614 }
6615
6616 if (buf_size_same) {
6617 if (!ring_buffer_expanded)
6618 r = sprintf(buf, "%lu (expanded: %lu)\n",
6619 size >> 10,
6620 trace_buf_size >> 10);
6621 else
6622 r = sprintf(buf, "%lu\n", size >> 10);
6623 } else
6624 r = sprintf(buf, "X\n");
6625 } else
6626 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
6627
6628 mutex_unlock(&trace_types_lock);
6629
6630 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6631 return ret;
6632}
6633
6634static ssize_t
6635tracing_entries_write(struct file *filp, const char __user *ubuf,
6636 size_t cnt, loff_t *ppos)
6637{
6638 struct inode *inode = file_inode(filp);
6639 struct trace_array *tr = inode->i_private;
6640 unsigned long val;
6641 int ret;
6642
6643 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6644 if (ret)
6645 return ret;
6646
6647 /* must have at least 1 entry */
6648 if (!val)
6649 return -EINVAL;
6650
6651 /* value is in KB */
6652 val <<= 10;
6653 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6654 if (ret < 0)
6655 return ret;
6656
6657 *ppos += cnt;
6658
6659 return cnt;
6660}
6661
6662static ssize_t
6663tracing_total_entries_read(struct file *filp, char __user *ubuf,
6664 size_t cnt, loff_t *ppos)
6665{
6666 struct trace_array *tr = filp->private_data;
6667 char buf[64];
6668 int r, cpu;
6669 unsigned long size = 0, expanded_size = 0;
6670
6671 mutex_lock(&trace_types_lock);
6672 for_each_tracing_cpu(cpu) {
6673 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
6674 if (!ring_buffer_expanded)
6675 expanded_size += trace_buf_size >> 10;
6676 }
6677 if (ring_buffer_expanded)
6678 r = sprintf(buf, "%lu\n", size);
6679 else
6680 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6681 mutex_unlock(&trace_types_lock);
6682
6683 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6684}
6685
6686static ssize_t
6687tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6688 size_t cnt, loff_t *ppos)
6689{
6690 /*
6691 * There is no need to read what the user has written, this function
6692 * is just to make sure that there is no error when "echo" is used
6693 */
6694
6695 *ppos += cnt;
6696
6697 return cnt;
6698}
6699
6700static int
6701tracing_free_buffer_release(struct inode *inode, struct file *filp)
6702{
6703 struct trace_array *tr = inode->i_private;
6704
6705 /* disable tracing ? */
6706 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6707 tracer_tracing_off(tr);
6708 /* resize the ring buffer to 0 */
6709 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6710
6711 trace_array_put(tr);
6712
6713 return 0;
6714}
6715
6716static ssize_t
6717tracing_mark_write(struct file *filp, const char __user *ubuf,
6718 size_t cnt, loff_t *fpos)
6719{
6720 struct trace_array *tr = filp->private_data;
6721 struct ring_buffer_event *event;
6722 enum event_trigger_type tt = ETT_NONE;
6723 struct trace_buffer *buffer;
6724 struct print_entry *entry;
6725 ssize_t written;
6726 int size;
6727 int len;
6728
6729/* Used in tracing_mark_raw_write() as well */
6730#define FAULTED_STR "<faulted>"
6731#define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6732
6733 if (tracing_disabled)
6734 return -EINVAL;
6735
6736 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6737 return -EINVAL;
6738
6739 if (cnt > TRACE_BUF_SIZE)
6740 cnt = TRACE_BUF_SIZE;
6741
6742 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6743
6744 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
6745
6746 /* If less than "<faulted>", then make sure we can still add that */
6747 if (cnt < FAULTED_SIZE)
6748 size += FAULTED_SIZE - cnt;
6749
6750 buffer = tr->array_buffer.buffer;
6751 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6752 tracing_gen_ctx());
6753 if (unlikely(!event))
6754 /* Ring buffer disabled, return as if not open for write */
6755 return -EBADF;
6756
6757 entry = ring_buffer_event_data(event);
6758 entry->ip = _THIS_IP_;
6759
6760 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6761 if (len) {
6762 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6763 cnt = FAULTED_SIZE;
6764 written = -EFAULT;
6765 } else
6766 written = cnt;
6767
6768 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6769 /* do not add \n before testing triggers, but add \0 */
6770 entry->buf[cnt] = '\0';
6771 tt = event_triggers_call(tr->trace_marker_file, entry, event);
6772 }
6773
6774 if (entry->buf[cnt - 1] != '\n') {
6775 entry->buf[cnt] = '\n';
6776 entry->buf[cnt + 1] = '\0';
6777 } else
6778 entry->buf[cnt] = '\0';
6779
6780 if (static_branch_unlikely(&trace_marker_exports_enabled))
6781 ftrace_exports(event, TRACE_EXPORT_MARKER);
6782 __buffer_unlock_commit(buffer, event);
6783
6784 if (tt)
6785 event_triggers_post_call(tr->trace_marker_file, tt);
6786
6787 if (written > 0)
6788 *fpos += written;
6789
6790 return written;
6791}
6792
6793/* Limit it for now to 3K (including tag) */
6794#define RAW_DATA_MAX_SIZE (1024*3)
6795
6796static ssize_t
6797tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6798 size_t cnt, loff_t *fpos)
6799{
6800 struct trace_array *tr = filp->private_data;
6801 struct ring_buffer_event *event;
6802 struct trace_buffer *buffer;
6803 struct raw_data_entry *entry;
6804 ssize_t written;
6805 int size;
6806 int len;
6807
6808#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6809
6810 if (tracing_disabled)
6811 return -EINVAL;
6812
6813 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6814 return -EINVAL;
6815
6816 /* The marker must at least have a tag id */
6817 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6818 return -EINVAL;
6819
6820 if (cnt > TRACE_BUF_SIZE)
6821 cnt = TRACE_BUF_SIZE;
6822
6823 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6824
6825 size = sizeof(*entry) + cnt;
6826 if (cnt < FAULT_SIZE_ID)
6827 size += FAULT_SIZE_ID - cnt;
6828
6829 buffer = tr->array_buffer.buffer;
6830 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6831 tracing_gen_ctx());
6832 if (!event)
6833 /* Ring buffer disabled, return as if not open for write */
6834 return -EBADF;
6835
6836 entry = ring_buffer_event_data(event);
6837
6838 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6839 if (len) {
6840 entry->id = -1;
6841 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6842 written = -EFAULT;
6843 } else
6844 written = cnt;
6845
6846 __buffer_unlock_commit(buffer, event);
6847
6848 if (written > 0)
6849 *fpos += written;
6850
6851 return written;
6852}
6853
6854static int tracing_clock_show(struct seq_file *m, void *v)
6855{
6856 struct trace_array *tr = m->private;
6857 int i;
6858
6859 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6860 seq_printf(m,
6861 "%s%s%s%s", i ? " " : "",
6862 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6863 i == tr->clock_id ? "]" : "");
6864 seq_putc(m, '\n');
6865
6866 return 0;
6867}
6868
6869int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6870{
6871 int i;
6872
6873 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6874 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6875 break;
6876 }
6877 if (i == ARRAY_SIZE(trace_clocks))
6878 return -EINVAL;
6879
6880 mutex_lock(&trace_types_lock);
6881
6882 tr->clock_id = i;
6883
6884 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
6885
6886 /*
6887 * New clock may not be consistent with the previous clock.
6888 * Reset the buffer so that it doesn't have incomparable timestamps.
6889 */
6890 tracing_reset_online_cpus(&tr->array_buffer);
6891
6892#ifdef CONFIG_TRACER_MAX_TRACE
6893 if (tr->max_buffer.buffer)
6894 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
6895 tracing_reset_online_cpus(&tr->max_buffer);
6896#endif
6897
6898 mutex_unlock(&trace_types_lock);
6899
6900 return 0;
6901}
6902
6903static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6904 size_t cnt, loff_t *fpos)
6905{
6906 struct seq_file *m = filp->private_data;
6907 struct trace_array *tr = m->private;
6908 char buf[64];
6909 const char *clockstr;
6910 int ret;
6911
6912 if (cnt >= sizeof(buf))
6913 return -EINVAL;
6914
6915 if (copy_from_user(buf, ubuf, cnt))
6916 return -EFAULT;
6917
6918 buf[cnt] = 0;
6919
6920 clockstr = strstrip(buf);
6921
6922 ret = tracing_set_clock(tr, clockstr);
6923 if (ret)
6924 return ret;
6925
6926 *fpos += cnt;
6927
6928 return cnt;
6929}
6930
6931static int tracing_clock_open(struct inode *inode, struct file *file)
6932{
6933 struct trace_array *tr = inode->i_private;
6934 int ret;
6935
6936 ret = tracing_check_open_get_tr(tr);
6937 if (ret)
6938 return ret;
6939
6940 ret = single_open(file, tracing_clock_show, inode->i_private);
6941 if (ret < 0)
6942 trace_array_put(tr);
6943
6944 return ret;
6945}
6946
6947static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6948{
6949 struct trace_array *tr = m->private;
6950
6951 mutex_lock(&trace_types_lock);
6952
6953 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
6954 seq_puts(m, "delta [absolute]\n");
6955 else
6956 seq_puts(m, "[delta] absolute\n");
6957
6958 mutex_unlock(&trace_types_lock);
6959
6960 return 0;
6961}
6962
6963static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6964{
6965 struct trace_array *tr = inode->i_private;
6966 int ret;
6967
6968 ret = tracing_check_open_get_tr(tr);
6969 if (ret)
6970 return ret;
6971
6972 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6973 if (ret < 0)
6974 trace_array_put(tr);
6975
6976 return ret;
6977}
6978
6979int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
6980{
6981 int ret = 0;
6982
6983 mutex_lock(&trace_types_lock);
6984
6985 if (abs && tr->time_stamp_abs_ref++)
6986 goto out;
6987
6988 if (!abs) {
6989 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
6990 ret = -EINVAL;
6991 goto out;
6992 }
6993
6994 if (--tr->time_stamp_abs_ref)
6995 goto out;
6996 }
6997
6998 ring_buffer_set_time_stamp_abs(tr->array_buffer.buffer, abs);
6999
7000#ifdef CONFIG_TRACER_MAX_TRACE
7001 if (tr->max_buffer.buffer)
7002 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
7003#endif
7004 out:
7005 mutex_unlock(&trace_types_lock);
7006
7007 return ret;
7008}
7009
7010struct ftrace_buffer_info {
7011 struct trace_iterator iter;
7012 void *spare;
7013 unsigned int spare_cpu;
7014 unsigned int read;
7015};
7016
7017#ifdef CONFIG_TRACER_SNAPSHOT
7018static int tracing_snapshot_open(struct inode *inode, struct file *file)
7019{
7020 struct trace_array *tr = inode->i_private;
7021 struct trace_iterator *iter;
7022 struct seq_file *m;
7023 int ret;
7024
7025 ret = tracing_check_open_get_tr(tr);
7026 if (ret)
7027 return ret;
7028
7029 if (file->f_mode & FMODE_READ) {
7030 iter = __tracing_open(inode, file, true);
7031 if (IS_ERR(iter))
7032 ret = PTR_ERR(iter);
7033 } else {
7034 /* Writes still need the seq_file to hold the private data */
7035 ret = -ENOMEM;
7036 m = kzalloc(sizeof(*m), GFP_KERNEL);
7037 if (!m)
7038 goto out;
7039 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
7040 if (!iter) {
7041 kfree(m);
7042 goto out;
7043 }
7044 ret = 0;
7045
7046 iter->tr = tr;
7047 iter->array_buffer = &tr->max_buffer;
7048 iter->cpu_file = tracing_get_cpu(inode);
7049 m->private = iter;
7050 file->private_data = m;
7051 }
7052out:
7053 if (ret < 0)
7054 trace_array_put(tr);
7055
7056 return ret;
7057}
7058
7059static ssize_t
7060tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
7061 loff_t *ppos)
7062{
7063 struct seq_file *m = filp->private_data;
7064 struct trace_iterator *iter = m->private;
7065 struct trace_array *tr = iter->tr;
7066 unsigned long val;
7067 int ret;
7068
7069 ret = tracing_update_buffers();
7070 if (ret < 0)
7071 return ret;
7072
7073 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7074 if (ret)
7075 return ret;
7076
7077 mutex_lock(&trace_types_lock);
7078
7079 if (tr->current_trace->use_max_tr) {
7080 ret = -EBUSY;
7081 goto out;
7082 }
7083
7084 arch_spin_lock(&tr->max_lock);
7085 if (tr->cond_snapshot)
7086 ret = -EBUSY;
7087 arch_spin_unlock(&tr->max_lock);
7088 if (ret)
7089 goto out;
7090
7091 switch (val) {
7092 case 0:
7093 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7094 ret = -EINVAL;
7095 break;
7096 }
7097 if (tr->allocated_snapshot)
7098 free_snapshot(tr);
7099 break;
7100 case 1:
7101/* Only allow per-cpu swap if the ring buffer supports it */
7102#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
7103 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7104 ret = -EINVAL;
7105 break;
7106 }
7107#endif
7108 if (tr->allocated_snapshot)
7109 ret = resize_buffer_duplicate_size(&tr->max_buffer,
7110 &tr->array_buffer, iter->cpu_file);
7111 else
7112 ret = tracing_alloc_snapshot_instance(tr);
7113 if (ret < 0)
7114 break;
7115 local_irq_disable();
7116 /* Now, we're going to swap */
7117 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7118 update_max_tr(tr, current, smp_processor_id(), NULL);
7119 else
7120 update_max_tr_single(tr, current, iter->cpu_file);
7121 local_irq_enable();
7122 break;
7123 default:
7124 if (tr->allocated_snapshot) {
7125 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7126 tracing_reset_online_cpus(&tr->max_buffer);
7127 else
7128 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
7129 }
7130 break;
7131 }
7132
7133 if (ret >= 0) {
7134 *ppos += cnt;
7135 ret = cnt;
7136 }
7137out:
7138 mutex_unlock(&trace_types_lock);
7139 return ret;
7140}
7141
7142static int tracing_snapshot_release(struct inode *inode, struct file *file)
7143{
7144 struct seq_file *m = file->private_data;
7145 int ret;
7146
7147 ret = tracing_release(inode, file);
7148
7149 if (file->f_mode & FMODE_READ)
7150 return ret;
7151
7152 /* If write only, the seq_file is just a stub */
7153 if (m)
7154 kfree(m->private);
7155 kfree(m);
7156
7157 return 0;
7158}
7159
7160static int tracing_buffers_open(struct inode *inode, struct file *filp);
7161static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7162 size_t count, loff_t *ppos);
7163static int tracing_buffers_release(struct inode *inode, struct file *file);
7164static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7165 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7166
7167static int snapshot_raw_open(struct inode *inode, struct file *filp)
7168{
7169 struct ftrace_buffer_info *info;
7170 int ret;
7171
7172 /* The following checks for tracefs lockdown */
7173 ret = tracing_buffers_open(inode, filp);
7174 if (ret < 0)
7175 return ret;
7176
7177 info = filp->private_data;
7178
7179 if (info->iter.trace->use_max_tr) {
7180 tracing_buffers_release(inode, filp);
7181 return -EBUSY;
7182 }
7183
7184 info->iter.snapshot = true;
7185 info->iter.array_buffer = &info->iter.tr->max_buffer;
7186
7187 return ret;
7188}
7189
7190#endif /* CONFIG_TRACER_SNAPSHOT */
7191
7192
7193static const struct file_operations tracing_thresh_fops = {
7194 .open = tracing_open_generic,
7195 .read = tracing_thresh_read,
7196 .write = tracing_thresh_write,
7197 .llseek = generic_file_llseek,
7198};
7199
7200#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7201static const struct file_operations tracing_max_lat_fops = {
7202 .open = tracing_open_generic,
7203 .read = tracing_max_lat_read,
7204 .write = tracing_max_lat_write,
7205 .llseek = generic_file_llseek,
7206};
7207#endif
7208
7209static const struct file_operations set_tracer_fops = {
7210 .open = tracing_open_generic,
7211 .read = tracing_set_trace_read,
7212 .write = tracing_set_trace_write,
7213 .llseek = generic_file_llseek,
7214};
7215
7216static const struct file_operations tracing_pipe_fops = {
7217 .open = tracing_open_pipe,
7218 .poll = tracing_poll_pipe,
7219 .read = tracing_read_pipe,
7220 .splice_read = tracing_splice_read_pipe,
7221 .release = tracing_release_pipe,
7222 .llseek = no_llseek,
7223};
7224
7225static const struct file_operations tracing_entries_fops = {
7226 .open = tracing_open_generic_tr,
7227 .read = tracing_entries_read,
7228 .write = tracing_entries_write,
7229 .llseek = generic_file_llseek,
7230 .release = tracing_release_generic_tr,
7231};
7232
7233static const struct file_operations tracing_total_entries_fops = {
7234 .open = tracing_open_generic_tr,
7235 .read = tracing_total_entries_read,
7236 .llseek = generic_file_llseek,
7237 .release = tracing_release_generic_tr,
7238};
7239
7240static const struct file_operations tracing_free_buffer_fops = {
7241 .open = tracing_open_generic_tr,
7242 .write = tracing_free_buffer_write,
7243 .release = tracing_free_buffer_release,
7244};
7245
7246static const struct file_operations tracing_mark_fops = {
7247 .open = tracing_open_generic_tr,
7248 .write = tracing_mark_write,
7249 .llseek = generic_file_llseek,
7250 .release = tracing_release_generic_tr,
7251};
7252
7253static const struct file_operations tracing_mark_raw_fops = {
7254 .open = tracing_open_generic_tr,
7255 .write = tracing_mark_raw_write,
7256 .llseek = generic_file_llseek,
7257 .release = tracing_release_generic_tr,
7258};
7259
7260static const struct file_operations trace_clock_fops = {
7261 .open = tracing_clock_open,
7262 .read = seq_read,
7263 .llseek = seq_lseek,
7264 .release = tracing_single_release_tr,
7265 .write = tracing_clock_write,
7266};
7267
7268static const struct file_operations trace_time_stamp_mode_fops = {
7269 .open = tracing_time_stamp_mode_open,
7270 .read = seq_read,
7271 .llseek = seq_lseek,
7272 .release = tracing_single_release_tr,
7273};
7274
7275#ifdef CONFIG_TRACER_SNAPSHOT
7276static const struct file_operations snapshot_fops = {
7277 .open = tracing_snapshot_open,
7278 .read = seq_read,
7279 .write = tracing_snapshot_write,
7280 .llseek = tracing_lseek,
7281 .release = tracing_snapshot_release,
7282};
7283
7284static const struct file_operations snapshot_raw_fops = {
7285 .open = snapshot_raw_open,
7286 .read = tracing_buffers_read,
7287 .release = tracing_buffers_release,
7288 .splice_read = tracing_buffers_splice_read,
7289 .llseek = no_llseek,
7290};
7291
7292#endif /* CONFIG_TRACER_SNAPSHOT */
7293
7294#define TRACING_LOG_ERRS_MAX 8
7295#define TRACING_LOG_LOC_MAX 128
7296
7297#define CMD_PREFIX " Command: "
7298
7299struct err_info {
7300 const char **errs; /* ptr to loc-specific array of err strings */
7301 u8 type; /* index into errs -> specific err string */
7302 u8 pos; /* MAX_FILTER_STR_VAL = 256 */
7303 u64 ts;
7304};
7305
7306struct tracing_log_err {
7307 struct list_head list;
7308 struct err_info info;
7309 char loc[TRACING_LOG_LOC_MAX]; /* err location */
7310 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
7311};
7312
7313static DEFINE_MUTEX(tracing_err_log_lock);
7314
7315static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
7316{
7317 struct tracing_log_err *err;
7318
7319 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7320 err = kzalloc(sizeof(*err), GFP_KERNEL);
7321 if (!err)
7322 err = ERR_PTR(-ENOMEM);
7323 tr->n_err_log_entries++;
7324
7325 return err;
7326 }
7327
7328 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7329 list_del(&err->list);
7330
7331 return err;
7332}
7333
7334/**
7335 * err_pos - find the position of a string within a command for error careting
7336 * @cmd: The tracing command that caused the error
7337 * @str: The string to position the caret at within @cmd
7338 *
7339 * Finds the position of the first occurence of @str within @cmd. The
7340 * return value can be passed to tracing_log_err() for caret placement
7341 * within @cmd.
7342 *
7343 * Returns the index within @cmd of the first occurence of @str or 0
7344 * if @str was not found.
7345 */
7346unsigned int err_pos(char *cmd, const char *str)
7347{
7348 char *found;
7349
7350 if (WARN_ON(!strlen(cmd)))
7351 return 0;
7352
7353 found = strstr(cmd, str);
7354 if (found)
7355 return found - cmd;
7356
7357 return 0;
7358}
7359
7360/**
7361 * tracing_log_err - write an error to the tracing error log
7362 * @tr: The associated trace array for the error (NULL for top level array)
7363 * @loc: A string describing where the error occurred
7364 * @cmd: The tracing command that caused the error
7365 * @errs: The array of loc-specific static error strings
7366 * @type: The index into errs[], which produces the specific static err string
7367 * @pos: The position the caret should be placed in the cmd
7368 *
7369 * Writes an error into tracing/error_log of the form:
7370 *
7371 * <loc>: error: <text>
7372 * Command: <cmd>
7373 * ^
7374 *
7375 * tracing/error_log is a small log file containing the last
7376 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7377 * unless there has been a tracing error, and the error log can be
7378 * cleared and have its memory freed by writing the empty string in
7379 * truncation mode to it i.e. echo > tracing/error_log.
7380 *
7381 * NOTE: the @errs array along with the @type param are used to
7382 * produce a static error string - this string is not copied and saved
7383 * when the error is logged - only a pointer to it is saved. See
7384 * existing callers for examples of how static strings are typically
7385 * defined for use with tracing_log_err().
7386 */
7387void tracing_log_err(struct trace_array *tr,
7388 const char *loc, const char *cmd,
7389 const char **errs, u8 type, u8 pos)
7390{
7391 struct tracing_log_err *err;
7392
7393 if (!tr)
7394 tr = &global_trace;
7395
7396 mutex_lock(&tracing_err_log_lock);
7397 err = get_tracing_log_err(tr);
7398 if (PTR_ERR(err) == -ENOMEM) {
7399 mutex_unlock(&tracing_err_log_lock);
7400 return;
7401 }
7402
7403 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7404 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
7405
7406 err->info.errs = errs;
7407 err->info.type = type;
7408 err->info.pos = pos;
7409 err->info.ts = local_clock();
7410
7411 list_add_tail(&err->list, &tr->err_log);
7412 mutex_unlock(&tracing_err_log_lock);
7413}
7414
7415static void clear_tracing_err_log(struct trace_array *tr)
7416{
7417 struct tracing_log_err *err, *next;
7418
7419 mutex_lock(&tracing_err_log_lock);
7420 list_for_each_entry_safe(err, next, &tr->err_log, list) {
7421 list_del(&err->list);
7422 kfree(err);
7423 }
7424
7425 tr->n_err_log_entries = 0;
7426 mutex_unlock(&tracing_err_log_lock);
7427}
7428
7429static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7430{
7431 struct trace_array *tr = m->private;
7432
7433 mutex_lock(&tracing_err_log_lock);
7434
7435 return seq_list_start(&tr->err_log, *pos);
7436}
7437
7438static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7439{
7440 struct trace_array *tr = m->private;
7441
7442 return seq_list_next(v, &tr->err_log, pos);
7443}
7444
7445static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7446{
7447 mutex_unlock(&tracing_err_log_lock);
7448}
7449
7450static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
7451{
7452 u8 i;
7453
7454 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7455 seq_putc(m, ' ');
7456 for (i = 0; i < pos; i++)
7457 seq_putc(m, ' ');
7458 seq_puts(m, "^\n");
7459}
7460
7461static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7462{
7463 struct tracing_log_err *err = v;
7464
7465 if (err) {
7466 const char *err_text = err->info.errs[err->info.type];
7467 u64 sec = err->info.ts;
7468 u32 nsec;
7469
7470 nsec = do_div(sec, NSEC_PER_SEC);
7471 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7472 err->loc, err_text);
7473 seq_printf(m, "%s", err->cmd);
7474 tracing_err_log_show_pos(m, err->info.pos);
7475 }
7476
7477 return 0;
7478}
7479
7480static const struct seq_operations tracing_err_log_seq_ops = {
7481 .start = tracing_err_log_seq_start,
7482 .next = tracing_err_log_seq_next,
7483 .stop = tracing_err_log_seq_stop,
7484 .show = tracing_err_log_seq_show
7485};
7486
7487static int tracing_err_log_open(struct inode *inode, struct file *file)
7488{
7489 struct trace_array *tr = inode->i_private;
7490 int ret = 0;
7491
7492 ret = tracing_check_open_get_tr(tr);
7493 if (ret)
7494 return ret;
7495
7496 /* If this file was opened for write, then erase contents */
7497 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7498 clear_tracing_err_log(tr);
7499
7500 if (file->f_mode & FMODE_READ) {
7501 ret = seq_open(file, &tracing_err_log_seq_ops);
7502 if (!ret) {
7503 struct seq_file *m = file->private_data;
7504 m->private = tr;
7505 } else {
7506 trace_array_put(tr);
7507 }
7508 }
7509 return ret;
7510}
7511
7512static ssize_t tracing_err_log_write(struct file *file,
7513 const char __user *buffer,
7514 size_t count, loff_t *ppos)
7515{
7516 return count;
7517}
7518
7519static int tracing_err_log_release(struct inode *inode, struct file *file)
7520{
7521 struct trace_array *tr = inode->i_private;
7522
7523 trace_array_put(tr);
7524
7525 if (file->f_mode & FMODE_READ)
7526 seq_release(inode, file);
7527
7528 return 0;
7529}
7530
7531static const struct file_operations tracing_err_log_fops = {
7532 .open = tracing_err_log_open,
7533 .write = tracing_err_log_write,
7534 .read = seq_read,
7535 .llseek = seq_lseek,
7536 .release = tracing_err_log_release,
7537};
7538
7539static int tracing_buffers_open(struct inode *inode, struct file *filp)
7540{
7541 struct trace_array *tr = inode->i_private;
7542 struct ftrace_buffer_info *info;
7543 int ret;
7544
7545 ret = tracing_check_open_get_tr(tr);
7546 if (ret)
7547 return ret;
7548
7549 info = kvzalloc(sizeof(*info), GFP_KERNEL);
7550 if (!info) {
7551 trace_array_put(tr);
7552 return -ENOMEM;
7553 }
7554
7555 mutex_lock(&trace_types_lock);
7556
7557 info->iter.tr = tr;
7558 info->iter.cpu_file = tracing_get_cpu(inode);
7559 info->iter.trace = tr->current_trace;
7560 info->iter.array_buffer = &tr->array_buffer;
7561 info->spare = NULL;
7562 /* Force reading ring buffer for first read */
7563 info->read = (unsigned int)-1;
7564
7565 filp->private_data = info;
7566
7567 tr->trace_ref++;
7568
7569 mutex_unlock(&trace_types_lock);
7570
7571 ret = nonseekable_open(inode, filp);
7572 if (ret < 0)
7573 trace_array_put(tr);
7574
7575 return ret;
7576}
7577
7578static __poll_t
7579tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7580{
7581 struct ftrace_buffer_info *info = filp->private_data;
7582 struct trace_iterator *iter = &info->iter;
7583
7584 return trace_poll(iter, filp, poll_table);
7585}
7586
7587static ssize_t
7588tracing_buffers_read(struct file *filp, char __user *ubuf,
7589 size_t count, loff_t *ppos)
7590{
7591 struct ftrace_buffer_info *info = filp->private_data;
7592 struct trace_iterator *iter = &info->iter;
7593 ssize_t ret = 0;
7594 ssize_t size;
7595
7596 if (!count)
7597 return 0;
7598
7599#ifdef CONFIG_TRACER_MAX_TRACE
7600 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7601 return -EBUSY;
7602#endif
7603
7604 if (!info->spare) {
7605 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
7606 iter->cpu_file);
7607 if (IS_ERR(info->spare)) {
7608 ret = PTR_ERR(info->spare);
7609 info->spare = NULL;
7610 } else {
7611 info->spare_cpu = iter->cpu_file;
7612 }
7613 }
7614 if (!info->spare)
7615 return ret;
7616
7617 /* Do we have previous read data to read? */
7618 if (info->read < PAGE_SIZE)
7619 goto read;
7620
7621 again:
7622 trace_access_lock(iter->cpu_file);
7623 ret = ring_buffer_read_page(iter->array_buffer->buffer,
7624 &info->spare,
7625 count,
7626 iter->cpu_file, 0);
7627 trace_access_unlock(iter->cpu_file);
7628
7629 if (ret < 0) {
7630 if (trace_empty(iter)) {
7631 if ((filp->f_flags & O_NONBLOCK))
7632 return -EAGAIN;
7633
7634 ret = wait_on_pipe(iter, 0);
7635 if (ret)
7636 return ret;
7637
7638 goto again;
7639 }
7640 return 0;
7641 }
7642
7643 info->read = 0;
7644 read:
7645 size = PAGE_SIZE - info->read;
7646 if (size > count)
7647 size = count;
7648
7649 ret = copy_to_user(ubuf, info->spare + info->read, size);
7650 if (ret == size)
7651 return -EFAULT;
7652
7653 size -= ret;
7654
7655 *ppos += size;
7656 info->read += size;
7657
7658 return size;
7659}
7660
7661static int tracing_buffers_release(struct inode *inode, struct file *file)
7662{
7663 struct ftrace_buffer_info *info = file->private_data;
7664 struct trace_iterator *iter = &info->iter;
7665
7666 mutex_lock(&trace_types_lock);
7667
7668 iter->tr->trace_ref--;
7669
7670 __trace_array_put(iter->tr);
7671
7672 if (info->spare)
7673 ring_buffer_free_read_page(iter->array_buffer->buffer,
7674 info->spare_cpu, info->spare);
7675 kvfree(info);
7676
7677 mutex_unlock(&trace_types_lock);
7678
7679 return 0;
7680}
7681
7682struct buffer_ref {
7683 struct trace_buffer *buffer;
7684 void *page;
7685 int cpu;
7686 refcount_t refcount;
7687};
7688
7689static void buffer_ref_release(struct buffer_ref *ref)
7690{
7691 if (!refcount_dec_and_test(&ref->refcount))
7692 return;
7693 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
7694 kfree(ref);
7695}
7696
7697static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7698 struct pipe_buffer *buf)
7699{
7700 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7701
7702 buffer_ref_release(ref);
7703 buf->private = 0;
7704}
7705
7706static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
7707 struct pipe_buffer *buf)
7708{
7709 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7710
7711 if (refcount_read(&ref->refcount) > INT_MAX/2)
7712 return false;
7713
7714 refcount_inc(&ref->refcount);
7715 return true;
7716}
7717
7718/* Pipe buffer operations for a buffer. */
7719static const struct pipe_buf_operations buffer_pipe_buf_ops = {
7720 .release = buffer_pipe_buf_release,
7721 .get = buffer_pipe_buf_get,
7722};
7723
7724/*
7725 * Callback from splice_to_pipe(), if we need to release some pages
7726 * at the end of the spd in case we error'ed out in filling the pipe.
7727 */
7728static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7729{
7730 struct buffer_ref *ref =
7731 (struct buffer_ref *)spd->partial[i].private;
7732
7733 buffer_ref_release(ref);
7734 spd->partial[i].private = 0;
7735}
7736
7737static ssize_t
7738tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7739 struct pipe_inode_info *pipe, size_t len,
7740 unsigned int flags)
7741{
7742 struct ftrace_buffer_info *info = file->private_data;
7743 struct trace_iterator *iter = &info->iter;
7744 struct partial_page partial_def[PIPE_DEF_BUFFERS];
7745 struct page *pages_def[PIPE_DEF_BUFFERS];
7746 struct splice_pipe_desc spd = {
7747 .pages = pages_def,
7748 .partial = partial_def,
7749 .nr_pages_max = PIPE_DEF_BUFFERS,
7750 .ops = &buffer_pipe_buf_ops,
7751 .spd_release = buffer_spd_release,
7752 };
7753 struct buffer_ref *ref;
7754 int entries, i;
7755 ssize_t ret = 0;
7756
7757#ifdef CONFIG_TRACER_MAX_TRACE
7758 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7759 return -EBUSY;
7760#endif
7761
7762 if (*ppos & (PAGE_SIZE - 1))
7763 return -EINVAL;
7764
7765 if (len & (PAGE_SIZE - 1)) {
7766 if (len < PAGE_SIZE)
7767 return -EINVAL;
7768 len &= PAGE_MASK;
7769 }
7770
7771 if (splice_grow_spd(pipe, &spd))
7772 return -ENOMEM;
7773
7774 again:
7775 trace_access_lock(iter->cpu_file);
7776 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7777
7778 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
7779 struct page *page;
7780 int r;
7781
7782 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
7783 if (!ref) {
7784 ret = -ENOMEM;
7785 break;
7786 }
7787
7788 refcount_set(&ref->refcount, 1);
7789 ref->buffer = iter->array_buffer->buffer;
7790 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
7791 if (IS_ERR(ref->page)) {
7792 ret = PTR_ERR(ref->page);
7793 ref->page = NULL;
7794 kfree(ref);
7795 break;
7796 }
7797 ref->cpu = iter->cpu_file;
7798
7799 r = ring_buffer_read_page(ref->buffer, &ref->page,
7800 len, iter->cpu_file, 1);
7801 if (r < 0) {
7802 ring_buffer_free_read_page(ref->buffer, ref->cpu,
7803 ref->page);
7804 kfree(ref);
7805 break;
7806 }
7807
7808 page = virt_to_page(ref->page);
7809
7810 spd.pages[i] = page;
7811 spd.partial[i].len = PAGE_SIZE;
7812 spd.partial[i].offset = 0;
7813 spd.partial[i].private = (unsigned long)ref;
7814 spd.nr_pages++;
7815 *ppos += PAGE_SIZE;
7816
7817 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7818 }
7819
7820 trace_access_unlock(iter->cpu_file);
7821 spd.nr_pages = i;
7822
7823 /* did we read anything? */
7824 if (!spd.nr_pages) {
7825 if (ret)
7826 goto out;
7827
7828 ret = -EAGAIN;
7829 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
7830 goto out;
7831
7832 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
7833 if (ret)
7834 goto out;
7835
7836 goto again;
7837 }
7838
7839 ret = splice_to_pipe(pipe, &spd);
7840out:
7841 splice_shrink_spd(&spd);
7842
7843 return ret;
7844}
7845
7846static const struct file_operations tracing_buffers_fops = {
7847 .open = tracing_buffers_open,
7848 .read = tracing_buffers_read,
7849 .poll = tracing_buffers_poll,
7850 .release = tracing_buffers_release,
7851 .splice_read = tracing_buffers_splice_read,
7852 .llseek = no_llseek,
7853};
7854
7855static ssize_t
7856tracing_stats_read(struct file *filp, char __user *ubuf,
7857 size_t count, loff_t *ppos)
7858{
7859 struct inode *inode = file_inode(filp);
7860 struct trace_array *tr = inode->i_private;
7861 struct array_buffer *trace_buf = &tr->array_buffer;
7862 int cpu = tracing_get_cpu(inode);
7863 struct trace_seq *s;
7864 unsigned long cnt;
7865 unsigned long long t;
7866 unsigned long usec_rem;
7867
7868 s = kmalloc(sizeof(*s), GFP_KERNEL);
7869 if (!s)
7870 return -ENOMEM;
7871
7872 trace_seq_init(s);
7873
7874 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
7875 trace_seq_printf(s, "entries: %ld\n", cnt);
7876
7877 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
7878 trace_seq_printf(s, "overrun: %ld\n", cnt);
7879
7880 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
7881 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7882
7883 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
7884 trace_seq_printf(s, "bytes: %ld\n", cnt);
7885
7886 if (trace_clocks[tr->clock_id].in_ns) {
7887 /* local or global for trace_clock */
7888 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7889 usec_rem = do_div(t, USEC_PER_SEC);
7890 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7891 t, usec_rem);
7892
7893 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
7894 usec_rem = do_div(t, USEC_PER_SEC);
7895 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7896 } else {
7897 /* counter or tsc mode for trace_clock */
7898 trace_seq_printf(s, "oldest event ts: %llu\n",
7899 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7900
7901 trace_seq_printf(s, "now ts: %llu\n",
7902 ring_buffer_time_stamp(trace_buf->buffer, cpu));
7903 }
7904
7905 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7906 trace_seq_printf(s, "dropped events: %ld\n", cnt);
7907
7908 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7909 trace_seq_printf(s, "read events: %ld\n", cnt);
7910
7911 count = simple_read_from_buffer(ubuf, count, ppos,
7912 s->buffer, trace_seq_used(s));
7913
7914 kfree(s);
7915
7916 return count;
7917}
7918
7919static const struct file_operations tracing_stats_fops = {
7920 .open = tracing_open_generic_tr,
7921 .read = tracing_stats_read,
7922 .llseek = generic_file_llseek,
7923 .release = tracing_release_generic_tr,
7924};
7925
7926#ifdef CONFIG_DYNAMIC_FTRACE
7927
7928static ssize_t
7929tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7930 size_t cnt, loff_t *ppos)
7931{
7932 ssize_t ret;
7933 char *buf;
7934 int r;
7935
7936 /* 256 should be plenty to hold the amount needed */
7937 buf = kmalloc(256, GFP_KERNEL);
7938 if (!buf)
7939 return -ENOMEM;
7940
7941 r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
7942 ftrace_update_tot_cnt,
7943 ftrace_number_of_pages,
7944 ftrace_number_of_groups);
7945
7946 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7947 kfree(buf);
7948 return ret;
7949}
7950
7951static const struct file_operations tracing_dyn_info_fops = {
7952 .open = tracing_open_generic,
7953 .read = tracing_read_dyn_info,
7954 .llseek = generic_file_llseek,
7955};
7956#endif /* CONFIG_DYNAMIC_FTRACE */
7957
7958#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7959static void
7960ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
7961 struct trace_array *tr, struct ftrace_probe_ops *ops,
7962 void *data)
7963{
7964 tracing_snapshot_instance(tr);
7965}
7966
7967static void
7968ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
7969 struct trace_array *tr, struct ftrace_probe_ops *ops,
7970 void *data)
7971{
7972 struct ftrace_func_mapper *mapper = data;
7973 long *count = NULL;
7974
7975 if (mapper)
7976 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7977
7978 if (count) {
7979
7980 if (*count <= 0)
7981 return;
7982
7983 (*count)--;
7984 }
7985
7986 tracing_snapshot_instance(tr);
7987}
7988
7989static int
7990ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7991 struct ftrace_probe_ops *ops, void *data)
7992{
7993 struct ftrace_func_mapper *mapper = data;
7994 long *count = NULL;
7995
7996 seq_printf(m, "%ps:", (void *)ip);
7997
7998 seq_puts(m, "snapshot");
7999
8000 if (mapper)
8001 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8002
8003 if (count)
8004 seq_printf(m, ":count=%ld\n", *count);
8005 else
8006 seq_puts(m, ":unlimited\n");
8007
8008 return 0;
8009}
8010
8011static int
8012ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
8013 unsigned long ip, void *init_data, void **data)
8014{
8015 struct ftrace_func_mapper *mapper = *data;
8016
8017 if (!mapper) {
8018 mapper = allocate_ftrace_func_mapper();
8019 if (!mapper)
8020 return -ENOMEM;
8021 *data = mapper;
8022 }
8023
8024 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
8025}
8026
8027static void
8028ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
8029 unsigned long ip, void *data)
8030{
8031 struct ftrace_func_mapper *mapper = data;
8032
8033 if (!ip) {
8034 if (!mapper)
8035 return;
8036 free_ftrace_func_mapper(mapper, NULL);
8037 return;
8038 }
8039
8040 ftrace_func_mapper_remove_ip(mapper, ip);
8041}
8042
8043static struct ftrace_probe_ops snapshot_probe_ops = {
8044 .func = ftrace_snapshot,
8045 .print = ftrace_snapshot_print,
8046};
8047
8048static struct ftrace_probe_ops snapshot_count_probe_ops = {
8049 .func = ftrace_count_snapshot,
8050 .print = ftrace_snapshot_print,
8051 .init = ftrace_snapshot_init,
8052 .free = ftrace_snapshot_free,
8053};
8054
8055static int
8056ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
8057 char *glob, char *cmd, char *param, int enable)
8058{
8059 struct ftrace_probe_ops *ops;
8060 void *count = (void *)-1;
8061 char *number;
8062 int ret;
8063
8064 if (!tr)
8065 return -ENODEV;
8066
8067 /* hash funcs only work with set_ftrace_filter */
8068 if (!enable)
8069 return -EINVAL;
8070
8071 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
8072
8073 if (glob[0] == '!')
8074 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
8075
8076 if (!param)
8077 goto out_reg;
8078
8079 number = strsep(&param, ":");
8080
8081 if (!strlen(number))
8082 goto out_reg;
8083
8084 /*
8085 * We use the callback data field (which is a pointer)
8086 * as our counter.
8087 */
8088 ret = kstrtoul(number, 0, (unsigned long *)&count);
8089 if (ret)
8090 return ret;
8091
8092 out_reg:
8093 ret = tracing_alloc_snapshot_instance(tr);
8094 if (ret < 0)
8095 goto out;
8096
8097 ret = register_ftrace_function_probe(glob, tr, ops, count);
8098
8099 out:
8100 return ret < 0 ? ret : 0;
8101}
8102
8103static struct ftrace_func_command ftrace_snapshot_cmd = {
8104 .name = "snapshot",
8105 .func = ftrace_trace_snapshot_callback,
8106};
8107
8108static __init int register_snapshot_cmd(void)
8109{
8110 return register_ftrace_command(&ftrace_snapshot_cmd);
8111}
8112#else
8113static inline __init int register_snapshot_cmd(void) { return 0; }
8114#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
8115
8116static struct dentry *tracing_get_dentry(struct trace_array *tr)
8117{
8118 if (WARN_ON(!tr->dir))
8119 return ERR_PTR(-ENODEV);
8120
8121 /* Top directory uses NULL as the parent */
8122 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
8123 return NULL;
8124
8125 /* All sub buffers have a descriptor */
8126 return tr->dir;
8127}
8128
8129static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
8130{
8131 struct dentry *d_tracer;
8132
8133 if (tr->percpu_dir)
8134 return tr->percpu_dir;
8135
8136 d_tracer = tracing_get_dentry(tr);
8137 if (IS_ERR(d_tracer))
8138 return NULL;
8139
8140 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
8141
8142 MEM_FAIL(!tr->percpu_dir,
8143 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8144
8145 return tr->percpu_dir;
8146}
8147
8148static struct dentry *
8149trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
8150 void *data, long cpu, const struct file_operations *fops)
8151{
8152 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8153
8154 if (ret) /* See tracing_get_cpu() */
8155 d_inode(ret)->i_cdev = (void *)(cpu + 1);
8156 return ret;
8157}
8158
8159static void
8160tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8161{
8162 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8163 struct dentry *d_cpu;
8164 char cpu_dir[30]; /* 30 characters should be more than enough */
8165
8166 if (!d_percpu)
8167 return;
8168
8169 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8170 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8171 if (!d_cpu) {
8172 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8173 return;
8174 }
8175
8176 /* per cpu trace_pipe */
8177 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
8178 tr, cpu, &tracing_pipe_fops);
8179
8180 /* per cpu trace */
8181 trace_create_cpu_file("trace", 0644, d_cpu,
8182 tr, cpu, &tracing_fops);
8183
8184 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
8185 tr, cpu, &tracing_buffers_fops);
8186
8187 trace_create_cpu_file("stats", 0444, d_cpu,
8188 tr, cpu, &tracing_stats_fops);
8189
8190 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
8191 tr, cpu, &tracing_entries_fops);
8192
8193#ifdef CONFIG_TRACER_SNAPSHOT
8194 trace_create_cpu_file("snapshot", 0644, d_cpu,
8195 tr, cpu, &snapshot_fops);
8196
8197 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
8198 tr, cpu, &snapshot_raw_fops);
8199#endif
8200}
8201
8202#ifdef CONFIG_FTRACE_SELFTEST
8203/* Let selftest have access to static functions in this file */
8204#include "trace_selftest.c"
8205#endif
8206
8207static ssize_t
8208trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
8209 loff_t *ppos)
8210{
8211 struct trace_option_dentry *topt = filp->private_data;
8212 char *buf;
8213
8214 if (topt->flags->val & topt->opt->bit)
8215 buf = "1\n";
8216 else
8217 buf = "0\n";
8218
8219 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8220}
8221
8222static ssize_t
8223trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
8224 loff_t *ppos)
8225{
8226 struct trace_option_dentry *topt = filp->private_data;
8227 unsigned long val;
8228 int ret;
8229
8230 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8231 if (ret)
8232 return ret;
8233
8234 if (val != 0 && val != 1)
8235 return -EINVAL;
8236
8237 if (!!(topt->flags->val & topt->opt->bit) != val) {
8238 mutex_lock(&trace_types_lock);
8239 ret = __set_tracer_option(topt->tr, topt->flags,
8240 topt->opt, !val);
8241 mutex_unlock(&trace_types_lock);
8242 if (ret)
8243 return ret;
8244 }
8245
8246 *ppos += cnt;
8247
8248 return cnt;
8249}
8250
8251
8252static const struct file_operations trace_options_fops = {
8253 .open = tracing_open_generic,
8254 .read = trace_options_read,
8255 .write = trace_options_write,
8256 .llseek = generic_file_llseek,
8257};
8258
8259/*
8260 * In order to pass in both the trace_array descriptor as well as the index
8261 * to the flag that the trace option file represents, the trace_array
8262 * has a character array of trace_flags_index[], which holds the index
8263 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8264 * The address of this character array is passed to the flag option file
8265 * read/write callbacks.
8266 *
8267 * In order to extract both the index and the trace_array descriptor,
8268 * get_tr_index() uses the following algorithm.
8269 *
8270 * idx = *ptr;
8271 *
8272 * As the pointer itself contains the address of the index (remember
8273 * index[1] == 1).
8274 *
8275 * Then to get the trace_array descriptor, by subtracting that index
8276 * from the ptr, we get to the start of the index itself.
8277 *
8278 * ptr - idx == &index[0]
8279 *
8280 * Then a simple container_of() from that pointer gets us to the
8281 * trace_array descriptor.
8282 */
8283static void get_tr_index(void *data, struct trace_array **ptr,
8284 unsigned int *pindex)
8285{
8286 *pindex = *(unsigned char *)data;
8287
8288 *ptr = container_of(data - *pindex, struct trace_array,
8289 trace_flags_index);
8290}
8291
8292static ssize_t
8293trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
8294 loff_t *ppos)
8295{
8296 void *tr_index = filp->private_data;
8297 struct trace_array *tr;
8298 unsigned int index;
8299 char *buf;
8300
8301 get_tr_index(tr_index, &tr, &index);
8302
8303 if (tr->trace_flags & (1 << index))
8304 buf = "1\n";
8305 else
8306 buf = "0\n";
8307
8308 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8309}
8310
8311static ssize_t
8312trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
8313 loff_t *ppos)
8314{
8315 void *tr_index = filp->private_data;
8316 struct trace_array *tr;
8317 unsigned int index;
8318 unsigned long val;
8319 int ret;
8320
8321 get_tr_index(tr_index, &tr, &index);
8322
8323 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8324 if (ret)
8325 return ret;
8326
8327 if (val != 0 && val != 1)
8328 return -EINVAL;
8329
8330 mutex_lock(&event_mutex);
8331 mutex_lock(&trace_types_lock);
8332 ret = set_tracer_flag(tr, 1 << index, val);
8333 mutex_unlock(&trace_types_lock);
8334 mutex_unlock(&event_mutex);
8335
8336 if (ret < 0)
8337 return ret;
8338
8339 *ppos += cnt;
8340
8341 return cnt;
8342}
8343
8344static const struct file_operations trace_options_core_fops = {
8345 .open = tracing_open_generic,
8346 .read = trace_options_core_read,
8347 .write = trace_options_core_write,
8348 .llseek = generic_file_llseek,
8349};
8350
8351struct dentry *trace_create_file(const char *name,
8352 umode_t mode,
8353 struct dentry *parent,
8354 void *data,
8355 const struct file_operations *fops)
8356{
8357 struct dentry *ret;
8358
8359 ret = tracefs_create_file(name, mode, parent, data, fops);
8360 if (!ret)
8361 pr_warn("Could not create tracefs '%s' entry\n", name);
8362
8363 return ret;
8364}
8365
8366
8367static struct dentry *trace_options_init_dentry(struct trace_array *tr)
8368{
8369 struct dentry *d_tracer;
8370
8371 if (tr->options)
8372 return tr->options;
8373
8374 d_tracer = tracing_get_dentry(tr);
8375 if (IS_ERR(d_tracer))
8376 return NULL;
8377
8378 tr->options = tracefs_create_dir("options", d_tracer);
8379 if (!tr->options) {
8380 pr_warn("Could not create tracefs directory 'options'\n");
8381 return NULL;
8382 }
8383
8384 return tr->options;
8385}
8386
8387static void
8388create_trace_option_file(struct trace_array *tr,
8389 struct trace_option_dentry *topt,
8390 struct tracer_flags *flags,
8391 struct tracer_opt *opt)
8392{
8393 struct dentry *t_options;
8394
8395 t_options = trace_options_init_dentry(tr);
8396 if (!t_options)
8397 return;
8398
8399 topt->flags = flags;
8400 topt->opt = opt;
8401 topt->tr = tr;
8402
8403 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
8404 &trace_options_fops);
8405
8406}
8407
8408static void
8409create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
8410{
8411 struct trace_option_dentry *topts;
8412 struct trace_options *tr_topts;
8413 struct tracer_flags *flags;
8414 struct tracer_opt *opts;
8415 int cnt;
8416 int i;
8417
8418 if (!tracer)
8419 return;
8420
8421 flags = tracer->flags;
8422
8423 if (!flags || !flags->opts)
8424 return;
8425
8426 /*
8427 * If this is an instance, only create flags for tracers
8428 * the instance may have.
8429 */
8430 if (!trace_ok_for_array(tracer, tr))
8431 return;
8432
8433 for (i = 0; i < tr->nr_topts; i++) {
8434 /* Make sure there's no duplicate flags. */
8435 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8436 return;
8437 }
8438
8439 opts = flags->opts;
8440
8441 for (cnt = 0; opts[cnt].name; cnt++)
8442 ;
8443
8444 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
8445 if (!topts)
8446 return;
8447
8448 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
8449 GFP_KERNEL);
8450 if (!tr_topts) {
8451 kfree(topts);
8452 return;
8453 }
8454
8455 tr->topts = tr_topts;
8456 tr->topts[tr->nr_topts].tracer = tracer;
8457 tr->topts[tr->nr_topts].topts = topts;
8458 tr->nr_topts++;
8459
8460 for (cnt = 0; opts[cnt].name; cnt++) {
8461 create_trace_option_file(tr, &topts[cnt], flags,
8462 &opts[cnt]);
8463 MEM_FAIL(topts[cnt].entry == NULL,
8464 "Failed to create trace option: %s",
8465 opts[cnt].name);
8466 }
8467}
8468
8469static struct dentry *
8470create_trace_option_core_file(struct trace_array *tr,
8471 const char *option, long index)
8472{
8473 struct dentry *t_options;
8474
8475 t_options = trace_options_init_dentry(tr);
8476 if (!t_options)
8477 return NULL;
8478
8479 return trace_create_file(option, 0644, t_options,
8480 (void *)&tr->trace_flags_index[index],
8481 &trace_options_core_fops);
8482}
8483
8484static void create_trace_options_dir(struct trace_array *tr)
8485{
8486 struct dentry *t_options;
8487 bool top_level = tr == &global_trace;
8488 int i;
8489
8490 t_options = trace_options_init_dentry(tr);
8491 if (!t_options)
8492 return;
8493
8494 for (i = 0; trace_options[i]; i++) {
8495 if (top_level ||
8496 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8497 create_trace_option_core_file(tr, trace_options[i], i);
8498 }
8499}
8500
8501static ssize_t
8502rb_simple_read(struct file *filp, char __user *ubuf,
8503 size_t cnt, loff_t *ppos)
8504{
8505 struct trace_array *tr = filp->private_data;
8506 char buf[64];
8507 int r;
8508
8509 r = tracer_tracing_is_on(tr);
8510 r = sprintf(buf, "%d\n", r);
8511
8512 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8513}
8514
8515static ssize_t
8516rb_simple_write(struct file *filp, const char __user *ubuf,
8517 size_t cnt, loff_t *ppos)
8518{
8519 struct trace_array *tr = filp->private_data;
8520 struct trace_buffer *buffer = tr->array_buffer.buffer;
8521 unsigned long val;
8522 int ret;
8523
8524 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8525 if (ret)
8526 return ret;
8527
8528 if (buffer) {
8529 mutex_lock(&trace_types_lock);
8530 if (!!val == tracer_tracing_is_on(tr)) {
8531 val = 0; /* do nothing */
8532 } else if (val) {
8533 tracer_tracing_on(tr);
8534 if (tr->current_trace->start)
8535 tr->current_trace->start(tr);
8536 } else {
8537 tracer_tracing_off(tr);
8538 if (tr->current_trace->stop)
8539 tr->current_trace->stop(tr);
8540 }
8541 mutex_unlock(&trace_types_lock);
8542 }
8543
8544 (*ppos)++;
8545
8546 return cnt;
8547}
8548
8549static const struct file_operations rb_simple_fops = {
8550 .open = tracing_open_generic_tr,
8551 .read = rb_simple_read,
8552 .write = rb_simple_write,
8553 .release = tracing_release_generic_tr,
8554 .llseek = default_llseek,
8555};
8556
8557static ssize_t
8558buffer_percent_read(struct file *filp, char __user *ubuf,
8559 size_t cnt, loff_t *ppos)
8560{
8561 struct trace_array *tr = filp->private_data;
8562 char buf[64];
8563 int r;
8564
8565 r = tr->buffer_percent;
8566 r = sprintf(buf, "%d\n", r);
8567
8568 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8569}
8570
8571static ssize_t
8572buffer_percent_write(struct file *filp, const char __user *ubuf,
8573 size_t cnt, loff_t *ppos)
8574{
8575 struct trace_array *tr = filp->private_data;
8576 unsigned long val;
8577 int ret;
8578
8579 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8580 if (ret)
8581 return ret;
8582
8583 if (val > 100)
8584 return -EINVAL;
8585
8586 if (!val)
8587 val = 1;
8588
8589 tr->buffer_percent = val;
8590
8591 (*ppos)++;
8592
8593 return cnt;
8594}
8595
8596static const struct file_operations buffer_percent_fops = {
8597 .open = tracing_open_generic_tr,
8598 .read = buffer_percent_read,
8599 .write = buffer_percent_write,
8600 .release = tracing_release_generic_tr,
8601 .llseek = default_llseek,
8602};
8603
8604static struct dentry *trace_instance_dir;
8605
8606static void
8607init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
8608
8609static int
8610allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
8611{
8612 enum ring_buffer_flags rb_flags;
8613
8614 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
8615
8616 buf->tr = tr;
8617
8618 buf->buffer = ring_buffer_alloc(size, rb_flags);
8619 if (!buf->buffer)
8620 return -ENOMEM;
8621
8622 buf->data = alloc_percpu(struct trace_array_cpu);
8623 if (!buf->data) {
8624 ring_buffer_free(buf->buffer);
8625 buf->buffer = NULL;
8626 return -ENOMEM;
8627 }
8628
8629 /* Allocate the first page for all buffers */
8630 set_buffer_entries(&tr->array_buffer,
8631 ring_buffer_size(tr->array_buffer.buffer, 0));
8632
8633 return 0;
8634}
8635
8636static int allocate_trace_buffers(struct trace_array *tr, int size)
8637{
8638 int ret;
8639
8640 ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
8641 if (ret)
8642 return ret;
8643
8644#ifdef CONFIG_TRACER_MAX_TRACE
8645 ret = allocate_trace_buffer(tr, &tr->max_buffer,
8646 allocate_snapshot ? size : 1);
8647 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
8648 ring_buffer_free(tr->array_buffer.buffer);
8649 tr->array_buffer.buffer = NULL;
8650 free_percpu(tr->array_buffer.data);
8651 tr->array_buffer.data = NULL;
8652 return -ENOMEM;
8653 }
8654 tr->allocated_snapshot = allocate_snapshot;
8655
8656 /*
8657 * Only the top level trace array gets its snapshot allocated
8658 * from the kernel command line.
8659 */
8660 allocate_snapshot = false;
8661#endif
8662
8663 return 0;
8664}
8665
8666static void free_trace_buffer(struct array_buffer *buf)
8667{
8668 if (buf->buffer) {
8669 ring_buffer_free(buf->buffer);
8670 buf->buffer = NULL;
8671 free_percpu(buf->data);
8672 buf->data = NULL;
8673 }
8674}
8675
8676static void free_trace_buffers(struct trace_array *tr)
8677{
8678 if (!tr)
8679 return;
8680
8681 free_trace_buffer(&tr->array_buffer);
8682
8683#ifdef CONFIG_TRACER_MAX_TRACE
8684 free_trace_buffer(&tr->max_buffer);
8685#endif
8686}
8687
8688static void init_trace_flags_index(struct trace_array *tr)
8689{
8690 int i;
8691
8692 /* Used by the trace options files */
8693 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8694 tr->trace_flags_index[i] = i;
8695}
8696
8697static void __update_tracer_options(struct trace_array *tr)
8698{
8699 struct tracer *t;
8700
8701 for (t = trace_types; t; t = t->next)
8702 add_tracer_options(tr, t);
8703}
8704
8705static void update_tracer_options(struct trace_array *tr)
8706{
8707 mutex_lock(&trace_types_lock);
8708 __update_tracer_options(tr);
8709 mutex_unlock(&trace_types_lock);
8710}
8711
8712/* Must have trace_types_lock held */
8713struct trace_array *trace_array_find(const char *instance)
8714{
8715 struct trace_array *tr, *found = NULL;
8716
8717 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8718 if (tr->name && strcmp(tr->name, instance) == 0) {
8719 found = tr;
8720 break;
8721 }
8722 }
8723
8724 return found;
8725}
8726
8727struct trace_array *trace_array_find_get(const char *instance)
8728{
8729 struct trace_array *tr;
8730
8731 mutex_lock(&trace_types_lock);
8732 tr = trace_array_find(instance);
8733 if (tr)
8734 tr->ref++;
8735 mutex_unlock(&trace_types_lock);
8736
8737 return tr;
8738}
8739
8740static int trace_array_create_dir(struct trace_array *tr)
8741{
8742 int ret;
8743
8744 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
8745 if (!tr->dir)
8746 return -EINVAL;
8747
8748 ret = event_trace_add_tracer(tr->dir, tr);
8749 if (ret)
8750 tracefs_remove(tr->dir);
8751
8752 init_tracer_tracefs(tr, tr->dir);
8753 __update_tracer_options(tr);
8754
8755 return ret;
8756}
8757
8758static struct trace_array *trace_array_create(const char *name)
8759{
8760 struct trace_array *tr;
8761 int ret;
8762
8763 ret = -ENOMEM;
8764 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
8765 if (!tr)
8766 return ERR_PTR(ret);
8767
8768 tr->name = kstrdup(name, GFP_KERNEL);
8769 if (!tr->name)
8770 goto out_free_tr;
8771
8772 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8773 goto out_free_tr;
8774
8775 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
8776
8777 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8778
8779 raw_spin_lock_init(&tr->start_lock);
8780
8781 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8782
8783 tr->current_trace = &nop_trace;
8784
8785 INIT_LIST_HEAD(&tr->systems);
8786 INIT_LIST_HEAD(&tr->events);
8787 INIT_LIST_HEAD(&tr->hist_vars);
8788 INIT_LIST_HEAD(&tr->err_log);
8789
8790 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
8791 goto out_free_tr;
8792
8793 if (ftrace_allocate_ftrace_ops(tr) < 0)
8794 goto out_free_tr;
8795
8796 ftrace_init_trace_array(tr);
8797
8798 init_trace_flags_index(tr);
8799
8800 if (trace_instance_dir) {
8801 ret = trace_array_create_dir(tr);
8802 if (ret)
8803 goto out_free_tr;
8804 } else
8805 __trace_early_add_events(tr);
8806
8807 list_add(&tr->list, &ftrace_trace_arrays);
8808
8809 tr->ref++;
8810
8811 return tr;
8812
8813 out_free_tr:
8814 ftrace_free_ftrace_ops(tr);
8815 free_trace_buffers(tr);
8816 free_cpumask_var(tr->tracing_cpumask);
8817 kfree(tr->name);
8818 kfree(tr);
8819
8820 return ERR_PTR(ret);
8821}
8822
8823static int instance_mkdir(const char *name)
8824{
8825 struct trace_array *tr;
8826 int ret;
8827
8828 mutex_lock(&event_mutex);
8829 mutex_lock(&trace_types_lock);
8830
8831 ret = -EEXIST;
8832 if (trace_array_find(name))
8833 goto out_unlock;
8834
8835 tr = trace_array_create(name);
8836
8837 ret = PTR_ERR_OR_ZERO(tr);
8838
8839out_unlock:
8840 mutex_unlock(&trace_types_lock);
8841 mutex_unlock(&event_mutex);
8842 return ret;
8843}
8844
8845/**
8846 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
8847 * @name: The name of the trace array to be looked up/created.
8848 *
8849 * Returns pointer to trace array with given name.
8850 * NULL, if it cannot be created.
8851 *
8852 * NOTE: This function increments the reference counter associated with the
8853 * trace array returned. This makes sure it cannot be freed while in use.
8854 * Use trace_array_put() once the trace array is no longer needed.
8855 * If the trace_array is to be freed, trace_array_destroy() needs to
8856 * be called after the trace_array_put(), or simply let user space delete
8857 * it from the tracefs instances directory. But until the
8858 * trace_array_put() is called, user space can not delete it.
8859 *
8860 */
8861struct trace_array *trace_array_get_by_name(const char *name)
8862{
8863 struct trace_array *tr;
8864
8865 mutex_lock(&event_mutex);
8866 mutex_lock(&trace_types_lock);
8867
8868 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8869 if (tr->name && strcmp(tr->name, name) == 0)
8870 goto out_unlock;
8871 }
8872
8873 tr = trace_array_create(name);
8874
8875 if (IS_ERR(tr))
8876 tr = NULL;
8877out_unlock:
8878 if (tr)
8879 tr->ref++;
8880
8881 mutex_unlock(&trace_types_lock);
8882 mutex_unlock(&event_mutex);
8883 return tr;
8884}
8885EXPORT_SYMBOL_GPL(trace_array_get_by_name);
8886
8887static int __remove_instance(struct trace_array *tr)
8888{
8889 int i;
8890
8891 /* Reference counter for a newly created trace array = 1. */
8892 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))
8893 return -EBUSY;
8894
8895 list_del(&tr->list);
8896
8897 /* Disable all the flags that were enabled coming in */
8898 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8899 if ((1 << i) & ZEROED_TRACE_FLAGS)
8900 set_tracer_flag(tr, 1 << i, 0);
8901 }
8902
8903 tracing_set_nop(tr);
8904 clear_ftrace_function_probes(tr);
8905 event_trace_del_tracer(tr);
8906 ftrace_clear_pids(tr);
8907 ftrace_destroy_function_files(tr);
8908 tracefs_remove(tr->dir);
8909 free_trace_buffers(tr);
8910
8911 for (i = 0; i < tr->nr_topts; i++) {
8912 kfree(tr->topts[i].topts);
8913 }
8914 kfree(tr->topts);
8915
8916 free_cpumask_var(tr->tracing_cpumask);
8917 kfree(tr->name);
8918 kfree(tr);
8919
8920 return 0;
8921}
8922
8923int trace_array_destroy(struct trace_array *this_tr)
8924{
8925 struct trace_array *tr;
8926 int ret;
8927
8928 if (!this_tr)
8929 return -EINVAL;
8930
8931 mutex_lock(&event_mutex);
8932 mutex_lock(&trace_types_lock);
8933
8934 ret = -ENODEV;
8935
8936 /* Making sure trace array exists before destroying it. */
8937 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8938 if (tr == this_tr) {
8939 ret = __remove_instance(tr);
8940 break;
8941 }
8942 }
8943
8944 mutex_unlock(&trace_types_lock);
8945 mutex_unlock(&event_mutex);
8946
8947 return ret;
8948}
8949EXPORT_SYMBOL_GPL(trace_array_destroy);
8950
8951static int instance_rmdir(const char *name)
8952{
8953 struct trace_array *tr;
8954 int ret;
8955
8956 mutex_lock(&event_mutex);
8957 mutex_lock(&trace_types_lock);
8958
8959 ret = -ENODEV;
8960 tr = trace_array_find(name);
8961 if (tr)
8962 ret = __remove_instance(tr);
8963
8964 mutex_unlock(&trace_types_lock);
8965 mutex_unlock(&event_mutex);
8966
8967 return ret;
8968}
8969
8970static __init void create_trace_instances(struct dentry *d_tracer)
8971{
8972 struct trace_array *tr;
8973
8974 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8975 instance_mkdir,
8976 instance_rmdir);
8977 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
8978 return;
8979
8980 mutex_lock(&event_mutex);
8981 mutex_lock(&trace_types_lock);
8982
8983 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8984 if (!tr->name)
8985 continue;
8986 if (MEM_FAIL(trace_array_create_dir(tr) < 0,
8987 "Failed to create instance directory\n"))
8988 break;
8989 }
8990
8991 mutex_unlock(&trace_types_lock);
8992 mutex_unlock(&event_mutex);
8993}
8994
8995static void
8996init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8997{
8998 struct trace_event_file *file;
8999 int cpu;
9000
9001 trace_create_file("available_tracers", 0444, d_tracer,
9002 tr, &show_traces_fops);
9003
9004 trace_create_file("current_tracer", 0644, d_tracer,
9005 tr, &set_tracer_fops);
9006
9007 trace_create_file("tracing_cpumask", 0644, d_tracer,
9008 tr, &tracing_cpumask_fops);
9009
9010 trace_create_file("trace_options", 0644, d_tracer,
9011 tr, &tracing_iter_fops);
9012
9013 trace_create_file("trace", 0644, d_tracer,
9014 tr, &tracing_fops);
9015
9016 trace_create_file("trace_pipe", 0444, d_tracer,
9017 tr, &tracing_pipe_fops);
9018
9019 trace_create_file("buffer_size_kb", 0644, d_tracer,
9020 tr, &tracing_entries_fops);
9021
9022 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
9023 tr, &tracing_total_entries_fops);
9024
9025 trace_create_file("free_buffer", 0200, d_tracer,
9026 tr, &tracing_free_buffer_fops);
9027
9028 trace_create_file("trace_marker", 0220, d_tracer,
9029 tr, &tracing_mark_fops);
9030
9031 file = __find_event_file(tr, "ftrace", "print");
9032 if (file && file->dir)
9033 trace_create_file("trigger", 0644, file->dir, file,
9034 &event_trigger_fops);
9035 tr->trace_marker_file = file;
9036
9037 trace_create_file("trace_marker_raw", 0220, d_tracer,
9038 tr, &tracing_mark_raw_fops);
9039
9040 trace_create_file("trace_clock", 0644, d_tracer, tr,
9041 &trace_clock_fops);
9042
9043 trace_create_file("tracing_on", 0644, d_tracer,
9044 tr, &rb_simple_fops);
9045
9046 trace_create_file("timestamp_mode", 0444, d_tracer, tr,
9047 &trace_time_stamp_mode_fops);
9048
9049 tr->buffer_percent = 50;
9050
9051 trace_create_file("buffer_percent", 0444, d_tracer,
9052 tr, &buffer_percent_fops);
9053
9054 create_trace_options_dir(tr);
9055
9056#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
9057 trace_create_maxlat_file(tr, d_tracer);
9058#endif
9059
9060 if (ftrace_create_function_files(tr, d_tracer))
9061 MEM_FAIL(1, "Could not allocate function filter files");
9062
9063#ifdef CONFIG_TRACER_SNAPSHOT
9064 trace_create_file("snapshot", 0644, d_tracer,
9065 tr, &snapshot_fops);
9066#endif
9067
9068 trace_create_file("error_log", 0644, d_tracer,
9069 tr, &tracing_err_log_fops);
9070
9071 for_each_tracing_cpu(cpu)
9072 tracing_init_tracefs_percpu(tr, cpu);
9073
9074 ftrace_init_tracefs(tr, d_tracer);
9075}
9076
9077static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
9078{
9079 struct vfsmount *mnt;
9080 struct file_system_type *type;
9081
9082 /*
9083 * To maintain backward compatibility for tools that mount
9084 * debugfs to get to the tracing facility, tracefs is automatically
9085 * mounted to the debugfs/tracing directory.
9086 */
9087 type = get_fs_type("tracefs");
9088 if (!type)
9089 return NULL;
9090 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
9091 put_filesystem(type);
9092 if (IS_ERR(mnt))
9093 return NULL;
9094 mntget(mnt);
9095
9096 return mnt;
9097}
9098
9099/**
9100 * tracing_init_dentry - initialize top level trace array
9101 *
9102 * This is called when creating files or directories in the tracing
9103 * directory. It is called via fs_initcall() by any of the boot up code
9104 * and expects to return the dentry of the top level tracing directory.
9105 */
9106int tracing_init_dentry(void)
9107{
9108 struct trace_array *tr = &global_trace;
9109
9110 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9111 pr_warn("Tracing disabled due to lockdown\n");
9112 return -EPERM;
9113 }
9114
9115 /* The top level trace array uses NULL as parent */
9116 if (tr->dir)
9117 return 0;
9118
9119 if (WARN_ON(!tracefs_initialized()))
9120 return -ENODEV;
9121
9122 /*
9123 * As there may still be users that expect the tracing
9124 * files to exist in debugfs/tracing, we must automount
9125 * the tracefs file system there, so older tools still
9126 * work with the newer kerenl.
9127 */
9128 tr->dir = debugfs_create_automount("tracing", NULL,
9129 trace_automount, NULL);
9130
9131 return 0;
9132}
9133
9134extern struct trace_eval_map *__start_ftrace_eval_maps[];
9135extern struct trace_eval_map *__stop_ftrace_eval_maps[];
9136
9137static struct workqueue_struct *eval_map_wq __initdata;
9138static struct work_struct eval_map_work __initdata;
9139
9140static void __init eval_map_work_func(struct work_struct *work)
9141{
9142 int len;
9143
9144 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
9145 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
9146}
9147
9148static int __init trace_eval_init(void)
9149{
9150 INIT_WORK(&eval_map_work, eval_map_work_func);
9151
9152 eval_map_wq = alloc_workqueue("eval_map_wq", WQ_UNBOUND, 0);
9153 if (!eval_map_wq) {
9154 pr_err("Unable to allocate eval_map_wq\n");
9155 /* Do work here */
9156 eval_map_work_func(&eval_map_work);
9157 return -ENOMEM;
9158 }
9159
9160 queue_work(eval_map_wq, &eval_map_work);
9161 return 0;
9162}
9163
9164static int __init trace_eval_sync(void)
9165{
9166 /* Make sure the eval map updates are finished */
9167 if (eval_map_wq)
9168 destroy_workqueue(eval_map_wq);
9169 return 0;
9170}
9171
9172late_initcall_sync(trace_eval_sync);
9173
9174
9175#ifdef CONFIG_MODULES
9176static void trace_module_add_evals(struct module *mod)
9177{
9178 if (!mod->num_trace_evals)
9179 return;
9180
9181 /*
9182 * Modules with bad taint do not have events created, do
9183 * not bother with enums either.
9184 */
9185 if (trace_module_has_bad_taint(mod))
9186 return;
9187
9188 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
9189}
9190
9191#ifdef CONFIG_TRACE_EVAL_MAP_FILE
9192static void trace_module_remove_evals(struct module *mod)
9193{
9194 union trace_eval_map_item *map;
9195 union trace_eval_map_item **last = &trace_eval_maps;
9196
9197 if (!mod->num_trace_evals)
9198 return;
9199
9200 mutex_lock(&trace_eval_mutex);
9201
9202 map = trace_eval_maps;
9203
9204 while (map) {
9205 if (map->head.mod == mod)
9206 break;
9207 map = trace_eval_jmp_to_tail(map);
9208 last = &map->tail.next;
9209 map = map->tail.next;
9210 }
9211 if (!map)
9212 goto out;
9213
9214 *last = trace_eval_jmp_to_tail(map)->tail.next;
9215 kfree(map);
9216 out:
9217 mutex_unlock(&trace_eval_mutex);
9218}
9219#else
9220static inline void trace_module_remove_evals(struct module *mod) { }
9221#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9222
9223static int trace_module_notify(struct notifier_block *self,
9224 unsigned long val, void *data)
9225{
9226 struct module *mod = data;
9227
9228 switch (val) {
9229 case MODULE_STATE_COMING:
9230 trace_module_add_evals(mod);
9231 break;
9232 case MODULE_STATE_GOING:
9233 trace_module_remove_evals(mod);
9234 break;
9235 }
9236
9237 return NOTIFY_OK;
9238}
9239
9240static struct notifier_block trace_module_nb = {
9241 .notifier_call = trace_module_notify,
9242 .priority = 0,
9243};
9244#endif /* CONFIG_MODULES */
9245
9246static __init int tracer_init_tracefs(void)
9247{
9248 int ret;
9249
9250 trace_access_lock_init();
9251
9252 ret = tracing_init_dentry();
9253 if (ret)
9254 return 0;
9255
9256 event_trace_init();
9257
9258 init_tracer_tracefs(&global_trace, NULL);
9259 ftrace_init_tracefs_toplevel(&global_trace, NULL);
9260
9261 trace_create_file("tracing_thresh", 0644, NULL,
9262 &global_trace, &tracing_thresh_fops);
9263
9264 trace_create_file("README", 0444, NULL,
9265 NULL, &tracing_readme_fops);
9266
9267 trace_create_file("saved_cmdlines", 0444, NULL,
9268 NULL, &tracing_saved_cmdlines_fops);
9269
9270 trace_create_file("saved_cmdlines_size", 0644, NULL,
9271 NULL, &tracing_saved_cmdlines_size_fops);
9272
9273 trace_create_file("saved_tgids", 0444, NULL,
9274 NULL, &tracing_saved_tgids_fops);
9275
9276 trace_eval_init();
9277
9278 trace_create_eval_file(NULL);
9279
9280#ifdef CONFIG_MODULES
9281 register_module_notifier(&trace_module_nb);
9282#endif
9283
9284#ifdef CONFIG_DYNAMIC_FTRACE
9285 trace_create_file("dyn_ftrace_total_info", 0444, NULL,
9286 NULL, &tracing_dyn_info_fops);
9287#endif
9288
9289 create_trace_instances(NULL);
9290
9291 update_tracer_options(&global_trace);
9292
9293 return 0;
9294}
9295
9296static int trace_panic_handler(struct notifier_block *this,
9297 unsigned long event, void *unused)
9298{
9299 if (ftrace_dump_on_oops)
9300 ftrace_dump(ftrace_dump_on_oops);
9301 return NOTIFY_OK;
9302}
9303
9304static struct notifier_block trace_panic_notifier = {
9305 .notifier_call = trace_panic_handler,
9306 .next = NULL,
9307 .priority = 150 /* priority: INT_MAX >= x >= 0 */
9308};
9309
9310static int trace_die_handler(struct notifier_block *self,
9311 unsigned long val,
9312 void *data)
9313{
9314 switch (val) {
9315 case DIE_OOPS:
9316 if (ftrace_dump_on_oops)
9317 ftrace_dump(ftrace_dump_on_oops);
9318 break;
9319 default:
9320 break;
9321 }
9322 return NOTIFY_OK;
9323}
9324
9325static struct notifier_block trace_die_notifier = {
9326 .notifier_call = trace_die_handler,
9327 .priority = 200
9328};
9329
9330/*
9331 * printk is set to max of 1024, we really don't need it that big.
9332 * Nothing should be printing 1000 characters anyway.
9333 */
9334#define TRACE_MAX_PRINT 1000
9335
9336/*
9337 * Define here KERN_TRACE so that we have one place to modify
9338 * it if we decide to change what log level the ftrace dump
9339 * should be at.
9340 */
9341#define KERN_TRACE KERN_EMERG
9342
9343void
9344trace_printk_seq(struct trace_seq *s)
9345{
9346 /* Probably should print a warning here. */
9347 if (s->seq.len >= TRACE_MAX_PRINT)
9348 s->seq.len = TRACE_MAX_PRINT;
9349
9350 /*
9351 * More paranoid code. Although the buffer size is set to
9352 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9353 * an extra layer of protection.
9354 */
9355 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9356 s->seq.len = s->seq.size - 1;
9357
9358 /* should be zero ended, but we are paranoid. */
9359 s->buffer[s->seq.len] = 0;
9360
9361 printk(KERN_TRACE "%s", s->buffer);
9362
9363 trace_seq_init(s);
9364}
9365
9366void trace_init_global_iter(struct trace_iterator *iter)
9367{
9368 iter->tr = &global_trace;
9369 iter->trace = iter->tr->current_trace;
9370 iter->cpu_file = RING_BUFFER_ALL_CPUS;
9371 iter->array_buffer = &global_trace.array_buffer;
9372
9373 if (iter->trace && iter->trace->open)
9374 iter->trace->open(iter);
9375
9376 /* Annotate start of buffers if we had overruns */
9377 if (ring_buffer_overruns(iter->array_buffer->buffer))
9378 iter->iter_flags |= TRACE_FILE_ANNOTATE;
9379
9380 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
9381 if (trace_clocks[iter->tr->clock_id].in_ns)
9382 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9383}
9384
9385void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
9386{
9387 /* use static because iter can be a bit big for the stack */
9388 static struct trace_iterator iter;
9389 static atomic_t dump_running;
9390 struct trace_array *tr = &global_trace;
9391 unsigned int old_userobj;
9392 unsigned long flags;
9393 int cnt = 0, cpu;
9394
9395 /* Only allow one dump user at a time. */
9396 if (atomic_inc_return(&dump_running) != 1) {
9397 atomic_dec(&dump_running);
9398 return;
9399 }
9400
9401 /*
9402 * Always turn off tracing when we dump.
9403 * We don't need to show trace output of what happens
9404 * between multiple crashes.
9405 *
9406 * If the user does a sysrq-z, then they can re-enable
9407 * tracing with echo 1 > tracing_on.
9408 */
9409 tracing_off();
9410
9411 local_irq_save(flags);
9412 printk_nmi_direct_enter();
9413
9414 /* Simulate the iterator */
9415 trace_init_global_iter(&iter);
9416 /* Can not use kmalloc for iter.temp and iter.fmt */
9417 iter.temp = static_temp_buf;
9418 iter.temp_size = STATIC_TEMP_BUF_SIZE;
9419 iter.fmt = static_fmt_buf;
9420 iter.fmt_size = STATIC_FMT_BUF_SIZE;
9421
9422 for_each_tracing_cpu(cpu) {
9423 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9424 }
9425
9426 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
9427
9428 /* don't look at user memory in panic mode */
9429 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
9430
9431 switch (oops_dump_mode) {
9432 case DUMP_ALL:
9433 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9434 break;
9435 case DUMP_ORIG:
9436 iter.cpu_file = raw_smp_processor_id();
9437 break;
9438 case DUMP_NONE:
9439 goto out_enable;
9440 default:
9441 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
9442 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9443 }
9444
9445 printk(KERN_TRACE "Dumping ftrace buffer:\n");
9446
9447 /* Did function tracer already get disabled? */
9448 if (ftrace_is_dead()) {
9449 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9450 printk("# MAY BE MISSING FUNCTION EVENTS\n");
9451 }
9452
9453 /*
9454 * We need to stop all tracing on all CPUS to read
9455 * the next buffer. This is a bit expensive, but is
9456 * not done often. We fill all what we can read,
9457 * and then release the locks again.
9458 */
9459
9460 while (!trace_empty(&iter)) {
9461
9462 if (!cnt)
9463 printk(KERN_TRACE "---------------------------------\n");
9464
9465 cnt++;
9466
9467 trace_iterator_reset(&iter);
9468 iter.iter_flags |= TRACE_FILE_LAT_FMT;
9469
9470 if (trace_find_next_entry_inc(&iter) != NULL) {
9471 int ret;
9472
9473 ret = print_trace_line(&iter);
9474 if (ret != TRACE_TYPE_NO_CONSUME)
9475 trace_consume(&iter);
9476 }
9477 touch_nmi_watchdog();
9478
9479 trace_printk_seq(&iter.seq);
9480 }
9481
9482 if (!cnt)
9483 printk(KERN_TRACE " (ftrace buffer empty)\n");
9484 else
9485 printk(KERN_TRACE "---------------------------------\n");
9486
9487 out_enable:
9488 tr->trace_flags |= old_userobj;
9489
9490 for_each_tracing_cpu(cpu) {
9491 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9492 }
9493 atomic_dec(&dump_running);
9494 printk_nmi_direct_exit();
9495 local_irq_restore(flags);
9496}
9497EXPORT_SYMBOL_GPL(ftrace_dump);
9498
9499#define WRITE_BUFSIZE 4096
9500
9501ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9502 size_t count, loff_t *ppos,
9503 int (*createfn)(const char *))
9504{
9505 char *kbuf, *buf, *tmp;
9506 int ret = 0;
9507 size_t done = 0;
9508 size_t size;
9509
9510 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9511 if (!kbuf)
9512 return -ENOMEM;
9513
9514 while (done < count) {
9515 size = count - done;
9516
9517 if (size >= WRITE_BUFSIZE)
9518 size = WRITE_BUFSIZE - 1;
9519
9520 if (copy_from_user(kbuf, buffer + done, size)) {
9521 ret = -EFAULT;
9522 goto out;
9523 }
9524 kbuf[size] = '\0';
9525 buf = kbuf;
9526 do {
9527 tmp = strchr(buf, '\n');
9528 if (tmp) {
9529 *tmp = '\0';
9530 size = tmp - buf + 1;
9531 } else {
9532 size = strlen(buf);
9533 if (done + size < count) {
9534 if (buf != kbuf)
9535 break;
9536 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9537 pr_warn("Line length is too long: Should be less than %d\n",
9538 WRITE_BUFSIZE - 2);
9539 ret = -EINVAL;
9540 goto out;
9541 }
9542 }
9543 done += size;
9544
9545 /* Remove comments */
9546 tmp = strchr(buf, '#');
9547
9548 if (tmp)
9549 *tmp = '\0';
9550
9551 ret = createfn(buf);
9552 if (ret)
9553 goto out;
9554 buf += size;
9555
9556 } while (done < count);
9557 }
9558 ret = done;
9559
9560out:
9561 kfree(kbuf);
9562
9563 return ret;
9564}
9565
9566__init static int tracer_alloc_buffers(void)
9567{
9568 int ring_buf_size;
9569 int ret = -ENOMEM;
9570
9571
9572 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9573 pr_warn("Tracing disabled due to lockdown\n");
9574 return -EPERM;
9575 }
9576
9577 /*
9578 * Make sure we don't accidentally add more trace options
9579 * than we have bits for.
9580 */
9581 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
9582
9583 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
9584 goto out;
9585
9586 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9587 goto out_free_buffer_mask;
9588
9589 /* Only allocate trace_printk buffers if a trace_printk exists */
9590 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
9591 /* Must be called before global_trace.buffer is allocated */
9592 trace_printk_init_buffers();
9593
9594 /* To save memory, keep the ring buffer size to its minimum */
9595 if (ring_buffer_expanded)
9596 ring_buf_size = trace_buf_size;
9597 else
9598 ring_buf_size = 1;
9599
9600 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
9601 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9602
9603 raw_spin_lock_init(&global_trace.start_lock);
9604
9605 /*
9606 * The prepare callbacks allocates some memory for the ring buffer. We
9607 * don't free the buffer if the CPU goes down. If we were to free
9608 * the buffer, then the user would lose any trace that was in the
9609 * buffer. The memory will be removed once the "instance" is removed.
9610 */
9611 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
9612 "trace/RB:preapre", trace_rb_cpu_prepare,
9613 NULL);
9614 if (ret < 0)
9615 goto out_free_cpumask;
9616 /* Used for event triggers */
9617 ret = -ENOMEM;
9618 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
9619 if (!temp_buffer)
9620 goto out_rm_hp_state;
9621
9622 if (trace_create_savedcmd() < 0)
9623 goto out_free_temp_buffer;
9624
9625 /* TODO: make the number of buffers hot pluggable with CPUS */
9626 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
9627 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
9628 goto out_free_savedcmd;
9629 }
9630
9631 if (global_trace.buffer_disabled)
9632 tracing_off();
9633
9634 if (trace_boot_clock) {
9635 ret = tracing_set_clock(&global_trace, trace_boot_clock);
9636 if (ret < 0)
9637 pr_warn("Trace clock %s not defined, going back to default\n",
9638 trace_boot_clock);
9639 }
9640
9641 /*
9642 * register_tracer() might reference current_trace, so it
9643 * needs to be set before we register anything. This is
9644 * just a bootstrap of current_trace anyway.
9645 */
9646 global_trace.current_trace = &nop_trace;
9647
9648 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9649
9650 ftrace_init_global_array_ops(&global_trace);
9651
9652 init_trace_flags_index(&global_trace);
9653
9654 register_tracer(&nop_trace);
9655
9656 /* Function tracing may start here (via kernel command line) */
9657 init_function_trace();
9658
9659 /* All seems OK, enable tracing */
9660 tracing_disabled = 0;
9661
9662 atomic_notifier_chain_register(&panic_notifier_list,
9663 &trace_panic_notifier);
9664
9665 register_die_notifier(&trace_die_notifier);
9666
9667 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
9668
9669 INIT_LIST_HEAD(&global_trace.systems);
9670 INIT_LIST_HEAD(&global_trace.events);
9671 INIT_LIST_HEAD(&global_trace.hist_vars);
9672 INIT_LIST_HEAD(&global_trace.err_log);
9673 list_add(&global_trace.list, &ftrace_trace_arrays);
9674
9675 apply_trace_boot_options();
9676
9677 register_snapshot_cmd();
9678
9679 return 0;
9680
9681out_free_savedcmd:
9682 free_saved_cmdlines_buffer(savedcmd);
9683out_free_temp_buffer:
9684 ring_buffer_free(temp_buffer);
9685out_rm_hp_state:
9686 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
9687out_free_cpumask:
9688 free_cpumask_var(global_trace.tracing_cpumask);
9689out_free_buffer_mask:
9690 free_cpumask_var(tracing_buffer_mask);
9691out:
9692 return ret;
9693}
9694
9695void __init early_trace_init(void)
9696{
9697 if (tracepoint_printk) {
9698 tracepoint_print_iter =
9699 kzalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
9700 if (MEM_FAIL(!tracepoint_print_iter,
9701 "Failed to allocate trace iterator\n"))
9702 tracepoint_printk = 0;
9703 else
9704 static_key_enable(&tracepoint_printk_key.key);
9705 }
9706 tracer_alloc_buffers();
9707}
9708
9709void __init trace_init(void)
9710{
9711 trace_event_init();
9712}
9713
9714__init static int clear_boot_tracer(void)
9715{
9716 /*
9717 * The default tracer at boot buffer is an init section.
9718 * This function is called in lateinit. If we did not
9719 * find the boot tracer, then clear it out, to prevent
9720 * later registration from accessing the buffer that is
9721 * about to be freed.
9722 */
9723 if (!default_bootup_tracer)
9724 return 0;
9725
9726 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
9727 default_bootup_tracer);
9728 default_bootup_tracer = NULL;
9729
9730 return 0;
9731}
9732
9733fs_initcall(tracer_init_tracefs);
9734late_initcall_sync(clear_boot_tracer);
9735
9736#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
9737__init static int tracing_set_default_clock(void)
9738{
9739 /* sched_clock_stable() is determined in late_initcall */
9740 if (!trace_boot_clock && !sched_clock_stable()) {
9741 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9742 pr_warn("Can not set tracing clock due to lockdown\n");
9743 return -EPERM;
9744 }
9745
9746 printk(KERN_WARNING
9747 "Unstable clock detected, switching default tracing clock to \"global\"\n"
9748 "If you want to keep using the local clock, then add:\n"
9749 " \"trace_clock=local\"\n"
9750 "on the kernel command line\n");
9751 tracing_set_clock(&global_trace, "global");
9752 }
9753
9754 return 0;
9755}
9756late_initcall_sync(tracing_set_default_clock);
9757#endif