linux - kernel/trace/trace.c

4261 lines · 95.2 KiB

1	/*
2	* ring buffer based function tracer
3	*
4	* Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
5	* Copyright (C) 2008 Ingo Molnar <[email protected]>
6	*
7	* Originally taken from the RT patch by:
8	* Arnaldo Carvalho de Melo <[email protected]>
9	*
10	* Based on code from the latency_tracer, that is:
11	* Copyright (C) 2004-2006 Ingo Molnar
12	* Copyright (C) 2004 William Lee Irwin III
13	*/
14	#include <linux/ring_buffer.h>
15	#include <linux/utsrelease.h>
16	#include <linux/stacktrace.h>
17	#include <linux/writeback.h>
18	#include <linux/kallsyms.h>
19	#include <linux/seq_file.h>
20	#include <linux/notifier.h>
21	#include <linux/irqflags.h>
22	#include <linux/debugfs.h>
23	#include <linux/pagemap.h>
24	#include <linux/hardirq.h>
25	#include <linux/linkage.h>
26	#include <linux/uaccess.h>
27	#include <linux/kprobes.h>
28	#include <linux/ftrace.h>
29	#include <linux/module.h>
30	#include <linux/percpu.h>
31	#include <linux/splice.h>
32	#include <linux/kdebug.h>
33	#include <linux/string.h>
34	#include <linux/ctype.h>
35	#include <linux/init.h>
36	#include <linux/poll.h>
37	#include <linux/gfp.h>
38	#include <linux/fs.h>
39
40	#include "trace.h"
41	#include "trace_output.h"
42
43	#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
44
45	unsigned long __read_mostly tracing_max_latency;
46	unsigned long __read_mostly tracing_thresh;
47
48	/*
49	* On boot up, the ring buffer is set to the minimum size, so that
50	* we do not waste memory on systems that are not using tracing.
51	*/
52	static int ring_buffer_expanded;
53
54	/*
55	* We need to change this state when a selftest is running.
56	* A selftest will lurk into the ring-buffer to count the
57	* entries inserted during the selftest although some concurrent
58	* insertions into the ring-buffer such as trace_printk could occurred
59	* at the same time, giving false positive or negative results.
60	*/
61	static bool __read_mostly tracing_selftest_running;
62
63	/*
64	* If a tracer is running, we do not want to run SELFTEST.
65	*/
66	static bool __read_mostly tracing_selftest_disabled;
67
68	/* For tracers that don't implement custom flags */
69	static struct tracer_opt dummy_tracer_opt[] = {
70	{ }
71	};
72
73	static struct tracer_flags dummy_tracer_flags = {
74	.val = 0,
75	.opts = dummy_tracer_opt
76	};
77
78	static int dummy_set_flag(u32 old_flags, u32 bit, int set)
79	{
80	return 0;
81	}
82
83	/*
84	* Kill all tracing for good (never come back).
85	* It is initialized to 1 but will turn to zero if the initialization
86	* of the tracer is successful. But that is the only place that sets
87	* this back to zero.
88	*/
89	static int tracing_disabled = 1;
90
91	static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
92
93	static inline void ftrace_disable_cpu(void)
94	{
95	preempt_disable();
96	local_inc(&__get_cpu_var(ftrace_cpu_disabled));
97	}
98
99	static inline void ftrace_enable_cpu(void)
100	{
101	local_dec(&__get_cpu_var(ftrace_cpu_disabled));
102	preempt_enable();
103	}
104
105	static cpumask_var_t __read_mostly tracing_buffer_mask;
106
107	/* Define which cpu buffers are currently read in trace_pipe */
108	static cpumask_var_t tracing_reader_cpumask;
109
110	#define for_each_tracing_cpu(cpu) \
111	for_each_cpu(cpu, tracing_buffer_mask)
112
113	/*
114	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
115	*
116	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
117	* is set, then ftrace_dump is called. This will output the contents
118	* of the ftrace buffers to the console. This is very useful for
119	* capturing traces that lead to crashes and outputing it to a
120	* serial console.
121	*
122	* It is default off, but you can enable it with either specifying
123	* "ftrace_dump_on_oops" in the kernel command line, or setting
124	* /proc/sys/kernel/ftrace_dump_on_oops to true.
125	*/
126	int ftrace_dump_on_oops;
127
128	static int tracing_set_tracer(const char *buf);
129
130	#define BOOTUP_TRACER_SIZE 100
131	static char bootup_tracer_buf[BOOTUP_TRACER_SIZE] __initdata;
132	static char *default_bootup_tracer;
133
134	static int __init set_ftrace(char *str)
135	{
136	strncpy(bootup_tracer_buf, str, BOOTUP_TRACER_SIZE);
137	default_bootup_tracer = bootup_tracer_buf;
138	/* We are using ftrace early, expand it */
139	ring_buffer_expanded = 1;
140	return 1;
141	}
142	__setup("ftrace=", set_ftrace);
143
144	static int __init set_ftrace_dump_on_oops(char *str)
145	{
146	ftrace_dump_on_oops = 1;
147	return 1;
148	}
149	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
150
151	unsigned long long ns2usecs(cycle_t nsec)
152	{
153	nsec += 500;
154	do_div(nsec, 1000);
155	return nsec;
156	}
157
158	/*
159	* The global_trace is the descriptor that holds the tracing
160	* buffers for the live tracing. For each CPU, it contains
161	* a link list of pages that will store trace entries. The
162	* page descriptor of the pages in the memory is used to hold
163	* the link list by linking the lru item in the page descriptor
164	* to each of the pages in the buffer per CPU.
165	*
166	* For each active CPU there is a data field that holds the
167	* pages for the buffer for that CPU. Each CPU has the same number
168	* of pages allocated for its buffer.
169	*/
170	static struct trace_array global_trace;
171
172	static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
173
174	cycle_t ftrace_now(int cpu)
175	{
176	u64 ts;
177
178	/* Early boot up does not have a buffer yet */
179	if (!global_trace.buffer)
180	return trace_clock_local();
181
182	ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
183	ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
184
185	return ts;
186	}
187
188	/*
189	* The max_tr is used to snapshot the global_trace when a maximum
190	* latency is reached. Some tracers will use this to store a maximum
191	* trace while it continues examining live traces.
192	*
193	* The buffers for the max_tr are set up the same as the global_trace.
194	* When a snapshot is taken, the link list of the max_tr is swapped
195	* with the link list of the global_trace and the buffers are reset for
196	* the global_trace so the tracing can continue.
197	*/
198	static struct trace_array max_tr;
199
200	static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
201
202	/* tracer_enabled is used to toggle activation of a tracer */
203	static int tracer_enabled = 1;
204
205	/**
206	* tracing_is_enabled - return tracer_enabled status
207	*
208	* This function is used by other tracers to know the status
209	* of the tracer_enabled flag. Tracers may use this function
210	* to know if it should enable their features when starting
211	* up. See irqsoff tracer for an example (start_irqsoff_tracer).
212	*/
213	int tracing_is_enabled(void)
214	{
215	return tracer_enabled;
216	}
217
218	/*
219	* trace_buf_size is the size in bytes that is allocated
220	* for a buffer. Note, the number of bytes is always rounded
221	* to page size.
222	*
223	* This number is purposely set to a low number of 16384.
224	* If the dump on oops happens, it will be much appreciated
225	* to not have to wait for all that output. Anyway this can be
226	* boot time and run time configurable.
227	*/
228	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
229
230	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
231
232	/* trace_types holds a link list of available tracers. */
233	static struct tracer *trace_types __read_mostly;
234
235	/* current_trace points to the tracer that is currently active */
236	static struct tracer *current_trace __read_mostly;
237
238	/*
239	* max_tracer_type_len is used to simplify the allocating of
240	* buffers to read userspace tracer names. We keep track of
241	* the longest tracer name registered.
242	*/
243	static int max_tracer_type_len;
244
245	/*
246	* trace_types_lock is used to protect the trace_types list.
247	* This lock is also used to keep user access serialized.
248	* Accesses from userspace will grab this lock while userspace
249	* activities happen inside the kernel.
250	*/
251	static DEFINE_MUTEX(trace_types_lock);
252
253	/* trace_wait is a waitqueue for tasks blocked on trace_poll */
254	static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
255
256	/* trace_flags holds trace_options default values */
257	unsigned long trace_flags = TRACE_ITER_PRINT_PARENT \| TRACE_ITER_PRINTK \|
258	TRACE_ITER_ANNOTATE \| TRACE_ITER_CONTEXT_INFO \| TRACE_ITER_SLEEP_TIME;
259
260	/**
261	* trace_wake_up - wake up tasks waiting for trace input
262	*
263	* Simply wakes up any task that is blocked on the trace_wait
264	* queue. These is used with trace_poll for tasks polling the trace.
265	*/
266	void trace_wake_up(void)
267	{
268	/*
269	* The runqueue_is_locked() can fail, but this is the best we
270	* have for now:
271	*/
272	if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
273	wake_up(&trace_wait);
274	}
275
276	static int __init set_buf_size(char *str)
277	{
278	unsigned long buf_size;
279	int ret;
280
281	if (!str)
282	return 0;
283	ret = strict_strtoul(str, 0, &buf_size);
284	/* nr_entries can not be zero */
285	if (ret < 0 \|\| buf_size == 0)
286	return 0;
287	trace_buf_size = buf_size;
288	return 1;
289	}
290	__setup("trace_buf_size=", set_buf_size);
291
292	unsigned long nsecs_to_usecs(unsigned long nsecs)
293	{
294	return nsecs / 1000;
295	}
296
297	/* These must match the bit postions in trace_iterator_flags */
298	static const char *trace_options[] = {
299	"print-parent",
300	"sym-offset",
301	"sym-addr",
302	"verbose",
303	"raw",
304	"hex",
305	"bin",
306	"block",
307	"stacktrace",
308	"sched-tree",
309	"trace_printk",
310	"ftrace_preempt",
311	"branch",
312	"annotate",
313	"userstacktrace",
314	"sym-userobj",
315	"printk-msg-only",
316	"context-info",
317	"latency-format",
318	"global-clock",
319	"sleep-time",
320	NULL
321	};
322
323	/*
324	* ftrace_max_lock is used to protect the swapping of buffers
325	* when taking a max snapshot. The buffers themselves are
326	* protected by per_cpu spinlocks. But the action of the swap
327	* needs its own lock.
328	*
329	* This is defined as a raw_spinlock_t in order to help
330	* with performance when lockdep debugging is enabled.
331	*/
332	static raw_spinlock_t ftrace_max_lock =
333	(raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
334
335	/*
336	* Copy the new maximum trace into the separate maximum-trace
337	* structure. (this way the maximum trace is permanently saved,
338	* for later retrieval via /debugfs/tracing/latency_trace)
339	*/
340	static void
341	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
342	{
343	struct trace_array_cpu *data = tr->data[cpu];
344
345	max_tr.cpu = cpu;
346	max_tr.time_start = data->preempt_timestamp;
347
348	data = max_tr.data[cpu];
349	data->saved_latency = tracing_max_latency;
350
351	memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
352	data->pid = tsk->pid;
353	data->uid = task_uid(tsk);
354	data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
355	data->policy = tsk->policy;
356	data->rt_priority = tsk->rt_priority;
357
358	/* record this tasks comm */
359	tracing_record_cmdline(tsk);
360	}
361
362	ssize_t trace_seq_to_user(struct trace_seq s, char __user ubuf, size_t cnt)
363	{
364	int len;
365	int ret;
366
367	if (!cnt)
368	return 0;
369
370	if (s->len <= s->readpos)
371	return -EBUSY;
372
373	len = s->len - s->readpos;
374	if (cnt > len)
375	cnt = len;
376	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
377	if (ret == cnt)
378	return -EFAULT;
379
380	cnt -= ret;
381
382	s->readpos += cnt;
383	return cnt;
384	}
385
386	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
387	{
388	int len;
389	void *ret;
390
391	if (s->len <= s->readpos)
392	return -EBUSY;
393
394	len = s->len - s->readpos;
395	if (cnt > len)
396	cnt = len;
397	ret = memcpy(buf, s->buffer + s->readpos, cnt);
398	if (!ret)
399	return -EFAULT;
400
401	s->readpos += cnt;
402	return cnt;
403	}
404
405	static void
406	trace_print_seq(struct seq_file m, struct trace_seq s)
407	{
408	int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
409
410	s->buffer[len] = 0;
411	seq_puts(m, s->buffer);
412
413	trace_seq_init(s);
414	}
415
416	/**
417	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
418	* @tr: tracer
419	* @tsk: the task with the latency
420	* @cpu: The cpu that initiated the trace.
421	*
422	* Flip the buffers between the @tr and the max_tr and record information
423	* about which task was the cause of this latency.
424	*/
425	void
426	update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
427	{
428	struct ring_buffer *buf = tr->buffer;
429
430	WARN_ON_ONCE(!irqs_disabled());
431	__raw_spin_lock(&ftrace_max_lock);
432
433	tr->buffer = max_tr.buffer;
434	max_tr.buffer = buf;
435
436	ftrace_disable_cpu();
437	ring_buffer_reset(tr->buffer);
438	ftrace_enable_cpu();
439
440	__update_max_tr(tr, tsk, cpu);
441	__raw_spin_unlock(&ftrace_max_lock);
442	}
443
444	/**
445	* update_max_tr_single - only copy one trace over, and reset the rest
446	* @tr - tracer
447	* @tsk - task with the latency
448	* @cpu - the cpu of the buffer to copy.
449	*
450	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
451	*/
452	void
453	update_max_tr_single(struct trace_array tr, struct task_struct tsk, int cpu)
454	{
455	int ret;
456
457	WARN_ON_ONCE(!irqs_disabled());
458	__raw_spin_lock(&ftrace_max_lock);
459
460	ftrace_disable_cpu();
461
462	ring_buffer_reset(max_tr.buffer);
463	ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
464
465	ftrace_enable_cpu();
466
467	WARN_ON_ONCE(ret && ret != -EAGAIN);
468
469	__update_max_tr(tr, tsk, cpu);
470	__raw_spin_unlock(&ftrace_max_lock);
471	}
472
473	/**
474	* register_tracer - register a tracer with the ftrace system.
475	* @type - the plugin for the tracer
476	*
477	* Register a new plugin tracer.
478	*/
479	int register_tracer(struct tracer *type)
480	__releases(kernel_lock)
481	__acquires(kernel_lock)
482	{
483	struct tracer *t;
484	int len;
485	int ret = 0;
486
487	if (!type->name) {
488	pr_info("Tracer must have a name\n");
489	return -1;
490	}
491
492	/*
493	* When this gets called we hold the BKL which means that
494	* preemption is disabled. Various trace selftests however
495	* need to disable and enable preemption for successful tests.
496	* So we drop the BKL here and grab it after the tests again.
497	*/
498	unlock_kernel();
499	mutex_lock(&trace_types_lock);
500
501	tracing_selftest_running = true;
502
503	for (t = trace_types; t; t = t->next) {
504	if (strcmp(type->name, t->name) == 0) {
505	/* already found */
506	pr_info("Trace %s already registered\n",
507	type->name);
508	ret = -1;
509	goto out;
510	}
511	}
512
513	if (!type->set_flag)
514	type->set_flag = &dummy_set_flag;
515	if (!type->flags)
516	type->flags = &dummy_tracer_flags;
517	else
518	if (!type->flags->opts)
519	type->flags->opts = dummy_tracer_opt;
520	if (!type->wait_pipe)
521	type->wait_pipe = default_wait_pipe;
522
523
524	#ifdef CONFIG_FTRACE_STARTUP_TEST
525	if (type->selftest && !tracing_selftest_disabled) {
526	struct tracer *saved_tracer = current_trace;
527	struct trace_array *tr = &global_trace;
528	int i;
529
530	/*
531	* Run a selftest on this tracer.
532	* Here we reset the trace buffer, and set the current
533	* tracer to be this tracer. The tracer can then run some
534	* internal tracing to verify that everything is in order.
535	* If we fail, we do not register this tracer.
536	*/
537	for_each_tracing_cpu(i)
538	tracing_reset(tr, i);
539
540	current_trace = type;
541	/* the test is responsible for initializing and enabling */
542	pr_info("Testing tracer %s: ", type->name);
543	ret = type->selftest(type, tr);
544	/* the test is responsible for resetting too */
545	current_trace = saved_tracer;
546	if (ret) {
547	printk(KERN_CONT "FAILED!\n");
548	goto out;
549	}
550	/* Only reset on passing, to avoid touching corrupted buffers */
551	for_each_tracing_cpu(i)
552	tracing_reset(tr, i);
553
554	printk(KERN_CONT "PASSED\n");
555	}
556	#endif
557
558	type->next = trace_types;
559	trace_types = type;
560	len = strlen(type->name);
561	if (len > max_tracer_type_len)
562	max_tracer_type_len = len;
563
564	out:
565	tracing_selftest_running = false;
566	mutex_unlock(&trace_types_lock);
567
568	if (ret \|\| !default_bootup_tracer)
569	goto out_unlock;
570
571	if (strncmp(default_bootup_tracer, type->name, BOOTUP_TRACER_SIZE))
572	goto out_unlock;
573
574	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
575	/* Do we want this tracer to start on bootup? */
576	tracing_set_tracer(type->name);
577	default_bootup_tracer = NULL;
578	/* disable other selftests, since this will break it. */
579	tracing_selftest_disabled = 1;
580	#ifdef CONFIG_FTRACE_STARTUP_TEST
581	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
582	type->name);
583	#endif
584
585	out_unlock:
586	lock_kernel();
587	return ret;
588	}
589
590	void unregister_tracer(struct tracer *type)
591	{
592	struct tracer **t;
593	int len;
594
595	mutex_lock(&trace_types_lock);
596	for (t = &trace_types; t; t = &(t)->next) {
597	if (*t == type)
598	goto found;
599	}
600	pr_info("Trace %s not registered\n", type->name);
601	goto out;
602
603	found:
604	t = (t)->next;
605
606	if (type == current_trace && tracer_enabled) {
607	tracer_enabled = 0;
608	tracing_stop();
609	if (current_trace->stop)
610	current_trace->stop(&global_trace);
611	current_trace = &nop_trace;
612	}
613
614	if (strlen(type->name) != max_tracer_type_len)
615	goto out;
616
617	max_tracer_type_len = 0;
618	for (t = &trace_types; t; t = &(t)->next) {
619	len = strlen((*t)->name);
620	if (len > max_tracer_type_len)
621	max_tracer_type_len = len;
622	}
623	out:
624	mutex_unlock(&trace_types_lock);
625	}
626
627	void tracing_reset(struct trace_array *tr, int cpu)
628	{
629	ftrace_disable_cpu();
630	ring_buffer_reset_cpu(tr->buffer, cpu);
631	ftrace_enable_cpu();
632	}
633
634	void tracing_reset_online_cpus(struct trace_array *tr)
635	{
636	int cpu;
637
638	tr->time_start = ftrace_now(tr->cpu);
639
640	for_each_online_cpu(cpu)
641	tracing_reset(tr, cpu);
642	}
643
644	#define SAVED_CMDLINES 128
645	#define NO_CMDLINE_MAP UINT_MAX
646	static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
647	static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
648	static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
649	static int cmdline_idx;
650	static raw_spinlock_t trace_cmdline_lock = __RAW_SPIN_LOCK_UNLOCKED;
651
652	/* temporary disable recording */
653	static atomic_t trace_record_cmdline_disabled __read_mostly;
654
655	static void trace_init_cmdlines(void)
656	{
657	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
658	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
659	cmdline_idx = 0;
660	}
661
662	static int trace_stop_count;
663	static DEFINE_SPINLOCK(tracing_start_lock);
664
665	/**
666	* ftrace_off_permanent - disable all ftrace code permanently
667	*
668	* This should only be called when a serious anomally has
669	* been detected. This will turn off the function tracing,
670	* ring buffers, and other tracing utilites. It takes no
671	* locks and can be called from any context.
672	*/
673	void ftrace_off_permanent(void)
674	{
675	tracing_disabled = 1;
676	ftrace_stop();
677	tracing_off_permanent();
678	}
679
680	/**
681	* tracing_start - quick start of the tracer
682	*
683	* If tracing is enabled but was stopped by tracing_stop,
684	* this will start the tracer back up.
685	*/
686	void tracing_start(void)
687	{
688	struct ring_buffer *buffer;
689	unsigned long flags;
690
691	if (tracing_disabled)
692	return;
693
694	spin_lock_irqsave(&tracing_start_lock, flags);
695	if (--trace_stop_count) {
696	if (trace_stop_count < 0) {
697	/* Someone screwed up their debugging */
698	WARN_ON_ONCE(1);
699	trace_stop_count = 0;
700	}
701	goto out;
702	}
703
704
705	buffer = global_trace.buffer;
706	if (buffer)
707	ring_buffer_record_enable(buffer);
708
709	buffer = max_tr.buffer;
710	if (buffer)
711	ring_buffer_record_enable(buffer);
712
713	ftrace_start();
714	out:
715	spin_unlock_irqrestore(&tracing_start_lock, flags);
716	}
717
718	/**
719	* tracing_stop - quick stop of the tracer
720	*
721	* Light weight way to stop tracing. Use in conjunction with
722	* tracing_start.
723	*/
724	void tracing_stop(void)
725	{
726	struct ring_buffer *buffer;
727	unsigned long flags;
728
729	ftrace_stop();
730	spin_lock_irqsave(&tracing_start_lock, flags);
731	if (trace_stop_count++)
732	goto out;
733
734	buffer = global_trace.buffer;
735	if (buffer)
736	ring_buffer_record_disable(buffer);
737
738	buffer = max_tr.buffer;
739	if (buffer)
740	ring_buffer_record_disable(buffer);
741
742	out:
743	spin_unlock_irqrestore(&tracing_start_lock, flags);
744	}
745
746	void trace_stop_cmdline_recording(void);
747
748	static void trace_save_cmdline(struct task_struct *tsk)
749	{
750	unsigned pid, idx;
751
752	if (!tsk->pid \|\| unlikely(tsk->pid > PID_MAX_DEFAULT))
753	return;
754
755	/*
756	* It's not the end of the world if we don't get
757	* the lock, but we also don't want to spin
758	* nor do we want to disable interrupts,
759	* so if we miss here, then better luck next time.
760	*/
761	if (!__raw_spin_trylock(&trace_cmdline_lock))
762	return;
763
764	idx = map_pid_to_cmdline[tsk->pid];
765	if (idx == NO_CMDLINE_MAP) {
766	idx = (cmdline_idx + 1) % SAVED_CMDLINES;
767
768	/*
769	* Check whether the cmdline buffer at idx has a pid
770	* mapped. We are going to overwrite that entry so we
771	* need to clear the map_pid_to_cmdline. Otherwise we
772	* would read the new comm for the old pid.
773	*/
774	pid = map_cmdline_to_pid[idx];
775	if (pid != NO_CMDLINE_MAP)
776	map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
777
778	map_cmdline_to_pid[idx] = tsk->pid;
779	map_pid_to_cmdline[tsk->pid] = idx;
780
781	cmdline_idx = idx;
782	}
783
784	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
785
786	__raw_spin_unlock(&trace_cmdline_lock);
787	}
788
789	void trace_find_cmdline(int pid, char comm[])
790	{
791	unsigned map;
792
793	if (!pid) {
794	strcpy(comm, "<idle>");
795	return;
796	}
797
798	if (pid > PID_MAX_DEFAULT) {
799	strcpy(comm, "<...>");
800	return;
801	}
802
803	__raw_spin_lock(&trace_cmdline_lock);
804	map = map_pid_to_cmdline[pid];
805	if (map != NO_CMDLINE_MAP)
806	strcpy(comm, saved_cmdlines[map]);
807	else
808	strcpy(comm, "<...>");
809
810	__raw_spin_unlock(&trace_cmdline_lock);
811	}
812
813	void tracing_record_cmdline(struct task_struct *tsk)
814	{
815	if (atomic_read(&trace_record_cmdline_disabled) \|\| !tracer_enabled \|\|
816	!tracing_is_on())
817	return;
818
819	trace_save_cmdline(tsk);
820	}
821
822	void
823	tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
824	int pc)
825	{
826	struct task_struct *tsk = current;
827
828	entry->preempt_count = pc & 0xff;
829	entry->pid = (tsk) ? tsk->pid : 0;
830	entry->tgid = (tsk) ? tsk->tgid : 0;
831	entry->flags =
832	#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
833	(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) \|
834	#else
835	TRACE_FLAG_IRQS_NOSUPPORT \|
836	#endif
837	((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) \|
838	((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) \|
839	(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
840	}
841
842	struct ring_buffer_event trace_buffer_lock_reserve(struct trace_array tr,
843	unsigned char type,
844	unsigned long len,
845	unsigned long flags, int pc)
846	{
847	struct ring_buffer_event *event;
848
849	event = ring_buffer_lock_reserve(tr->buffer, len);
850	if (event != NULL) {
851	struct trace_entry *ent = ring_buffer_event_data(event);
852
853	tracing_generic_entry_update(ent, flags, pc);
854	ent->type = type;
855	}
856
857	return event;
858	}
859	static void ftrace_trace_stack(struct trace_array *tr,
860	unsigned long flags, int skip, int pc);
861	static void ftrace_trace_userstack(struct trace_array *tr,
862	unsigned long flags, int pc);
863
864	static inline void __trace_buffer_unlock_commit(struct trace_array *tr,
865	struct ring_buffer_event *event,
866	unsigned long flags, int pc,
867	int wake)
868	{
869	ring_buffer_unlock_commit(tr->buffer, event);
870
871	ftrace_trace_stack(tr, flags, 6, pc);
872	ftrace_trace_userstack(tr, flags, pc);
873
874	if (wake)
875	trace_wake_up();
876	}
877
878	void trace_buffer_unlock_commit(struct trace_array *tr,
879	struct ring_buffer_event *event,
880	unsigned long flags, int pc)
881	{
882	__trace_buffer_unlock_commit(tr, event, flags, pc, 1);
883	}
884
885	struct ring_buffer_event *
886	trace_current_buffer_lock_reserve(unsigned char type, unsigned long len,
887	unsigned long flags, int pc)
888	{
889	return trace_buffer_lock_reserve(&global_trace,
890	type, len, flags, pc);
891	}
892
893	void trace_current_buffer_unlock_commit(struct ring_buffer_event *event,
894	unsigned long flags, int pc)
895	{
896	return __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 1);
897	}
898
899	void trace_nowake_buffer_unlock_commit(struct ring_buffer_event *event,
900	unsigned long flags, int pc)
901	{
902	return __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 0);
903	}
904
905	void
906	trace_function(struct trace_array *tr,
907	unsigned long ip, unsigned long parent_ip, unsigned long flags,
908	int pc)
909	{
910	struct ring_buffer_event *event;
911	struct ftrace_entry *entry;
912
913	/* If we are reading the ring buffer, don't trace */
914	if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
915	return;
916
917	event = trace_buffer_lock_reserve(tr, TRACE_FN, sizeof(*entry),
918	flags, pc);
919	if (!event)
920	return;
921	entry = ring_buffer_event_data(event);
922	entry->ip = ip;
923	entry->parent_ip = parent_ip;
924	ring_buffer_unlock_commit(tr->buffer, event);
925	}
926
927	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
928	static int __trace_graph_entry(struct trace_array *tr,
929	struct ftrace_graph_ent *trace,
930	unsigned long flags,
931	int pc)
932	{
933	struct ring_buffer_event *event;
934	struct ftrace_graph_ent_entry *entry;
935
936	if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
937	return 0;
938
939	event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_ENT,
940	sizeof(*entry), flags, pc);
941	if (!event)
942	return 0;
943	entry = ring_buffer_event_data(event);
944	entry->graph_ent = *trace;
945	ring_buffer_unlock_commit(global_trace.buffer, event);
946
947	return 1;
948	}
949
950	static void __trace_graph_return(struct trace_array *tr,
951	struct ftrace_graph_ret *trace,
952	unsigned long flags,
953	int pc)
954	{
955	struct ring_buffer_event *event;
956	struct ftrace_graph_ret_entry *entry;
957
958	if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
959	return;
960
961	event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_RET,
962	sizeof(*entry), flags, pc);
963	if (!event)
964	return;
965	entry = ring_buffer_event_data(event);
966	entry->ret = *trace;
967	ring_buffer_unlock_commit(global_trace.buffer, event);
968	}
969	#endif
970
971	void
972	ftrace(struct trace_array tr, struct trace_array_cpu data,
973	unsigned long ip, unsigned long parent_ip, unsigned long flags,
974	int pc)
975	{
976	if (likely(!atomic_read(&data->disabled)))
977	trace_function(tr, ip, parent_ip, flags, pc);
978	}
979
980	static void __ftrace_trace_stack(struct trace_array *tr,
981	unsigned long flags,
982	int skip, int pc)
983	{
984	#ifdef CONFIG_STACKTRACE
985	struct ring_buffer_event *event;
986	struct stack_entry *entry;
987	struct stack_trace trace;
988
989	event = trace_buffer_lock_reserve(tr, TRACE_STACK,
990	sizeof(*entry), flags, pc);
991	if (!event)
992	return;
993	entry = ring_buffer_event_data(event);
994	memset(&entry->caller, 0, sizeof(entry->caller));
995
996	trace.nr_entries = 0;
997	trace.max_entries = FTRACE_STACK_ENTRIES;
998	trace.skip = skip;
999	trace.entries = entry->caller;
1000
1001	save_stack_trace(&trace);
1002	ring_buffer_unlock_commit(tr->buffer, event);
1003	#endif
1004	}
1005
1006	static void ftrace_trace_stack(struct trace_array *tr,
1007	unsigned long flags,
1008	int skip, int pc)
1009	{
1010	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1011	return;
1012
1013	__ftrace_trace_stack(tr, flags, skip, pc);
1014	}
1015
1016	void __trace_stack(struct trace_array *tr,
1017	unsigned long flags,
1018	int skip, int pc)
1019	{
1020	__ftrace_trace_stack(tr, flags, skip, pc);
1021	}
1022
1023	static void ftrace_trace_userstack(struct trace_array *tr,
1024	unsigned long flags, int pc)
1025	{
1026	#ifdef CONFIG_STACKTRACE
1027	struct ring_buffer_event *event;
1028	struct userstack_entry *entry;
1029	struct stack_trace trace;
1030
1031	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1032	return;
1033
1034	event = trace_buffer_lock_reserve(tr, TRACE_USER_STACK,
1035	sizeof(*entry), flags, pc);
1036	if (!event)
1037	return;
1038	entry = ring_buffer_event_data(event);
1039
1040	memset(&entry->caller, 0, sizeof(entry->caller));
1041
1042	trace.nr_entries = 0;
1043	trace.max_entries = FTRACE_STACK_ENTRIES;
1044	trace.skip = 0;
1045	trace.entries = entry->caller;
1046
1047	save_stack_trace_user(&trace);
1048	ring_buffer_unlock_commit(tr->buffer, event);
1049	#endif
1050	}
1051
1052	#ifdef UNUSED
1053	static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1054	{
1055	ftrace_trace_userstack(tr, flags, preempt_count());
1056	}
1057	#endif /* UNUSED */
1058
1059	static void
1060	ftrace_trace_special(void *__tr,
1061	unsigned long arg1, unsigned long arg2, unsigned long arg3,
1062	int pc)
1063	{
1064	struct ring_buffer_event *event;
1065	struct trace_array *tr = __tr;
1066	struct special_entry *entry;
1067
1068	event = trace_buffer_lock_reserve(tr, TRACE_SPECIAL,
1069	sizeof(*entry), 0, pc);
1070	if (!event)
1071	return;
1072	entry = ring_buffer_event_data(event);
1073	entry->arg1 = arg1;
1074	entry->arg2 = arg2;
1075	entry->arg3 = arg3;
1076	trace_buffer_unlock_commit(tr, event, 0, pc);
1077	}
1078
1079	void
1080	__trace_special(void __tr, void __data,
1081	unsigned long arg1, unsigned long arg2, unsigned long arg3)
1082	{
1083	ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
1084	}
1085
1086	void
1087	tracing_sched_switch_trace(struct trace_array *tr,
1088	struct task_struct *prev,
1089	struct task_struct *next,
1090	unsigned long flags, int pc)
1091	{
1092	struct ring_buffer_event *event;
1093	struct ctx_switch_entry *entry;
1094
1095	event = trace_buffer_lock_reserve(tr, TRACE_CTX,
1096	sizeof(*entry), flags, pc);
1097	if (!event)
1098	return;
1099	entry = ring_buffer_event_data(event);
1100	entry->prev_pid = prev->pid;
1101	entry->prev_prio = prev->prio;
1102	entry->prev_state = prev->state;
1103	entry->next_pid = next->pid;
1104	entry->next_prio = next->prio;
1105	entry->next_state = next->state;
1106	entry->next_cpu = task_cpu(next);
1107	trace_buffer_unlock_commit(tr, event, flags, pc);
1108	}
1109
1110	void
1111	tracing_sched_wakeup_trace(struct trace_array *tr,
1112	struct task_struct *wakee,
1113	struct task_struct *curr,
1114	unsigned long flags, int pc)
1115	{
1116	struct ring_buffer_event *event;
1117	struct ctx_switch_entry *entry;
1118
1119	event = trace_buffer_lock_reserve(tr, TRACE_WAKE,
1120	sizeof(*entry), flags, pc);
1121	if (!event)
1122	return;
1123	entry = ring_buffer_event_data(event);
1124	entry->prev_pid = curr->pid;
1125	entry->prev_prio = curr->prio;
1126	entry->prev_state = curr->state;
1127	entry->next_pid = wakee->pid;
1128	entry->next_prio = wakee->prio;
1129	entry->next_state = wakee->state;
1130	entry->next_cpu = task_cpu(wakee);
1131
1132	ring_buffer_unlock_commit(tr->buffer, event);
1133	ftrace_trace_stack(tr, flags, 6, pc);
1134	ftrace_trace_userstack(tr, flags, pc);
1135	}
1136
1137	void
1138	ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1139	{
1140	struct trace_array *tr = &global_trace;
1141	struct trace_array_cpu *data;
1142	unsigned long flags;
1143	int cpu;
1144	int pc;
1145
1146	if (tracing_disabled)
1147	return;
1148
1149	pc = preempt_count();
1150	local_irq_save(flags);
1151	cpu = raw_smp_processor_id();
1152	data = tr->data[cpu];
1153
1154	if (likely(atomic_inc_return(&data->disabled) == 1))
1155	ftrace_trace_special(tr, arg1, arg2, arg3, pc);
1156
1157	atomic_dec(&data->disabled);
1158	local_irq_restore(flags);
1159	}
1160
1161	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1162	int trace_graph_entry(struct ftrace_graph_ent *trace)
1163	{
1164	struct trace_array *tr = &global_trace;
1165	struct trace_array_cpu *data;
1166	unsigned long flags;
1167	long disabled;
1168	int ret;
1169	int cpu;
1170	int pc;
1171
1172	if (!ftrace_trace_task(current))
1173	return 0;
1174
1175	if (!ftrace_graph_addr(trace->func))
1176	return 0;
1177
1178	local_irq_save(flags);
1179	cpu = raw_smp_processor_id();
1180	data = tr->data[cpu];
1181	disabled = atomic_inc_return(&data->disabled);
1182	if (likely(disabled == 1)) {
1183	pc = preempt_count();
1184	ret = __trace_graph_entry(tr, trace, flags, pc);
1185	} else {
1186	ret = 0;
1187	}
1188	/* Only do the atomic if it is not already set */
1189	if (!test_tsk_trace_graph(current))
1190	set_tsk_trace_graph(current);
1191
1192	atomic_dec(&data->disabled);
1193	local_irq_restore(flags);
1194
1195	return ret;
1196	}
1197
1198	void trace_graph_return(struct ftrace_graph_ret *trace)
1199	{
1200	struct trace_array *tr = &global_trace;
1201	struct trace_array_cpu *data;
1202	unsigned long flags;
1203	long disabled;
1204	int cpu;
1205	int pc;
1206
1207	local_irq_save(flags);
1208	cpu = raw_smp_processor_id();
1209	data = tr->data[cpu];
1210	disabled = atomic_inc_return(&data->disabled);
1211	if (likely(disabled == 1)) {
1212	pc = preempt_count();
1213	__trace_graph_return(tr, trace, flags, pc);
1214	}
1215	if (!trace->depth)
1216	clear_tsk_trace_graph(current);
1217	atomic_dec(&data->disabled);
1218	local_irq_restore(flags);
1219	}
1220	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1221
1222
1223	/**
1224	* trace_vbprintk - write binary msg to tracing buffer
1225	*
1226	*/
1227	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1228	{
1229	static raw_spinlock_t trace_buf_lock =
1230	(raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
1231	static u32 trace_buf[TRACE_BUF_SIZE];
1232
1233	struct ring_buffer_event *event;
1234	struct trace_array *tr = &global_trace;
1235	struct trace_array_cpu *data;
1236	struct bprint_entry *entry;
1237	unsigned long flags;
1238	int resched;
1239	int cpu, len = 0, size, pc;
1240
1241	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
1242	return 0;
1243
1244	/* Don't pollute graph traces with trace_vprintk internals */
1245	pause_graph_tracing();
1246
1247	pc = preempt_count();
1248	resched = ftrace_preempt_disable();
1249	cpu = raw_smp_processor_id();
1250	data = tr->data[cpu];
1251
1252	if (unlikely(atomic_read(&data->disabled)))
1253	goto out;
1254
1255	/* Lockdep uses trace_printk for lock tracing */
1256	local_irq_save(flags);
1257	__raw_spin_lock(&trace_buf_lock);
1258	len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1259
1260	if (len > TRACE_BUF_SIZE \|\| len < 0)
1261	goto out_unlock;
1262
1263	size = sizeof(entry) + sizeof(u32) len;
1264	event = trace_buffer_lock_reserve(tr, TRACE_BPRINT, size, flags, pc);
1265	if (!event)
1266	goto out_unlock;
1267	entry = ring_buffer_event_data(event);
1268	entry->ip = ip;
1269	entry->fmt = fmt;
1270
1271	memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1272	ring_buffer_unlock_commit(tr->buffer, event);
1273
1274	out_unlock:
1275	__raw_spin_unlock(&trace_buf_lock);
1276	local_irq_restore(flags);
1277
1278	out:
1279	ftrace_preempt_enable(resched);
1280	unpause_graph_tracing();
1281
1282	return len;
1283	}
1284	EXPORT_SYMBOL_GPL(trace_vbprintk);
1285
1286	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1287	{
1288	static raw_spinlock_t trace_buf_lock = __RAW_SPIN_LOCK_UNLOCKED;
1289	static char trace_buf[TRACE_BUF_SIZE];
1290
1291	struct ring_buffer_event *event;
1292	struct trace_array *tr = &global_trace;
1293	struct trace_array_cpu *data;
1294	int cpu, len = 0, size, pc;
1295	struct print_entry *entry;
1296	unsigned long irq_flags;
1297
1298	if (tracing_disabled \|\| tracing_selftest_running)
1299	return 0;
1300
1301	pc = preempt_count();
1302	preempt_disable_notrace();
1303	cpu = raw_smp_processor_id();
1304	data = tr->data[cpu];
1305
1306	if (unlikely(atomic_read(&data->disabled)))
1307	goto out;
1308
1309	pause_graph_tracing();
1310	raw_local_irq_save(irq_flags);
1311	__raw_spin_lock(&trace_buf_lock);
1312	len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1313
1314	len = min(len, TRACE_BUF_SIZE-1);
1315	trace_buf[len] = 0;
1316
1317	size = sizeof(*entry) + len + 1;
1318	event = trace_buffer_lock_reserve(tr, TRACE_PRINT, size, irq_flags, pc);
1319	if (!event)
1320	goto out_unlock;
1321	entry = ring_buffer_event_data(event);
1322	entry->ip = ip;
1323
1324	memcpy(&entry->buf, trace_buf, len);
1325	entry->buf[len] = 0;
1326	ring_buffer_unlock_commit(tr->buffer, event);
1327
1328	out_unlock:
1329	__raw_spin_unlock(&trace_buf_lock);
1330	raw_local_irq_restore(irq_flags);
1331	unpause_graph_tracing();
1332	out:
1333	preempt_enable_notrace();
1334
1335	return len;
1336	}
1337	EXPORT_SYMBOL_GPL(trace_vprintk);
1338
1339	enum trace_file_type {
1340	TRACE_FILE_LAT_FMT = 1,
1341	TRACE_FILE_ANNOTATE = 2,
1342	};
1343
1344	static void trace_iterator_increment(struct trace_iterator *iter)
1345	{
1346	/* Don't allow ftrace to trace into the ring buffers */
1347	ftrace_disable_cpu();
1348
1349	iter->idx++;
1350	if (iter->buffer_iter[iter->cpu])
1351	ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1352
1353	ftrace_enable_cpu();
1354	}
1355
1356	static struct trace_entry *
1357	peek_next_entry(struct trace_iterator iter, int cpu, u64 ts)
1358	{
1359	struct ring_buffer_event *event;
1360	struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1361
1362	/* Don't allow ftrace to trace into the ring buffers */
1363	ftrace_disable_cpu();
1364
1365	if (buf_iter)
1366	event = ring_buffer_iter_peek(buf_iter, ts);
1367	else
1368	event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1369
1370	ftrace_enable_cpu();
1371
1372	return event ? ring_buffer_event_data(event) : NULL;
1373	}
1374
1375	static struct trace_entry *
1376	__find_next_entry(struct trace_iterator iter, int ent_cpu, u64 *ent_ts)
1377	{
1378	struct ring_buffer *buffer = iter->tr->buffer;
1379	struct trace_entry ent, next = NULL;
1380	int cpu_file = iter->cpu_file;
1381	u64 next_ts = 0, ts;
1382	int next_cpu = -1;
1383	int cpu;
1384
1385	/*
1386	* If we are in a per_cpu trace file, don't bother by iterating over
1387	* all cpu and peek directly.
1388	*/
1389	if (cpu_file > TRACE_PIPE_ALL_CPU) {
1390	if (ring_buffer_empty_cpu(buffer, cpu_file))
1391	return NULL;
1392	ent = peek_next_entry(iter, cpu_file, ent_ts);
1393	if (ent_cpu)
1394	*ent_cpu = cpu_file;
1395
1396	return ent;
1397	}
1398
1399	for_each_tracing_cpu(cpu) {
1400
1401	if (ring_buffer_empty_cpu(buffer, cpu))
1402	continue;
1403
1404	ent = peek_next_entry(iter, cpu, &ts);
1405
1406	/*
1407	* Pick the entry with the smallest timestamp:
1408	*/
1409	if (ent && (!next \|\| ts < next_ts)) {
1410	next = ent;
1411	next_cpu = cpu;
1412	next_ts = ts;
1413	}
1414	}
1415
1416	if (ent_cpu)
1417	*ent_cpu = next_cpu;
1418
1419	if (ent_ts)
1420	*ent_ts = next_ts;
1421
1422	return next;
1423	}
1424
1425	/* Find the next real entry, without updating the iterator itself */
1426	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
1427	int ent_cpu, u64 ent_ts)
1428	{
1429	return __find_next_entry(iter, ent_cpu, ent_ts);
1430	}
1431
1432	/* Find the next real entry, and increment the iterator to the next entry */
1433	static void find_next_entry_inc(struct trace_iterator iter)
1434	{
1435	iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1436
1437	if (iter->ent)
1438	trace_iterator_increment(iter);
1439
1440	return iter->ent ? iter : NULL;
1441	}
1442
1443	static void trace_consume(struct trace_iterator *iter)
1444	{
1445	/* Don't allow ftrace to trace into the ring buffers */
1446	ftrace_disable_cpu();
1447	ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1448	ftrace_enable_cpu();
1449	}
1450
1451	static void s_next(struct seq_file m, void v, loff_t pos)
1452	{
1453	struct trace_iterator *iter = m->private;
1454	int i = (int)*pos;
1455	void *ent;
1456
1457	(*pos)++;
1458
1459	/* can't go backwards */
1460	if (iter->idx > i)
1461	return NULL;
1462
1463	if (iter->idx < 0)
1464	ent = find_next_entry_inc(iter);
1465	else
1466	ent = iter;
1467
1468	while (ent && iter->idx < i)
1469	ent = find_next_entry_inc(iter);
1470
1471	iter->pos = *pos;
1472
1473	return ent;
1474	}
1475
1476	/*
1477	* No necessary locking here. The worst thing which can
1478	* happen is loosing events consumed at the same time
1479	* by a trace_pipe reader.
1480	* Other than that, we don't risk to crash the ring buffer
1481	* because it serializes the readers.
1482	*
1483	* The current tracer is copied to avoid a global locking
1484	* all around.
1485	*/
1486	static void s_start(struct seq_file m, loff_t *pos)
1487	{
1488	struct trace_iterator *iter = m->private;
1489	static struct tracer *old_tracer;
1490	int cpu_file = iter->cpu_file;
1491	void *p = NULL;
1492	loff_t l = 0;
1493	int cpu;
1494
1495	/* copy the tracer to avoid using a global lock all around */
1496	mutex_lock(&trace_types_lock);
1497	if (unlikely(old_tracer != current_trace && current_trace)) {
1498	old_tracer = current_trace;
1499	iter->trace = current_trace;
1500	}
1501	mutex_unlock(&trace_types_lock);
1502
1503	atomic_inc(&trace_record_cmdline_disabled);
1504
1505	if (*pos != iter->pos) {
1506	iter->ent = NULL;
1507	iter->cpu = 0;
1508	iter->idx = -1;
1509
1510	ftrace_disable_cpu();
1511
1512	if (cpu_file == TRACE_PIPE_ALL_CPU) {
1513	for_each_tracing_cpu(cpu)
1514	ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1515	} else
1516	ring_buffer_iter_reset(iter->buffer_iter[cpu_file]);
1517
1518
1519	ftrace_enable_cpu();
1520
1521	for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1522	;
1523
1524	} else {
1525	l = *pos - 1;
1526	p = s_next(m, p, &l);
1527	}
1528
1529	return p;
1530	}
1531
1532	static void s_stop(struct seq_file m, void p)
1533	{
1534	atomic_dec(&trace_record_cmdline_disabled);
1535	}
1536
1537	static void print_lat_help_header(struct seq_file *m)
1538	{
1539	seq_puts(m, "# _------=> CPU# \n");
1540	seq_puts(m, "# / _-----=> irqs-off \n");
1541	seq_puts(m, "# \| / _----=> need-resched \n");
1542	seq_puts(m, "# \|\| / _---=> hardirq/softirq \n");
1543	seq_puts(m, "# \|\|\| / _--=> preempt-depth \n");
1544	seq_puts(m, "# \|\|\|\| / \n");
1545	seq_puts(m, "# \|\|\|\|\| delay \n");
1546	seq_puts(m, "# cmd pid \|\|\|\|\| time \| caller \n");
1547	seq_puts(m, "# \\ / \|\|\|\|\| \\ \| / \n");
1548	}
1549
1550	static void print_func_help_header(struct seq_file *m)
1551	{
1552	seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1553	seq_puts(m, "# \| \| \| \| \|\n");
1554	}
1555
1556
1557	static void
1558	print_trace_header(struct seq_file m, struct trace_iterator iter)
1559	{
1560	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1561	struct trace_array *tr = iter->tr;
1562	struct trace_array_cpu *data = tr->data[tr->cpu];
1563	struct tracer *type = current_trace;
1564	unsigned long total;
1565	unsigned long entries;
1566	const char *name = "preemption";
1567
1568	if (type)
1569	name = type->name;
1570
1571	entries = ring_buffer_entries(iter->tr->buffer);
1572	total = entries +
1573	ring_buffer_overruns(iter->tr->buffer);
1574
1575	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1576	name, UTS_RELEASE);
1577	seq_puts(m, "# -----------------------------------"
1578	"---------------------------------\n");
1579	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d \|"
1580	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1581	nsecs_to_usecs(data->saved_latency),
1582	entries,
1583	total,
1584	tr->cpu,
1585	#if defined(CONFIG_PREEMPT_NONE)
1586	"server",
1587	#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1588	"desktop",
1589	#elif defined(CONFIG_PREEMPT)
1590	"preempt",
1591	#else
1592	"unknown",
1593	#endif
1594	/* These are reserved for later use */
1595	0, 0, 0, 0);
1596	#ifdef CONFIG_SMP
1597	seq_printf(m, " #P:%d)\n", num_online_cpus());
1598	#else
1599	seq_puts(m, ")\n");
1600	#endif
1601	seq_puts(m, "# -----------------\n");
1602	seq_printf(m, "# \| task: %.16s-%d "
1603	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1604	data->comm, data->pid, data->uid, data->nice,
1605	data->policy, data->rt_priority);
1606	seq_puts(m, "# -----------------\n");
1607
1608	if (data->critical_start) {
1609	seq_puts(m, "# => started at: ");
1610	seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1611	trace_print_seq(m, &iter->seq);
1612	seq_puts(m, "\n# => ended at: ");
1613	seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1614	trace_print_seq(m, &iter->seq);
1615	seq_puts(m, "#\n");
1616	}
1617
1618	seq_puts(m, "#\n");
1619	}
1620
1621	static void test_cpu_buff_start(struct trace_iterator *iter)
1622	{
1623	struct trace_seq *s = &iter->seq;
1624
1625	if (!(trace_flags & TRACE_ITER_ANNOTATE))
1626	return;
1627
1628	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1629	return;
1630
1631	if (cpumask_test_cpu(iter->cpu, iter->started))
1632	return;
1633
1634	cpumask_set_cpu(iter->cpu, iter->started);
1635
1636	/* Don't print started cpu buffer for the first entry of the trace */
1637	if (iter->idx > 1)
1638	trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1639	iter->cpu);
1640	}
1641
1642	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1643	{
1644	struct trace_seq *s = &iter->seq;
1645	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1646	struct trace_entry *entry;
1647	struct trace_event *event;
1648
1649	entry = iter->ent;
1650
1651	test_cpu_buff_start(iter);
1652
1653	event = ftrace_find_event(entry->type);
1654
1655	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1656	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1657	if (!trace_print_lat_context(iter))
1658	goto partial;
1659	} else {
1660	if (!trace_print_context(iter))
1661	goto partial;
1662	}
1663	}
1664
1665	if (event)
1666	return event->trace(iter, sym_flags);
1667
1668	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1669	goto partial;
1670
1671	return TRACE_TYPE_HANDLED;
1672	partial:
1673	return TRACE_TYPE_PARTIAL_LINE;
1674	}
1675
1676	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1677	{
1678	struct trace_seq *s = &iter->seq;
1679	struct trace_entry *entry;
1680	struct trace_event *event;
1681
1682	entry = iter->ent;
1683
1684	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1685	if (!trace_seq_printf(s, "%d %d %llu ",
1686	entry->pid, iter->cpu, iter->ts))
1687	goto partial;
1688	}
1689
1690	event = ftrace_find_event(entry->type);
1691	if (event)
1692	return event->raw(iter, 0);
1693
1694	if (!trace_seq_printf(s, "%d ?\n", entry->type))
1695	goto partial;
1696
1697	return TRACE_TYPE_HANDLED;
1698	partial:
1699	return TRACE_TYPE_PARTIAL_LINE;
1700	}
1701
1702	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1703	{
1704	struct trace_seq *s = &iter->seq;
1705	unsigned char newline = '\n';
1706	struct trace_entry *entry;
1707	struct trace_event *event;
1708
1709	entry = iter->ent;
1710
1711	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1712	SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1713	SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1714	SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1715	}
1716
1717	event = ftrace_find_event(entry->type);
1718	if (event) {
1719	enum print_line_t ret = event->hex(iter, 0);
1720	if (ret != TRACE_TYPE_HANDLED)
1721	return ret;
1722	}
1723
1724	SEQ_PUT_FIELD_RET(s, newline);
1725
1726	return TRACE_TYPE_HANDLED;
1727	}
1728
1729	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1730	{
1731	struct trace_seq *s = &iter->seq;
1732	struct trace_entry *entry;
1733	struct trace_event *event;
1734
1735	entry = iter->ent;
1736
1737	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1738	SEQ_PUT_FIELD_RET(s, entry->pid);
1739	SEQ_PUT_FIELD_RET(s, iter->cpu);
1740	SEQ_PUT_FIELD_RET(s, iter->ts);
1741	}
1742
1743	event = ftrace_find_event(entry->type);
1744	return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
1745	}
1746
1747	static int trace_empty(struct trace_iterator *iter)
1748	{
1749	int cpu;
1750
1751	/* If we are looking at one CPU buffer, only check that one */
1752	if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
1753	cpu = iter->cpu_file;
1754	if (iter->buffer_iter[cpu]) {
1755	if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1756	return 0;
1757	} else {
1758	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1759	return 0;
1760	}
1761	return 1;
1762	}
1763
1764	for_each_tracing_cpu(cpu) {
1765	if (iter->buffer_iter[cpu]) {
1766	if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1767	return 0;
1768	} else {
1769	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1770	return 0;
1771	}
1772	}
1773
1774	return 1;
1775	}
1776
1777	static enum print_line_t print_trace_line(struct trace_iterator *iter)
1778	{
1779	enum print_line_t ret;
1780
1781	if (iter->trace && iter->trace->print_line) {
1782	ret = iter->trace->print_line(iter);
1783	if (ret != TRACE_TYPE_UNHANDLED)
1784	return ret;
1785	}
1786
1787	if (iter->ent->type == TRACE_BPRINT &&
1788	trace_flags & TRACE_ITER_PRINTK &&
1789	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
1790	return trace_print_bprintk_msg_only(iter);
1791
1792	if (iter->ent->type == TRACE_PRINT &&
1793	trace_flags & TRACE_ITER_PRINTK &&
1794	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
1795	return trace_print_printk_msg_only(iter);
1796
1797	if (trace_flags & TRACE_ITER_BIN)
1798	return print_bin_fmt(iter);
1799
1800	if (trace_flags & TRACE_ITER_HEX)
1801	return print_hex_fmt(iter);
1802
1803	if (trace_flags & TRACE_ITER_RAW)
1804	return print_raw_fmt(iter);
1805
1806	return print_trace_fmt(iter);
1807	}
1808
1809	static int s_show(struct seq_file m, void v)
1810	{
1811	struct trace_iterator *iter = v;
1812
1813	if (iter->ent == NULL) {
1814	if (iter->tr) {
1815	seq_printf(m, "# tracer: %s\n", iter->trace->name);
1816	seq_puts(m, "#\n");
1817	}
1818	if (iter->trace && iter->trace->print_header)
1819	iter->trace->print_header(m);
1820	else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1821	/* print nothing if the buffers are empty */
1822	if (trace_empty(iter))
1823	return 0;
1824	print_trace_header(m, iter);
1825	if (!(trace_flags & TRACE_ITER_VERBOSE))
1826	print_lat_help_header(m);
1827	} else {
1828	if (!(trace_flags & TRACE_ITER_VERBOSE))
1829	print_func_help_header(m);
1830	}
1831	} else {
1832	print_trace_line(iter);
1833	trace_print_seq(m, &iter->seq);
1834	}
1835
1836	return 0;
1837	}
1838
1839	static struct seq_operations tracer_seq_ops = {
1840	.start = s_start,
1841	.next = s_next,
1842	.stop = s_stop,
1843	.show = s_show,
1844	};
1845
1846	static struct trace_iterator *
1847	__tracing_open(struct inode inode, struct file file)
1848	{
1849	long cpu_file = (long) inode->i_private;
1850	void *fail_ret = ERR_PTR(-ENOMEM);
1851	struct trace_iterator *iter;
1852	struct seq_file *m;
1853	int cpu, ret;
1854
1855	if (tracing_disabled)
1856	return ERR_PTR(-ENODEV);
1857
1858	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1859	if (!iter)
1860	return ERR_PTR(-ENOMEM);
1861
1862	/*
1863	* We make a copy of the current tracer to avoid concurrent
1864	* changes on it while we are reading.
1865	*/
1866	mutex_lock(&trace_types_lock);
1867	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
1868	if (!iter->trace)
1869	goto fail;
1870
1871	if (current_trace)
1872	iter->trace = current_trace;
1873
1874	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL))
1875	goto fail;
1876
1877	cpumask_clear(iter->started);
1878
1879	if (current_trace && current_trace->print_max)
1880	iter->tr = &max_tr;
1881	else
1882	iter->tr = &global_trace;
1883	iter->pos = -1;
1884	mutex_init(&iter->mutex);
1885	iter->cpu_file = cpu_file;
1886
1887	/* Notify the tracer early; before we stop tracing. */
1888	if (iter->trace && iter->trace->open)
1889	iter->trace->open(iter);
1890
1891	/* Annotate start of buffers if we had overruns */
1892	if (ring_buffer_overruns(iter->tr->buffer))
1893	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
1894
1895	if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
1896	for_each_tracing_cpu(cpu) {
1897
1898	iter->buffer_iter[cpu] =
1899	ring_buffer_read_start(iter->tr->buffer, cpu);
1900	}
1901	} else {
1902	cpu = iter->cpu_file;
1903	iter->buffer_iter[cpu] =
1904	ring_buffer_read_start(iter->tr->buffer, cpu);
1905	}
1906
1907	/* TODO stop tracer */
1908	ret = seq_open(file, &tracer_seq_ops);
1909	if (ret < 0) {
1910	fail_ret = ERR_PTR(ret);
1911	goto fail_buffer;
1912	}
1913
1914	m = file->private_data;
1915	m->private = iter;
1916
1917	/* stop the trace while dumping */
1918	tracing_stop();
1919
1920	mutex_unlock(&trace_types_lock);
1921
1922	return iter;
1923
1924	fail_buffer:
1925	for_each_tracing_cpu(cpu) {
1926	if (iter->buffer_iter[cpu])
1927	ring_buffer_read_finish(iter->buffer_iter[cpu]);
1928	}
1929	free_cpumask_var(iter->started);
1930	fail:
1931	mutex_unlock(&trace_types_lock);
1932	kfree(iter->trace);
1933	kfree(iter);
1934
1935	return fail_ret;
1936	}
1937
1938	int tracing_open_generic(struct inode inode, struct file filp)
1939	{
1940	if (tracing_disabled)
1941	return -ENODEV;
1942
1943	filp->private_data = inode->i_private;
1944	return 0;
1945	}
1946
1947	static int tracing_release(struct inode inode, struct file file)
1948	{
1949	struct seq_file m = (struct seq_file )file->private_data;
1950	struct trace_iterator *iter;
1951	int cpu;
1952
1953	if (!(file->f_mode & FMODE_READ))
1954	return 0;
1955
1956	iter = m->private;
1957
1958	mutex_lock(&trace_types_lock);
1959	for_each_tracing_cpu(cpu) {
1960	if (iter->buffer_iter[cpu])
1961	ring_buffer_read_finish(iter->buffer_iter[cpu]);
1962	}
1963
1964	if (iter->trace && iter->trace->close)
1965	iter->trace->close(iter);
1966
1967	/* reenable tracing if it was previously enabled */
1968	tracing_start();
1969	mutex_unlock(&trace_types_lock);
1970
1971	seq_release(inode, file);
1972	mutex_destroy(&iter->mutex);
1973	free_cpumask_var(iter->started);
1974	kfree(iter->trace);
1975	kfree(iter);
1976	return 0;
1977	}
1978
1979	static int tracing_open(struct inode inode, struct file file)
1980	{
1981	struct trace_iterator *iter;
1982	int ret = 0;
1983
1984	/* If this file was open for write, then erase contents */
1985	if ((file->f_mode & FMODE_WRITE) &&
1986	!(file->f_flags & O_APPEND)) {
1987	long cpu = (long) inode->i_private;
1988
1989	if (cpu == TRACE_PIPE_ALL_CPU)
1990	tracing_reset_online_cpus(&global_trace);
1991	else
1992	tracing_reset(&global_trace, cpu);
1993	}
1994
1995	if (file->f_mode & FMODE_READ) {
1996	iter = __tracing_open(inode, file);
1997	if (IS_ERR(iter))
1998	ret = PTR_ERR(iter);
1999	else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2000	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
2001	}
2002	return ret;
2003	}
2004
2005	static void *
2006	t_next(struct seq_file m, void v, loff_t *pos)
2007	{
2008	struct tracer *t = m->private;
2009
2010	(*pos)++;
2011
2012	if (t)
2013	t = t->next;
2014
2015	m->private = t;
2016
2017	return t;
2018	}
2019
2020	static void t_start(struct seq_file m, loff_t *pos)
2021	{
2022	struct tracer *t = m->private;
2023	loff_t l = 0;
2024
2025	mutex_lock(&trace_types_lock);
2026	for (; t && l < *pos; t = t_next(m, t, &l))
2027	;
2028
2029	return t;
2030	}
2031
2032	static void t_stop(struct seq_file m, void p)
2033	{
2034	mutex_unlock(&trace_types_lock);
2035	}
2036
2037	static int t_show(struct seq_file m, void v)
2038	{
2039	struct tracer *t = v;
2040
2041	if (!t)
2042	return 0;
2043
2044	seq_printf(m, "%s", t->name);
2045	if (t->next)
2046	seq_putc(m, ' ');
2047	else
2048	seq_putc(m, '\n');
2049
2050	return 0;
2051	}
2052
2053	static struct seq_operations show_traces_seq_ops = {
2054	.start = t_start,
2055	.next = t_next,
2056	.stop = t_stop,
2057	.show = t_show,
2058	};
2059
2060	static int show_traces_open(struct inode inode, struct file file)
2061	{
2062	int ret;
2063
2064	if (tracing_disabled)
2065	return -ENODEV;
2066
2067	ret = seq_open(file, &show_traces_seq_ops);
2068	if (!ret) {
2069	struct seq_file *m = file->private_data;
2070	m->private = trace_types;
2071	}
2072
2073	return ret;
2074	}
2075
2076	static ssize_t
2077	tracing_write_stub(struct file filp, const char __user ubuf,
2078	size_t count, loff_t *ppos)
2079	{
2080	return count;
2081	}
2082
2083	static const struct file_operations tracing_fops = {
2084	.open = tracing_open,
2085	.read = seq_read,
2086	.write = tracing_write_stub,
2087	.llseek = seq_lseek,
2088	.release = tracing_release,
2089	};
2090
2091	static const struct file_operations show_traces_fops = {
2092	.open = show_traces_open,
2093	.read = seq_read,
2094	.release = seq_release,
2095	};
2096
2097	/*
2098	* Only trace on a CPU if the bitmask is set:
2099	*/
2100	static cpumask_var_t tracing_cpumask;
2101
2102	/*
2103	* The tracer itself will not take this lock, but still we want
2104	* to provide a consistent cpumask to user-space:
2105	*/
2106	static DEFINE_MUTEX(tracing_cpumask_update_lock);
2107
2108	/*
2109	* Temporary storage for the character representation of the
2110	* CPU bitmask (and one more byte for the newline):
2111	*/
2112	static char mask_str[NR_CPUS + 1];
2113
2114	static ssize_t
2115	tracing_cpumask_read(struct file filp, char __user ubuf,
2116	size_t count, loff_t *ppos)
2117	{
2118	int len;
2119
2120	mutex_lock(&tracing_cpumask_update_lock);
2121
2122	len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2123	if (count - len < 2) {
2124	count = -EINVAL;
2125	goto out_err;
2126	}
2127	len += sprintf(mask_str + len, "\n");
2128	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2129
2130	out_err:
2131	mutex_unlock(&tracing_cpumask_update_lock);
2132
2133	return count;
2134	}
2135
2136	static ssize_t
2137	tracing_cpumask_write(struct file filp, const char __user ubuf,
2138	size_t count, loff_t *ppos)
2139	{
2140	int err, cpu;
2141	cpumask_var_t tracing_cpumask_new;
2142
2143	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2144	return -ENOMEM;
2145
2146	mutex_lock(&tracing_cpumask_update_lock);
2147	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2148	if (err)
2149	goto err_unlock;
2150
2151	local_irq_disable();
2152	__raw_spin_lock(&ftrace_max_lock);
2153	for_each_tracing_cpu(cpu) {
2154	/*
2155	* Increase/decrease the disabled counter if we are
2156	* about to flip a bit in the cpumask:
2157	*/
2158	if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2159	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2160	atomic_inc(&global_trace.data[cpu]->disabled);
2161	}
2162	if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2163	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2164	atomic_dec(&global_trace.data[cpu]->disabled);
2165	}
2166	}
2167	__raw_spin_unlock(&ftrace_max_lock);
2168	local_irq_enable();
2169
2170	cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2171
2172	mutex_unlock(&tracing_cpumask_update_lock);
2173	free_cpumask_var(tracing_cpumask_new);
2174
2175	return count;
2176
2177	err_unlock:
2178	mutex_unlock(&tracing_cpumask_update_lock);
2179	free_cpumask_var(tracing_cpumask);
2180
2181	return err;
2182	}
2183
2184	static const struct file_operations tracing_cpumask_fops = {
2185	.open = tracing_open_generic,
2186	.read = tracing_cpumask_read,
2187	.write = tracing_cpumask_write,
2188	};
2189
2190	static ssize_t
2191	tracing_trace_options_read(struct file filp, char __user ubuf,
2192	size_t cnt, loff_t *ppos)
2193	{
2194	struct tracer_opt *trace_opts;
2195	u32 tracer_flags;
2196	int len = 0;
2197	char *buf;
2198	int r = 0;
2199	int i;
2200
2201
2202	/* calculate max size */
2203	for (i = 0; trace_options[i]; i++) {
2204	len += strlen(trace_options[i]);
2205	len += 3; /* "no" and newline */
2206	}
2207
2208	mutex_lock(&trace_types_lock);
2209	tracer_flags = current_trace->flags->val;
2210	trace_opts = current_trace->flags->opts;
2211
2212	/*
2213	* Increase the size with names of options specific
2214	* of the current tracer.
2215	*/
2216	for (i = 0; trace_opts[i].name; i++) {
2217	len += strlen(trace_opts[i].name);
2218	len += 3; /* "no" and newline */
2219	}
2220
2221	/* +2 for \n and \0 */
2222	buf = kmalloc(len + 2, GFP_KERNEL);
2223	if (!buf) {
2224	mutex_unlock(&trace_types_lock);
2225	return -ENOMEM;
2226	}
2227
2228	for (i = 0; trace_options[i]; i++) {
2229	if (trace_flags & (1 << i))
2230	r += sprintf(buf + r, "%s\n", trace_options[i]);
2231	else
2232	r += sprintf(buf + r, "no%s\n", trace_options[i]);
2233	}
2234
2235	for (i = 0; trace_opts[i].name; i++) {
2236	if (tracer_flags & trace_opts[i].bit)
2237	r += sprintf(buf + r, "%s\n",
2238	trace_opts[i].name);
2239	else
2240	r += sprintf(buf + r, "no%s\n",
2241	trace_opts[i].name);
2242	}
2243	mutex_unlock(&trace_types_lock);
2244
2245	WARN_ON(r >= len + 2);
2246
2247	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2248
2249	kfree(buf);
2250	return r;
2251	}
2252
2253	/* Try to assign a tracer specific option */
2254	static int set_tracer_option(struct tracer trace, char cmp, int neg)
2255	{
2256	struct tracer_flags *trace_flags = trace->flags;
2257	struct tracer_opt *opts = NULL;
2258	int ret = 0, i = 0;
2259	int len;
2260
2261	for (i = 0; trace_flags->opts[i].name; i++) {
2262	opts = &trace_flags->opts[i];
2263	len = strlen(opts->name);
2264
2265	if (strncmp(cmp, opts->name, len) == 0) {
2266	ret = trace->set_flag(trace_flags->val,
2267	opts->bit, !neg);
2268	break;
2269	}
2270	}
2271	/* Not found */
2272	if (!trace_flags->opts[i].name)
2273	return -EINVAL;
2274
2275	/* Refused to handle */
2276	if (ret)
2277	return ret;
2278
2279	if (neg)
2280	trace_flags->val &= ~opts->bit;
2281	else
2282	trace_flags->val \|= opts->bit;
2283
2284	return 0;
2285	}
2286
2287	static void set_tracer_flags(unsigned int mask, int enabled)
2288	{
2289	/* do nothing if flag is already set */
2290	if (!!(trace_flags & mask) == !!enabled)
2291	return;
2292
2293	if (enabled)
2294	trace_flags \|= mask;
2295	else
2296	trace_flags &= ~mask;
2297
2298	if (mask == TRACE_ITER_GLOBAL_CLK) {
2299	u64 (*func)(void);
2300
2301	if (enabled)
2302	func = trace_clock_global;
2303	else
2304	func = trace_clock_local;
2305
2306	mutex_lock(&trace_types_lock);
2307	ring_buffer_set_clock(global_trace.buffer, func);
2308
2309	if (max_tr.buffer)
2310	ring_buffer_set_clock(max_tr.buffer, func);
2311	mutex_unlock(&trace_types_lock);
2312	}
2313	}
2314
2315	static ssize_t
2316	tracing_trace_options_write(struct file filp, const char __user ubuf,
2317	size_t cnt, loff_t *ppos)
2318	{
2319	char buf[64];
2320	char *cmp = buf;
2321	int neg = 0;
2322	int ret;
2323	int i;
2324
2325	if (cnt >= sizeof(buf))
2326	return -EINVAL;
2327
2328	if (copy_from_user(&buf, ubuf, cnt))
2329	return -EFAULT;
2330
2331	buf[cnt] = 0;
2332
2333	if (strncmp(buf, "no", 2) == 0) {
2334	neg = 1;
2335	cmp += 2;
2336	}
2337
2338	for (i = 0; trace_options[i]; i++) {
2339	int len = strlen(trace_options[i]);
2340
2341	if (strncmp(cmp, trace_options[i], len) == 0) {
2342	set_tracer_flags(1 << i, !neg);
2343	break;
2344	}
2345	}
2346
2347	/* If no option could be set, test the specific tracer options */
2348	if (!trace_options[i]) {
2349	mutex_lock(&trace_types_lock);
2350	ret = set_tracer_option(current_trace, cmp, neg);
2351	mutex_unlock(&trace_types_lock);
2352	if (ret)
2353	return ret;
2354	}
2355
2356	filp->f_pos += cnt;
2357
2358	return cnt;
2359	}
2360
2361	static const struct file_operations tracing_iter_fops = {
2362	.open = tracing_open_generic,
2363	.read = tracing_trace_options_read,
2364	.write = tracing_trace_options_write,
2365	};
2366
2367	static const char readme_msg[] =
2368	"tracing mini-HOWTO:\n\n"
2369	"# mkdir /debug\n"
2370	"# mount -t debugfs nodev /debug\n\n"
2371	"# cat /debug/tracing/available_tracers\n"
2372	"wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
2373	"# cat /debug/tracing/current_tracer\n"
2374	"nop\n"
2375	"# echo sched_switch > /debug/tracing/current_tracer\n"
2376	"# cat /debug/tracing/current_tracer\n"
2377	"sched_switch\n"
2378	"# cat /debug/tracing/trace_options\n"
2379	"noprint-parent nosym-offset nosym-addr noverbose\n"
2380	"# echo print-parent > /debug/tracing/trace_options\n"
2381	"# echo 1 > /debug/tracing/tracing_enabled\n"
2382	"# cat /debug/tracing/trace > /tmp/trace.txt\n"
2383	"# echo 0 > /debug/tracing/tracing_enabled\n"
2384	;
2385
2386	static ssize_t
2387	tracing_readme_read(struct file filp, char __user ubuf,
2388	size_t cnt, loff_t *ppos)
2389	{
2390	return simple_read_from_buffer(ubuf, cnt, ppos,
2391	readme_msg, strlen(readme_msg));
2392	}
2393
2394	static const struct file_operations tracing_readme_fops = {
2395	.open = tracing_open_generic,
2396	.read = tracing_readme_read,
2397	};
2398
2399	static ssize_t
2400	tracing_ctrl_read(struct file filp, char __user ubuf,
2401	size_t cnt, loff_t *ppos)
2402	{
2403	char buf[64];
2404	int r;
2405
2406	r = sprintf(buf, "%u\n", tracer_enabled);
2407	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2408	}
2409
2410	static ssize_t
2411	tracing_ctrl_write(struct file filp, const char __user ubuf,
2412	size_t cnt, loff_t *ppos)
2413	{
2414	struct trace_array *tr = filp->private_data;
2415	char buf[64];
2416	unsigned long val;
2417	int ret;
2418
2419	if (cnt >= sizeof(buf))
2420	return -EINVAL;
2421
2422	if (copy_from_user(&buf, ubuf, cnt))
2423	return -EFAULT;
2424
2425	buf[cnt] = 0;
2426
2427	ret = strict_strtoul(buf, 10, &val);
2428	if (ret < 0)
2429	return ret;
2430
2431	val = !!val;
2432
2433	mutex_lock(&trace_types_lock);
2434	if (tracer_enabled ^ val) {
2435	if (val) {
2436	tracer_enabled = 1;
2437	if (current_trace->start)
2438	current_trace->start(tr);
2439	tracing_start();
2440	} else {
2441	tracer_enabled = 0;
2442	tracing_stop();
2443	if (current_trace->stop)
2444	current_trace->stop(tr);
2445	}
2446	}
2447	mutex_unlock(&trace_types_lock);
2448
2449	filp->f_pos += cnt;
2450
2451	return cnt;
2452	}
2453
2454	static ssize_t
2455	tracing_set_trace_read(struct file filp, char __user ubuf,
2456	size_t cnt, loff_t *ppos)
2457	{
2458	char buf[max_tracer_type_len+2];
2459	int r;
2460
2461	mutex_lock(&trace_types_lock);
2462	if (current_trace)
2463	r = sprintf(buf, "%s\n", current_trace->name);
2464	else
2465	r = sprintf(buf, "\n");
2466	mutex_unlock(&trace_types_lock);
2467
2468	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2469	}
2470
2471	int tracer_init(struct tracer t, struct trace_array tr)
2472	{
2473	tracing_reset_online_cpus(tr);
2474	return t->init(tr);
2475	}
2476
2477	static int tracing_resize_ring_buffer(unsigned long size)
2478	{
2479	int ret;
2480
2481	/*
2482	* If kernel or user changes the size of the ring buffer
2483	* we use the size that was given, and we can forget about
2484	* expanding it later.
2485	*/
2486	ring_buffer_expanded = 1;
2487
2488	ret = ring_buffer_resize(global_trace.buffer, size);
2489	if (ret < 0)
2490	return ret;
2491
2492	ret = ring_buffer_resize(max_tr.buffer, size);
2493	if (ret < 0) {
2494	int r;
2495
2496	r = ring_buffer_resize(global_trace.buffer,
2497	global_trace.entries);
2498	if (r < 0) {
2499	/*
2500	* AARGH! We are left with different
2501	* size max buffer!!!!
2502	* The max buffer is our "snapshot" buffer.
2503	* When a tracer needs a snapshot (one of the
2504	* latency tracers), it swaps the max buffer
2505	* with the saved snap shot. We succeeded to
2506	* update the size of the main buffer, but failed to
2507	* update the size of the max buffer. But when we tried
2508	* to reset the main buffer to the original size, we
2509	* failed there too. This is very unlikely to
2510	* happen, but if it does, warn and kill all
2511	* tracing.
2512	*/
2513	WARN_ON(1);
2514	tracing_disabled = 1;
2515	}
2516	return ret;
2517	}
2518
2519	global_trace.entries = size;
2520
2521	return ret;
2522	}
2523
2524	/**
2525	* tracing_update_buffers - used by tracing facility to expand ring buffers
2526	*
2527	* To save on memory when the tracing is never used on a system with it
2528	* configured in. The ring buffers are set to a minimum size. But once
2529	* a user starts to use the tracing facility, then they need to grow
2530	* to their default size.
2531	*
2532	* This function is to be called when a tracer is about to be used.
2533	*/
2534	int tracing_update_buffers(void)
2535	{
2536	int ret = 0;
2537
2538	mutex_lock(&trace_types_lock);
2539	if (!ring_buffer_expanded)
2540	ret = tracing_resize_ring_buffer(trace_buf_size);
2541	mutex_unlock(&trace_types_lock);
2542
2543	return ret;
2544	}
2545
2546	struct trace_option_dentry;
2547
2548	static struct trace_option_dentry *
2549	create_trace_option_files(struct tracer *tracer);
2550
2551	static void
2552	destroy_trace_option_files(struct trace_option_dentry *topts);
2553
2554	static int tracing_set_tracer(const char *buf)
2555	{
2556	static struct trace_option_dentry *topts;
2557	struct trace_array *tr = &global_trace;
2558	struct tracer *t;
2559	int ret = 0;
2560
2561	mutex_lock(&trace_types_lock);
2562
2563	if (!ring_buffer_expanded) {
2564	ret = tracing_resize_ring_buffer(trace_buf_size);
2565	if (ret < 0)
2566	goto out;
2567	ret = 0;
2568	}
2569
2570	for (t = trace_types; t; t = t->next) {
2571	if (strcmp(t->name, buf) == 0)
2572	break;
2573	}
2574	if (!t) {
2575	ret = -EINVAL;
2576	goto out;
2577	}
2578	if (t == current_trace)
2579	goto out;
2580
2581	trace_branch_disable();
2582	if (current_trace && current_trace->reset)
2583	current_trace->reset(tr);
2584
2585	destroy_trace_option_files(topts);
2586
2587	current_trace = t;
2588
2589	topts = create_trace_option_files(current_trace);
2590
2591	if (t->init) {
2592	ret = tracer_init(t, tr);
2593	if (ret)
2594	goto out;
2595	}
2596
2597	trace_branch_enable(tr);
2598	out:
2599	mutex_unlock(&trace_types_lock);
2600
2601	return ret;
2602	}
2603
2604	static ssize_t
2605	tracing_set_trace_write(struct file filp, const char __user ubuf,
2606	size_t cnt, loff_t *ppos)
2607	{
2608	char buf[max_tracer_type_len+1];
2609	int i;
2610	size_t ret;
2611	int err;
2612
2613	ret = cnt;
2614
2615	if (cnt > max_tracer_type_len)
2616	cnt = max_tracer_type_len;
2617
2618	if (copy_from_user(&buf, ubuf, cnt))
2619	return -EFAULT;
2620
2621	buf[cnt] = 0;
2622
2623	/* strip ending whitespace. */
2624	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2625	buf[i] = 0;
2626
2627	err = tracing_set_tracer(buf);
2628	if (err)
2629	return err;
2630
2631	filp->f_pos += ret;
2632
2633	return ret;
2634	}
2635
2636	static ssize_t
2637	tracing_max_lat_read(struct file filp, char __user ubuf,
2638	size_t cnt, loff_t *ppos)
2639	{
2640	unsigned long *ptr = filp->private_data;
2641	char buf[64];
2642	int r;
2643
2644	r = snprintf(buf, sizeof(buf), "%ld\n",
2645	ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(ptr));
2646	if (r > sizeof(buf))
2647	r = sizeof(buf);
2648	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2649	}
2650
2651	static ssize_t
2652	tracing_max_lat_write(struct file filp, const char __user ubuf,
2653	size_t cnt, loff_t *ppos)
2654	{
2655	unsigned long *ptr = filp->private_data;
2656	char buf[64];
2657	unsigned long val;
2658	int ret;
2659
2660	if (cnt >= sizeof(buf))
2661	return -EINVAL;
2662
2663	if (copy_from_user(&buf, ubuf, cnt))
2664	return -EFAULT;
2665
2666	buf[cnt] = 0;
2667
2668	ret = strict_strtoul(buf, 10, &val);
2669	if (ret < 0)
2670	return ret;
2671
2672	ptr = val 1000;
2673
2674	return cnt;
2675	}
2676
2677	static int tracing_open_pipe(struct inode inode, struct file filp)
2678	{
2679	long cpu_file = (long) inode->i_private;
2680	struct trace_iterator *iter;
2681	int ret = 0;
2682
2683	if (tracing_disabled)
2684	return -ENODEV;
2685
2686	mutex_lock(&trace_types_lock);
2687
2688	/* We only allow one reader per cpu */
2689	if (cpu_file == TRACE_PIPE_ALL_CPU) {
2690	if (!cpumask_empty(tracing_reader_cpumask)) {
2691	ret = -EBUSY;
2692	goto out;
2693	}
2694	cpumask_setall(tracing_reader_cpumask);
2695	} else {
2696	if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask))
2697	cpumask_set_cpu(cpu_file, tracing_reader_cpumask);
2698	else {
2699	ret = -EBUSY;
2700	goto out;
2701	}
2702	}
2703
2704	/* create a buffer to store the information to pass to userspace */
2705	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2706	if (!iter) {
2707	ret = -ENOMEM;
2708	goto out;
2709	}
2710
2711	/*
2712	* We make a copy of the current tracer to avoid concurrent
2713	* changes on it while we are reading.
2714	*/
2715	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
2716	if (!iter->trace) {
2717	ret = -ENOMEM;
2718	goto fail;
2719	}
2720	if (current_trace)
2721	iter->trace = current_trace;
2722
2723	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
2724	ret = -ENOMEM;
2725	goto fail;
2726	}
2727
2728	/* trace pipe does not show start of buffer */
2729	cpumask_setall(iter->started);
2730
2731	iter->cpu_file = cpu_file;
2732	iter->tr = &global_trace;
2733	mutex_init(&iter->mutex);
2734	filp->private_data = iter;
2735
2736	if (iter->trace->pipe_open)
2737	iter->trace->pipe_open(iter);
2738
2739	out:
2740	mutex_unlock(&trace_types_lock);
2741	return ret;
2742
2743	fail:
2744	kfree(iter->trace);
2745	kfree(iter);
2746	mutex_unlock(&trace_types_lock);
2747	return ret;
2748	}
2749
2750	static int tracing_release_pipe(struct inode inode, struct file file)
2751	{
2752	struct trace_iterator *iter = file->private_data;
2753
2754	mutex_lock(&trace_types_lock);
2755
2756	if (iter->cpu_file == TRACE_PIPE_ALL_CPU)
2757	cpumask_clear(tracing_reader_cpumask);
2758	else
2759	cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask);
2760
2761	mutex_unlock(&trace_types_lock);
2762
2763	free_cpumask_var(iter->started);
2764	mutex_destroy(&iter->mutex);
2765	kfree(iter->trace);
2766	kfree(iter);
2767
2768	return 0;
2769	}
2770
2771	static unsigned int
2772	tracing_poll_pipe(struct file filp, poll_table poll_table)
2773	{
2774	struct trace_iterator *iter = filp->private_data;
2775
2776	if (trace_flags & TRACE_ITER_BLOCK) {
2777	/*
2778	* Always select as readable when in blocking mode
2779	*/
2780	return POLLIN \| POLLRDNORM;
2781	} else {
2782	if (!trace_empty(iter))
2783	return POLLIN \| POLLRDNORM;
2784	poll_wait(filp, &trace_wait, poll_table);
2785	if (!trace_empty(iter))
2786	return POLLIN \| POLLRDNORM;
2787
2788	return 0;
2789	}
2790	}
2791
2792
2793	void default_wait_pipe(struct trace_iterator *iter)
2794	{
2795	DEFINE_WAIT(wait);
2796
2797	prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
2798
2799	if (trace_empty(iter))
2800	schedule();
2801
2802	finish_wait(&trace_wait, &wait);
2803	}
2804
2805	/*
2806	* This is a make-shift waitqueue.
2807	* A tracer might use this callback on some rare cases:
2808	*
2809	* 1) the current tracer might hold the runqueue lock when it wakes up
2810	* a reader, hence a deadlock (sched, function, and function graph tracers)
2811	* 2) the function tracers, trace all functions, we don't want
2812	* the overhead of calling wake_up and friends
2813	* (and tracing them too)
2814	*
2815	* Anyway, this is really very primitive wakeup.
2816	*/
2817	void poll_wait_pipe(struct trace_iterator *iter)
2818	{
2819	set_current_state(TASK_INTERRUPTIBLE);
2820	/* sleep for 100 msecs, and try again. */
2821	schedule_timeout(HZ / 10);
2822	}
2823
2824	/* Must be called with trace_types_lock mutex held. */
2825	static int tracing_wait_pipe(struct file *filp)
2826	{
2827	struct trace_iterator *iter = filp->private_data;
2828
2829	while (trace_empty(iter)) {
2830
2831	if ((filp->f_flags & O_NONBLOCK)) {
2832	return -EAGAIN;
2833	}
2834
2835	mutex_unlock(&iter->mutex);
2836
2837	iter->trace->wait_pipe(iter);
2838
2839	mutex_lock(&iter->mutex);
2840
2841	if (signal_pending(current))
2842	return -EINTR;
2843
2844	/*
2845	* We block until we read something and tracing is disabled.
2846	* We still block if tracing is disabled, but we have never
2847	* read anything. This allows a user to cat this file, and
2848	* then enable tracing. But after we have read something,
2849	* we give an EOF when tracing is again disabled.
2850	*
2851	* iter->pos will be 0 if we haven't read anything.
2852	*/
2853	if (!tracer_enabled && iter->pos)
2854	break;
2855	}
2856
2857	return 1;
2858	}
2859
2860	/*
2861	* Consumer reader.
2862	*/
2863	static ssize_t
2864	tracing_read_pipe(struct file filp, char __user ubuf,
2865	size_t cnt, loff_t *ppos)
2866	{
2867	struct trace_iterator *iter = filp->private_data;
2868	static struct tracer *old_tracer;
2869	ssize_t sret;
2870
2871	/* return any leftover data */
2872	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2873	if (sret != -EBUSY)
2874	return sret;
2875
2876	trace_seq_init(&iter->seq);
2877
2878	/* copy the tracer to avoid using a global lock all around */
2879	mutex_lock(&trace_types_lock);
2880	if (unlikely(old_tracer != current_trace && current_trace)) {
2881	old_tracer = current_trace;
2882	iter->trace = current_trace;
2883	}
2884	mutex_unlock(&trace_types_lock);
2885
2886	/*
2887	* Avoid more than one consumer on a single file descriptor
2888	* This is just a matter of traces coherency, the ring buffer itself
2889	* is protected.
2890	*/
2891	mutex_lock(&iter->mutex);
2892	if (iter->trace->read) {
2893	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2894	if (sret)
2895	goto out;
2896	}
2897
2898	waitagain:
2899	sret = tracing_wait_pipe(filp);
2900	if (sret <= 0)
2901	goto out;
2902
2903	/* stop when tracing is finished */
2904	if (trace_empty(iter)) {
2905	sret = 0;
2906	goto out;
2907	}
2908
2909	if (cnt >= PAGE_SIZE)
2910	cnt = PAGE_SIZE - 1;
2911
2912	/* reset all but tr, trace, and overruns */
2913	memset(&iter->seq, 0,
2914	sizeof(struct trace_iterator) -
2915	offsetof(struct trace_iterator, seq));
2916	iter->pos = -1;
2917
2918	while (find_next_entry_inc(iter) != NULL) {
2919	enum print_line_t ret;
2920	int len = iter->seq.len;
2921
2922	ret = print_trace_line(iter);
2923	if (ret == TRACE_TYPE_PARTIAL_LINE) {
2924	/* don't print partial lines */
2925	iter->seq.len = len;
2926	break;
2927	}
2928	if (ret != TRACE_TYPE_NO_CONSUME)
2929	trace_consume(iter);
2930
2931	if (iter->seq.len >= cnt)
2932	break;
2933	}
2934
2935	/* Now copy what we have to the user */
2936	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2937	if (iter->seq.readpos >= iter->seq.len)
2938	trace_seq_init(&iter->seq);
2939
2940	/*
2941	* If there was nothing to send to user, inspite of consuming trace
2942	* entries, go back to wait for more entries.
2943	*/
2944	if (sret == -EBUSY)
2945	goto waitagain;
2946
2947	out:
2948	mutex_unlock(&iter->mutex);
2949
2950	return sret;
2951	}
2952
2953	static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
2954	struct pipe_buffer *buf)
2955	{
2956	__free_page(buf->page);
2957	}
2958
2959	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
2960	unsigned int idx)
2961	{
2962	__free_page(spd->pages[idx]);
2963	}
2964
2965	static struct pipe_buf_operations tracing_pipe_buf_ops = {
2966	.can_merge = 0,
2967	.map = generic_pipe_buf_map,
2968	.unmap = generic_pipe_buf_unmap,
2969	.confirm = generic_pipe_buf_confirm,
2970	.release = tracing_pipe_buf_release,
2971	.steal = generic_pipe_buf_steal,
2972	.get = generic_pipe_buf_get,
2973	};
2974
2975	static size_t
2976	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
2977	{
2978	size_t count;
2979	int ret;
2980
2981	/* Seq buffer is page-sized, exactly what we need. */
2982	for (;;) {
2983	count = iter->seq.len;
2984	ret = print_trace_line(iter);
2985	count = iter->seq.len - count;
2986	if (rem < count) {
2987	rem = 0;
2988	iter->seq.len -= count;
2989	break;
2990	}
2991	if (ret == TRACE_TYPE_PARTIAL_LINE) {
2992	iter->seq.len -= count;
2993	break;
2994	}
2995
2996	trace_consume(iter);
2997	rem -= count;
2998	if (!find_next_entry_inc(iter)) {
2999	rem = 0;
3000	iter->ent = NULL;
3001	break;
3002	}
3003	}
3004
3005	return rem;
3006	}
3007
3008	static ssize_t tracing_splice_read_pipe(struct file *filp,
3009	loff_t *ppos,
3010	struct pipe_inode_info *pipe,
3011	size_t len,
3012	unsigned int flags)
3013	{
3014	struct page *pages[PIPE_BUFFERS];
3015	struct partial_page partial[PIPE_BUFFERS];
3016	struct trace_iterator *iter = filp->private_data;
3017	struct splice_pipe_desc spd = {
3018	.pages = pages,
3019	.partial = partial,
3020	.nr_pages = 0, /* This gets updated below. */
3021	.flags = flags,
3022	.ops = &tracing_pipe_buf_ops,
3023	.spd_release = tracing_spd_release_pipe,
3024	};
3025	static struct tracer *old_tracer;
3026	ssize_t ret;
3027	size_t rem;
3028	unsigned int i;
3029
3030	/* copy the tracer to avoid using a global lock all around */
3031	mutex_lock(&trace_types_lock);
3032	if (unlikely(old_tracer != current_trace && current_trace)) {
3033	old_tracer = current_trace;
3034	iter->trace = current_trace;
3035	}
3036	mutex_unlock(&trace_types_lock);
3037
3038	mutex_lock(&iter->mutex);
3039
3040	if (iter->trace->splice_read) {
3041	ret = iter->trace->splice_read(iter, filp,
3042	ppos, pipe, len, flags);
3043	if (ret)
3044	goto out_err;
3045	}
3046
3047	ret = tracing_wait_pipe(filp);
3048	if (ret <= 0)
3049	goto out_err;
3050
3051	if (!iter->ent && !find_next_entry_inc(iter)) {
3052	ret = -EFAULT;
3053	goto out_err;
3054	}
3055
3056	/* Fill as many pages as possible. */
3057	for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
3058	pages[i] = alloc_page(GFP_KERNEL);
3059	if (!pages[i])
3060	break;
3061
3062	rem = tracing_fill_pipe_page(rem, iter);
3063
3064	/* Copy the data into the page, so we can start over. */
3065	ret = trace_seq_to_buffer(&iter->seq,
3066	page_address(pages[i]),
3067	iter->seq.len);
3068	if (ret < 0) {
3069	__free_page(pages[i]);
3070	break;
3071	}
3072	partial[i].offset = 0;
3073	partial[i].len = iter->seq.len;
3074
3075	trace_seq_init(&iter->seq);
3076	}
3077
3078	mutex_unlock(&iter->mutex);
3079
3080	spd.nr_pages = i;
3081
3082	return splice_to_pipe(pipe, &spd);
3083
3084	out_err:
3085	mutex_unlock(&iter->mutex);
3086
3087	return ret;
3088	}
3089
3090	static ssize_t
3091	tracing_entries_read(struct file filp, char __user ubuf,
3092	size_t cnt, loff_t *ppos)
3093	{
3094	struct trace_array *tr = filp->private_data;
3095	char buf[96];
3096	int r;
3097
3098	mutex_lock(&trace_types_lock);
3099	if (!ring_buffer_expanded)
3100	r = sprintf(buf, "%lu (expanded: %lu)\n",
3101	tr->entries >> 10,
3102	trace_buf_size >> 10);
3103	else
3104	r = sprintf(buf, "%lu\n", tr->entries >> 10);
3105	mutex_unlock(&trace_types_lock);
3106
3107	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3108	}
3109
3110	static ssize_t
3111	tracing_entries_write(struct file filp, const char __user ubuf,
3112	size_t cnt, loff_t *ppos)
3113	{
3114	unsigned long val;
3115	char buf[64];
3116	int ret, cpu;
3117
3118	if (cnt >= sizeof(buf))
3119	return -EINVAL;
3120
3121	if (copy_from_user(&buf, ubuf, cnt))
3122	return -EFAULT;
3123
3124	buf[cnt] = 0;
3125
3126	ret = strict_strtoul(buf, 10, &val);
3127	if (ret < 0)
3128	return ret;
3129
3130	/* must have at least 1 entry */
3131	if (!val)
3132	return -EINVAL;
3133
3134	mutex_lock(&trace_types_lock);
3135
3136	tracing_stop();
3137
3138	/* disable all cpu buffers */
3139	for_each_tracing_cpu(cpu) {
3140	if (global_trace.data[cpu])
3141	atomic_inc(&global_trace.data[cpu]->disabled);
3142	if (max_tr.data[cpu])
3143	atomic_inc(&max_tr.data[cpu]->disabled);
3144	}
3145
3146	/* value is in KB */
3147	val <<= 10;
3148
3149	if (val != global_trace.entries) {
3150	ret = tracing_resize_ring_buffer(val);
3151	if (ret < 0) {
3152	cnt = ret;
3153	goto out;
3154	}
3155	}
3156
3157	filp->f_pos += cnt;
3158
3159	/* If check pages failed, return ENOMEM */
3160	if (tracing_disabled)
3161	cnt = -ENOMEM;
3162	out:
3163	for_each_tracing_cpu(cpu) {
3164	if (global_trace.data[cpu])
3165	atomic_dec(&global_trace.data[cpu]->disabled);
3166	if (max_tr.data[cpu])
3167	atomic_dec(&max_tr.data[cpu]->disabled);
3168	}
3169
3170	tracing_start();
3171	max_tr.entries = global_trace.entries;
3172	mutex_unlock(&trace_types_lock);
3173
3174	return cnt;
3175	}
3176
3177	static int mark_printk(const char *fmt, ...)
3178	{
3179	int ret;
3180	va_list args;
3181	va_start(args, fmt);
3182	ret = trace_vprintk(0, fmt, args);
3183	va_end(args);
3184	return ret;
3185	}
3186
3187	static ssize_t
3188	tracing_mark_write(struct file filp, const char __user ubuf,
3189	size_t cnt, loff_t *fpos)
3190	{
3191	char *buf;
3192	char *end;
3193
3194	if (tracing_disabled)
3195	return -EINVAL;
3196
3197	if (cnt > TRACE_BUF_SIZE)
3198	cnt = TRACE_BUF_SIZE;
3199
3200	buf = kmalloc(cnt + 1, GFP_KERNEL);
3201	if (buf == NULL)
3202	return -ENOMEM;
3203
3204	if (copy_from_user(buf, ubuf, cnt)) {
3205	kfree(buf);
3206	return -EFAULT;
3207	}
3208
3209	/* Cut from the first nil or newline. */
3210	buf[cnt] = '\0';
3211	end = strchr(buf, '\n');
3212	if (end)
3213	*end = '\0';
3214
3215	cnt = mark_printk("%s\n", buf);
3216	kfree(buf);
3217	*fpos += cnt;
3218
3219	return cnt;
3220	}
3221
3222	static const struct file_operations tracing_max_lat_fops = {
3223	.open = tracing_open_generic,
3224	.read = tracing_max_lat_read,
3225	.write = tracing_max_lat_write,
3226	};
3227
3228	static const struct file_operations tracing_ctrl_fops = {
3229	.open = tracing_open_generic,
3230	.read = tracing_ctrl_read,
3231	.write = tracing_ctrl_write,
3232	};
3233
3234	static const struct file_operations set_tracer_fops = {
3235	.open = tracing_open_generic,
3236	.read = tracing_set_trace_read,
3237	.write = tracing_set_trace_write,
3238	};
3239
3240	static const struct file_operations tracing_pipe_fops = {
3241	.open = tracing_open_pipe,
3242	.poll = tracing_poll_pipe,
3243	.read = tracing_read_pipe,
3244	.splice_read = tracing_splice_read_pipe,
3245	.release = tracing_release_pipe,
3246	};
3247
3248	static const struct file_operations tracing_entries_fops = {
3249	.open = tracing_open_generic,
3250	.read = tracing_entries_read,
3251	.write = tracing_entries_write,
3252	};
3253
3254	static const struct file_operations tracing_mark_fops = {
3255	.open = tracing_open_generic,
3256	.write = tracing_mark_write,
3257	};
3258
3259	struct ftrace_buffer_info {
3260	struct trace_array *tr;
3261	void *spare;
3262	int cpu;
3263	unsigned int read;
3264	};
3265
3266	static int tracing_buffers_open(struct inode inode, struct file filp)
3267	{
3268	int cpu = (int)(long)inode->i_private;
3269	struct ftrace_buffer_info *info;
3270
3271	if (tracing_disabled)
3272	return -ENODEV;
3273
3274	info = kzalloc(sizeof(*info), GFP_KERNEL);
3275	if (!info)
3276	return -ENOMEM;
3277
3278	info->tr = &global_trace;
3279	info->cpu = cpu;
3280	info->spare = NULL;
3281	/* Force reading ring buffer for first read */
3282	info->read = (unsigned int)-1;
3283
3284	filp->private_data = info;
3285
3286	return nonseekable_open(inode, filp);
3287	}
3288
3289	static ssize_t
3290	tracing_buffers_read(struct file filp, char __user ubuf,
3291	size_t count, loff_t *ppos)
3292	{
3293	struct ftrace_buffer_info *info = filp->private_data;
3294	unsigned int pos;
3295	ssize_t ret;
3296	size_t size;
3297
3298	if (!count)
3299	return 0;
3300
3301	if (!info->spare)
3302	info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3303	if (!info->spare)
3304	return -ENOMEM;
3305
3306	/* Do we have previous read data to read? */
3307	if (info->read < PAGE_SIZE)
3308	goto read;
3309
3310	info->read = 0;
3311
3312	ret = ring_buffer_read_page(info->tr->buffer,
3313	&info->spare,
3314	count,
3315	info->cpu, 0);
3316	if (ret < 0)
3317	return 0;
3318
3319	pos = ring_buffer_page_len(info->spare);
3320
3321	if (pos < PAGE_SIZE)
3322	memset(info->spare + pos, 0, PAGE_SIZE - pos);
3323
3324	read:
3325	size = PAGE_SIZE - info->read;
3326	if (size > count)
3327	size = count;
3328
3329	ret = copy_to_user(ubuf, info->spare + info->read, size);
3330	if (ret == size)
3331	return -EFAULT;
3332	size -= ret;
3333
3334	*ppos += size;
3335	info->read += size;
3336
3337	return size;
3338	}
3339
3340	static int tracing_buffers_release(struct inode inode, struct file file)
3341	{
3342	struct ftrace_buffer_info *info = file->private_data;
3343
3344	if (info->spare)
3345	ring_buffer_free_read_page(info->tr->buffer, info->spare);
3346	kfree(info);
3347
3348	return 0;
3349	}
3350
3351	struct buffer_ref {
3352	struct ring_buffer *buffer;
3353	void *page;
3354	int ref;
3355	};
3356
3357	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3358	struct pipe_buffer *buf)
3359	{
3360	struct buffer_ref ref = (struct buffer_ref )buf->private;
3361
3362	if (--ref->ref)
3363	return;
3364
3365	ring_buffer_free_read_page(ref->buffer, ref->page);
3366	kfree(ref);
3367	buf->private = 0;
3368	}
3369
3370	static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3371	struct pipe_buffer *buf)
3372	{
3373	return 1;
3374	}
3375
3376	static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3377	struct pipe_buffer *buf)
3378	{
3379	struct buffer_ref ref = (struct buffer_ref )buf->private;
3380
3381	ref->ref++;
3382	}
3383
3384	/* Pipe buffer operations for a buffer. */
3385	static struct pipe_buf_operations buffer_pipe_buf_ops = {
3386	.can_merge = 0,
3387	.map = generic_pipe_buf_map,
3388	.unmap = generic_pipe_buf_unmap,
3389	.confirm = generic_pipe_buf_confirm,
3390	.release = buffer_pipe_buf_release,
3391	.steal = buffer_pipe_buf_steal,
3392	.get = buffer_pipe_buf_get,
3393	};
3394
3395	/*
3396	* Callback from splice_to_pipe(), if we need to release some pages
3397	* at the end of the spd in case we error'ed out in filling the pipe.
3398	*/
3399	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3400	{
3401	struct buffer_ref *ref =
3402	(struct buffer_ref *)spd->partial[i].private;
3403
3404	if (--ref->ref)
3405	return;
3406
3407	ring_buffer_free_read_page(ref->buffer, ref->page);
3408	kfree(ref);
3409	spd->partial[i].private = 0;
3410	}
3411
3412	static ssize_t
3413	tracing_buffers_splice_read(struct file file, loff_t ppos,
3414	struct pipe_inode_info *pipe, size_t len,
3415	unsigned int flags)
3416	{
3417	struct ftrace_buffer_info *info = file->private_data;
3418	struct partial_page partial[PIPE_BUFFERS];
3419	struct page *pages[PIPE_BUFFERS];
3420	struct splice_pipe_desc spd = {
3421	.pages = pages,
3422	.partial = partial,
3423	.flags = flags,
3424	.ops = &buffer_pipe_buf_ops,
3425	.spd_release = buffer_spd_release,
3426	};
3427	struct buffer_ref *ref;
3428	int size, i;
3429	size_t ret;
3430
3431	if (*ppos & (PAGE_SIZE - 1)) {
3432	WARN_ONCE(1, "Ftrace: previous read must page-align\n");
3433	return -EINVAL;
3434	}
3435
3436	if (len & (PAGE_SIZE - 1)) {
3437	WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
3438	if (len < PAGE_SIZE)
3439	return -EINVAL;
3440	len &= PAGE_MASK;
3441	}
3442
3443	for (i = 0; i < PIPE_BUFFERS && len; i++, len -= PAGE_SIZE) {
3444	struct page *page;
3445	int r;
3446
3447	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3448	if (!ref)
3449	break;
3450
3451	ref->ref = 1;
3452	ref->buffer = info->tr->buffer;
3453	ref->page = ring_buffer_alloc_read_page(ref->buffer);
3454	if (!ref->page) {
3455	kfree(ref);
3456	break;
3457	}
3458
3459	r = ring_buffer_read_page(ref->buffer, &ref->page,
3460	len, info->cpu, 0);
3461	if (r < 0) {
3462	ring_buffer_free_read_page(ref->buffer,
3463	ref->page);
3464	kfree(ref);
3465	break;
3466	}
3467
3468	/*
3469	* zero out any left over data, this is going to
3470	* user land.
3471	*/
3472	size = ring_buffer_page_len(ref->page);
3473	if (size < PAGE_SIZE)
3474	memset(ref->page + size, 0, PAGE_SIZE - size);
3475
3476	page = virt_to_page(ref->page);
3477
3478	spd.pages[i] = page;
3479	spd.partial[i].len = PAGE_SIZE;
3480	spd.partial[i].offset = 0;
3481	spd.partial[i].private = (unsigned long)ref;
3482	spd.nr_pages++;
3483	*ppos += PAGE_SIZE;
3484	}
3485
3486	spd.nr_pages = i;
3487
3488	/* did we read anything? */
3489	if (!spd.nr_pages) {
3490	if (flags & SPLICE_F_NONBLOCK)
3491	ret = -EAGAIN;
3492	else
3493	ret = 0;
3494	/* TODO: block */
3495	return ret;
3496	}
3497
3498	ret = splice_to_pipe(pipe, &spd);
3499
3500	return ret;
3501	}
3502
3503	static const struct file_operations tracing_buffers_fops = {
3504	.open = tracing_buffers_open,
3505	.read = tracing_buffers_read,
3506	.release = tracing_buffers_release,
3507	.splice_read = tracing_buffers_splice_read,
3508	.llseek = no_llseek,
3509	};
3510
3511	#ifdef CONFIG_DYNAMIC_FTRACE
3512
3513	int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3514	{
3515	return 0;
3516	}
3517
3518	static ssize_t
3519	tracing_read_dyn_info(struct file filp, char __user ubuf,
3520	size_t cnt, loff_t *ppos)
3521	{
3522	static char ftrace_dyn_info_buffer[1024];
3523	static DEFINE_MUTEX(dyn_info_mutex);
3524	unsigned long *p = filp->private_data;
3525	char *buf = ftrace_dyn_info_buffer;
3526	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
3527	int r;
3528
3529	mutex_lock(&dyn_info_mutex);
3530	r = sprintf(buf, "%ld ", *p);
3531
3532	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
3533	buf[r++] = '\n';
3534
3535	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3536
3537	mutex_unlock(&dyn_info_mutex);
3538
3539	return r;
3540	}
3541
3542	static const struct file_operations tracing_dyn_info_fops = {
3543	.open = tracing_open_generic,
3544	.read = tracing_read_dyn_info,
3545	};
3546	#endif
3547
3548	static struct dentry *d_tracer;
3549
3550	struct dentry *tracing_init_dentry(void)
3551	{
3552	static int once;
3553
3554	if (d_tracer)
3555	return d_tracer;
3556
3557	if (!debugfs_initialized())
3558	return NULL;
3559
3560	d_tracer = debugfs_create_dir("tracing", NULL);
3561
3562	if (!d_tracer && !once) {
3563	once = 1;
3564	pr_warning("Could not create debugfs directory 'tracing'\n");
3565	return NULL;
3566	}
3567
3568	return d_tracer;
3569	}
3570
3571	static struct dentry *d_percpu;
3572
3573	struct dentry *tracing_dentry_percpu(void)
3574	{
3575	static int once;
3576	struct dentry *d_tracer;
3577
3578	if (d_percpu)
3579	return d_percpu;
3580
3581	d_tracer = tracing_init_dentry();
3582
3583	if (!d_tracer)
3584	return NULL;
3585
3586	d_percpu = debugfs_create_dir("per_cpu", d_tracer);
3587
3588	if (!d_percpu && !once) {
3589	once = 1;
3590	pr_warning("Could not create debugfs directory 'per_cpu'\n");
3591	return NULL;
3592	}
3593
3594	return d_percpu;
3595	}
3596
3597	static void tracing_init_debugfs_percpu(long cpu)
3598	{
3599	struct dentry *d_percpu = tracing_dentry_percpu();
3600	struct dentry entry, d_cpu;
3601	/* strlen(cpu) + MAX(log10(cpu)) + '\0' */
3602	char cpu_dir[7];
3603
3604	if (cpu > 999 \|\| cpu < 0)
3605	return;
3606
3607	sprintf(cpu_dir, "cpu%ld", cpu);
3608	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
3609	if (!d_cpu) {
3610	pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
3611	return;
3612	}
3613
3614	/* per cpu trace_pipe */
3615	entry = debugfs_create_file("trace_pipe", 0444, d_cpu,
3616	(void *) cpu, &tracing_pipe_fops);
3617	if (!entry)
3618	pr_warning("Could not create debugfs 'trace_pipe' entry\n");
3619
3620	/* per cpu trace */
3621	entry = debugfs_create_file("trace", 0644, d_cpu,
3622	(void *) cpu, &tracing_fops);
3623	if (!entry)
3624	pr_warning("Could not create debugfs 'trace' entry\n");
3625
3626	entry = debugfs_create_file("trace_pipe_raw", 0444, d_cpu,
3627	(void *) cpu, &tracing_buffers_fops);
3628	if (!entry)
3629	pr_warning("Could not create debugfs 'trace_pipe_raw' entry\n");
3630	}
3631
3632	#ifdef CONFIG_FTRACE_SELFTEST
3633	/* Let selftest have access to static functions in this file */
3634	#include "trace_selftest.c"
3635	#endif
3636
3637	struct trace_option_dentry {
3638	struct tracer_opt *opt;
3639	struct tracer_flags *flags;
3640	struct dentry *entry;
3641	};
3642
3643	static ssize_t
3644	trace_options_read(struct file filp, char __user ubuf, size_t cnt,
3645	loff_t *ppos)
3646	{
3647	struct trace_option_dentry *topt = filp->private_data;
3648	char *buf;
3649
3650	if (topt->flags->val & topt->opt->bit)
3651	buf = "1\n";
3652	else
3653	buf = "0\n";
3654
3655	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3656	}
3657
3658	static ssize_t
3659	trace_options_write(struct file filp, const char __user ubuf, size_t cnt,
3660	loff_t *ppos)
3661	{
3662	struct trace_option_dentry *topt = filp->private_data;
3663	unsigned long val;
3664	char buf[64];
3665	int ret;
3666
3667	if (cnt >= sizeof(buf))
3668	return -EINVAL;
3669
3670	if (copy_from_user(&buf, ubuf, cnt))
3671	return -EFAULT;
3672
3673	buf[cnt] = 0;
3674
3675	ret = strict_strtoul(buf, 10, &val);
3676	if (ret < 0)
3677	return ret;
3678
3679	ret = 0;
3680	switch (val) {
3681	case 0:
3682	/* do nothing if already cleared */
3683	if (!(topt->flags->val & topt->opt->bit))
3684	break;
3685
3686	mutex_lock(&trace_types_lock);
3687	if (current_trace->set_flag)
3688	ret = current_trace->set_flag(topt->flags->val,
3689	topt->opt->bit, 0);
3690	mutex_unlock(&trace_types_lock);
3691	if (ret)
3692	return ret;
3693	topt->flags->val &= ~topt->opt->bit;
3694	break;
3695	case 1:
3696	/* do nothing if already set */
3697	if (topt->flags->val & topt->opt->bit)
3698	break;
3699
3700	mutex_lock(&trace_types_lock);
3701	if (current_trace->set_flag)
3702	ret = current_trace->set_flag(topt->flags->val,
3703	topt->opt->bit, 1);
3704	mutex_unlock(&trace_types_lock);
3705	if (ret)
3706	return ret;
3707	topt->flags->val \|= topt->opt->bit;
3708	break;
3709
3710	default:
3711	return -EINVAL;
3712	}
3713
3714	*ppos += cnt;
3715
3716	return cnt;
3717	}
3718
3719
3720	static const struct file_operations trace_options_fops = {
3721	.open = tracing_open_generic,
3722	.read = trace_options_read,
3723	.write = trace_options_write,
3724	};
3725
3726	static ssize_t
3727	trace_options_core_read(struct file filp, char __user ubuf, size_t cnt,
3728	loff_t *ppos)
3729	{
3730	long index = (long)filp->private_data;
3731	char *buf;
3732
3733	if (trace_flags & (1 << index))
3734	buf = "1\n";
3735	else
3736	buf = "0\n";
3737
3738	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3739	}
3740
3741	static ssize_t
3742	trace_options_core_write(struct file filp, const char __user ubuf, size_t cnt,
3743	loff_t *ppos)
3744	{
3745	long index = (long)filp->private_data;
3746	char buf[64];
3747	unsigned long val;
3748	int ret;
3749
3750	if (cnt >= sizeof(buf))
3751	return -EINVAL;
3752
3753	if (copy_from_user(&buf, ubuf, cnt))
3754	return -EFAULT;
3755
3756	buf[cnt] = 0;
3757
3758	ret = strict_strtoul(buf, 10, &val);
3759	if (ret < 0)
3760	return ret;
3761
3762	switch (val) {
3763	case 0:
3764	trace_flags &= ~(1 << index);
3765	break;
3766	case 1:
3767	trace_flags \|= 1 << index;
3768	break;
3769
3770	default:
3771	return -EINVAL;
3772	}
3773
3774	*ppos += cnt;
3775
3776	return cnt;
3777	}
3778
3779	static const struct file_operations trace_options_core_fops = {
3780	.open = tracing_open_generic,
3781	.read = trace_options_core_read,
3782	.write = trace_options_core_write,
3783	};
3784
3785	static struct dentry *trace_options_init_dentry(void)
3786	{
3787	struct dentry *d_tracer;
3788	static struct dentry *t_options;
3789
3790	if (t_options)
3791	return t_options;
3792
3793	d_tracer = tracing_init_dentry();
3794	if (!d_tracer)
3795	return NULL;
3796
3797	t_options = debugfs_create_dir("options", d_tracer);
3798	if (!t_options) {
3799	pr_warning("Could not create debugfs directory 'options'\n");
3800	return NULL;
3801	}
3802
3803	return t_options;
3804	}
3805
3806	static void
3807	create_trace_option_file(struct trace_option_dentry *topt,
3808	struct tracer_flags *flags,
3809	struct tracer_opt *opt)
3810	{
3811	struct dentry *t_options;
3812	struct dentry *entry;
3813
3814	t_options = trace_options_init_dentry();
3815	if (!t_options)
3816	return;
3817
3818	topt->flags = flags;
3819	topt->opt = opt;
3820
3821	entry = debugfs_create_file(opt->name, 0644, t_options, topt,
3822	&trace_options_fops);
3823
3824	topt->entry = entry;
3825
3826	}
3827
3828	static struct trace_option_dentry *
3829	create_trace_option_files(struct tracer *tracer)
3830	{
3831	struct trace_option_dentry *topts;
3832	struct tracer_flags *flags;
3833	struct tracer_opt *opts;
3834	int cnt;
3835
3836	if (!tracer)
3837	return NULL;
3838
3839	flags = tracer->flags;
3840
3841	if (!flags \|\| !flags->opts)
3842	return NULL;
3843
3844	opts = flags->opts;
3845
3846	for (cnt = 0; opts[cnt].name; cnt++)
3847	;
3848
3849	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
3850	if (!topts)
3851	return NULL;
3852
3853	for (cnt = 0; opts[cnt].name; cnt++)
3854	create_trace_option_file(&topts[cnt], flags,
3855	&opts[cnt]);
3856
3857	return topts;
3858	}
3859
3860	static void
3861	destroy_trace_option_files(struct trace_option_dentry *topts)
3862	{
3863	int cnt;
3864
3865	if (!topts)
3866	return;
3867
3868	for (cnt = 0; topts[cnt].opt; cnt++) {
3869	if (topts[cnt].entry)
3870	debugfs_remove(topts[cnt].entry);
3871	}
3872
3873	kfree(topts);
3874	}
3875
3876	static struct dentry *
3877	create_trace_option_core_file(const char *option, long index)
3878	{
3879	struct dentry *t_options;
3880	struct dentry *entry;
3881
3882	t_options = trace_options_init_dentry();
3883	if (!t_options)
3884	return NULL;
3885
3886	entry = debugfs_create_file(option, 0644, t_options, (void *)index,
3887	&trace_options_core_fops);
3888
3889	return entry;
3890	}
3891
3892	static __init void create_trace_options_dir(void)
3893	{
3894	struct dentry *t_options;
3895	struct dentry *entry;
3896	int i;
3897
3898	t_options = trace_options_init_dentry();
3899	if (!t_options)
3900	return;
3901
3902	for (i = 0; trace_options[i]; i++) {
3903	entry = create_trace_option_core_file(trace_options[i], i);
3904	if (!entry)
3905	pr_warning("Could not create debugfs %s entry\n",
3906	trace_options[i]);
3907	}
3908	}
3909
3910	static __init int tracer_init_debugfs(void)
3911	{
3912	struct dentry *d_tracer;
3913	struct dentry *entry;
3914	int cpu;
3915
3916	d_tracer = tracing_init_dentry();
3917
3918	entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
3919	&global_trace, &tracing_ctrl_fops);
3920	if (!entry)
3921	pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
3922
3923	entry = debugfs_create_file("trace_options", 0644, d_tracer,
3924	NULL, &tracing_iter_fops);
3925	if (!entry)
3926	pr_warning("Could not create debugfs 'trace_options' entry\n");
3927
3928	create_trace_options_dir();
3929
3930	entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
3931	NULL, &tracing_cpumask_fops);
3932	if (!entry)
3933	pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
3934
3935	entry = debugfs_create_file("trace", 0644, d_tracer,
3936	(void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
3937	if (!entry)
3938	pr_warning("Could not create debugfs 'trace' entry\n");
3939
3940	entry = debugfs_create_file("available_tracers", 0444, d_tracer,
3941	&global_trace, &show_traces_fops);
3942	if (!entry)
3943	pr_warning("Could not create debugfs 'available_tracers' entry\n");
3944
3945	entry = debugfs_create_file("current_tracer", 0444, d_tracer,
3946	&global_trace, &set_tracer_fops);
3947	if (!entry)
3948	pr_warning("Could not create debugfs 'current_tracer' entry\n");
3949
3950	entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
3951	&tracing_max_latency,
3952	&tracing_max_lat_fops);
3953	if (!entry)
3954	pr_warning("Could not create debugfs "
3955	"'tracing_max_latency' entry\n");
3956
3957	entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
3958	&tracing_thresh, &tracing_max_lat_fops);
3959	if (!entry)
3960	pr_warning("Could not create debugfs "
3961	"'tracing_thresh' entry\n");
3962	entry = debugfs_create_file("README", 0644, d_tracer,
3963	NULL, &tracing_readme_fops);
3964	if (!entry)
3965	pr_warning("Could not create debugfs 'README' entry\n");
3966
3967	entry = debugfs_create_file("trace_pipe", 0444, d_tracer,
3968	(void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
3969	if (!entry)
3970	pr_warning("Could not create debugfs "
3971	"'trace_pipe' entry\n");
3972
3973	entry = debugfs_create_file("buffer_size_kb", 0644, d_tracer,
3974	&global_trace, &tracing_entries_fops);
3975	if (!entry)
3976	pr_warning("Could not create debugfs "
3977	"'buffer_size_kb' entry\n");
3978
3979	entry = debugfs_create_file("trace_marker", 0220, d_tracer,
3980	NULL, &tracing_mark_fops);
3981	if (!entry)
3982	pr_warning("Could not create debugfs "
3983	"'trace_marker' entry\n");
3984
3985	#ifdef CONFIG_DYNAMIC_FTRACE
3986	entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
3987	&ftrace_update_tot_cnt,
3988	&tracing_dyn_info_fops);
3989	if (!entry)
3990	pr_warning("Could not create debugfs "
3991	"'dyn_ftrace_total_info' entry\n");
3992	#endif
3993	#ifdef CONFIG_SYSPROF_TRACER
3994	init_tracer_sysprof_debugfs(d_tracer);
3995	#endif
3996
3997	for_each_tracing_cpu(cpu)
3998	tracing_init_debugfs_percpu(cpu);
3999
4000	return 0;
4001	}
4002
4003	static int trace_panic_handler(struct notifier_block *this,
4004	unsigned long event, void *unused)
4005	{
4006	if (ftrace_dump_on_oops)
4007	ftrace_dump();
4008	return NOTIFY_OK;
4009	}
4010
4011	static struct notifier_block trace_panic_notifier = {
4012	.notifier_call = trace_panic_handler,
4013	.next = NULL,
4014	.priority = 150 /* priority: INT_MAX >= x >= 0 */
4015	};
4016
4017	static int trace_die_handler(struct notifier_block *self,
4018	unsigned long val,
4019	void *data)
4020	{
4021	switch (val) {
4022	case DIE_OOPS:
4023	if (ftrace_dump_on_oops)
4024	ftrace_dump();
4025	break;
4026	default:
4027	break;
4028	}
4029	return NOTIFY_OK;
4030	}
4031
4032	static struct notifier_block trace_die_notifier = {
4033	.notifier_call = trace_die_handler,
4034	.priority = 200
4035	};
4036
4037	/*
4038	* printk is set to max of 1024, we really don't need it that big.
4039	* Nothing should be printing 1000 characters anyway.
4040	*/
4041	#define TRACE_MAX_PRINT 1000
4042
4043	/*
4044	* Define here KERN_TRACE so that we have one place to modify
4045	* it if we decide to change what log level the ftrace dump
4046	* should be at.
4047	*/
4048	#define KERN_TRACE KERN_EMERG
4049
4050	static void
4051	trace_printk_seq(struct trace_seq *s)
4052	{
4053	/* Probably should print a warning here. */
4054	if (s->len >= 1000)
4055	s->len = 1000;
4056
4057	/* should be zero ended, but we are paranoid. */
4058	s->buffer[s->len] = 0;
4059
4060	printk(KERN_TRACE "%s", s->buffer);
4061
4062	trace_seq_init(s);
4063	}
4064
4065	static void __ftrace_dump(bool disable_tracing)
4066	{
4067	static DEFINE_SPINLOCK(ftrace_dump_lock);
4068	/* use static because iter can be a bit big for the stack */
4069	static struct trace_iterator iter;
4070	unsigned int old_userobj;
4071	static int dump_ran;
4072	unsigned long flags;
4073	int cnt = 0, cpu;
4074
4075	/* only one dump */
4076	spin_lock_irqsave(&ftrace_dump_lock, flags);
4077	if (dump_ran)
4078	goto out;
4079
4080	dump_ran = 1;
4081
4082	tracing_off();
4083
4084	if (disable_tracing)
4085	ftrace_kill();
4086
4087	for_each_tracing_cpu(cpu) {
4088	atomic_inc(&global_trace.data[cpu]->disabled);
4089	}
4090
4091	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4092
4093	/* don't look at user memory in panic mode */
4094	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4095
4096	printk(KERN_TRACE "Dumping ftrace buffer:\n");
4097
4098	/* Simulate the iterator */
4099	iter.tr = &global_trace;
4100	iter.trace = current_trace;
4101	iter.cpu_file = TRACE_PIPE_ALL_CPU;
4102
4103	/*
4104	* We need to stop all tracing on all CPUS to read the
4105	* the next buffer. This is a bit expensive, but is
4106	* not done often. We fill all what we can read,
4107	* and then release the locks again.
4108	*/
4109
4110	while (!trace_empty(&iter)) {
4111
4112	if (!cnt)
4113	printk(KERN_TRACE "---------------------------------\n");
4114
4115	cnt++;
4116
4117	/* reset all but tr, trace, and overruns */
4118	memset(&iter.seq, 0,
4119	sizeof(struct trace_iterator) -
4120	offsetof(struct trace_iterator, seq));
4121	iter.iter_flags \|= TRACE_FILE_LAT_FMT;
4122	iter.pos = -1;
4123
4124	if (find_next_entry_inc(&iter) != NULL) {
4125	print_trace_line(&iter);
4126	trace_consume(&iter);
4127	}
4128
4129	trace_printk_seq(&iter.seq);
4130	}
4131
4132	if (!cnt)
4133	printk(KERN_TRACE " (ftrace buffer empty)\n");
4134	else
4135	printk(KERN_TRACE "---------------------------------\n");
4136
4137	/* Re-enable tracing if requested */
4138	if (!disable_tracing) {
4139	trace_flags \|= old_userobj;
4140
4141	for_each_tracing_cpu(cpu) {
4142	atomic_dec(&global_trace.data[cpu]->disabled);
4143	}
4144	tracing_on();
4145	}
4146
4147	out:
4148	spin_unlock_irqrestore(&ftrace_dump_lock, flags);
4149	}
4150
4151	/* By default: disable tracing after the dump */
4152	void ftrace_dump(void)
4153	{
4154	__ftrace_dump(true);
4155	}
4156
4157	__init static int tracer_alloc_buffers(void)
4158	{
4159	struct trace_array_cpu *data;
4160	int ring_buf_size;
4161	int i;
4162	int ret = -ENOMEM;
4163
4164	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4165	goto out;
4166
4167	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4168	goto out_free_buffer_mask;
4169
4170	if (!alloc_cpumask_var(&tracing_reader_cpumask, GFP_KERNEL))
4171	goto out_free_tracing_cpumask;
4172
4173	/* To save memory, keep the ring buffer size to its minimum */
4174	if (ring_buffer_expanded)
4175	ring_buf_size = trace_buf_size;
4176	else
4177	ring_buf_size = 1;
4178
4179	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4180	cpumask_copy(tracing_cpumask, cpu_all_mask);
4181	cpumask_clear(tracing_reader_cpumask);
4182
4183	/* TODO: make the number of buffers hot pluggable with CPUS */
4184	global_trace.buffer = ring_buffer_alloc(ring_buf_size,
4185	TRACE_BUFFER_FLAGS);
4186	if (!global_trace.buffer) {
4187	printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4188	WARN_ON(1);
4189	goto out_free_cpumask;
4190	}
4191	global_trace.entries = ring_buffer_size(global_trace.buffer);
4192
4193
4194	#ifdef CONFIG_TRACER_MAX_TRACE
4195	max_tr.buffer = ring_buffer_alloc(ring_buf_size,
4196	TRACE_BUFFER_FLAGS);
4197	if (!max_tr.buffer) {
4198	printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4199	WARN_ON(1);
4200	ring_buffer_free(global_trace.buffer);
4201	goto out_free_cpumask;
4202	}
4203	max_tr.entries = ring_buffer_size(max_tr.buffer);
4204	WARN_ON(max_tr.entries != global_trace.entries);
4205	#endif
4206
4207	/* Allocate the first page for all buffers */
4208	for_each_tracing_cpu(i) {
4209	data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
4210	max_tr.data[i] = &per_cpu(max_data, i);
4211	}
4212
4213	trace_init_cmdlines();
4214
4215	register_tracer(&nop_trace);
4216	current_trace = &nop_trace;
4217	#ifdef CONFIG_BOOT_TRACER
4218	register_tracer(&boot_tracer);
4219	#endif
4220	/* All seems OK, enable tracing */
4221	tracing_disabled = 0;
4222
4223	atomic_notifier_chain_register(&panic_notifier_list,
4224	&trace_panic_notifier);
4225
4226	register_die_notifier(&trace_die_notifier);
4227
4228	return 0;
4229
4230	out_free_cpumask:
4231	free_cpumask_var(tracing_reader_cpumask);
4232	out_free_tracing_cpumask:
4233	free_cpumask_var(tracing_cpumask);
4234	out_free_buffer_mask:
4235	free_cpumask_var(tracing_buffer_mask);
4236	out:
4237	return ret;
4238	}
4239
4240	__init static int clear_boot_tracer(void)
4241	{
4242	/*
4243	* The default tracer at boot buffer is an init section.
4244	* This function is called in lateinit. If we did not
4245	* find the boot tracer, then clear it out, to prevent
4246	* later registration from accessing the buffer that is
4247	* about to be freed.
4248	*/
4249	if (!default_bootup_tracer)
4250	return 0;
4251
4252	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4253	default_bootup_tracer);
4254	default_bootup_tracer = NULL;
4255
4256	return 0;
4257	}
4258
4259	early_initcall(tracer_alloc_buffers);
4260	fs_initcall(tracer_init_debugfs);
4261	late_initcall(clear_boot_tracer);