linux - kernel/trace/trace.c

5181 lines · 117.5 KiB

1	/*
2	* ring buffer based function tracer
3	*
4	* Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
5	* Copyright (C) 2008 Ingo Molnar <[email protected]>
6	*
7	* Originally taken from the RT patch by:
8	* Arnaldo Carvalho de Melo <[email protected]>
9	*
10	* Based on code from the latency_tracer, that is:
11	* Copyright (C) 2004-2006 Ingo Molnar
12	* Copyright (C) 2004 William Lee Irwin III
13	*/
14	#include <linux/ring_buffer.h>
15	#include <generated/utsrelease.h>
16	#include <linux/stacktrace.h>
17	#include <linux/writeback.h>
18	#include <linux/kallsyms.h>
19	#include <linux/seq_file.h>
20	#include <linux/notifier.h>
21	#include <linux/irqflags.h>
22	#include <linux/debugfs.h>
23	#include <linux/pagemap.h>
24	#include <linux/hardirq.h>
25	#include <linux/linkage.h>
26	#include <linux/uaccess.h>
27	#include <linux/kprobes.h>
28	#include <linux/ftrace.h>
29	#include <linux/module.h>
30	#include <linux/percpu.h>
31	#include <linux/splice.h>
32	#include <linux/kdebug.h>
33	#include <linux/string.h>
34	#include <linux/rwsem.h>
35	#include <linux/slab.h>
36	#include <linux/ctype.h>
37	#include <linux/init.h>
38	#include <linux/poll.h>
39	#include <linux/nmi.h>
40	#include <linux/fs.h>
41
42	#include "trace.h"
43	#include "trace_output.h"
44
45	/*
46	* On boot up, the ring buffer is set to the minimum size, so that
47	* we do not waste memory on systems that are not using tracing.
48	*/
49	int ring_buffer_expanded;
50
51	/*
52	* We need to change this state when a selftest is running.
53	* A selftest will lurk into the ring-buffer to count the
54	* entries inserted during the selftest although some concurrent
55	* insertions into the ring-buffer such as trace_printk could occurred
56	* at the same time, giving false positive or negative results.
57	*/
58	static bool __read_mostly tracing_selftest_running;
59
60	/*
61	* If a tracer is running, we do not want to run SELFTEST.
62	*/
63	bool __read_mostly tracing_selftest_disabled;
64
65	/* For tracers that don't implement custom flags */
66	static struct tracer_opt dummy_tracer_opt[] = {
67	{ }
68	};
69
70	static struct tracer_flags dummy_tracer_flags = {
71	.val = 0,
72	.opts = dummy_tracer_opt
73	};
74
75	static int dummy_set_flag(u32 old_flags, u32 bit, int set)
76	{
77	return 0;
78	}
79
80	/*
81	* Kill all tracing for good (never come back).
82	* It is initialized to 1 but will turn to zero if the initialization
83	* of the tracer is successful. But that is the only place that sets
84	* this back to zero.
85	*/
86	static int tracing_disabled = 1;
87
88	DEFINE_PER_CPU(int, ftrace_cpu_disabled);
89
90	cpumask_var_t __read_mostly tracing_buffer_mask;
91
92	/*
93	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
94	*
95	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
96	* is set, then ftrace_dump is called. This will output the contents
97	* of the ftrace buffers to the console. This is very useful for
98	* capturing traces that lead to crashes and outputing it to a
99	* serial console.
100	*
101	* It is default off, but you can enable it with either specifying
102	* "ftrace_dump_on_oops" in the kernel command line, or setting
103	* /proc/sys/kernel/ftrace_dump_on_oops
104	* Set 1 if you want to dump buffers of all CPUs
105	* Set 2 if you want to dump the buffer of the CPU that triggered oops
106	*/
107
108	enum ftrace_dump_mode ftrace_dump_on_oops;
109
110	static int tracing_set_tracer(const char *buf);
111
112	#define MAX_TRACER_SIZE 100
113	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
114	static char *default_bootup_tracer;
115
116	static int __init set_cmdline_ftrace(char *str)
117	{
118	strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
119	default_bootup_tracer = bootup_tracer_buf;
120	/* We are using ftrace early, expand it */
121	ring_buffer_expanded = 1;
122	return 1;
123	}
124	__setup("ftrace=", set_cmdline_ftrace);
125
126	static int __init set_ftrace_dump_on_oops(char *str)
127	{
128	if (str++ != '=' \|\| !str) {
129	ftrace_dump_on_oops = DUMP_ALL;
130	return 1;
131	}
132
133	if (!strcmp("orig_cpu", str)) {
134	ftrace_dump_on_oops = DUMP_ORIG;
135	return 1;
136	}
137
138	return 0;
139	}
140	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
141
142	unsigned long long ns2usecs(cycle_t nsec)
143	{
144	nsec += 500;
145	do_div(nsec, 1000);
146	return nsec;
147	}
148
149	/*
150	* The global_trace is the descriptor that holds the tracing
151	* buffers for the live tracing. For each CPU, it contains
152	* a link list of pages that will store trace entries. The
153	* page descriptor of the pages in the memory is used to hold
154	* the link list by linking the lru item in the page descriptor
155	* to each of the pages in the buffer per CPU.
156	*
157	* For each active CPU there is a data field that holds the
158	* pages for the buffer for that CPU. Each CPU has the same number
159	* of pages allocated for its buffer.
160	*/
161	static struct trace_array global_trace;
162
163	static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
164
165	int filter_current_check_discard(struct ring_buffer *buffer,
166	struct ftrace_event_call call, void rec,
167	struct ring_buffer_event *event)
168	{
169	return filter_check_discard(call, rec, buffer, event);
170	}
171	EXPORT_SYMBOL_GPL(filter_current_check_discard);
172
173	cycle_t ftrace_now(int cpu)
174	{
175	u64 ts;
176
177	/* Early boot up does not have a buffer yet */
178	if (!global_trace.buffer)
179	return trace_clock_local();
180
181	ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
182	ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
183
184	return ts;
185	}
186
187	/*
188	* The max_tr is used to snapshot the global_trace when a maximum
189	* latency is reached. Some tracers will use this to store a maximum
190	* trace while it continues examining live traces.
191	*
192	* The buffers for the max_tr are set up the same as the global_trace.
193	* When a snapshot is taken, the link list of the max_tr is swapped
194	* with the link list of the global_trace and the buffers are reset for
195	* the global_trace so the tracing can continue.
196	*/
197	static struct trace_array max_tr;
198
199	static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
200
201	/* tracer_enabled is used to toggle activation of a tracer */
202	static int tracer_enabled = 1;
203
204	/**
205	* tracing_is_enabled - return tracer_enabled status
206	*
207	* This function is used by other tracers to know the status
208	* of the tracer_enabled flag. Tracers may use this function
209	* to know if it should enable their features when starting
210	* up. See irqsoff tracer for an example (start_irqsoff_tracer).
211	*/
212	int tracing_is_enabled(void)
213	{
214	return tracer_enabled;
215	}
216
217	/*
218	* trace_buf_size is the size in bytes that is allocated
219	* for a buffer. Note, the number of bytes is always rounded
220	* to page size.
221	*
222	* This number is purposely set to a low number of 16384.
223	* If the dump on oops happens, it will be much appreciated
224	* to not have to wait for all that output. Anyway this can be
225	* boot time and run time configurable.
226	*/
227	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
228
229	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
230
231	/* trace_types holds a link list of available tracers. */
232	static struct tracer *trace_types __read_mostly;
233
234	/* current_trace points to the tracer that is currently active */
235	static struct tracer *current_trace __read_mostly;
236
237	/*
238	* trace_types_lock is used to protect the trace_types list.
239	*/
240	static DEFINE_MUTEX(trace_types_lock);
241
242	/*
243	* serialize the access of the ring buffer
244	*
245	* ring buffer serializes readers, but it is low level protection.
246	* The validity of the events (which returns by ring_buffer_peek() ..etc)
247	* are not protected by ring buffer.
248	*
249	* The content of events may become garbage if we allow other process consumes
250	* these events concurrently:
251	* A) the page of the consumed events may become a normal page
252	* (not reader page) in ring buffer, and this page will be rewrited
253	* by events producer.
254	* B) The page of the consumed events may become a page for splice_read,
255	* and this page will be returned to system.
256	*
257	* These primitives allow multi process access to different cpu ring buffer
258	* concurrently.
259	*
260	* These primitives don't distinguish read-only and read-consume access.
261	* Multi read-only access are also serialized.
262	*/
263
264	#ifdef CONFIG_SMP
265	static DECLARE_RWSEM(all_cpu_access_lock);
266	static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
267
268	static inline void trace_access_lock(int cpu)
269	{
270	if (cpu == TRACE_PIPE_ALL_CPU) {
271	/* gain it for accessing the whole ring buffer. */
272	down_write(&all_cpu_access_lock);
273	} else {
274	/* gain it for accessing a cpu ring buffer. */
275
276	/* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
277	down_read(&all_cpu_access_lock);
278
279	/* Secondly block other access to this @cpu ring buffer. */
280	mutex_lock(&per_cpu(cpu_access_lock, cpu));
281	}
282	}
283
284	static inline void trace_access_unlock(int cpu)
285	{
286	if (cpu == TRACE_PIPE_ALL_CPU) {
287	up_write(&all_cpu_access_lock);
288	} else {
289	mutex_unlock(&per_cpu(cpu_access_lock, cpu));
290	up_read(&all_cpu_access_lock);
291	}
292	}
293
294	static inline void trace_access_lock_init(void)
295	{
296	int cpu;
297
298	for_each_possible_cpu(cpu)
299	mutex_init(&per_cpu(cpu_access_lock, cpu));
300	}
301
302	#else
303
304	static DEFINE_MUTEX(access_lock);
305
306	static inline void trace_access_lock(int cpu)
307	{
308	(void)cpu;
309	mutex_lock(&access_lock);
310	}
311
312	static inline void trace_access_unlock(int cpu)
313	{
314	(void)cpu;
315	mutex_unlock(&access_lock);
316	}
317
318	static inline void trace_access_lock_init(void)
319	{
320	}
321
322	#endif
323
324	/* trace_wait is a waitqueue for tasks blocked on trace_poll */
325	static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
326
327	/* trace_flags holds trace_options default values */
328	unsigned long trace_flags = TRACE_ITER_PRINT_PARENT \| TRACE_ITER_PRINTK \|
329	TRACE_ITER_ANNOTATE \| TRACE_ITER_CONTEXT_INFO \| TRACE_ITER_SLEEP_TIME \|
330	TRACE_ITER_GRAPH_TIME \| TRACE_ITER_RECORD_CMD \| TRACE_ITER_OVERWRITE \|
331	TRACE_ITER_IRQ_INFO \| TRACE_ITER_MARKERS;
332
333	static int trace_stop_count;
334	static DEFINE_RAW_SPINLOCK(tracing_start_lock);
335
336	static void wakeup_work_handler(struct work_struct *work)
337	{
338	wake_up(&trace_wait);
339	}
340
341	static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
342
343	/**
344	* tracing_on - enable tracing buffers
345	*
346	* This function enables tracing buffers that may have been
347	* disabled with tracing_off.
348	*/
349	void tracing_on(void)
350	{
351	if (global_trace.buffer)
352	ring_buffer_record_on(global_trace.buffer);
353	/*
354	* This flag is only looked at when buffers haven't been
355	* allocated yet. We don't really care about the race
356	* between setting this flag and actually turning
357	* on the buffer.
358	*/
359	global_trace.buffer_disabled = 0;
360	}
361	EXPORT_SYMBOL_GPL(tracing_on);
362
363	/**
364	* tracing_off - turn off tracing buffers
365	*
366	* This function stops the tracing buffers from recording data.
367	* It does not disable any overhead the tracers themselves may
368	* be causing. This function simply causes all recording to
369	* the ring buffers to fail.
370	*/
371	void tracing_off(void)
372	{
373	if (global_trace.buffer)
374	ring_buffer_record_off(global_trace.buffer);
375	/*
376	* This flag is only looked at when buffers haven't been
377	* allocated yet. We don't really care about the race
378	* between setting this flag and actually turning
379	* on the buffer.
380	*/
381	global_trace.buffer_disabled = 1;
382	}
383	EXPORT_SYMBOL_GPL(tracing_off);
384
385	/**
386	* tracing_is_on - show state of ring buffers enabled
387	*/
388	int tracing_is_on(void)
389	{
390	if (global_trace.buffer)
391	return ring_buffer_record_is_on(global_trace.buffer);
392	return !global_trace.buffer_disabled;
393	}
394	EXPORT_SYMBOL_GPL(tracing_is_on);
395
396	/**
397	* trace_wake_up - wake up tasks waiting for trace input
398	*
399	* Schedules a delayed work to wake up any task that is blocked on the
400	* trace_wait queue. These is used with trace_poll for tasks polling the
401	* trace.
402	*/
403	void trace_wake_up(void)
404	{
405	const unsigned long delay = msecs_to_jiffies(2);
406
407	if (trace_flags & TRACE_ITER_BLOCK)
408	return;
409	schedule_delayed_work(&wakeup_work, delay);
410	}
411
412	static int __init set_buf_size(char *str)
413	{
414	unsigned long buf_size;
415
416	if (!str)
417	return 0;
418	buf_size = memparse(str, &str);
419	/* nr_entries can not be zero */
420	if (buf_size == 0)
421	return 0;
422	trace_buf_size = buf_size;
423	return 1;
424	}
425	__setup("trace_buf_size=", set_buf_size);
426
427	static int __init set_tracing_thresh(char *str)
428	{
429	unsigned long threshold;
430	int ret;
431
432	if (!str)
433	return 0;
434	ret = strict_strtoul(str, 0, &threshold);
435	if (ret < 0)
436	return 0;
437	tracing_thresh = threshold * 1000;
438	return 1;
439	}
440	__setup("tracing_thresh=", set_tracing_thresh);
441
442	unsigned long nsecs_to_usecs(unsigned long nsecs)
443	{
444	return nsecs / 1000;
445	}
446
447	/* These must match the bit postions in trace_iterator_flags */
448	static const char *trace_options[] = {
449	"print-parent",
450	"sym-offset",
451	"sym-addr",
452	"verbose",
453	"raw",
454	"hex",
455	"bin",
456	"block",
457	"stacktrace",
458	"trace_printk",
459	"ftrace_preempt",
460	"branch",
461	"annotate",
462	"userstacktrace",
463	"sym-userobj",
464	"printk-msg-only",
465	"context-info",
466	"latency-format",
467	"sleep-time",
468	"graph-time",
469	"record-cmd",
470	"overwrite",
471	"disable_on_free",
472	"irq-info",
473	"markers",
474	NULL
475	};
476
477	static struct {
478	u64 (*func)(void);
479	const char *name;
480	} trace_clocks[] = {
481	{ trace_clock_local, "local" },
482	{ trace_clock_global, "global" },
483	{ trace_clock_counter, "counter" },
484	};
485
486	int trace_clock_id;
487
488	/*
489	* trace_parser_get_init - gets the buffer for trace parser
490	*/
491	int trace_parser_get_init(struct trace_parser *parser, int size)
492	{
493	memset(parser, 0, sizeof(*parser));
494
495	parser->buffer = kmalloc(size, GFP_KERNEL);
496	if (!parser->buffer)
497	return 1;
498
499	parser->size = size;
500	return 0;
501	}
502
503	/*
504	* trace_parser_put - frees the buffer for trace parser
505	*/
506	void trace_parser_put(struct trace_parser *parser)
507	{
508	kfree(parser->buffer);
509	}
510
511	/*
512	* trace_get_user - reads the user input string separated by space
513	* (matched by isspace(ch))
514	*
515	* For each string found the 'struct trace_parser' is updated,
516	* and the function returns.
517	*
518	* Returns number of bytes read.
519	*
520	* See kernel/trace/trace.h for 'struct trace_parser' details.
521	*/
522	int trace_get_user(struct trace_parser parser, const char __user ubuf,
523	size_t cnt, loff_t *ppos)
524	{
525	char ch;
526	size_t read = 0;
527	ssize_t ret;
528
529	if (!*ppos)
530	trace_parser_clear(parser);
531
532	ret = get_user(ch, ubuf++);
533	if (ret)
534	goto out;
535
536	read++;
537	cnt--;
538
539	/*
540	* The parser is not finished with the last write,
541	* continue reading the user input without skipping spaces.
542	*/
543	if (!parser->cont) {
544	/* skip white space */
545	while (cnt && isspace(ch)) {
546	ret = get_user(ch, ubuf++);
547	if (ret)
548	goto out;
549	read++;
550	cnt--;
551	}
552
553	/* only spaces were written */
554	if (isspace(ch)) {
555	*ppos += read;
556	ret = read;
557	goto out;
558	}
559
560	parser->idx = 0;
561	}
562
563	/* read the non-space input */
564	while (cnt && !isspace(ch)) {
565	if (parser->idx < parser->size - 1)
566	parser->buffer[parser->idx++] = ch;
567	else {
568	ret = -EINVAL;
569	goto out;
570	}
571	ret = get_user(ch, ubuf++);
572	if (ret)
573	goto out;
574	read++;
575	cnt--;
576	}
577
578	/* We either got finished input or we have to wait for another call. */
579	if (isspace(ch)) {
580	parser->buffer[parser->idx] = 0;
581	parser->cont = false;
582	} else {
583	parser->cont = true;
584	parser->buffer[parser->idx++] = ch;
585	}
586
587	*ppos += read;
588	ret = read;
589
590	out:
591	return ret;
592	}
593
594	ssize_t trace_seq_to_user(struct trace_seq s, char __user ubuf, size_t cnt)
595	{
596	int len;
597	int ret;
598
599	if (!cnt)
600	return 0;
601
602	if (s->len <= s->readpos)
603	return -EBUSY;
604
605	len = s->len - s->readpos;
606	if (cnt > len)
607	cnt = len;
608	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
609	if (ret == cnt)
610	return -EFAULT;
611
612	cnt -= ret;
613
614	s->readpos += cnt;
615	return cnt;
616	}
617
618	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
619	{
620	int len;
621
622	if (s->len <= s->readpos)
623	return -EBUSY;
624
625	len = s->len - s->readpos;
626	if (cnt > len)
627	cnt = len;
628	memcpy(buf, s->buffer + s->readpos, cnt);
629
630	s->readpos += cnt;
631	return cnt;
632	}
633
634	/*
635	* ftrace_max_lock is used to protect the swapping of buffers
636	* when taking a max snapshot. The buffers themselves are
637	* protected by per_cpu spinlocks. But the action of the swap
638	* needs its own lock.
639	*
640	* This is defined as a arch_spinlock_t in order to help
641	* with performance when lockdep debugging is enabled.
642	*
643	* It is also used in other places outside the update_max_tr
644	* so it needs to be defined outside of the
645	* CONFIG_TRACER_MAX_TRACE.
646	*/
647	static arch_spinlock_t ftrace_max_lock =
648	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
649
650	unsigned long __read_mostly tracing_thresh;
651
652	#ifdef CONFIG_TRACER_MAX_TRACE
653	unsigned long __read_mostly tracing_max_latency;
654
655	/*
656	* Copy the new maximum trace into the separate maximum-trace
657	* structure. (this way the maximum trace is permanently saved,
658	* for later retrieval via /sys/kernel/debug/tracing/latency_trace)
659	*/
660	static void
661	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
662	{
663	struct trace_array_cpu *data = tr->data[cpu];
664	struct trace_array_cpu *max_data;
665
666	max_tr.cpu = cpu;
667	max_tr.time_start = data->preempt_timestamp;
668
669	max_data = max_tr.data[cpu];
670	max_data->saved_latency = tracing_max_latency;
671	max_data->critical_start = data->critical_start;
672	max_data->critical_end = data->critical_end;
673
674	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
675	max_data->pid = tsk->pid;
676	max_data->uid = task_uid(tsk);
677	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
678	max_data->policy = tsk->policy;
679	max_data->rt_priority = tsk->rt_priority;
680
681	/* record this tasks comm */
682	tracing_record_cmdline(tsk);
683	}
684
685	/**
686	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
687	* @tr: tracer
688	* @tsk: the task with the latency
689	* @cpu: The cpu that initiated the trace.
690	*
691	* Flip the buffers between the @tr and the max_tr and record information
692	* about which task was the cause of this latency.
693	*/
694	void
695	update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
696	{
697	struct ring_buffer *buf = tr->buffer;
698
699	if (trace_stop_count)
700	return;
701
702	WARN_ON_ONCE(!irqs_disabled());
703	if (!current_trace->use_max_tr) {
704	WARN_ON_ONCE(1);
705	return;
706	}
707	arch_spin_lock(&ftrace_max_lock);
708
709	tr->buffer = max_tr.buffer;
710	max_tr.buffer = buf;
711
712	__update_max_tr(tr, tsk, cpu);
713	arch_spin_unlock(&ftrace_max_lock);
714	}
715
716	/**
717	* update_max_tr_single - only copy one trace over, and reset the rest
718	* @tr - tracer
719	* @tsk - task with the latency
720	* @cpu - the cpu of the buffer to copy.
721	*
722	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
723	*/
724	void
725	update_max_tr_single(struct trace_array tr, struct task_struct tsk, int cpu)
726	{
727	int ret;
728
729	if (trace_stop_count)
730	return;
731
732	WARN_ON_ONCE(!irqs_disabled());
733	if (!current_trace->use_max_tr) {
734	WARN_ON_ONCE(1);
735	return;
736	}
737
738	arch_spin_lock(&ftrace_max_lock);
739
740	ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
741
742	if (ret == -EBUSY) {
743	/*
744	* We failed to swap the buffer due to a commit taking
745	* place on this CPU. We fail to record, but we reset
746	* the max trace buffer (no one writes directly to it)
747	* and flag that it failed.
748	*/
749	trace_array_printk(&max_tr, _THIS_IP_,
750	"Failed to swap buffers due to commit in progress\n");
751	}
752
753	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
754
755	__update_max_tr(tr, tsk, cpu);
756	arch_spin_unlock(&ftrace_max_lock);
757	}
758	#endif /* CONFIG_TRACER_MAX_TRACE */
759
760	/**
761	* register_tracer - register a tracer with the ftrace system.
762	* @type - the plugin for the tracer
763	*
764	* Register a new plugin tracer.
765	*/
766	int register_tracer(struct tracer *type)
767	{
768	struct tracer *t;
769	int ret = 0;
770
771	if (!type->name) {
772	pr_info("Tracer must have a name\n");
773	return -1;
774	}
775
776	if (strlen(type->name) >= MAX_TRACER_SIZE) {
777	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
778	return -1;
779	}
780
781	mutex_lock(&trace_types_lock);
782
783	tracing_selftest_running = true;
784
785	for (t = trace_types; t; t = t->next) {
786	if (strcmp(type->name, t->name) == 0) {
787	/* already found */
788	pr_info("Tracer %s already registered\n",
789	type->name);
790	ret = -1;
791	goto out;
792	}
793	}
794
795	if (!type->set_flag)
796	type->set_flag = &dummy_set_flag;
797	if (!type->flags)
798	type->flags = &dummy_tracer_flags;
799	else
800	if (!type->flags->opts)
801	type->flags->opts = dummy_tracer_opt;
802	if (!type->wait_pipe)
803	type->wait_pipe = default_wait_pipe;
804
805
806	#ifdef CONFIG_FTRACE_STARTUP_TEST
807	if (type->selftest && !tracing_selftest_disabled) {
808	struct tracer *saved_tracer = current_trace;
809	struct trace_array *tr = &global_trace;
810
811	/*
812	* Run a selftest on this tracer.
813	* Here we reset the trace buffer, and set the current
814	* tracer to be this tracer. The tracer can then run some
815	* internal tracing to verify that everything is in order.
816	* If we fail, we do not register this tracer.
817	*/
818	tracing_reset_online_cpus(tr);
819
820	current_trace = type;
821
822	/* If we expanded the buffers, make sure the max is expanded too */
823	if (ring_buffer_expanded && type->use_max_tr)
824	ring_buffer_resize(max_tr.buffer, trace_buf_size,
825	RING_BUFFER_ALL_CPUS);
826
827	/* the test is responsible for initializing and enabling */
828	pr_info("Testing tracer %s: ", type->name);
829	ret = type->selftest(type, tr);
830	/* the test is responsible for resetting too */
831	current_trace = saved_tracer;
832	if (ret) {
833	printk(KERN_CONT "FAILED!\n");
834	/* Add the warning after printing 'FAILED' */
835	WARN_ON(1);
836	goto out;
837	}
838	/* Only reset on passing, to avoid touching corrupted buffers */
839	tracing_reset_online_cpus(tr);
840
841	/* Shrink the max buffer again */
842	if (ring_buffer_expanded && type->use_max_tr)
843	ring_buffer_resize(max_tr.buffer, 1,
844	RING_BUFFER_ALL_CPUS);
845
846	printk(KERN_CONT "PASSED\n");
847	}
848	#endif
849
850	type->next = trace_types;
851	trace_types = type;
852
853	out:
854	tracing_selftest_running = false;
855	mutex_unlock(&trace_types_lock);
856
857	if (ret \|\| !default_bootup_tracer)
858	goto out_unlock;
859
860	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
861	goto out_unlock;
862
863	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
864	/* Do we want this tracer to start on bootup? */
865	tracing_set_tracer(type->name);
866	default_bootup_tracer = NULL;
867	/* disable other selftests, since this will break it. */
868	tracing_selftest_disabled = 1;
869	#ifdef CONFIG_FTRACE_STARTUP_TEST
870	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
871	type->name);
872	#endif
873
874	out_unlock:
875	return ret;
876	}
877
878	void unregister_tracer(struct tracer *type)
879	{
880	struct tracer **t;
881
882	mutex_lock(&trace_types_lock);
883	for (t = &trace_types; t; t = &(t)->next) {
884	if (*t == type)
885	goto found;
886	}
887	pr_info("Tracer %s not registered\n", type->name);
888	goto out;
889
890	found:
891	t = (t)->next;
892
893	if (type == current_trace && tracer_enabled) {
894	tracer_enabled = 0;
895	tracing_stop();
896	if (current_trace->stop)
897	current_trace->stop(&global_trace);
898	current_trace = &nop_trace;
899	}
900	out:
901	mutex_unlock(&trace_types_lock);
902	}
903
904	void tracing_reset(struct trace_array *tr, int cpu)
905	{
906	struct ring_buffer *buffer = tr->buffer;
907
908	ring_buffer_record_disable(buffer);
909
910	/* Make sure all commits have finished */
911	synchronize_sched();
912	ring_buffer_reset_cpu(buffer, cpu);
913
914	ring_buffer_record_enable(buffer);
915	}
916
917	void tracing_reset_online_cpus(struct trace_array *tr)
918	{
919	struct ring_buffer *buffer = tr->buffer;
920	int cpu;
921
922	ring_buffer_record_disable(buffer);
923
924	/* Make sure all commits have finished */
925	synchronize_sched();
926
927	tr->time_start = ftrace_now(tr->cpu);
928
929	for_each_online_cpu(cpu)
930	ring_buffer_reset_cpu(buffer, cpu);
931
932	ring_buffer_record_enable(buffer);
933	}
934
935	void tracing_reset_current(int cpu)
936	{
937	tracing_reset(&global_trace, cpu);
938	}
939
940	void tracing_reset_current_online_cpus(void)
941	{
942	tracing_reset_online_cpus(&global_trace);
943	}
944
945	#define SAVED_CMDLINES 128
946	#define NO_CMDLINE_MAP UINT_MAX
947	static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
948	static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
949	static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
950	static int cmdline_idx;
951	static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
952
953	/* temporary disable recording */
954	static atomic_t trace_record_cmdline_disabled __read_mostly;
955
956	static void trace_init_cmdlines(void)
957	{
958	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
959	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
960	cmdline_idx = 0;
961	}
962
963	int is_tracing_stopped(void)
964	{
965	return trace_stop_count;
966	}
967
968	/**
969	* ftrace_off_permanent - disable all ftrace code permanently
970	*
971	* This should only be called when a serious anomally has
972	* been detected. This will turn off the function tracing,
973	* ring buffers, and other tracing utilites. It takes no
974	* locks and can be called from any context.
975	*/
976	void ftrace_off_permanent(void)
977	{
978	tracing_disabled = 1;
979	ftrace_stop();
980	tracing_off_permanent();
981	}
982
983	/**
984	* tracing_start - quick start of the tracer
985	*
986	* If tracing is enabled but was stopped by tracing_stop,
987	* this will start the tracer back up.
988	*/
989	void tracing_start(void)
990	{
991	struct ring_buffer *buffer;
992	unsigned long flags;
993
994	if (tracing_disabled)
995	return;
996
997	raw_spin_lock_irqsave(&tracing_start_lock, flags);
998	if (--trace_stop_count) {
999	if (trace_stop_count < 0) {
1000	/* Someone screwed up their debugging */
1001	WARN_ON_ONCE(1);
1002	trace_stop_count = 0;
1003	}
1004	goto out;
1005	}
1006
1007	/* Prevent the buffers from switching */
1008	arch_spin_lock(&ftrace_max_lock);
1009
1010	buffer = global_trace.buffer;
1011	if (buffer)
1012	ring_buffer_record_enable(buffer);
1013
1014	buffer = max_tr.buffer;
1015	if (buffer)
1016	ring_buffer_record_enable(buffer);
1017
1018	arch_spin_unlock(&ftrace_max_lock);
1019
1020	ftrace_start();
1021	out:
1022	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1023	}
1024
1025	/**
1026	* tracing_stop - quick stop of the tracer
1027	*
1028	* Light weight way to stop tracing. Use in conjunction with
1029	* tracing_start.
1030	*/
1031	void tracing_stop(void)
1032	{
1033	struct ring_buffer *buffer;
1034	unsigned long flags;
1035
1036	ftrace_stop();
1037	raw_spin_lock_irqsave(&tracing_start_lock, flags);
1038	if (trace_stop_count++)
1039	goto out;
1040
1041	/* Prevent the buffers from switching */
1042	arch_spin_lock(&ftrace_max_lock);
1043
1044	buffer = global_trace.buffer;
1045	if (buffer)
1046	ring_buffer_record_disable(buffer);
1047
1048	buffer = max_tr.buffer;
1049	if (buffer)
1050	ring_buffer_record_disable(buffer);
1051
1052	arch_spin_unlock(&ftrace_max_lock);
1053
1054	out:
1055	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1056	}
1057
1058	void trace_stop_cmdline_recording(void);
1059
1060	static void trace_save_cmdline(struct task_struct *tsk)
1061	{
1062	unsigned pid, idx;
1063
1064	if (!tsk->pid \|\| unlikely(tsk->pid > PID_MAX_DEFAULT))
1065	return;
1066
1067	/*
1068	* It's not the end of the world if we don't get
1069	* the lock, but we also don't want to spin
1070	* nor do we want to disable interrupts,
1071	* so if we miss here, then better luck next time.
1072	*/
1073	if (!arch_spin_trylock(&trace_cmdline_lock))
1074	return;
1075
1076	idx = map_pid_to_cmdline[tsk->pid];
1077	if (idx == NO_CMDLINE_MAP) {
1078	idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1079
1080	/*
1081	* Check whether the cmdline buffer at idx has a pid
1082	* mapped. We are going to overwrite that entry so we
1083	* need to clear the map_pid_to_cmdline. Otherwise we
1084	* would read the new comm for the old pid.
1085	*/
1086	pid = map_cmdline_to_pid[idx];
1087	if (pid != NO_CMDLINE_MAP)
1088	map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1089
1090	map_cmdline_to_pid[idx] = tsk->pid;
1091	map_pid_to_cmdline[tsk->pid] = idx;
1092
1093	cmdline_idx = idx;
1094	}
1095
1096	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1097
1098	arch_spin_unlock(&trace_cmdline_lock);
1099	}
1100
1101	void trace_find_cmdline(int pid, char comm[])
1102	{
1103	unsigned map;
1104
1105	if (!pid) {
1106	strcpy(comm, "<idle>");
1107	return;
1108	}
1109
1110	if (WARN_ON_ONCE(pid < 0)) {
1111	strcpy(comm, "<XXX>");
1112	return;
1113	}
1114
1115	if (pid > PID_MAX_DEFAULT) {
1116	strcpy(comm, "<...>");
1117	return;
1118	}
1119
1120	preempt_disable();
1121	arch_spin_lock(&trace_cmdline_lock);
1122	map = map_pid_to_cmdline[pid];
1123	if (map != NO_CMDLINE_MAP)
1124	strcpy(comm, saved_cmdlines[map]);
1125	else
1126	strcpy(comm, "<...>");
1127
1128	arch_spin_unlock(&trace_cmdline_lock);
1129	preempt_enable();
1130	}
1131
1132	void tracing_record_cmdline(struct task_struct *tsk)
1133	{
1134	if (atomic_read(&trace_record_cmdline_disabled) \|\| !tracer_enabled \|\|
1135	!tracing_is_on())
1136	return;
1137
1138	trace_save_cmdline(tsk);
1139	}
1140
1141	void
1142	tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1143	int pc)
1144	{
1145	struct task_struct *tsk = current;
1146
1147	entry->preempt_count = pc & 0xff;
1148	entry->pid = (tsk) ? tsk->pid : 0;
1149	entry->padding = 0;
1150	entry->flags =
1151	#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1152	(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) \|
1153	#else
1154	TRACE_FLAG_IRQS_NOSUPPORT \|
1155	#endif
1156	((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) \|
1157	((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) \|
1158	(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1159	}
1160	EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1161
1162	struct ring_buffer_event *
1163	trace_buffer_lock_reserve(struct ring_buffer *buffer,
1164	int type,
1165	unsigned long len,
1166	unsigned long flags, int pc)
1167	{
1168	struct ring_buffer_event *event;
1169
1170	event = ring_buffer_lock_reserve(buffer, len);
1171	if (event != NULL) {
1172	struct trace_entry *ent = ring_buffer_event_data(event);
1173
1174	tracing_generic_entry_update(ent, flags, pc);
1175	ent->type = type;
1176	}
1177
1178	return event;
1179	}
1180
1181	static inline void
1182	__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1183	struct ring_buffer_event *event,
1184	unsigned long flags, int pc,
1185	int wake)
1186	{
1187	ring_buffer_unlock_commit(buffer, event);
1188
1189	ftrace_trace_stack(buffer, flags, 6, pc);
1190	ftrace_trace_userstack(buffer, flags, pc);
1191
1192	if (wake)
1193	trace_wake_up();
1194	}
1195
1196	void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1197	struct ring_buffer_event *event,
1198	unsigned long flags, int pc)
1199	{
1200	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1201	}
1202
1203	struct ring_buffer_event *
1204	trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1205	int type, unsigned long len,
1206	unsigned long flags, int pc)
1207	{
1208	*current_rb = global_trace.buffer;
1209	return trace_buffer_lock_reserve(*current_rb,
1210	type, len, flags, pc);
1211	}
1212	EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1213
1214	void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1215	struct ring_buffer_event *event,
1216	unsigned long flags, int pc)
1217	{
1218	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1219	}
1220	EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1221
1222	void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1223	struct ring_buffer_event *event,
1224	unsigned long flags, int pc)
1225	{
1226	__trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1227	}
1228	EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1229
1230	void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1231	struct ring_buffer_event *event,
1232	unsigned long flags, int pc,
1233	struct pt_regs *regs)
1234	{
1235	ring_buffer_unlock_commit(buffer, event);
1236
1237	ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1238	ftrace_trace_userstack(buffer, flags, pc);
1239	}
1240	EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1241
1242	void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1243	struct ring_buffer_event *event)
1244	{
1245	ring_buffer_discard_commit(buffer, event);
1246	}
1247	EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1248
1249	void
1250	trace_function(struct trace_array *tr,
1251	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1252	int pc)
1253	{
1254	struct ftrace_event_call *call = &event_function;
1255	struct ring_buffer *buffer = tr->buffer;
1256	struct ring_buffer_event *event;
1257	struct ftrace_entry *entry;
1258
1259	/* If we are reading the ring buffer, don't trace */
1260	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1261	return;
1262
1263	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1264	flags, pc);
1265	if (!event)
1266	return;
1267	entry = ring_buffer_event_data(event);
1268	entry->ip = ip;
1269	entry->parent_ip = parent_ip;
1270
1271	if (!filter_check_discard(call, entry, buffer, event))
1272	ring_buffer_unlock_commit(buffer, event);
1273	}
1274
1275	void
1276	ftrace(struct trace_array tr, struct trace_array_cpu data,
1277	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1278	int pc)
1279	{
1280	if (likely(!atomic_read(&data->disabled)))
1281	trace_function(tr, ip, parent_ip, flags, pc);
1282	}
1283
1284	#ifdef CONFIG_STACKTRACE
1285
1286	#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1287	struct ftrace_stack {
1288	unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1289	};
1290
1291	static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1292	static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1293
1294	static void __ftrace_trace_stack(struct ring_buffer *buffer,
1295	unsigned long flags,
1296	int skip, int pc, struct pt_regs *regs)
1297	{
1298	struct ftrace_event_call *call = &event_kernel_stack;
1299	struct ring_buffer_event *event;
1300	struct stack_entry *entry;
1301	struct stack_trace trace;
1302	int use_stack;
1303	int size = FTRACE_STACK_ENTRIES;
1304
1305	trace.nr_entries = 0;
1306	trace.skip = skip;
1307
1308	/*
1309	* Since events can happen in NMIs there's no safe way to
1310	* use the per cpu ftrace_stacks. We reserve it and if an interrupt
1311	* or NMI comes in, it will just have to use the default
1312	* FTRACE_STACK_SIZE.
1313	*/
1314	preempt_disable_notrace();
1315
1316	use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1317	/*
1318	* We don't need any atomic variables, just a barrier.
1319	* If an interrupt comes in, we don't care, because it would
1320	* have exited and put the counter back to what we want.
1321	* We just need a barrier to keep gcc from moving things
1322	* around.
1323	*/
1324	barrier();
1325	if (use_stack == 1) {
1326	trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1327	trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1328
1329	if (regs)
1330	save_stack_trace_regs(regs, &trace);
1331	else
1332	save_stack_trace(&trace);
1333
1334	if (trace.nr_entries > size)
1335	size = trace.nr_entries;
1336	} else
1337	/* From now on, use_stack is a boolean */
1338	use_stack = 0;
1339
1340	size *= sizeof(unsigned long);
1341
1342	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1343	sizeof(*entry) + size, flags, pc);
1344	if (!event)
1345	goto out;
1346	entry = ring_buffer_event_data(event);
1347
1348	memset(&entry->caller, 0, size);
1349
1350	if (use_stack)
1351	memcpy(&entry->caller, trace.entries,
1352	trace.nr_entries * sizeof(unsigned long));
1353	else {
1354	trace.max_entries = FTRACE_STACK_ENTRIES;
1355	trace.entries = entry->caller;
1356	if (regs)
1357	save_stack_trace_regs(regs, &trace);
1358	else
1359	save_stack_trace(&trace);
1360	}
1361
1362	entry->size = trace.nr_entries;
1363
1364	if (!filter_check_discard(call, entry, buffer, event))
1365	ring_buffer_unlock_commit(buffer, event);
1366
1367	out:
1368	/* Again, don't let gcc optimize things here */
1369	barrier();
1370	__get_cpu_var(ftrace_stack_reserve)--;
1371	preempt_enable_notrace();
1372
1373	}
1374
1375	void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1376	int skip, int pc, struct pt_regs *regs)
1377	{
1378	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1379	return;
1380
1381	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
1382	}
1383
1384	void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1385	int skip, int pc)
1386	{
1387	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1388	return;
1389
1390	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1391	}
1392
1393	void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1394	int pc)
1395	{
1396	__ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1397	}
1398
1399	/**
1400	* trace_dump_stack - record a stack back trace in the trace buffer
1401	*/
1402	void trace_dump_stack(void)
1403	{
1404	unsigned long flags;
1405
1406	if (tracing_disabled \|\| tracing_selftest_running)
1407	return;
1408
1409	local_save_flags(flags);
1410
1411	/* skipping 3 traces, seems to get us at the caller of this function */
1412	__ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1413	}
1414
1415	static DEFINE_PER_CPU(int, user_stack_count);
1416
1417	void
1418	ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1419	{
1420	struct ftrace_event_call *call = &event_user_stack;
1421	struct ring_buffer_event *event;
1422	struct userstack_entry *entry;
1423	struct stack_trace trace;
1424
1425	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1426	return;
1427
1428	/*
1429	* NMIs can not handle page faults, even with fix ups.
1430	* The save user stack can (and often does) fault.
1431	*/
1432	if (unlikely(in_nmi()))
1433	return;
1434
1435	/*
1436	* prevent recursion, since the user stack tracing may
1437	* trigger other kernel events.
1438	*/
1439	preempt_disable();
1440	if (__this_cpu_read(user_stack_count))
1441	goto out;
1442
1443	__this_cpu_inc(user_stack_count);
1444
1445	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1446	sizeof(*entry), flags, pc);
1447	if (!event)
1448	goto out_drop_count;
1449	entry = ring_buffer_event_data(event);
1450
1451	entry->tgid = current->tgid;
1452	memset(&entry->caller, 0, sizeof(entry->caller));
1453
1454	trace.nr_entries = 0;
1455	trace.max_entries = FTRACE_STACK_ENTRIES;
1456	trace.skip = 0;
1457	trace.entries = entry->caller;
1458
1459	save_stack_trace_user(&trace);
1460	if (!filter_check_discard(call, entry, buffer, event))
1461	ring_buffer_unlock_commit(buffer, event);
1462
1463	out_drop_count:
1464	__this_cpu_dec(user_stack_count);
1465	out:
1466	preempt_enable();
1467	}
1468
1469	#ifdef UNUSED
1470	static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1471	{
1472	ftrace_trace_userstack(tr, flags, preempt_count());
1473	}
1474	#endif /* UNUSED */
1475
1476	#endif /* CONFIG_STACKTRACE */
1477
1478	/* created for use with alloc_percpu */
1479	struct trace_buffer_struct {
1480	char buffer[TRACE_BUF_SIZE];
1481	};
1482
1483	static struct trace_buffer_struct *trace_percpu_buffer;
1484	static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1485	static struct trace_buffer_struct *trace_percpu_irq_buffer;
1486	static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1487
1488	/*
1489	* The buffer used is dependent on the context. There is a per cpu
1490	* buffer for normal context, softirq contex, hard irq context and
1491	* for NMI context. Thise allows for lockless recording.
1492	*
1493	* Note, if the buffers failed to be allocated, then this returns NULL
1494	*/
1495	static char *get_trace_buf(void)
1496	{
1497	struct trace_buffer_struct *percpu_buffer;
1498	struct trace_buffer_struct *buffer;
1499
1500	/*
1501	* If we have allocated per cpu buffers, then we do not
1502	* need to do any locking.
1503	*/
1504	if (in_nmi())
1505	percpu_buffer = trace_percpu_nmi_buffer;
1506	else if (in_irq())
1507	percpu_buffer = trace_percpu_irq_buffer;
1508	else if (in_softirq())
1509	percpu_buffer = trace_percpu_sirq_buffer;
1510	else
1511	percpu_buffer = trace_percpu_buffer;
1512
1513	if (!percpu_buffer)
1514	return NULL;
1515
1516	buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1517
1518	return buffer->buffer;
1519	}
1520
1521	static int alloc_percpu_trace_buffer(void)
1522	{
1523	struct trace_buffer_struct *buffers;
1524	struct trace_buffer_struct *sirq_buffers;
1525	struct trace_buffer_struct *irq_buffers;
1526	struct trace_buffer_struct *nmi_buffers;
1527
1528	buffers = alloc_percpu(struct trace_buffer_struct);
1529	if (!buffers)
1530	goto err_warn;
1531
1532	sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1533	if (!sirq_buffers)
1534	goto err_sirq;
1535
1536	irq_buffers = alloc_percpu(struct trace_buffer_struct);
1537	if (!irq_buffers)
1538	goto err_irq;
1539
1540	nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1541	if (!nmi_buffers)
1542	goto err_nmi;
1543
1544	trace_percpu_buffer = buffers;
1545	trace_percpu_sirq_buffer = sirq_buffers;
1546	trace_percpu_irq_buffer = irq_buffers;
1547	trace_percpu_nmi_buffer = nmi_buffers;
1548
1549	return 0;
1550
1551	err_nmi:
1552	free_percpu(irq_buffers);
1553	err_irq:
1554	free_percpu(sirq_buffers);
1555	err_sirq:
1556	free_percpu(buffers);
1557	err_warn:
1558	WARN(1, "Could not allocate percpu trace_printk buffer");
1559	return -ENOMEM;
1560	}
1561
1562	void trace_printk_init_buffers(void)
1563	{
1564	static int buffers_allocated;
1565
1566	if (buffers_allocated)
1567	return;
1568
1569	if (alloc_percpu_trace_buffer())
1570	return;
1571
1572	pr_info("ftrace: Allocated trace_printk buffers\n");
1573
1574	buffers_allocated = 1;
1575	}
1576
1577	/**
1578	* trace_vbprintk - write binary msg to tracing buffer
1579	*
1580	*/
1581	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1582	{
1583	struct ftrace_event_call *call = &event_bprint;
1584	struct ring_buffer_event *event;
1585	struct ring_buffer *buffer;
1586	struct trace_array *tr = &global_trace;
1587	struct bprint_entry *entry;
1588	unsigned long flags;
1589	char *tbuffer;
1590	int len = 0, size, pc;
1591
1592	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
1593	return 0;
1594
1595	/* Don't pollute graph traces with trace_vprintk internals */
1596	pause_graph_tracing();
1597
1598	pc = preempt_count();
1599	preempt_disable_notrace();
1600
1601	tbuffer = get_trace_buf();
1602	if (!tbuffer) {
1603	len = 0;
1604	goto out;
1605	}
1606
1607	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1608
1609	if (len > TRACE_BUF_SIZE/sizeof(int) \|\| len < 0)
1610	goto out;
1611
1612	local_save_flags(flags);
1613	size = sizeof(entry) + sizeof(u32) len;
1614	buffer = tr->buffer;
1615	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1616	flags, pc);
1617	if (!event)
1618	goto out;
1619	entry = ring_buffer_event_data(event);
1620	entry->ip = ip;
1621	entry->fmt = fmt;
1622
1623	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1624	if (!filter_check_discard(call, entry, buffer, event)) {
1625	ring_buffer_unlock_commit(buffer, event);
1626	ftrace_trace_stack(buffer, flags, 6, pc);
1627	}
1628
1629	out:
1630	preempt_enable_notrace();
1631	unpause_graph_tracing();
1632
1633	return len;
1634	}
1635	EXPORT_SYMBOL_GPL(trace_vbprintk);
1636
1637	int trace_array_printk(struct trace_array *tr,
1638	unsigned long ip, const char *fmt, ...)
1639	{
1640	int ret;
1641	va_list ap;
1642
1643	if (!(trace_flags & TRACE_ITER_PRINTK))
1644	return 0;
1645
1646	va_start(ap, fmt);
1647	ret = trace_array_vprintk(tr, ip, fmt, ap);
1648	va_end(ap);
1649	return ret;
1650	}
1651
1652	int trace_array_vprintk(struct trace_array *tr,
1653	unsigned long ip, const char *fmt, va_list args)
1654	{
1655	struct ftrace_event_call *call = &event_print;
1656	struct ring_buffer_event *event;
1657	struct ring_buffer *buffer;
1658	int len = 0, size, pc;
1659	struct print_entry *entry;
1660	unsigned long flags;
1661	char *tbuffer;
1662
1663	if (tracing_disabled \|\| tracing_selftest_running)
1664	return 0;
1665
1666	/* Don't pollute graph traces with trace_vprintk internals */
1667	pause_graph_tracing();
1668
1669	pc = preempt_count();
1670	preempt_disable_notrace();
1671
1672
1673	tbuffer = get_trace_buf();
1674	if (!tbuffer) {
1675	len = 0;
1676	goto out;
1677	}
1678
1679	len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1680	if (len > TRACE_BUF_SIZE)
1681	goto out;
1682
1683	local_save_flags(flags);
1684	size = sizeof(*entry) + len + 1;
1685	buffer = tr->buffer;
1686	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1687	flags, pc);
1688	if (!event)
1689	goto out;
1690	entry = ring_buffer_event_data(event);
1691	entry->ip = ip;
1692
1693	memcpy(&entry->buf, tbuffer, len);
1694	entry->buf[len] = '\0';
1695	if (!filter_check_discard(call, entry, buffer, event)) {
1696	ring_buffer_unlock_commit(buffer, event);
1697	ftrace_trace_stack(buffer, flags, 6, pc);
1698	}
1699	out:
1700	preempt_enable_notrace();
1701	unpause_graph_tracing();
1702
1703	return len;
1704	}
1705
1706	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1707	{
1708	return trace_array_vprintk(&global_trace, ip, fmt, args);
1709	}
1710	EXPORT_SYMBOL_GPL(trace_vprintk);
1711
1712	static void trace_iterator_increment(struct trace_iterator *iter)
1713	{
1714	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1715
1716	iter->idx++;
1717	if (buf_iter)
1718	ring_buffer_read(buf_iter, NULL);
1719	}
1720
1721	static struct trace_entry *
1722	peek_next_entry(struct trace_iterator iter, int cpu, u64 ts,
1723	unsigned long *lost_events)
1724	{
1725	struct ring_buffer_event *event;
1726	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
1727
1728	if (buf_iter)
1729	event = ring_buffer_iter_peek(buf_iter, ts);
1730	else
1731	event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1732	lost_events);
1733
1734	if (event) {
1735	iter->ent_size = ring_buffer_event_length(event);
1736	return ring_buffer_event_data(event);
1737	}
1738	iter->ent_size = 0;
1739	return NULL;
1740	}
1741
1742	static struct trace_entry *
1743	__find_next_entry(struct trace_iterator iter, int ent_cpu,
1744	unsigned long missing_events, u64 ent_ts)
1745	{
1746	struct ring_buffer *buffer = iter->tr->buffer;
1747	struct trace_entry ent, next = NULL;
1748	unsigned long lost_events = 0, next_lost = 0;
1749	int cpu_file = iter->cpu_file;
1750	u64 next_ts = 0, ts;
1751	int next_cpu = -1;
1752	int next_size = 0;
1753	int cpu;
1754
1755	/*
1756	* If we are in a per_cpu trace file, don't bother by iterating over
1757	* all cpu and peek directly.
1758	*/
1759	if (cpu_file > TRACE_PIPE_ALL_CPU) {
1760	if (ring_buffer_empty_cpu(buffer, cpu_file))
1761	return NULL;
1762	ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1763	if (ent_cpu)
1764	*ent_cpu = cpu_file;
1765
1766	return ent;
1767	}
1768
1769	for_each_tracing_cpu(cpu) {
1770
1771	if (ring_buffer_empty_cpu(buffer, cpu))
1772	continue;
1773
1774	ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1775
1776	/*
1777	* Pick the entry with the smallest timestamp:
1778	*/
1779	if (ent && (!next \|\| ts < next_ts)) {
1780	next = ent;
1781	next_cpu = cpu;
1782	next_ts = ts;
1783	next_lost = lost_events;
1784	next_size = iter->ent_size;
1785	}
1786	}
1787
1788	iter->ent_size = next_size;
1789
1790	if (ent_cpu)
1791	*ent_cpu = next_cpu;
1792
1793	if (ent_ts)
1794	*ent_ts = next_ts;
1795
1796	if (missing_events)
1797	*missing_events = next_lost;
1798
1799	return next;
1800	}
1801
1802	/* Find the next real entry, without updating the iterator itself */
1803	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
1804	int ent_cpu, u64 ent_ts)
1805	{
1806	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1807	}
1808
1809	/* Find the next real entry, and increment the iterator to the next entry */
1810	void trace_find_next_entry_inc(struct trace_iterator iter)
1811	{
1812	iter->ent = __find_next_entry(iter, &iter->cpu,
1813	&iter->lost_events, &iter->ts);
1814
1815	if (iter->ent)
1816	trace_iterator_increment(iter);
1817
1818	return iter->ent ? iter : NULL;
1819	}
1820
1821	static void trace_consume(struct trace_iterator *iter)
1822	{
1823	ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1824	&iter->lost_events);
1825	}
1826
1827	static void s_next(struct seq_file m, void v, loff_t pos)
1828	{
1829	struct trace_iterator *iter = m->private;
1830	int i = (int)*pos;
1831	void *ent;
1832
1833	WARN_ON_ONCE(iter->leftover);
1834
1835	(*pos)++;
1836
1837	/* can't go backwards */
1838	if (iter->idx > i)
1839	return NULL;
1840
1841	if (iter->idx < 0)
1842	ent = trace_find_next_entry_inc(iter);
1843	else
1844	ent = iter;
1845
1846	while (ent && iter->idx < i)
1847	ent = trace_find_next_entry_inc(iter);
1848
1849	iter->pos = *pos;
1850
1851	return ent;
1852	}
1853
1854	void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1855	{
1856	struct trace_array *tr = iter->tr;
1857	struct ring_buffer_event *event;
1858	struct ring_buffer_iter *buf_iter;
1859	unsigned long entries = 0;
1860	u64 ts;
1861
1862	tr->data[cpu]->skipped_entries = 0;
1863
1864	buf_iter = trace_buffer_iter(iter, cpu);
1865	if (!buf_iter)
1866	return;
1867
1868	ring_buffer_iter_reset(buf_iter);
1869
1870	/*
1871	* We could have the case with the max latency tracers
1872	* that a reset never took place on a cpu. This is evident
1873	* by the timestamp being before the start of the buffer.
1874	*/
1875	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1876	if (ts >= iter->tr->time_start)
1877	break;
1878	entries++;
1879	ring_buffer_read(buf_iter, NULL);
1880	}
1881
1882	tr->data[cpu]->skipped_entries = entries;
1883	}
1884
1885	/*
1886	* The current tracer is copied to avoid a global locking
1887	* all around.
1888	*/
1889	static void s_start(struct seq_file m, loff_t *pos)
1890	{
1891	struct trace_iterator *iter = m->private;
1892	static struct tracer *old_tracer;
1893	int cpu_file = iter->cpu_file;
1894	void *p = NULL;
1895	loff_t l = 0;
1896	int cpu;
1897
1898	/* copy the tracer to avoid using a global lock all around */
1899	mutex_lock(&trace_types_lock);
1900	if (unlikely(old_tracer != current_trace && current_trace)) {
1901	old_tracer = current_trace;
1902	iter->trace = current_trace;
1903	}
1904	mutex_unlock(&trace_types_lock);
1905
1906	atomic_inc(&trace_record_cmdline_disabled);
1907
1908	if (*pos != iter->pos) {
1909	iter->ent = NULL;
1910	iter->cpu = 0;
1911	iter->idx = -1;
1912
1913	if (cpu_file == TRACE_PIPE_ALL_CPU) {
1914	for_each_tracing_cpu(cpu)
1915	tracing_iter_reset(iter, cpu);
1916	} else
1917	tracing_iter_reset(iter, cpu_file);
1918
1919	iter->leftover = 0;
1920	for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1921	;
1922
1923	} else {
1924	/*
1925	* If we overflowed the seq_file before, then we want
1926	* to just reuse the trace_seq buffer again.
1927	*/
1928	if (iter->leftover)
1929	p = iter;
1930	else {
1931	l = *pos - 1;
1932	p = s_next(m, p, &l);
1933	}
1934	}
1935
1936	trace_event_read_lock();
1937	trace_access_lock(cpu_file);
1938	return p;
1939	}
1940
1941	static void s_stop(struct seq_file m, void p)
1942	{
1943	struct trace_iterator *iter = m->private;
1944
1945	atomic_dec(&trace_record_cmdline_disabled);
1946	trace_access_unlock(iter->cpu_file);
1947	trace_event_read_unlock();
1948	}
1949
1950	static void
1951	get_total_entries(struct trace_array tr, unsigned long total, unsigned long *entries)
1952	{
1953	unsigned long count;
1954	int cpu;
1955
1956	*total = 0;
1957	*entries = 0;
1958
1959	for_each_tracing_cpu(cpu) {
1960	count = ring_buffer_entries_cpu(tr->buffer, cpu);
1961	/*
1962	* If this buffer has skipped entries, then we hold all
1963	* entries for the trace and we need to ignore the
1964	* ones before the time stamp.
1965	*/
1966	if (tr->data[cpu]->skipped_entries) {
1967	count -= tr->data[cpu]->skipped_entries;
1968	/* total is the same as the entries */
1969	*total += count;
1970	} else
1971	*total += count +
1972	ring_buffer_overrun_cpu(tr->buffer, cpu);
1973	*entries += count;
1974	}
1975	}
1976
1977	static void print_lat_help_header(struct seq_file *m)
1978	{
1979	seq_puts(m, "# _------=> CPU# \n");
1980	seq_puts(m, "# / _-----=> irqs-off \n");
1981	seq_puts(m, "# \| / _----=> need-resched \n");
1982	seq_puts(m, "# \|\| / _---=> hardirq/softirq \n");
1983	seq_puts(m, "# \|\|\| / _--=> preempt-depth \n");
1984	seq_puts(m, "# \|\|\|\| / delay \n");
1985	seq_puts(m, "# cmd pid \|\|\|\|\| time \| caller \n");
1986	seq_puts(m, "# \\ / \|\|\|\|\| \\ \| / \n");
1987	}
1988
1989	static void print_event_info(struct trace_array tr, struct seq_file m)
1990	{
1991	unsigned long total;
1992	unsigned long entries;
1993
1994	get_total_entries(tr, &total, &entries);
1995	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
1996	entries, total, num_online_cpus());
1997	seq_puts(m, "#\n");
1998	}
1999
2000	static void print_func_help_header(struct trace_array tr, struct seq_file m)
2001	{
2002	print_event_info(tr, m);
2003	seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2004	seq_puts(m, "# \| \| \| \| \|\n");
2005	}
2006
2007	static void print_func_help_header_irq(struct trace_array tr, struct seq_file m)
2008	{
2009	print_event_info(tr, m);
2010	seq_puts(m, "# _-----=> irqs-off\n");
2011	seq_puts(m, "# / _----=> need-resched\n");
2012	seq_puts(m, "# \| / _---=> hardirq/softirq\n");
2013	seq_puts(m, "# \|\| / _--=> preempt-depth\n");
2014	seq_puts(m, "# \|\|\| / delay\n");
2015	seq_puts(m, "# TASK-PID CPU# \|\|\|\| TIMESTAMP FUNCTION\n");
2016	seq_puts(m, "# \| \| \| \|\|\|\| \| \|\n");
2017	}
2018
2019	void
2020	print_trace_header(struct seq_file m, struct trace_iterator iter)
2021	{
2022	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2023	struct trace_array *tr = iter->tr;
2024	struct trace_array_cpu *data = tr->data[tr->cpu];
2025	struct tracer *type = current_trace;
2026	unsigned long entries;
2027	unsigned long total;
2028	const char *name = "preemption";
2029
2030	if (type)
2031	name = type->name;
2032
2033	get_total_entries(tr, &total, &entries);
2034
2035	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2036	name, UTS_RELEASE);
2037	seq_puts(m, "# -----------------------------------"
2038	"---------------------------------\n");
2039	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d \|"
2040	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2041	nsecs_to_usecs(data->saved_latency),
2042	entries,
2043	total,
2044	tr->cpu,
2045	#if defined(CONFIG_PREEMPT_NONE)
2046	"server",
2047	#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2048	"desktop",
2049	#elif defined(CONFIG_PREEMPT)
2050	"preempt",
2051	#else
2052	"unknown",
2053	#endif
2054	/* These are reserved for later use */
2055	0, 0, 0, 0);
2056	#ifdef CONFIG_SMP
2057	seq_printf(m, " #P:%d)\n", num_online_cpus());
2058	#else
2059	seq_puts(m, ")\n");
2060	#endif
2061	seq_puts(m, "# -----------------\n");
2062	seq_printf(m, "# \| task: %.16s-%d "
2063	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2064	data->comm, data->pid,
2065	from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2066	data->policy, data->rt_priority);
2067	seq_puts(m, "# -----------------\n");
2068
2069	if (data->critical_start) {
2070	seq_puts(m, "# => started at: ");
2071	seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2072	trace_print_seq(m, &iter->seq);
2073	seq_puts(m, "\n# => ended at: ");
2074	seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2075	trace_print_seq(m, &iter->seq);
2076	seq_puts(m, "\n#\n");
2077	}
2078
2079	seq_puts(m, "#\n");
2080	}
2081
2082	static void test_cpu_buff_start(struct trace_iterator *iter)
2083	{
2084	struct trace_seq *s = &iter->seq;
2085
2086	if (!(trace_flags & TRACE_ITER_ANNOTATE))
2087	return;
2088
2089	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2090	return;
2091
2092	if (cpumask_test_cpu(iter->cpu, iter->started))
2093	return;
2094
2095	if (iter->tr->data[iter->cpu]->skipped_entries)
2096	return;
2097
2098	cpumask_set_cpu(iter->cpu, iter->started);
2099
2100	/* Don't print started cpu buffer for the first entry of the trace */
2101	if (iter->idx > 1)
2102	trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2103	iter->cpu);
2104	}
2105
2106	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2107	{
2108	struct trace_seq *s = &iter->seq;
2109	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2110	struct trace_entry *entry;
2111	struct trace_event *event;
2112
2113	entry = iter->ent;
2114
2115	test_cpu_buff_start(iter);
2116
2117	event = ftrace_find_event(entry->type);
2118
2119	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2120	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2121	if (!trace_print_lat_context(iter))
2122	goto partial;
2123	} else {
2124	if (!trace_print_context(iter))
2125	goto partial;
2126	}
2127	}
2128
2129	if (event)
2130	return event->funcs->trace(iter, sym_flags, event);
2131
2132	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2133	goto partial;
2134
2135	return TRACE_TYPE_HANDLED;
2136	partial:
2137	return TRACE_TYPE_PARTIAL_LINE;
2138	}
2139
2140	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2141	{
2142	struct trace_seq *s = &iter->seq;
2143	struct trace_entry *entry;
2144	struct trace_event *event;
2145
2146	entry = iter->ent;
2147
2148	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2149	if (!trace_seq_printf(s, "%d %d %llu ",
2150	entry->pid, iter->cpu, iter->ts))
2151	goto partial;
2152	}
2153
2154	event = ftrace_find_event(entry->type);
2155	if (event)
2156	return event->funcs->raw(iter, 0, event);
2157
2158	if (!trace_seq_printf(s, "%d ?\n", entry->type))
2159	goto partial;
2160
2161	return TRACE_TYPE_HANDLED;
2162	partial:
2163	return TRACE_TYPE_PARTIAL_LINE;
2164	}
2165
2166	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2167	{
2168	struct trace_seq *s = &iter->seq;
2169	unsigned char newline = '\n';
2170	struct trace_entry *entry;
2171	struct trace_event *event;
2172
2173	entry = iter->ent;
2174
2175	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2176	SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2177	SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2178	SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2179	}
2180
2181	event = ftrace_find_event(entry->type);
2182	if (event) {
2183	enum print_line_t ret = event->funcs->hex(iter, 0, event);
2184	if (ret != TRACE_TYPE_HANDLED)
2185	return ret;
2186	}
2187
2188	SEQ_PUT_FIELD_RET(s, newline);
2189
2190	return TRACE_TYPE_HANDLED;
2191	}
2192
2193	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2194	{
2195	struct trace_seq *s = &iter->seq;
2196	struct trace_entry *entry;
2197	struct trace_event *event;
2198
2199	entry = iter->ent;
2200
2201	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2202	SEQ_PUT_FIELD_RET(s, entry->pid);
2203	SEQ_PUT_FIELD_RET(s, iter->cpu);
2204	SEQ_PUT_FIELD_RET(s, iter->ts);
2205	}
2206
2207	event = ftrace_find_event(entry->type);
2208	return event ? event->funcs->binary(iter, 0, event) :
2209	TRACE_TYPE_HANDLED;
2210	}
2211
2212	int trace_empty(struct trace_iterator *iter)
2213	{
2214	struct ring_buffer_iter *buf_iter;
2215	int cpu;
2216
2217	/* If we are looking at one CPU buffer, only check that one */
2218	if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2219	cpu = iter->cpu_file;
2220	buf_iter = trace_buffer_iter(iter, cpu);
2221	if (buf_iter) {
2222	if (!ring_buffer_iter_empty(buf_iter))
2223	return 0;
2224	} else {
2225	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2226	return 0;
2227	}
2228	return 1;
2229	}
2230
2231	for_each_tracing_cpu(cpu) {
2232	buf_iter = trace_buffer_iter(iter, cpu);
2233	if (buf_iter) {
2234	if (!ring_buffer_iter_empty(buf_iter))
2235	return 0;
2236	} else {
2237	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2238	return 0;
2239	}
2240	}
2241
2242	return 1;
2243	}
2244
2245	/* Called with trace_event_read_lock() held. */
2246	enum print_line_t print_trace_line(struct trace_iterator *iter)
2247	{
2248	enum print_line_t ret;
2249
2250	if (iter->lost_events &&
2251	!trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2252	iter->cpu, iter->lost_events))
2253	return TRACE_TYPE_PARTIAL_LINE;
2254
2255	if (iter->trace && iter->trace->print_line) {
2256	ret = iter->trace->print_line(iter);
2257	if (ret != TRACE_TYPE_UNHANDLED)
2258	return ret;
2259	}
2260
2261	if (iter->ent->type == TRACE_BPRINT &&
2262	trace_flags & TRACE_ITER_PRINTK &&
2263	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2264	return trace_print_bprintk_msg_only(iter);
2265
2266	if (iter->ent->type == TRACE_PRINT &&
2267	trace_flags & TRACE_ITER_PRINTK &&
2268	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2269	return trace_print_printk_msg_only(iter);
2270
2271	if (trace_flags & TRACE_ITER_BIN)
2272	return print_bin_fmt(iter);
2273
2274	if (trace_flags & TRACE_ITER_HEX)
2275	return print_hex_fmt(iter);
2276
2277	if (trace_flags & TRACE_ITER_RAW)
2278	return print_raw_fmt(iter);
2279
2280	return print_trace_fmt(iter);
2281	}
2282
2283	void trace_latency_header(struct seq_file *m)
2284	{
2285	struct trace_iterator *iter = m->private;
2286
2287	/* print nothing if the buffers are empty */
2288	if (trace_empty(iter))
2289	return;
2290
2291	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2292	print_trace_header(m, iter);
2293
2294	if (!(trace_flags & TRACE_ITER_VERBOSE))
2295	print_lat_help_header(m);
2296	}
2297
2298	void trace_default_header(struct seq_file *m)
2299	{
2300	struct trace_iterator *iter = m->private;
2301
2302	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2303	return;
2304
2305	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2306	/* print nothing if the buffers are empty */
2307	if (trace_empty(iter))
2308	return;
2309	print_trace_header(m, iter);
2310	if (!(trace_flags & TRACE_ITER_VERBOSE))
2311	print_lat_help_header(m);
2312	} else {
2313	if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2314	if (trace_flags & TRACE_ITER_IRQ_INFO)
2315	print_func_help_header_irq(iter->tr, m);
2316	else
2317	print_func_help_header(iter->tr, m);
2318	}
2319	}
2320	}
2321
2322	static void test_ftrace_alive(struct seq_file *m)
2323	{
2324	if (!ftrace_is_dead())
2325	return;
2326	seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2327	seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2328	}
2329
2330	static int s_show(struct seq_file m, void v)
2331	{
2332	struct trace_iterator *iter = v;
2333	int ret;
2334
2335	if (iter->ent == NULL) {
2336	if (iter->tr) {
2337	seq_printf(m, "# tracer: %s\n", iter->trace->name);
2338	seq_puts(m, "#\n");
2339	test_ftrace_alive(m);
2340	}
2341	if (iter->trace && iter->trace->print_header)
2342	iter->trace->print_header(m);
2343	else
2344	trace_default_header(m);
2345
2346	} else if (iter->leftover) {
2347	/*
2348	* If we filled the seq_file buffer earlier, we
2349	* want to just show it now.
2350	*/
2351	ret = trace_print_seq(m, &iter->seq);
2352
2353	/* ret should this time be zero, but you never know */
2354	iter->leftover = ret;
2355
2356	} else {
2357	print_trace_line(iter);
2358	ret = trace_print_seq(m, &iter->seq);
2359	/*
2360	* If we overflow the seq_file buffer, then it will
2361	* ask us for this data again at start up.
2362	* Use that instead.
2363	* ret is 0 if seq_file write succeeded.
2364	* -1 otherwise.
2365	*/
2366	iter->leftover = ret;
2367	}
2368
2369	return 0;
2370	}
2371
2372	static const struct seq_operations tracer_seq_ops = {
2373	.start = s_start,
2374	.next = s_next,
2375	.stop = s_stop,
2376	.show = s_show,
2377	};
2378
2379	static struct trace_iterator *
2380	__tracing_open(struct inode inode, struct file file)
2381	{
2382	long cpu_file = (long) inode->i_private;
2383	struct trace_iterator *iter;
2384	int cpu;
2385
2386	if (tracing_disabled)
2387	return ERR_PTR(-ENODEV);
2388
2389	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2390	if (!iter)
2391	return ERR_PTR(-ENOMEM);
2392
2393	iter->buffer_iter = kzalloc(sizeof(iter->buffer_iter) num_possible_cpus(),
2394	GFP_KERNEL);
2395	if (!iter->buffer_iter)
2396	goto release;
2397
2398	/*
2399	* We make a copy of the current tracer to avoid concurrent
2400	* changes on it while we are reading.
2401	*/
2402	mutex_lock(&trace_types_lock);
2403	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2404	if (!iter->trace)
2405	goto fail;
2406
2407	if (current_trace)
2408	iter->trace = current_trace;
2409
2410	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2411	goto fail;
2412
2413	if (current_trace && current_trace->print_max)
2414	iter->tr = &max_tr;
2415	else
2416	iter->tr = &global_trace;
2417	iter->pos = -1;
2418	mutex_init(&iter->mutex);
2419	iter->cpu_file = cpu_file;
2420
2421	/* Notify the tracer early; before we stop tracing. */
2422	if (iter->trace && iter->trace->open)
2423	iter->trace->open(iter);
2424
2425	/* Annotate start of buffers if we had overruns */
2426	if (ring_buffer_overruns(iter->tr->buffer))
2427	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
2428
2429	/* stop the trace while dumping */
2430	tracing_stop();
2431
2432	if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2433	for_each_tracing_cpu(cpu) {
2434	iter->buffer_iter[cpu] =
2435	ring_buffer_read_prepare(iter->tr->buffer, cpu);
2436	}
2437	ring_buffer_read_prepare_sync();
2438	for_each_tracing_cpu(cpu) {
2439	ring_buffer_read_start(iter->buffer_iter[cpu]);
2440	tracing_iter_reset(iter, cpu);
2441	}
2442	} else {
2443	cpu = iter->cpu_file;
2444	iter->buffer_iter[cpu] =
2445	ring_buffer_read_prepare(iter->tr->buffer, cpu);
2446	ring_buffer_read_prepare_sync();
2447	ring_buffer_read_start(iter->buffer_iter[cpu]);
2448	tracing_iter_reset(iter, cpu);
2449	}
2450
2451	mutex_unlock(&trace_types_lock);
2452
2453	return iter;
2454
2455	fail:
2456	mutex_unlock(&trace_types_lock);
2457	kfree(iter->trace);
2458	kfree(iter->buffer_iter);
2459	release:
2460	seq_release_private(inode, file);
2461	return ERR_PTR(-ENOMEM);
2462	}
2463
2464	int tracing_open_generic(struct inode inode, struct file filp)
2465	{
2466	if (tracing_disabled)
2467	return -ENODEV;
2468
2469	filp->private_data = inode->i_private;
2470	return 0;
2471	}
2472
2473	static int tracing_release(struct inode inode, struct file file)
2474	{
2475	struct seq_file *m = file->private_data;
2476	struct trace_iterator *iter;
2477	int cpu;
2478
2479	if (!(file->f_mode & FMODE_READ))
2480	return 0;
2481
2482	iter = m->private;
2483
2484	mutex_lock(&trace_types_lock);
2485	for_each_tracing_cpu(cpu) {
2486	if (iter->buffer_iter[cpu])
2487	ring_buffer_read_finish(iter->buffer_iter[cpu]);
2488	}
2489
2490	if (iter->trace && iter->trace->close)
2491	iter->trace->close(iter);
2492
2493	/* reenable tracing if it was previously enabled */
2494	tracing_start();
2495	mutex_unlock(&trace_types_lock);
2496
2497	mutex_destroy(&iter->mutex);
2498	free_cpumask_var(iter->started);
2499	kfree(iter->trace);
2500	kfree(iter->buffer_iter);
2501	seq_release_private(inode, file);
2502	return 0;
2503	}
2504
2505	static int tracing_open(struct inode inode, struct file file)
2506	{
2507	struct trace_iterator *iter;
2508	int ret = 0;
2509
2510	/* If this file was open for write, then erase contents */
2511	if ((file->f_mode & FMODE_WRITE) &&
2512	(file->f_flags & O_TRUNC)) {
2513	long cpu = (long) inode->i_private;
2514
2515	if (cpu == TRACE_PIPE_ALL_CPU)
2516	tracing_reset_online_cpus(&global_trace);
2517	else
2518	tracing_reset(&global_trace, cpu);
2519	}
2520
2521	if (file->f_mode & FMODE_READ) {
2522	iter = __tracing_open(inode, file);
2523	if (IS_ERR(iter))
2524	ret = PTR_ERR(iter);
2525	else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2526	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
2527	}
2528	return ret;
2529	}
2530
2531	static void *
2532	t_next(struct seq_file m, void v, loff_t *pos)
2533	{
2534	struct tracer *t = v;
2535
2536	(*pos)++;
2537
2538	if (t)
2539	t = t->next;
2540
2541	return t;
2542	}
2543
2544	static void t_start(struct seq_file m, loff_t *pos)
2545	{
2546	struct tracer *t;
2547	loff_t l = 0;
2548
2549	mutex_lock(&trace_types_lock);
2550	for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2551	;
2552
2553	return t;
2554	}
2555
2556	static void t_stop(struct seq_file m, void p)
2557	{
2558	mutex_unlock(&trace_types_lock);
2559	}
2560
2561	static int t_show(struct seq_file m, void v)
2562	{
2563	struct tracer *t = v;
2564
2565	if (!t)
2566	return 0;
2567
2568	seq_printf(m, "%s", t->name);
2569	if (t->next)
2570	seq_putc(m, ' ');
2571	else
2572	seq_putc(m, '\n');
2573
2574	return 0;
2575	}
2576
2577	static const struct seq_operations show_traces_seq_ops = {
2578	.start = t_start,
2579	.next = t_next,
2580	.stop = t_stop,
2581	.show = t_show,
2582	};
2583
2584	static int show_traces_open(struct inode inode, struct file file)
2585	{
2586	if (tracing_disabled)
2587	return -ENODEV;
2588
2589	return seq_open(file, &show_traces_seq_ops);
2590	}
2591
2592	static ssize_t
2593	tracing_write_stub(struct file filp, const char __user ubuf,
2594	size_t count, loff_t *ppos)
2595	{
2596	return count;
2597	}
2598
2599	static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2600	{
2601	if (file->f_mode & FMODE_READ)
2602	return seq_lseek(file, offset, origin);
2603	else
2604	return 0;
2605	}
2606
2607	static const struct file_operations tracing_fops = {
2608	.open = tracing_open,
2609	.read = seq_read,
2610	.write = tracing_write_stub,
2611	.llseek = tracing_seek,
2612	.release = tracing_release,
2613	};
2614
2615	static const struct file_operations show_traces_fops = {
2616	.open = show_traces_open,
2617	.read = seq_read,
2618	.release = seq_release,
2619	.llseek = seq_lseek,
2620	};
2621
2622	/*
2623	* Only trace on a CPU if the bitmask is set:
2624	*/
2625	static cpumask_var_t tracing_cpumask;
2626
2627	/*
2628	* The tracer itself will not take this lock, but still we want
2629	* to provide a consistent cpumask to user-space:
2630	*/
2631	static DEFINE_MUTEX(tracing_cpumask_update_lock);
2632
2633	/*
2634	* Temporary storage for the character representation of the
2635	* CPU bitmask (and one more byte for the newline):
2636	*/
2637	static char mask_str[NR_CPUS + 1];
2638
2639	static ssize_t
2640	tracing_cpumask_read(struct file filp, char __user ubuf,
2641	size_t count, loff_t *ppos)
2642	{
2643	int len;
2644
2645	mutex_lock(&tracing_cpumask_update_lock);
2646
2647	len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2648	if (count - len < 2) {
2649	count = -EINVAL;
2650	goto out_err;
2651	}
2652	len += sprintf(mask_str + len, "\n");
2653	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2654
2655	out_err:
2656	mutex_unlock(&tracing_cpumask_update_lock);
2657
2658	return count;
2659	}
2660
2661	static ssize_t
2662	tracing_cpumask_write(struct file filp, const char __user ubuf,
2663	size_t count, loff_t *ppos)
2664	{
2665	int err, cpu;
2666	cpumask_var_t tracing_cpumask_new;
2667
2668	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2669	return -ENOMEM;
2670
2671	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2672	if (err)
2673	goto err_unlock;
2674
2675	mutex_lock(&tracing_cpumask_update_lock);
2676
2677	local_irq_disable();
2678	arch_spin_lock(&ftrace_max_lock);
2679	for_each_tracing_cpu(cpu) {
2680	/*
2681	* Increase/decrease the disabled counter if we are
2682	* about to flip a bit in the cpumask:
2683	*/
2684	if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2685	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2686	atomic_inc(&global_trace.data[cpu]->disabled);
2687	ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2688	}
2689	if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2690	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2691	atomic_dec(&global_trace.data[cpu]->disabled);
2692	ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2693	}
2694	}
2695	arch_spin_unlock(&ftrace_max_lock);
2696	local_irq_enable();
2697
2698	cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2699
2700	mutex_unlock(&tracing_cpumask_update_lock);
2701	free_cpumask_var(tracing_cpumask_new);
2702
2703	return count;
2704
2705	err_unlock:
2706	free_cpumask_var(tracing_cpumask_new);
2707
2708	return err;
2709	}
2710
2711	static const struct file_operations tracing_cpumask_fops = {
2712	.open = tracing_open_generic,
2713	.read = tracing_cpumask_read,
2714	.write = tracing_cpumask_write,
2715	.llseek = generic_file_llseek,
2716	};
2717
2718	static int tracing_trace_options_show(struct seq_file m, void v)
2719	{
2720	struct tracer_opt *trace_opts;
2721	u32 tracer_flags;
2722	int i;
2723
2724	mutex_lock(&trace_types_lock);
2725	tracer_flags = current_trace->flags->val;
2726	trace_opts = current_trace->flags->opts;
2727
2728	for (i = 0; trace_options[i]; i++) {
2729	if (trace_flags & (1 << i))
2730	seq_printf(m, "%s\n", trace_options[i]);
2731	else
2732	seq_printf(m, "no%s\n", trace_options[i]);
2733	}
2734
2735	for (i = 0; trace_opts[i].name; i++) {
2736	if (tracer_flags & trace_opts[i].bit)
2737	seq_printf(m, "%s\n", trace_opts[i].name);
2738	else
2739	seq_printf(m, "no%s\n", trace_opts[i].name);
2740	}
2741	mutex_unlock(&trace_types_lock);
2742
2743	return 0;
2744	}
2745
2746	static int __set_tracer_option(struct tracer *trace,
2747	struct tracer_flags *tracer_flags,
2748	struct tracer_opt *opts, int neg)
2749	{
2750	int ret;
2751
2752	ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2753	if (ret)
2754	return ret;
2755
2756	if (neg)
2757	tracer_flags->val &= ~opts->bit;
2758	else
2759	tracer_flags->val \|= opts->bit;
2760	return 0;
2761	}
2762
2763	/* Try to assign a tracer specific option */
2764	static int set_tracer_option(struct tracer trace, char cmp, int neg)
2765	{
2766	struct tracer_flags *tracer_flags = trace->flags;
2767	struct tracer_opt *opts = NULL;
2768	int i;
2769
2770	for (i = 0; tracer_flags->opts[i].name; i++) {
2771	opts = &tracer_flags->opts[i];
2772
2773	if (strcmp(cmp, opts->name) == 0)
2774	return __set_tracer_option(trace, trace->flags,
2775	opts, neg);
2776	}
2777
2778	return -EINVAL;
2779	}
2780
2781	static void set_tracer_flags(unsigned int mask, int enabled)
2782	{
2783	/* do nothing if flag is already set */
2784	if (!!(trace_flags & mask) == !!enabled)
2785	return;
2786
2787	if (enabled)
2788	trace_flags \|= mask;
2789	else
2790	trace_flags &= ~mask;
2791
2792	if (mask == TRACE_ITER_RECORD_CMD)
2793	trace_event_enable_cmd_record(enabled);
2794
2795	if (mask == TRACE_ITER_OVERWRITE)
2796	ring_buffer_change_overwrite(global_trace.buffer, enabled);
2797	}
2798
2799	static ssize_t
2800	tracing_trace_options_write(struct file filp, const char __user ubuf,
2801	size_t cnt, loff_t *ppos)
2802	{
2803	char buf[64];
2804	char *cmp;
2805	int neg = 0;
2806	int ret;
2807	int i;
2808
2809	if (cnt >= sizeof(buf))
2810	return -EINVAL;
2811
2812	if (copy_from_user(&buf, ubuf, cnt))
2813	return -EFAULT;
2814
2815	buf[cnt] = 0;
2816	cmp = strstrip(buf);
2817
2818	if (strncmp(cmp, "no", 2) == 0) {
2819	neg = 1;
2820	cmp += 2;
2821	}
2822
2823	for (i = 0; trace_options[i]; i++) {
2824	if (strcmp(cmp, trace_options[i]) == 0) {
2825	set_tracer_flags(1 << i, !neg);
2826	break;
2827	}
2828	}
2829
2830	/* If no option could be set, test the specific tracer options */
2831	if (!trace_options[i]) {
2832	mutex_lock(&trace_types_lock);
2833	ret = set_tracer_option(current_trace, cmp, neg);
2834	mutex_unlock(&trace_types_lock);
2835	if (ret)
2836	return ret;
2837	}
2838
2839	*ppos += cnt;
2840
2841	return cnt;
2842	}
2843
2844	static int tracing_trace_options_open(struct inode inode, struct file file)
2845	{
2846	if (tracing_disabled)
2847	return -ENODEV;
2848	return single_open(file, tracing_trace_options_show, NULL);
2849	}
2850
2851	static const struct file_operations tracing_iter_fops = {
2852	.open = tracing_trace_options_open,
2853	.read = seq_read,
2854	.llseek = seq_lseek,
2855	.release = single_release,
2856	.write = tracing_trace_options_write,
2857	};
2858
2859	static const char readme_msg[] =
2860	"tracing mini-HOWTO:\n\n"
2861	"# mount -t debugfs nodev /sys/kernel/debug\n\n"
2862	"# cat /sys/kernel/debug/tracing/available_tracers\n"
2863	"wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
2864	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2865	"nop\n"
2866	"# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
2867	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2868	"wakeup\n"
2869	"# cat /sys/kernel/debug/tracing/trace_options\n"
2870	"noprint-parent nosym-offset nosym-addr noverbose\n"
2871	"# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2872	"# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2873	"# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2874	"# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
2875	;
2876
2877	static ssize_t
2878	tracing_readme_read(struct file filp, char __user ubuf,
2879	size_t cnt, loff_t *ppos)
2880	{
2881	return simple_read_from_buffer(ubuf, cnt, ppos,
2882	readme_msg, strlen(readme_msg));
2883	}
2884
2885	static const struct file_operations tracing_readme_fops = {
2886	.open = tracing_open_generic,
2887	.read = tracing_readme_read,
2888	.llseek = generic_file_llseek,
2889	};
2890
2891	static ssize_t
2892	tracing_saved_cmdlines_read(struct file file, char __user ubuf,
2893	size_t cnt, loff_t *ppos)
2894	{
2895	char *buf_comm;
2896	char *file_buf;
2897	char *buf;
2898	int len = 0;
2899	int pid;
2900	int i;
2901
2902	file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2903	if (!file_buf)
2904	return -ENOMEM;
2905
2906	buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2907	if (!buf_comm) {
2908	kfree(file_buf);
2909	return -ENOMEM;
2910	}
2911
2912	buf = file_buf;
2913
2914	for (i = 0; i < SAVED_CMDLINES; i++) {
2915	int r;
2916
2917	pid = map_cmdline_to_pid[i];
2918	if (pid == -1 \|\| pid == NO_CMDLINE_MAP)
2919	continue;
2920
2921	trace_find_cmdline(pid, buf_comm);
2922	r = sprintf(buf, "%d %s\n", pid, buf_comm);
2923	buf += r;
2924	len += r;
2925	}
2926
2927	len = simple_read_from_buffer(ubuf, cnt, ppos,
2928	file_buf, len);
2929
2930	kfree(file_buf);
2931	kfree(buf_comm);
2932
2933	return len;
2934	}
2935
2936	static const struct file_operations tracing_saved_cmdlines_fops = {
2937	.open = tracing_open_generic,
2938	.read = tracing_saved_cmdlines_read,
2939	.llseek = generic_file_llseek,
2940	};
2941
2942	static ssize_t
2943	tracing_ctrl_read(struct file filp, char __user ubuf,
2944	size_t cnt, loff_t *ppos)
2945	{
2946	char buf[64];
2947	int r;
2948
2949	r = sprintf(buf, "%u\n", tracer_enabled);
2950	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2951	}
2952
2953	static ssize_t
2954	tracing_ctrl_write(struct file filp, const char __user ubuf,
2955	size_t cnt, loff_t *ppos)
2956	{
2957	struct trace_array *tr = filp->private_data;
2958	unsigned long val;
2959	int ret;
2960
2961	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2962	if (ret)
2963	return ret;
2964
2965	val = !!val;
2966
2967	mutex_lock(&trace_types_lock);
2968	if (tracer_enabled ^ val) {
2969
2970	/* Only need to warn if this is used to change the state */
2971	WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2972
2973	if (val) {
2974	tracer_enabled = 1;
2975	if (current_trace->start)
2976	current_trace->start(tr);
2977	tracing_start();
2978	} else {
2979	tracer_enabled = 0;
2980	tracing_stop();
2981	if (current_trace->stop)
2982	current_trace->stop(tr);
2983	}
2984	}
2985	mutex_unlock(&trace_types_lock);
2986
2987	*ppos += cnt;
2988
2989	return cnt;
2990	}
2991
2992	static ssize_t
2993	tracing_set_trace_read(struct file filp, char __user ubuf,
2994	size_t cnt, loff_t *ppos)
2995	{
2996	char buf[MAX_TRACER_SIZE+2];
2997	int r;
2998
2999	mutex_lock(&trace_types_lock);
3000	if (current_trace)
3001	r = sprintf(buf, "%s\n", current_trace->name);
3002	else
3003	r = sprintf(buf, "\n");
3004	mutex_unlock(&trace_types_lock);
3005
3006	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3007	}
3008
3009	int tracer_init(struct tracer t, struct trace_array tr)
3010	{
3011	tracing_reset_online_cpus(tr);
3012	return t->init(tr);
3013	}
3014
3015	static void set_buffer_entries(struct trace_array *tr, unsigned long val)
3016	{
3017	int cpu;
3018	for_each_tracing_cpu(cpu)
3019	tr->data[cpu]->entries = val;
3020	}
3021
3022	static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
3023	{
3024	int ret;
3025
3026	/*
3027	* If kernel or user changes the size of the ring buffer
3028	* we use the size that was given, and we can forget about
3029	* expanding it later.
3030	*/
3031	ring_buffer_expanded = 1;
3032
3033	ret = ring_buffer_resize(global_trace.buffer, size, cpu);
3034	if (ret < 0)
3035	return ret;
3036
3037	if (!current_trace->use_max_tr)
3038	goto out;
3039
3040	ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3041	if (ret < 0) {
3042	int r = 0;
3043
3044	if (cpu == RING_BUFFER_ALL_CPUS) {
3045	int i;
3046	for_each_tracing_cpu(i) {
3047	r = ring_buffer_resize(global_trace.buffer,
3048	global_trace.data[i]->entries,
3049	i);
3050	if (r < 0)
3051	break;
3052	}
3053	} else {
3054	r = ring_buffer_resize(global_trace.buffer,
3055	global_trace.data[cpu]->entries,
3056	cpu);
3057	}
3058
3059	if (r < 0) {
3060	/*
3061	* AARGH! We are left with different
3062	* size max buffer!!!!
3063	* The max buffer is our "snapshot" buffer.
3064	* When a tracer needs a snapshot (one of the
3065	* latency tracers), it swaps the max buffer
3066	* with the saved snap shot. We succeeded to
3067	* update the size of the main buffer, but failed to
3068	* update the size of the max buffer. But when we tried
3069	* to reset the main buffer to the original size, we
3070	* failed there too. This is very unlikely to
3071	* happen, but if it does, warn and kill all
3072	* tracing.
3073	*/
3074	WARN_ON(1);
3075	tracing_disabled = 1;
3076	}
3077	return ret;
3078	}
3079
3080	if (cpu == RING_BUFFER_ALL_CPUS)
3081	set_buffer_entries(&max_tr, size);
3082	else
3083	max_tr.data[cpu]->entries = size;
3084
3085	out:
3086	if (cpu == RING_BUFFER_ALL_CPUS)
3087	set_buffer_entries(&global_trace, size);
3088	else
3089	global_trace.data[cpu]->entries = size;
3090
3091	return ret;
3092	}
3093
3094	static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3095	{
3096	int ret = size;
3097
3098	mutex_lock(&trace_types_lock);
3099
3100	if (cpu_id != RING_BUFFER_ALL_CPUS) {
3101	/* make sure, this cpu is enabled in the mask */
3102	if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3103	ret = -EINVAL;
3104	goto out;
3105	}
3106	}
3107
3108	ret = __tracing_resize_ring_buffer(size, cpu_id);
3109	if (ret < 0)
3110	ret = -ENOMEM;
3111
3112	out:
3113	mutex_unlock(&trace_types_lock);
3114
3115	return ret;
3116	}
3117
3118
3119	/**
3120	* tracing_update_buffers - used by tracing facility to expand ring buffers
3121	*
3122	* To save on memory when the tracing is never used on a system with it
3123	* configured in. The ring buffers are set to a minimum size. But once
3124	* a user starts to use the tracing facility, then they need to grow
3125	* to their default size.
3126	*
3127	* This function is to be called when a tracer is about to be used.
3128	*/
3129	int tracing_update_buffers(void)
3130	{
3131	int ret = 0;
3132
3133	mutex_lock(&trace_types_lock);
3134	if (!ring_buffer_expanded)
3135	ret = __tracing_resize_ring_buffer(trace_buf_size,
3136	RING_BUFFER_ALL_CPUS);
3137	mutex_unlock(&trace_types_lock);
3138
3139	return ret;
3140	}
3141
3142	struct trace_option_dentry;
3143
3144	static struct trace_option_dentry *
3145	create_trace_option_files(struct tracer *tracer);
3146
3147	static void
3148	destroy_trace_option_files(struct trace_option_dentry *topts);
3149
3150	static int tracing_set_tracer(const char *buf)
3151	{
3152	static struct trace_option_dentry *topts;
3153	struct trace_array *tr = &global_trace;
3154	struct tracer *t;
3155	int ret = 0;
3156
3157	mutex_lock(&trace_types_lock);
3158
3159	if (!ring_buffer_expanded) {
3160	ret = __tracing_resize_ring_buffer(trace_buf_size,
3161	RING_BUFFER_ALL_CPUS);
3162	if (ret < 0)
3163	goto out;
3164	ret = 0;
3165	}
3166
3167	for (t = trace_types; t; t = t->next) {
3168	if (strcmp(t->name, buf) == 0)
3169	break;
3170	}
3171	if (!t) {
3172	ret = -EINVAL;
3173	goto out;
3174	}
3175	if (t == current_trace)
3176	goto out;
3177
3178	trace_branch_disable();
3179	if (current_trace && current_trace->reset)
3180	current_trace->reset(tr);
3181	if (current_trace && current_trace->use_max_tr) {
3182	/*
3183	* We don't free the ring buffer. instead, resize it because
3184	* The max_tr ring buffer has some state (e.g. ring->clock) and
3185	* we want preserve it.
3186	*/
3187	ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3188	set_buffer_entries(&max_tr, 1);
3189	}
3190	destroy_trace_option_files(topts);
3191
3192	current_trace = &nop_trace;
3193
3194	topts = create_trace_option_files(t);
3195	if (t->use_max_tr) {
3196	int cpu;
3197	/* we need to make per cpu buffer sizes equivalent */
3198	for_each_tracing_cpu(cpu) {
3199	ret = ring_buffer_resize(max_tr.buffer,
3200	global_trace.data[cpu]->entries,
3201	cpu);
3202	if (ret < 0)
3203	goto out;
3204	max_tr.data[cpu]->entries =
3205	global_trace.data[cpu]->entries;
3206	}
3207	}
3208
3209	if (t->init) {
3210	ret = tracer_init(t, tr);
3211	if (ret)
3212	goto out;
3213	}
3214
3215	current_trace = t;
3216	trace_branch_enable(tr);
3217	out:
3218	mutex_unlock(&trace_types_lock);
3219
3220	return ret;
3221	}
3222
3223	static ssize_t
3224	tracing_set_trace_write(struct file filp, const char __user ubuf,
3225	size_t cnt, loff_t *ppos)
3226	{
3227	char buf[MAX_TRACER_SIZE+1];
3228	int i;
3229	size_t ret;
3230	int err;
3231
3232	ret = cnt;
3233
3234	if (cnt > MAX_TRACER_SIZE)
3235	cnt = MAX_TRACER_SIZE;
3236
3237	if (copy_from_user(&buf, ubuf, cnt))
3238	return -EFAULT;
3239
3240	buf[cnt] = 0;
3241
3242	/* strip ending whitespace. */
3243	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3244	buf[i] = 0;
3245
3246	err = tracing_set_tracer(buf);
3247	if (err)
3248	return err;
3249
3250	*ppos += ret;
3251
3252	return ret;
3253	}
3254
3255	static ssize_t
3256	tracing_max_lat_read(struct file filp, char __user ubuf,
3257	size_t cnt, loff_t *ppos)
3258	{
3259	unsigned long *ptr = filp->private_data;
3260	char buf[64];
3261	int r;
3262
3263	r = snprintf(buf, sizeof(buf), "%ld\n",
3264	ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(ptr));
3265	if (r > sizeof(buf))
3266	r = sizeof(buf);
3267	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3268	}
3269
3270	static ssize_t
3271	tracing_max_lat_write(struct file filp, const char __user ubuf,
3272	size_t cnt, loff_t *ppos)
3273	{
3274	unsigned long *ptr = filp->private_data;
3275	unsigned long val;
3276	int ret;
3277
3278	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3279	if (ret)
3280	return ret;
3281
3282	ptr = val 1000;
3283
3284	return cnt;
3285	}
3286
3287	static int tracing_open_pipe(struct inode inode, struct file filp)
3288	{
3289	long cpu_file = (long) inode->i_private;
3290	struct trace_iterator *iter;
3291	int ret = 0;
3292
3293	if (tracing_disabled)
3294	return -ENODEV;
3295
3296	mutex_lock(&trace_types_lock);
3297
3298	/* create a buffer to store the information to pass to userspace */
3299	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3300	if (!iter) {
3301	ret = -ENOMEM;
3302	goto out;
3303	}
3304
3305	/*
3306	* We make a copy of the current tracer to avoid concurrent
3307	* changes on it while we are reading.
3308	*/
3309	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3310	if (!iter->trace) {
3311	ret = -ENOMEM;
3312	goto fail;
3313	}
3314	if (current_trace)
3315	iter->trace = current_trace;
3316
3317	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3318	ret = -ENOMEM;
3319	goto fail;
3320	}
3321
3322	/* trace pipe does not show start of buffer */
3323	cpumask_setall(iter->started);
3324
3325	if (trace_flags & TRACE_ITER_LATENCY_FMT)
3326	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
3327
3328	iter->cpu_file = cpu_file;
3329	iter->tr = &global_trace;
3330	mutex_init(&iter->mutex);
3331	filp->private_data = iter;
3332
3333	if (iter->trace->pipe_open)
3334	iter->trace->pipe_open(iter);
3335
3336	nonseekable_open(inode, filp);
3337	out:
3338	mutex_unlock(&trace_types_lock);
3339	return ret;
3340
3341	fail:
3342	kfree(iter->trace);
3343	kfree(iter);
3344	mutex_unlock(&trace_types_lock);
3345	return ret;
3346	}
3347
3348	static int tracing_release_pipe(struct inode inode, struct file file)
3349	{
3350	struct trace_iterator *iter = file->private_data;
3351
3352	mutex_lock(&trace_types_lock);
3353
3354	if (iter->trace->pipe_close)
3355	iter->trace->pipe_close(iter);
3356
3357	mutex_unlock(&trace_types_lock);
3358
3359	free_cpumask_var(iter->started);
3360	mutex_destroy(&iter->mutex);
3361	kfree(iter->trace);
3362	kfree(iter);
3363
3364	return 0;
3365	}
3366
3367	static unsigned int
3368	tracing_poll_pipe(struct file filp, poll_table poll_table)
3369	{
3370	struct trace_iterator *iter = filp->private_data;
3371
3372	if (trace_flags & TRACE_ITER_BLOCK) {
3373	/*
3374	* Always select as readable when in blocking mode
3375	*/
3376	return POLLIN \| POLLRDNORM;
3377	} else {
3378	if (!trace_empty(iter))
3379	return POLLIN \| POLLRDNORM;
3380	poll_wait(filp, &trace_wait, poll_table);
3381	if (!trace_empty(iter))
3382	return POLLIN \| POLLRDNORM;
3383
3384	return 0;
3385	}
3386	}
3387
3388
3389	void default_wait_pipe(struct trace_iterator *iter)
3390	{
3391	DEFINE_WAIT(wait);
3392
3393	prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3394
3395	if (trace_empty(iter))
3396	schedule();
3397
3398	finish_wait(&trace_wait, &wait);
3399	}
3400
3401	/*
3402	* This is a make-shift waitqueue.
3403	* A tracer might use this callback on some rare cases:
3404	*
3405	* 1) the current tracer might hold the runqueue lock when it wakes up
3406	* a reader, hence a deadlock (sched, function, and function graph tracers)
3407	* 2) the function tracers, trace all functions, we don't want
3408	* the overhead of calling wake_up and friends
3409	* (and tracing them too)
3410	*
3411	* Anyway, this is really very primitive wakeup.
3412	*/
3413	void poll_wait_pipe(struct trace_iterator *iter)
3414	{
3415	set_current_state(TASK_INTERRUPTIBLE);
3416	/* sleep for 100 msecs, and try again. */
3417	schedule_timeout(HZ / 10);
3418	}
3419
3420	/* Must be called with trace_types_lock mutex held. */
3421	static int tracing_wait_pipe(struct file *filp)
3422	{
3423	struct trace_iterator *iter = filp->private_data;
3424
3425	while (trace_empty(iter)) {
3426
3427	if ((filp->f_flags & O_NONBLOCK)) {
3428	return -EAGAIN;
3429	}
3430
3431	mutex_unlock(&iter->mutex);
3432
3433	iter->trace->wait_pipe(iter);
3434
3435	mutex_lock(&iter->mutex);
3436
3437	if (signal_pending(current))
3438	return -EINTR;
3439
3440	/*
3441	* We block until we read something and tracing is disabled.
3442	* We still block if tracing is disabled, but we have never
3443	* read anything. This allows a user to cat this file, and
3444	* then enable tracing. But after we have read something,
3445	* we give an EOF when tracing is again disabled.
3446	*
3447	* iter->pos will be 0 if we haven't read anything.
3448	*/
3449	if (!tracer_enabled && iter->pos)
3450	break;
3451	}
3452
3453	return 1;
3454	}
3455
3456	/*
3457	* Consumer reader.
3458	*/
3459	static ssize_t
3460	tracing_read_pipe(struct file filp, char __user ubuf,
3461	size_t cnt, loff_t *ppos)
3462	{
3463	struct trace_iterator *iter = filp->private_data;
3464	static struct tracer *old_tracer;
3465	ssize_t sret;
3466
3467	/* return any leftover data */
3468	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3469	if (sret != -EBUSY)
3470	return sret;
3471
3472	trace_seq_init(&iter->seq);
3473
3474	/* copy the tracer to avoid using a global lock all around */
3475	mutex_lock(&trace_types_lock);
3476	if (unlikely(old_tracer != current_trace && current_trace)) {
3477	old_tracer = current_trace;
3478	iter->trace = current_trace;
3479	}
3480	mutex_unlock(&trace_types_lock);
3481
3482	/*
3483	* Avoid more than one consumer on a single file descriptor
3484	* This is just a matter of traces coherency, the ring buffer itself
3485	* is protected.
3486	*/
3487	mutex_lock(&iter->mutex);
3488	if (iter->trace->read) {
3489	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3490	if (sret)
3491	goto out;
3492	}
3493
3494	waitagain:
3495	sret = tracing_wait_pipe(filp);
3496	if (sret <= 0)
3497	goto out;
3498
3499	/* stop when tracing is finished */
3500	if (trace_empty(iter)) {
3501	sret = 0;
3502	goto out;
3503	}
3504
3505	if (cnt >= PAGE_SIZE)
3506	cnt = PAGE_SIZE - 1;
3507
3508	/* reset all but tr, trace, and overruns */
3509	memset(&iter->seq, 0,
3510	sizeof(struct trace_iterator) -
3511	offsetof(struct trace_iterator, seq));
3512	iter->pos = -1;
3513
3514	trace_event_read_lock();
3515	trace_access_lock(iter->cpu_file);
3516	while (trace_find_next_entry_inc(iter) != NULL) {
3517	enum print_line_t ret;
3518	int len = iter->seq.len;
3519
3520	ret = print_trace_line(iter);
3521	if (ret == TRACE_TYPE_PARTIAL_LINE) {
3522	/* don't print partial lines */
3523	iter->seq.len = len;
3524	break;
3525	}
3526	if (ret != TRACE_TYPE_NO_CONSUME)
3527	trace_consume(iter);
3528
3529	if (iter->seq.len >= cnt)
3530	break;
3531
3532	/*
3533	* Setting the full flag means we reached the trace_seq buffer
3534	* size and we should leave by partial output condition above.
3535	* One of the trace_seq_* functions is not used properly.
3536	*/
3537	WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3538	iter->ent->type);
3539	}
3540	trace_access_unlock(iter->cpu_file);
3541	trace_event_read_unlock();
3542
3543	/* Now copy what we have to the user */
3544	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3545	if (iter->seq.readpos >= iter->seq.len)
3546	trace_seq_init(&iter->seq);
3547
3548	/*
3549	* If there was nothing to send to user, in spite of consuming trace
3550	* entries, go back to wait for more entries.
3551	*/
3552	if (sret == -EBUSY)
3553	goto waitagain;
3554
3555	out:
3556	mutex_unlock(&iter->mutex);
3557
3558	return sret;
3559	}
3560
3561	static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3562	struct pipe_buffer *buf)
3563	{
3564	__free_page(buf->page);
3565	}
3566
3567	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3568	unsigned int idx)
3569	{
3570	__free_page(spd->pages[idx]);
3571	}
3572
3573	static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3574	.can_merge = 0,
3575	.map = generic_pipe_buf_map,
3576	.unmap = generic_pipe_buf_unmap,
3577	.confirm = generic_pipe_buf_confirm,
3578	.release = tracing_pipe_buf_release,
3579	.steal = generic_pipe_buf_steal,
3580	.get = generic_pipe_buf_get,
3581	};
3582
3583	static size_t
3584	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3585	{
3586	size_t count;
3587	int ret;
3588
3589	/* Seq buffer is page-sized, exactly what we need. */
3590	for (;;) {
3591	count = iter->seq.len;
3592	ret = print_trace_line(iter);
3593	count = iter->seq.len - count;
3594	if (rem < count) {
3595	rem = 0;
3596	iter->seq.len -= count;
3597	break;
3598	}
3599	if (ret == TRACE_TYPE_PARTIAL_LINE) {
3600	iter->seq.len -= count;
3601	break;
3602	}
3603
3604	if (ret != TRACE_TYPE_NO_CONSUME)
3605	trace_consume(iter);
3606	rem -= count;
3607	if (!trace_find_next_entry_inc(iter)) {
3608	rem = 0;
3609	iter->ent = NULL;
3610	break;
3611	}
3612	}
3613
3614	return rem;
3615	}
3616
3617	static ssize_t tracing_splice_read_pipe(struct file *filp,
3618	loff_t *ppos,
3619	struct pipe_inode_info *pipe,
3620	size_t len,
3621	unsigned int flags)
3622	{
3623	struct page *pages_def[PIPE_DEF_BUFFERS];
3624	struct partial_page partial_def[PIPE_DEF_BUFFERS];
3625	struct trace_iterator *iter = filp->private_data;
3626	struct splice_pipe_desc spd = {
3627	.pages = pages_def,
3628	.partial = partial_def,
3629	.nr_pages = 0, /* This gets updated below. */
3630	.nr_pages_max = PIPE_DEF_BUFFERS,
3631	.flags = flags,
3632	.ops = &tracing_pipe_buf_ops,
3633	.spd_release = tracing_spd_release_pipe,
3634	};
3635	static struct tracer *old_tracer;
3636	ssize_t ret;
3637	size_t rem;
3638	unsigned int i;
3639
3640	if (splice_grow_spd(pipe, &spd))
3641	return -ENOMEM;
3642
3643	/* copy the tracer to avoid using a global lock all around */
3644	mutex_lock(&trace_types_lock);
3645	if (unlikely(old_tracer != current_trace && current_trace)) {
3646	old_tracer = current_trace;
3647	iter->trace = current_trace;
3648	}
3649	mutex_unlock(&trace_types_lock);
3650
3651	mutex_lock(&iter->mutex);
3652
3653	if (iter->trace->splice_read) {
3654	ret = iter->trace->splice_read(iter, filp,
3655	ppos, pipe, len, flags);
3656	if (ret)
3657	goto out_err;
3658	}
3659
3660	ret = tracing_wait_pipe(filp);
3661	if (ret <= 0)
3662	goto out_err;
3663
3664	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3665	ret = -EFAULT;
3666	goto out_err;
3667	}
3668
3669	trace_event_read_lock();
3670	trace_access_lock(iter->cpu_file);
3671
3672	/* Fill as many pages as possible. */
3673	for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3674	spd.pages[i] = alloc_page(GFP_KERNEL);
3675	if (!spd.pages[i])
3676	break;
3677
3678	rem = tracing_fill_pipe_page(rem, iter);
3679
3680	/* Copy the data into the page, so we can start over. */
3681	ret = trace_seq_to_buffer(&iter->seq,
3682	page_address(spd.pages[i]),
3683	iter->seq.len);
3684	if (ret < 0) {
3685	__free_page(spd.pages[i]);
3686	break;
3687	}
3688	spd.partial[i].offset = 0;
3689	spd.partial[i].len = iter->seq.len;
3690
3691	trace_seq_init(&iter->seq);
3692	}
3693
3694	trace_access_unlock(iter->cpu_file);
3695	trace_event_read_unlock();
3696	mutex_unlock(&iter->mutex);
3697
3698	spd.nr_pages = i;
3699
3700	ret = splice_to_pipe(pipe, &spd);
3701	out:
3702	splice_shrink_spd(&spd);
3703	return ret;
3704
3705	out_err:
3706	mutex_unlock(&iter->mutex);
3707	goto out;
3708	}
3709
3710	struct ftrace_entries_info {
3711	struct trace_array *tr;
3712	int cpu;
3713	};
3714
3715	static int tracing_entries_open(struct inode inode, struct file filp)
3716	{
3717	struct ftrace_entries_info *info;
3718
3719	if (tracing_disabled)
3720	return -ENODEV;
3721
3722	info = kzalloc(sizeof(*info), GFP_KERNEL);
3723	if (!info)
3724	return -ENOMEM;
3725
3726	info->tr = &global_trace;
3727	info->cpu = (unsigned long)inode->i_private;
3728
3729	filp->private_data = info;
3730
3731	return 0;
3732	}
3733
3734	static ssize_t
3735	tracing_entries_read(struct file filp, char __user ubuf,
3736	size_t cnt, loff_t *ppos)
3737	{
3738	struct ftrace_entries_info *info = filp->private_data;
3739	struct trace_array *tr = info->tr;
3740	char buf[64];
3741	int r = 0;
3742	ssize_t ret;
3743
3744	mutex_lock(&trace_types_lock);
3745
3746	if (info->cpu == RING_BUFFER_ALL_CPUS) {
3747	int cpu, buf_size_same;
3748	unsigned long size;
3749
3750	size = 0;
3751	buf_size_same = 1;
3752	/* check if all cpu sizes are same */
3753	for_each_tracing_cpu(cpu) {
3754	/* fill in the size from first enabled cpu */
3755	if (size == 0)
3756	size = tr->data[cpu]->entries;
3757	if (size != tr->data[cpu]->entries) {
3758	buf_size_same = 0;
3759	break;
3760	}
3761	}
3762
3763	if (buf_size_same) {
3764	if (!ring_buffer_expanded)
3765	r = sprintf(buf, "%lu (expanded: %lu)\n",
3766	size >> 10,
3767	trace_buf_size >> 10);
3768	else
3769	r = sprintf(buf, "%lu\n", size >> 10);
3770	} else
3771	r = sprintf(buf, "X\n");
3772	} else
3773	r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3774
3775	mutex_unlock(&trace_types_lock);
3776
3777	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3778	return ret;
3779	}
3780
3781	static ssize_t
3782	tracing_entries_write(struct file filp, const char __user ubuf,
3783	size_t cnt, loff_t *ppos)
3784	{
3785	struct ftrace_entries_info *info = filp->private_data;
3786	unsigned long val;
3787	int ret;
3788
3789	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3790	if (ret)
3791	return ret;
3792
3793	/* must have at least 1 entry */
3794	if (!val)
3795	return -EINVAL;
3796
3797	/* value is in KB */
3798	val <<= 10;
3799
3800	ret = tracing_resize_ring_buffer(val, info->cpu);
3801	if (ret < 0)
3802	return ret;
3803
3804	*ppos += cnt;
3805
3806	return cnt;
3807	}
3808
3809	static int
3810	tracing_entries_release(struct inode inode, struct file filp)
3811	{
3812	struct ftrace_entries_info *info = filp->private_data;
3813
3814	kfree(info);
3815
3816	return 0;
3817	}
3818
3819	static ssize_t
3820	tracing_total_entries_read(struct file filp, char __user ubuf,
3821	size_t cnt, loff_t *ppos)
3822	{
3823	struct trace_array *tr = filp->private_data;
3824	char buf[64];
3825	int r, cpu;
3826	unsigned long size = 0, expanded_size = 0;
3827
3828	mutex_lock(&trace_types_lock);
3829	for_each_tracing_cpu(cpu) {
3830	size += tr->data[cpu]->entries >> 10;
3831	if (!ring_buffer_expanded)
3832	expanded_size += trace_buf_size >> 10;
3833	}
3834	if (ring_buffer_expanded)
3835	r = sprintf(buf, "%lu\n", size);
3836	else
3837	r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3838	mutex_unlock(&trace_types_lock);
3839
3840	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3841	}
3842
3843	static ssize_t
3844	tracing_free_buffer_write(struct file filp, const char __user ubuf,
3845	size_t cnt, loff_t *ppos)
3846	{
3847	/*
3848	* There is no need to read what the user has written, this function
3849	* is just to make sure that there is no error when "echo" is used
3850	*/
3851
3852	*ppos += cnt;
3853
3854	return cnt;
3855	}
3856
3857	static int
3858	tracing_free_buffer_release(struct inode inode, struct file filp)
3859	{
3860	/* disable tracing ? */
3861	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3862	tracing_off();
3863	/* resize the ring buffer to 0 */
3864	tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
3865
3866	return 0;
3867	}
3868
3869	static ssize_t
3870	tracing_mark_write(struct file filp, const char __user ubuf,
3871	size_t cnt, loff_t *fpos)
3872	{
3873	unsigned long addr = (unsigned long)ubuf;
3874	struct ring_buffer_event *event;
3875	struct ring_buffer *buffer;
3876	struct print_entry *entry;
3877	unsigned long irq_flags;
3878	struct page *pages[2];
3879	void *map_page[2];
3880	int nr_pages = 1;
3881	ssize_t written;
3882	int offset;
3883	int size;
3884	int len;
3885	int ret;
3886	int i;
3887
3888	if (tracing_disabled)
3889	return -EINVAL;
3890
3891	if (!(trace_flags & TRACE_ITER_MARKERS))
3892	return -EINVAL;
3893
3894	if (cnt > TRACE_BUF_SIZE)
3895	cnt = TRACE_BUF_SIZE;
3896
3897	/*
3898	* Userspace is injecting traces into the kernel trace buffer.
3899	* We want to be as non intrusive as possible.
3900	* To do so, we do not want to allocate any special buffers
3901	* or take any locks, but instead write the userspace data
3902	* straight into the ring buffer.
3903	*
3904	* First we need to pin the userspace buffer into memory,
3905	* which, most likely it is, because it just referenced it.
3906	* But there's no guarantee that it is. By using get_user_pages_fast()
3907	* and kmap_atomic/kunmap_atomic() we can get access to the
3908	* pages directly. We then write the data directly into the
3909	* ring buffer.
3910	*/
3911	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
3912
3913	/* check if we cross pages */
3914	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
3915	nr_pages = 2;
3916
3917	offset = addr & (PAGE_SIZE - 1);
3918	addr &= PAGE_MASK;
3919
3920	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
3921	if (ret < nr_pages) {
3922	while (--ret >= 0)
3923	put_page(pages[ret]);
3924	written = -EFAULT;
3925	goto out;
3926	}
3927
3928	for (i = 0; i < nr_pages; i++)
3929	map_page[i] = kmap_atomic(pages[i]);
3930
3931	local_save_flags(irq_flags);
3932	size = sizeof(entry) + cnt + 2; / possible \n added */
3933	buffer = global_trace.buffer;
3934	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3935	irq_flags, preempt_count());
3936	if (!event) {
3937	/* Ring buffer disabled, return as if not open for write */
3938	written = -EBADF;
3939	goto out_unlock;
3940	}
3941
3942	entry = ring_buffer_event_data(event);
3943	entry->ip = _THIS_IP_;
3944
3945	if (nr_pages == 2) {
3946	len = PAGE_SIZE - offset;
3947	memcpy(&entry->buf, map_page[0] + offset, len);
3948	memcpy(&entry->buf[len], map_page[1], cnt - len);
3949	} else
3950	memcpy(&entry->buf, map_page[0] + offset, cnt);
3951
3952	if (entry->buf[cnt - 1] != '\n') {
3953	entry->buf[cnt] = '\n';
3954	entry->buf[cnt + 1] = '\0';
3955	} else
3956	entry->buf[cnt] = '\0';
3957
3958	ring_buffer_unlock_commit(buffer, event);
3959
3960	written = cnt;
3961
3962	*fpos += written;
3963
3964	out_unlock:
3965	for (i = 0; i < nr_pages; i++){
3966	kunmap_atomic(map_page[i]);
3967	put_page(pages[i]);
3968	}
3969	out:
3970	return written;
3971	}
3972
3973	static int tracing_clock_show(struct seq_file m, void v)
3974	{
3975	int i;
3976
3977	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3978	seq_printf(m,
3979	"%s%s%s%s", i ? " " : "",
3980	i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3981	i == trace_clock_id ? "]" : "");
3982	seq_putc(m, '\n');
3983
3984	return 0;
3985	}
3986
3987	static ssize_t tracing_clock_write(struct file filp, const char __user ubuf,
3988	size_t cnt, loff_t *fpos)
3989	{
3990	char buf[64];
3991	const char *clockstr;
3992	int i;
3993
3994	if (cnt >= sizeof(buf))
3995	return -EINVAL;
3996
3997	if (copy_from_user(&buf, ubuf, cnt))
3998	return -EFAULT;
3999
4000	buf[cnt] = 0;
4001
4002	clockstr = strstrip(buf);
4003
4004	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4005	if (strcmp(trace_clocks[i].name, clockstr) == 0)
4006	break;
4007	}
4008	if (i == ARRAY_SIZE(trace_clocks))
4009	return -EINVAL;
4010
4011	trace_clock_id = i;
4012
4013	mutex_lock(&trace_types_lock);
4014
4015	ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
4016	if (max_tr.buffer)
4017	ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
4018
4019	mutex_unlock(&trace_types_lock);
4020
4021	*fpos += cnt;
4022
4023	return cnt;
4024	}
4025
4026	static int tracing_clock_open(struct inode inode, struct file file)
4027	{
4028	if (tracing_disabled)
4029	return -ENODEV;
4030	return single_open(file, tracing_clock_show, NULL);
4031	}
4032
4033	static const struct file_operations tracing_max_lat_fops = {
4034	.open = tracing_open_generic,
4035	.read = tracing_max_lat_read,
4036	.write = tracing_max_lat_write,
4037	.llseek = generic_file_llseek,
4038	};
4039
4040	static const struct file_operations tracing_ctrl_fops = {
4041	.open = tracing_open_generic,
4042	.read = tracing_ctrl_read,
4043	.write = tracing_ctrl_write,
4044	.llseek = generic_file_llseek,
4045	};
4046
4047	static const struct file_operations set_tracer_fops = {
4048	.open = tracing_open_generic,
4049	.read = tracing_set_trace_read,
4050	.write = tracing_set_trace_write,
4051	.llseek = generic_file_llseek,
4052	};
4053
4054	static const struct file_operations tracing_pipe_fops = {
4055	.open = tracing_open_pipe,
4056	.poll = tracing_poll_pipe,
4057	.read = tracing_read_pipe,
4058	.splice_read = tracing_splice_read_pipe,
4059	.release = tracing_release_pipe,
4060	.llseek = no_llseek,
4061	};
4062
4063	static const struct file_operations tracing_entries_fops = {
4064	.open = tracing_entries_open,
4065	.read = tracing_entries_read,
4066	.write = tracing_entries_write,
4067	.release = tracing_entries_release,
4068	.llseek = generic_file_llseek,
4069	};
4070
4071	static const struct file_operations tracing_total_entries_fops = {
4072	.open = tracing_open_generic,
4073	.read = tracing_total_entries_read,
4074	.llseek = generic_file_llseek,
4075	};
4076
4077	static const struct file_operations tracing_free_buffer_fops = {
4078	.write = tracing_free_buffer_write,
4079	.release = tracing_free_buffer_release,
4080	};
4081
4082	static const struct file_operations tracing_mark_fops = {
4083	.open = tracing_open_generic,
4084	.write = tracing_mark_write,
4085	.llseek = generic_file_llseek,
4086	};
4087
4088	static const struct file_operations trace_clock_fops = {
4089	.open = tracing_clock_open,
4090	.read = seq_read,
4091	.llseek = seq_lseek,
4092	.release = single_release,
4093	.write = tracing_clock_write,
4094	};
4095
4096	struct ftrace_buffer_info {
4097	struct trace_array *tr;
4098	void *spare;
4099	int cpu;
4100	unsigned int read;
4101	};
4102
4103	static int tracing_buffers_open(struct inode inode, struct file filp)
4104	{
4105	int cpu = (int)(long)inode->i_private;
4106	struct ftrace_buffer_info *info;
4107
4108	if (tracing_disabled)
4109	return -ENODEV;
4110
4111	info = kzalloc(sizeof(*info), GFP_KERNEL);
4112	if (!info)
4113	return -ENOMEM;
4114
4115	info->tr = &global_trace;
4116	info->cpu = cpu;
4117	info->spare = NULL;
4118	/* Force reading ring buffer for first read */
4119	info->read = (unsigned int)-1;
4120
4121	filp->private_data = info;
4122
4123	return nonseekable_open(inode, filp);
4124	}
4125
4126	static ssize_t
4127	tracing_buffers_read(struct file filp, char __user ubuf,
4128	size_t count, loff_t *ppos)
4129	{
4130	struct ftrace_buffer_info *info = filp->private_data;
4131	ssize_t ret;
4132	size_t size;
4133
4134	if (!count)
4135	return 0;
4136
4137	if (!info->spare)
4138	info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4139	if (!info->spare)
4140	return -ENOMEM;
4141
4142	/* Do we have previous read data to read? */
4143	if (info->read < PAGE_SIZE)
4144	goto read;
4145
4146	trace_access_lock(info->cpu);
4147	ret = ring_buffer_read_page(info->tr->buffer,
4148	&info->spare,
4149	count,
4150	info->cpu, 0);
4151	trace_access_unlock(info->cpu);
4152	if (ret < 0)
4153	return 0;
4154
4155	info->read = 0;
4156
4157	read:
4158	size = PAGE_SIZE - info->read;
4159	if (size > count)
4160	size = count;
4161
4162	ret = copy_to_user(ubuf, info->spare + info->read, size);
4163	if (ret == size)
4164	return -EFAULT;
4165	size -= ret;
4166
4167	*ppos += size;
4168	info->read += size;
4169
4170	return size;
4171	}
4172
4173	static int tracing_buffers_release(struct inode inode, struct file file)
4174	{
4175	struct ftrace_buffer_info *info = file->private_data;
4176
4177	if (info->spare)
4178	ring_buffer_free_read_page(info->tr->buffer, info->spare);
4179	kfree(info);
4180
4181	return 0;
4182	}
4183
4184	struct buffer_ref {
4185	struct ring_buffer *buffer;
4186	void *page;
4187	int ref;
4188	};
4189
4190	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4191	struct pipe_buffer *buf)
4192	{
4193	struct buffer_ref ref = (struct buffer_ref )buf->private;
4194
4195	if (--ref->ref)
4196	return;
4197
4198	ring_buffer_free_read_page(ref->buffer, ref->page);
4199	kfree(ref);
4200	buf->private = 0;
4201	}
4202
4203	static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4204	struct pipe_buffer *buf)
4205	{
4206	struct buffer_ref ref = (struct buffer_ref )buf->private;
4207
4208	ref->ref++;
4209	}
4210
4211	/* Pipe buffer operations for a buffer. */
4212	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4213	.can_merge = 0,
4214	.map = generic_pipe_buf_map,
4215	.unmap = generic_pipe_buf_unmap,
4216	.confirm = generic_pipe_buf_confirm,
4217	.release = buffer_pipe_buf_release,
4218	.steal = generic_pipe_buf_steal,
4219	.get = buffer_pipe_buf_get,
4220	};
4221
4222	/*
4223	* Callback from splice_to_pipe(), if we need to release some pages
4224	* at the end of the spd in case we error'ed out in filling the pipe.
4225	*/
4226	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4227	{
4228	struct buffer_ref *ref =
4229	(struct buffer_ref *)spd->partial[i].private;
4230
4231	if (--ref->ref)
4232	return;
4233
4234	ring_buffer_free_read_page(ref->buffer, ref->page);
4235	kfree(ref);
4236	spd->partial[i].private = 0;
4237	}
4238
4239	static ssize_t
4240	tracing_buffers_splice_read(struct file file, loff_t ppos,
4241	struct pipe_inode_info *pipe, size_t len,
4242	unsigned int flags)
4243	{
4244	struct ftrace_buffer_info *info = file->private_data;
4245	struct partial_page partial_def[PIPE_DEF_BUFFERS];
4246	struct page *pages_def[PIPE_DEF_BUFFERS];
4247	struct splice_pipe_desc spd = {
4248	.pages = pages_def,
4249	.partial = partial_def,
4250	.nr_pages_max = PIPE_DEF_BUFFERS,
4251	.flags = flags,
4252	.ops = &buffer_pipe_buf_ops,
4253	.spd_release = buffer_spd_release,
4254	};
4255	struct buffer_ref *ref;
4256	int entries, size, i;
4257	size_t ret;
4258
4259	if (splice_grow_spd(pipe, &spd))
4260	return -ENOMEM;
4261
4262	if (*ppos & (PAGE_SIZE - 1)) {
4263	WARN_ONCE(1, "Ftrace: previous read must page-align\n");
4264	ret = -EINVAL;
4265	goto out;
4266	}
4267
4268	if (len & (PAGE_SIZE - 1)) {
4269	WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
4270	if (len < PAGE_SIZE) {
4271	ret = -EINVAL;
4272	goto out;
4273	}
4274	len &= PAGE_MASK;
4275	}
4276
4277	trace_access_lock(info->cpu);
4278	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4279
4280	for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4281	struct page *page;
4282	int r;
4283
4284	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4285	if (!ref)
4286	break;
4287
4288	ref->ref = 1;
4289	ref->buffer = info->tr->buffer;
4290	ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4291	if (!ref->page) {
4292	kfree(ref);
4293	break;
4294	}
4295
4296	r = ring_buffer_read_page(ref->buffer, &ref->page,
4297	len, info->cpu, 1);
4298	if (r < 0) {
4299	ring_buffer_free_read_page(ref->buffer, ref->page);
4300	kfree(ref);
4301	break;
4302	}
4303
4304	/*
4305	* zero out any left over data, this is going to
4306	* user land.
4307	*/
4308	size = ring_buffer_page_len(ref->page);
4309	if (size < PAGE_SIZE)
4310	memset(ref->page + size, 0, PAGE_SIZE - size);
4311
4312	page = virt_to_page(ref->page);
4313
4314	spd.pages[i] = page;
4315	spd.partial[i].len = PAGE_SIZE;
4316	spd.partial[i].offset = 0;
4317	spd.partial[i].private = (unsigned long)ref;
4318	spd.nr_pages++;
4319	*ppos += PAGE_SIZE;
4320
4321	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4322	}
4323
4324	trace_access_unlock(info->cpu);
4325	spd.nr_pages = i;
4326
4327	/* did we read anything? */
4328	if (!spd.nr_pages) {
4329	if (flags & SPLICE_F_NONBLOCK)
4330	ret = -EAGAIN;
4331	else
4332	ret = 0;
4333	/* TODO: block */
4334	goto out;
4335	}
4336
4337	ret = splice_to_pipe(pipe, &spd);
4338	splice_shrink_spd(&spd);
4339	out:
4340	return ret;
4341	}
4342
4343	static const struct file_operations tracing_buffers_fops = {
4344	.open = tracing_buffers_open,
4345	.read = tracing_buffers_read,
4346	.release = tracing_buffers_release,
4347	.splice_read = tracing_buffers_splice_read,
4348	.llseek = no_llseek,
4349	};
4350
4351	static ssize_t
4352	tracing_stats_read(struct file filp, char __user ubuf,
4353	size_t count, loff_t *ppos)
4354	{
4355	unsigned long cpu = (unsigned long)filp->private_data;
4356	struct trace_array *tr = &global_trace;
4357	struct trace_seq *s;
4358	unsigned long cnt;
4359	unsigned long long t;
4360	unsigned long usec_rem;
4361
4362	s = kmalloc(sizeof(*s), GFP_KERNEL);
4363	if (!s)
4364	return -ENOMEM;
4365
4366	trace_seq_init(s);
4367
4368	cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4369	trace_seq_printf(s, "entries: %ld\n", cnt);
4370
4371	cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4372	trace_seq_printf(s, "overrun: %ld\n", cnt);
4373
4374	cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4375	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4376
4377	cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4378	trace_seq_printf(s, "bytes: %ld\n", cnt);
4379
4380	t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4381	usec_rem = do_div(t, USEC_PER_SEC);
4382	trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);
4383
4384	t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4385	usec_rem = do_div(t, USEC_PER_SEC);
4386	trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4387
4388	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4389
4390	kfree(s);
4391
4392	return count;
4393	}
4394
4395	static const struct file_operations tracing_stats_fops = {
4396	.open = tracing_open_generic,
4397	.read = tracing_stats_read,
4398	.llseek = generic_file_llseek,
4399	};
4400
4401	#ifdef CONFIG_DYNAMIC_FTRACE
4402
4403	int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4404	{
4405	return 0;
4406	}
4407
4408	static ssize_t
4409	tracing_read_dyn_info(struct file filp, char __user ubuf,
4410	size_t cnt, loff_t *ppos)
4411	{
4412	static char ftrace_dyn_info_buffer[1024];
4413	static DEFINE_MUTEX(dyn_info_mutex);
4414	unsigned long *p = filp->private_data;
4415	char *buf = ftrace_dyn_info_buffer;
4416	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4417	int r;
4418
4419	mutex_lock(&dyn_info_mutex);
4420	r = sprintf(buf, "%ld ", *p);
4421
4422	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
4423	buf[r++] = '\n';
4424
4425	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4426
4427	mutex_unlock(&dyn_info_mutex);
4428
4429	return r;
4430	}
4431
4432	static const struct file_operations tracing_dyn_info_fops = {
4433	.open = tracing_open_generic,
4434	.read = tracing_read_dyn_info,
4435	.llseek = generic_file_llseek,
4436	};
4437	#endif
4438
4439	static struct dentry *d_tracer;
4440
4441	struct dentry *tracing_init_dentry(void)
4442	{
4443	static int once;
4444
4445	if (d_tracer)
4446	return d_tracer;
4447
4448	if (!debugfs_initialized())
4449	return NULL;
4450
4451	d_tracer = debugfs_create_dir("tracing", NULL);
4452
4453	if (!d_tracer && !once) {
4454	once = 1;
4455	pr_warning("Could not create debugfs directory 'tracing'\n");
4456	return NULL;
4457	}
4458
4459	return d_tracer;
4460	}
4461
4462	static struct dentry *d_percpu;
4463
4464	struct dentry *tracing_dentry_percpu(void)
4465	{
4466	static int once;
4467	struct dentry *d_tracer;
4468
4469	if (d_percpu)
4470	return d_percpu;
4471
4472	d_tracer = tracing_init_dentry();
4473
4474	if (!d_tracer)
4475	return NULL;
4476
4477	d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4478
4479	if (!d_percpu && !once) {
4480	once = 1;
4481	pr_warning("Could not create debugfs directory 'per_cpu'\n");
4482	return NULL;
4483	}
4484
4485	return d_percpu;
4486	}
4487
4488	static void tracing_init_debugfs_percpu(long cpu)
4489	{
4490	struct dentry *d_percpu = tracing_dentry_percpu();
4491	struct dentry *d_cpu;
4492	char cpu_dir[30]; /* 30 characters should be more than enough */
4493
4494	if (!d_percpu)
4495	return;
4496
4497	snprintf(cpu_dir, 30, "cpu%ld", cpu);
4498	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4499	if (!d_cpu) {
4500	pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4501	return;
4502	}
4503
4504	/* per cpu trace_pipe */
4505	trace_create_file("trace_pipe", 0444, d_cpu,
4506	(void *) cpu, &tracing_pipe_fops);
4507
4508	/* per cpu trace */
4509	trace_create_file("trace", 0644, d_cpu,
4510	(void *) cpu, &tracing_fops);
4511
4512	trace_create_file("trace_pipe_raw", 0444, d_cpu,
4513	(void *) cpu, &tracing_buffers_fops);
4514
4515	trace_create_file("stats", 0444, d_cpu,
4516	(void *) cpu, &tracing_stats_fops);
4517
4518	trace_create_file("buffer_size_kb", 0444, d_cpu,
4519	(void *) cpu, &tracing_entries_fops);
4520	}
4521
4522	#ifdef CONFIG_FTRACE_SELFTEST
4523	/* Let selftest have access to static functions in this file */
4524	#include "trace_selftest.c"
4525	#endif
4526
4527	struct trace_option_dentry {
4528	struct tracer_opt *opt;
4529	struct tracer_flags *flags;
4530	struct dentry *entry;
4531	};
4532
4533	static ssize_t
4534	trace_options_read(struct file filp, char __user ubuf, size_t cnt,
4535	loff_t *ppos)
4536	{
4537	struct trace_option_dentry *topt = filp->private_data;
4538	char *buf;
4539
4540	if (topt->flags->val & topt->opt->bit)
4541	buf = "1\n";
4542	else
4543	buf = "0\n";
4544
4545	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4546	}
4547
4548	static ssize_t
4549	trace_options_write(struct file filp, const char __user ubuf, size_t cnt,
4550	loff_t *ppos)
4551	{
4552	struct trace_option_dentry *topt = filp->private_data;
4553	unsigned long val;
4554	int ret;
4555
4556	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4557	if (ret)
4558	return ret;
4559
4560	if (val != 0 && val != 1)
4561	return -EINVAL;
4562
4563	if (!!(topt->flags->val & topt->opt->bit) != val) {
4564	mutex_lock(&trace_types_lock);
4565	ret = __set_tracer_option(current_trace, topt->flags,
4566	topt->opt, !val);
4567	mutex_unlock(&trace_types_lock);
4568	if (ret)
4569	return ret;
4570	}
4571
4572	*ppos += cnt;
4573
4574	return cnt;
4575	}
4576
4577
4578	static const struct file_operations trace_options_fops = {
4579	.open = tracing_open_generic,
4580	.read = trace_options_read,
4581	.write = trace_options_write,
4582	.llseek = generic_file_llseek,
4583	};
4584
4585	static ssize_t
4586	trace_options_core_read(struct file filp, char __user ubuf, size_t cnt,
4587	loff_t *ppos)
4588	{
4589	long index = (long)filp->private_data;
4590	char *buf;
4591
4592	if (trace_flags & (1 << index))
4593	buf = "1\n";
4594	else
4595	buf = "0\n";
4596
4597	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4598	}
4599
4600	static ssize_t
4601	trace_options_core_write(struct file filp, const char __user ubuf, size_t cnt,
4602	loff_t *ppos)
4603	{
4604	long index = (long)filp->private_data;
4605	unsigned long val;
4606	int ret;
4607
4608	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4609	if (ret)
4610	return ret;
4611
4612	if (val != 0 && val != 1)
4613	return -EINVAL;
4614	set_tracer_flags(1 << index, val);
4615
4616	*ppos += cnt;
4617
4618	return cnt;
4619	}
4620
4621	static const struct file_operations trace_options_core_fops = {
4622	.open = tracing_open_generic,
4623	.read = trace_options_core_read,
4624	.write = trace_options_core_write,
4625	.llseek = generic_file_llseek,
4626	};
4627
4628	struct dentry trace_create_file(const char name,
4629	umode_t mode,
4630	struct dentry *parent,
4631	void *data,
4632	const struct file_operations *fops)
4633	{
4634	struct dentry *ret;
4635
4636	ret = debugfs_create_file(name, mode, parent, data, fops);
4637	if (!ret)
4638	pr_warning("Could not create debugfs '%s' entry\n", name);
4639
4640	return ret;
4641	}
4642
4643
4644	static struct dentry *trace_options_init_dentry(void)
4645	{
4646	struct dentry *d_tracer;
4647	static struct dentry *t_options;
4648
4649	if (t_options)
4650	return t_options;
4651
4652	d_tracer = tracing_init_dentry();
4653	if (!d_tracer)
4654	return NULL;
4655
4656	t_options = debugfs_create_dir("options", d_tracer);
4657	if (!t_options) {
4658	pr_warning("Could not create debugfs directory 'options'\n");
4659	return NULL;
4660	}
4661
4662	return t_options;
4663	}
4664
4665	static void
4666	create_trace_option_file(struct trace_option_dentry *topt,
4667	struct tracer_flags *flags,
4668	struct tracer_opt *opt)
4669	{
4670	struct dentry *t_options;
4671
4672	t_options = trace_options_init_dentry();
4673	if (!t_options)
4674	return;
4675
4676	topt->flags = flags;
4677	topt->opt = opt;
4678
4679	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4680	&trace_options_fops);
4681
4682	}
4683
4684	static struct trace_option_dentry *
4685	create_trace_option_files(struct tracer *tracer)
4686	{
4687	struct trace_option_dentry *topts;
4688	struct tracer_flags *flags;
4689	struct tracer_opt *opts;
4690	int cnt;
4691
4692	if (!tracer)
4693	return NULL;
4694
4695	flags = tracer->flags;
4696
4697	if (!flags \|\| !flags->opts)
4698	return NULL;
4699
4700	opts = flags->opts;
4701
4702	for (cnt = 0; opts[cnt].name; cnt++)
4703	;
4704
4705	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4706	if (!topts)
4707	return NULL;
4708
4709	for (cnt = 0; opts[cnt].name; cnt++)
4710	create_trace_option_file(&topts[cnt], flags,
4711	&opts[cnt]);
4712
4713	return topts;
4714	}
4715
4716	static void
4717	destroy_trace_option_files(struct trace_option_dentry *topts)
4718	{
4719	int cnt;
4720
4721	if (!topts)
4722	return;
4723
4724	for (cnt = 0; topts[cnt].opt; cnt++) {
4725	if (topts[cnt].entry)
4726	debugfs_remove(topts[cnt].entry);
4727	}
4728
4729	kfree(topts);
4730	}
4731
4732	static struct dentry *
4733	create_trace_option_core_file(const char *option, long index)
4734	{
4735	struct dentry *t_options;
4736
4737	t_options = trace_options_init_dentry();
4738	if (!t_options)
4739	return NULL;
4740
4741	return trace_create_file(option, 0644, t_options, (void *)index,
4742	&trace_options_core_fops);
4743	}
4744
4745	static __init void create_trace_options_dir(void)
4746	{
4747	struct dentry *t_options;
4748	int i;
4749
4750	t_options = trace_options_init_dentry();
4751	if (!t_options)
4752	return;
4753
4754	for (i = 0; trace_options[i]; i++)
4755	create_trace_option_core_file(trace_options[i], i);
4756	}
4757
4758	static ssize_t
4759	rb_simple_read(struct file filp, char __user ubuf,
4760	size_t cnt, loff_t *ppos)
4761	{
4762	struct trace_array *tr = filp->private_data;
4763	struct ring_buffer *buffer = tr->buffer;
4764	char buf[64];
4765	int r;
4766
4767	if (buffer)
4768	r = ring_buffer_record_is_on(buffer);
4769	else
4770	r = 0;
4771
4772	r = sprintf(buf, "%d\n", r);
4773
4774	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4775	}
4776
4777	static ssize_t
4778	rb_simple_write(struct file filp, const char __user ubuf,
4779	size_t cnt, loff_t *ppos)
4780	{
4781	struct trace_array *tr = filp->private_data;
4782	struct ring_buffer *buffer = tr->buffer;
4783	unsigned long val;
4784	int ret;
4785
4786	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4787	if (ret)
4788	return ret;
4789
4790	if (buffer) {
4791	if (val)
4792	ring_buffer_record_on(buffer);
4793	else
4794	ring_buffer_record_off(buffer);
4795	}
4796
4797	(*ppos)++;
4798
4799	return cnt;
4800	}
4801
4802	static const struct file_operations rb_simple_fops = {
4803	.open = tracing_open_generic,
4804	.read = rb_simple_read,
4805	.write = rb_simple_write,
4806	.llseek = default_llseek,
4807	};
4808
4809	static __init int tracer_init_debugfs(void)
4810	{
4811	struct dentry *d_tracer;
4812	int cpu;
4813
4814	trace_access_lock_init();
4815
4816	d_tracer = tracing_init_dentry();
4817
4818	trace_create_file("tracing_enabled", 0644, d_tracer,
4819	&global_trace, &tracing_ctrl_fops);
4820
4821	trace_create_file("trace_options", 0644, d_tracer,
4822	NULL, &tracing_iter_fops);
4823
4824	trace_create_file("tracing_cpumask", 0644, d_tracer,
4825	NULL, &tracing_cpumask_fops);
4826
4827	trace_create_file("trace", 0644, d_tracer,
4828	(void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4829
4830	trace_create_file("available_tracers", 0444, d_tracer,
4831	&global_trace, &show_traces_fops);
4832
4833	trace_create_file("current_tracer", 0644, d_tracer,
4834	&global_trace, &set_tracer_fops);
4835
4836	#ifdef CONFIG_TRACER_MAX_TRACE
4837	trace_create_file("tracing_max_latency", 0644, d_tracer,
4838	&tracing_max_latency, &tracing_max_lat_fops);
4839	#endif
4840
4841	trace_create_file("tracing_thresh", 0644, d_tracer,
4842	&tracing_thresh, &tracing_max_lat_fops);
4843
4844	trace_create_file("README", 0444, d_tracer,
4845	NULL, &tracing_readme_fops);
4846
4847	trace_create_file("trace_pipe", 0444, d_tracer,
4848	(void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4849
4850	trace_create_file("buffer_size_kb", 0644, d_tracer,
4851	(void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
4852
4853	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4854	&global_trace, &tracing_total_entries_fops);
4855
4856	trace_create_file("free_buffer", 0644, d_tracer,
4857	&global_trace, &tracing_free_buffer_fops);
4858
4859	trace_create_file("trace_marker", 0220, d_tracer,
4860	NULL, &tracing_mark_fops);
4861
4862	trace_create_file("saved_cmdlines", 0444, d_tracer,
4863	NULL, &tracing_saved_cmdlines_fops);
4864
4865	trace_create_file("trace_clock", 0644, d_tracer, NULL,
4866	&trace_clock_fops);
4867
4868	trace_create_file("tracing_on", 0644, d_tracer,
4869	&global_trace, &rb_simple_fops);
4870
4871	#ifdef CONFIG_DYNAMIC_FTRACE
4872	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4873	&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4874	#endif
4875
4876	create_trace_options_dir();
4877
4878	for_each_tracing_cpu(cpu)
4879	tracing_init_debugfs_percpu(cpu);
4880
4881	return 0;
4882	}
4883
4884	static int trace_panic_handler(struct notifier_block *this,
4885	unsigned long event, void *unused)
4886	{
4887	if (ftrace_dump_on_oops)
4888	ftrace_dump(ftrace_dump_on_oops);
4889	return NOTIFY_OK;
4890	}
4891
4892	static struct notifier_block trace_panic_notifier = {
4893	.notifier_call = trace_panic_handler,
4894	.next = NULL,
4895	.priority = 150 /* priority: INT_MAX >= x >= 0 */
4896	};
4897
4898	static int trace_die_handler(struct notifier_block *self,
4899	unsigned long val,
4900	void *data)
4901	{
4902	switch (val) {
4903	case DIE_OOPS:
4904	if (ftrace_dump_on_oops)
4905	ftrace_dump(ftrace_dump_on_oops);
4906	break;
4907	default:
4908	break;
4909	}
4910	return NOTIFY_OK;
4911	}
4912
4913	static struct notifier_block trace_die_notifier = {
4914	.notifier_call = trace_die_handler,
4915	.priority = 200
4916	};
4917
4918	/*
4919	* printk is set to max of 1024, we really don't need it that big.
4920	* Nothing should be printing 1000 characters anyway.
4921	*/
4922	#define TRACE_MAX_PRINT 1000
4923
4924	/*
4925	* Define here KERN_TRACE so that we have one place to modify
4926	* it if we decide to change what log level the ftrace dump
4927	* should be at.
4928	*/
4929	#define KERN_TRACE KERN_EMERG
4930
4931	void
4932	trace_printk_seq(struct trace_seq *s)
4933	{
4934	/* Probably should print a warning here. */
4935	if (s->len >= 1000)
4936	s->len = 1000;
4937
4938	/* should be zero ended, but we are paranoid. */
4939	s->buffer[s->len] = 0;
4940
4941	printk(KERN_TRACE "%s", s->buffer);
4942
4943	trace_seq_init(s);
4944	}
4945
4946	void trace_init_global_iter(struct trace_iterator *iter)
4947	{
4948	iter->tr = &global_trace;
4949	iter->trace = current_trace;
4950	iter->cpu_file = TRACE_PIPE_ALL_CPU;
4951	}
4952
4953	static void
4954	__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4955	{
4956	static arch_spinlock_t ftrace_dump_lock =
4957	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4958	/* use static because iter can be a bit big for the stack */
4959	static struct trace_iterator iter;
4960	unsigned int old_userobj;
4961	static int dump_ran;
4962	unsigned long flags;
4963	int cnt = 0, cpu;
4964
4965	/* only one dump */
4966	local_irq_save(flags);
4967	arch_spin_lock(&ftrace_dump_lock);
4968	if (dump_ran)
4969	goto out;
4970
4971	dump_ran = 1;
4972
4973	tracing_off();
4974
4975	/* Did function tracer already get disabled? */
4976	if (ftrace_is_dead()) {
4977	printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
4978	printk("# MAY BE MISSING FUNCTION EVENTS\n");
4979	}
4980
4981	if (disable_tracing)
4982	ftrace_kill();
4983
4984	trace_init_global_iter(&iter);
4985
4986	for_each_tracing_cpu(cpu) {
4987	atomic_inc(&iter.tr->data[cpu]->disabled);
4988	}
4989
4990	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4991
4992	/* don't look at user memory in panic mode */
4993	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4994
4995	/* Simulate the iterator */
4996	iter.tr = &global_trace;
4997	iter.trace = current_trace;
4998
4999	switch (oops_dump_mode) {
5000	case DUMP_ALL:
5001	iter.cpu_file = TRACE_PIPE_ALL_CPU;
5002	break;
5003	case DUMP_ORIG:
5004	iter.cpu_file = raw_smp_processor_id();
5005	break;
5006	case DUMP_NONE:
5007	goto out_enable;
5008	default:
5009	printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
5010	iter.cpu_file = TRACE_PIPE_ALL_CPU;
5011	}
5012
5013	printk(KERN_TRACE "Dumping ftrace buffer:\n");
5014
5015	/*
5016	* We need to stop all tracing on all CPUS to read the
5017	* the next buffer. This is a bit expensive, but is
5018	* not done often. We fill all what we can read,
5019	* and then release the locks again.
5020	*/
5021
5022	while (!trace_empty(&iter)) {
5023
5024	if (!cnt)
5025	printk(KERN_TRACE "---------------------------------\n");
5026
5027	cnt++;
5028
5029	/* reset all but tr, trace, and overruns */
5030	memset(&iter.seq, 0,
5031	sizeof(struct trace_iterator) -
5032	offsetof(struct trace_iterator, seq));
5033	iter.iter_flags \|= TRACE_FILE_LAT_FMT;
5034	iter.pos = -1;
5035
5036	if (trace_find_next_entry_inc(&iter) != NULL) {
5037	int ret;
5038
5039	ret = print_trace_line(&iter);
5040	if (ret != TRACE_TYPE_NO_CONSUME)
5041	trace_consume(&iter);
5042	}
5043	touch_nmi_watchdog();
5044
5045	trace_printk_seq(&iter.seq);
5046	}
5047
5048	if (!cnt)
5049	printk(KERN_TRACE " (ftrace buffer empty)\n");
5050	else
5051	printk(KERN_TRACE "---------------------------------\n");
5052
5053	out_enable:
5054	/* Re-enable tracing if requested */
5055	if (!disable_tracing) {
5056	trace_flags \|= old_userobj;
5057
5058	for_each_tracing_cpu(cpu) {
5059	atomic_dec(&iter.tr->data[cpu]->disabled);
5060	}
5061	tracing_on();
5062	}
5063
5064	out:
5065	arch_spin_unlock(&ftrace_dump_lock);
5066	local_irq_restore(flags);
5067	}
5068
5069	/* By default: disable tracing after the dump */
5070	void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
5071	{
5072	__ftrace_dump(true, oops_dump_mode);
5073	}
5074	EXPORT_SYMBOL_GPL(ftrace_dump);
5075
5076	__init static int tracer_alloc_buffers(void)
5077	{
5078	int ring_buf_size;
5079	enum ring_buffer_flags rb_flags;
5080	int i;
5081	int ret = -ENOMEM;
5082
5083
5084	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5085	goto out;
5086
5087	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5088	goto out_free_buffer_mask;
5089
5090	/* Only allocate trace_printk buffers if a trace_printk exists */
5091	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5092	trace_printk_init_buffers();
5093
5094	/* To save memory, keep the ring buffer size to its minimum */
5095	if (ring_buffer_expanded)
5096	ring_buf_size = trace_buf_size;
5097	else
5098	ring_buf_size = 1;
5099
5100	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5101
5102	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5103	cpumask_copy(tracing_cpumask, cpu_all_mask);
5104
5105	/* TODO: make the number of buffers hot pluggable with CPUS */
5106	global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5107	if (!global_trace.buffer) {
5108	printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5109	WARN_ON(1);
5110	goto out_free_cpumask;
5111	}
5112	if (global_trace.buffer_disabled)
5113	tracing_off();
5114
5115
5116	#ifdef CONFIG_TRACER_MAX_TRACE
5117	max_tr.buffer = ring_buffer_alloc(1, rb_flags);
5118	if (!max_tr.buffer) {
5119	printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5120	WARN_ON(1);
5121	ring_buffer_free(global_trace.buffer);
5122	goto out_free_cpumask;
5123	}
5124	#endif
5125
5126	/* Allocate the first page for all buffers */
5127	for_each_tracing_cpu(i) {
5128	global_trace.data[i] = &per_cpu(global_trace_cpu, i);
5129	max_tr.data[i] = &per_cpu(max_tr_data, i);
5130	}
5131
5132	set_buffer_entries(&global_trace,
5133	ring_buffer_size(global_trace.buffer, 0));
5134	#ifdef CONFIG_TRACER_MAX_TRACE
5135	set_buffer_entries(&max_tr, 1);
5136	#endif
5137
5138	trace_init_cmdlines();
5139
5140	register_tracer(&nop_trace);
5141	current_trace = &nop_trace;
5142	/* All seems OK, enable tracing */
5143	tracing_disabled = 0;
5144
5145	atomic_notifier_chain_register(&panic_notifier_list,
5146	&trace_panic_notifier);
5147
5148	register_die_notifier(&trace_die_notifier);
5149
5150	return 0;
5151
5152	out_free_cpumask:
5153	free_cpumask_var(tracing_cpumask);
5154	out_free_buffer_mask:
5155	free_cpumask_var(tracing_buffer_mask);
5156	out:
5157	return ret;
5158	}
5159
5160	__init static int clear_boot_tracer(void)
5161	{
5162	/*
5163	* The default tracer at boot buffer is an init section.
5164	* This function is called in lateinit. If we did not
5165	* find the boot tracer, then clear it out, to prevent
5166	* later registration from accessing the buffer that is
5167	* about to be freed.
5168	*/
5169	if (!default_bootup_tracer)
5170	return 0;
5171
5172	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5173	default_bootup_tracer);
5174	default_bootup_tracer = NULL;
5175
5176	return 0;
5177	}
5178
5179	early_initcall(tracer_alloc_buffers);
5180	fs_initcall(tracer_init_debugfs);
5181	late_initcall(clear_boot_tracer);