linux - kernel/trace/trace.c

5223 lines · 119.3 KiB

1	/*
2	* ring buffer based function tracer
3	*
4	* Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
5	* Copyright (C) 2008 Ingo Molnar <[email protected]>
6	*
7	* Originally taken from the RT patch by:
8	* Arnaldo Carvalho de Melo <[email protected]>
9	*
10	* Based on code from the latency_tracer, that is:
11	* Copyright (C) 2004-2006 Ingo Molnar
12	* Copyright (C) 2004 Nadia Yvette Chambers
13	*/
14	#include <linux/ring_buffer.h>
15	#include <generated/utsrelease.h>
16	#include <linux/stacktrace.h>
17	#include <linux/writeback.h>
18	#include <linux/kallsyms.h>
19	#include <linux/seq_file.h>
20	#include <linux/notifier.h>
21	#include <linux/irqflags.h>
22	#include <linux/irq_work.h>
23	#include <linux/debugfs.h>
24	#include <linux/pagemap.h>
25	#include <linux/hardirq.h>
26	#include <linux/linkage.h>
27	#include <linux/uaccess.h>
28	#include <linux/kprobes.h>
29	#include <linux/ftrace.h>
30	#include <linux/module.h>
31	#include <linux/percpu.h>
32	#include <linux/splice.h>
33	#include <linux/kdebug.h>
34	#include <linux/string.h>
35	#include <linux/rwsem.h>
36	#include <linux/slab.h>
37	#include <linux/ctype.h>
38	#include <linux/init.h>
39	#include <linux/poll.h>
40	#include <linux/nmi.h>
41	#include <linux/fs.h>
42
43	#include "trace.h"
44	#include "trace_output.h"
45
46	/*
47	* On boot up, the ring buffer is set to the minimum size, so that
48	* we do not waste memory on systems that are not using tracing.
49	*/
50	int ring_buffer_expanded;
51
52	/*
53	* We need to change this state when a selftest is running.
54	* A selftest will lurk into the ring-buffer to count the
55	* entries inserted during the selftest although some concurrent
56	* insertions into the ring-buffer such as trace_printk could occurred
57	* at the same time, giving false positive or negative results.
58	*/
59	static bool __read_mostly tracing_selftest_running;
60
61	/*
62	* If a tracer is running, we do not want to run SELFTEST.
63	*/
64	bool __read_mostly tracing_selftest_disabled;
65
66	/* For tracers that don't implement custom flags */
67	static struct tracer_opt dummy_tracer_opt[] = {
68	{ }
69	};
70
71	static struct tracer_flags dummy_tracer_flags = {
72	.val = 0,
73	.opts = dummy_tracer_opt
74	};
75
76	static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77	{
78	return 0;
79	}
80
81	/*
82	* To prevent the comm cache from being overwritten when no
83	* tracing is active, only save the comm when a trace event
84	* occurred.
85	*/
86	static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88	/*
89	* When a reader is waiting for data, then this variable is
90	* set to true.
91	*/
92	static bool trace_wakeup_needed;
93
94	static struct irq_work trace_work_wakeup;
95
96	/*
97	* Kill all tracing for good (never come back).
98	* It is initialized to 1 but will turn to zero if the initialization
99	* of the tracer is successful. But that is the only place that sets
100	* this back to zero.
101	*/
102	static int tracing_disabled = 1;
103
104	DEFINE_PER_CPU(int, ftrace_cpu_disabled);
105
106	cpumask_var_t __read_mostly tracing_buffer_mask;
107
108	/*
109	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
110	*
111	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
112	* is set, then ftrace_dump is called. This will output the contents
113	* of the ftrace buffers to the console. This is very useful for
114	* capturing traces that lead to crashes and outputing it to a
115	* serial console.
116	*
117	* It is default off, but you can enable it with either specifying
118	* "ftrace_dump_on_oops" in the kernel command line, or setting
119	* /proc/sys/kernel/ftrace_dump_on_oops
120	* Set 1 if you want to dump buffers of all CPUs
121	* Set 2 if you want to dump the buffer of the CPU that triggered oops
122	*/
123
124	enum ftrace_dump_mode ftrace_dump_on_oops;
125
126	static int tracing_set_tracer(const char *buf);
127
128	#define MAX_TRACER_SIZE 100
129	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
130	static char *default_bootup_tracer;
131
132	static int __init set_cmdline_ftrace(char *str)
133	{
134	strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
135	default_bootup_tracer = bootup_tracer_buf;
136	/* We are using ftrace early, expand it */
137	ring_buffer_expanded = 1;
138	return 1;
139	}
140	__setup("ftrace=", set_cmdline_ftrace);
141
142	static int __init set_ftrace_dump_on_oops(char *str)
143	{
144	if (str++ != '=' \|\| !str) {
145	ftrace_dump_on_oops = DUMP_ALL;
146	return 1;
147	}
148
149	if (!strcmp("orig_cpu", str)) {
150	ftrace_dump_on_oops = DUMP_ORIG;
151	return 1;
152	}
153
154	return 0;
155	}
156	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
157
158
159	static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
160	static char *trace_boot_options __initdata;
161
162	static int __init set_trace_boot_options(char *str)
163	{
164	strncpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
165	trace_boot_options = trace_boot_options_buf;
166	return 0;
167	}
168	__setup("trace_options=", set_trace_boot_options);
169
170	unsigned long long ns2usecs(cycle_t nsec)
171	{
172	nsec += 500;
173	do_div(nsec, 1000);
174	return nsec;
175	}
176
177	/*
178	* The global_trace is the descriptor that holds the tracing
179	* buffers for the live tracing. For each CPU, it contains
180	* a link list of pages that will store trace entries. The
181	* page descriptor of the pages in the memory is used to hold
182	* the link list by linking the lru item in the page descriptor
183	* to each of the pages in the buffer per CPU.
184	*
185	* For each active CPU there is a data field that holds the
186	* pages for the buffer for that CPU. Each CPU has the same number
187	* of pages allocated for its buffer.
188	*/
189	static struct trace_array global_trace;
190
191	static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
192
193	int filter_current_check_discard(struct ring_buffer *buffer,
194	struct ftrace_event_call call, void rec,
195	struct ring_buffer_event *event)
196	{
197	return filter_check_discard(call, rec, buffer, event);
198	}
199	EXPORT_SYMBOL_GPL(filter_current_check_discard);
200
201	cycle_t ftrace_now(int cpu)
202	{
203	u64 ts;
204
205	/* Early boot up does not have a buffer yet */
206	if (!global_trace.buffer)
207	return trace_clock_local();
208
209	ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
210	ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
211
212	return ts;
213	}
214
215	/*
216	* The max_tr is used to snapshot the global_trace when a maximum
217	* latency is reached. Some tracers will use this to store a maximum
218	* trace while it continues examining live traces.
219	*
220	* The buffers for the max_tr are set up the same as the global_trace.
221	* When a snapshot is taken, the link list of the max_tr is swapped
222	* with the link list of the global_trace and the buffers are reset for
223	* the global_trace so the tracing can continue.
224	*/
225	static struct trace_array max_tr;
226
227	static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
228
229	int tracing_is_enabled(void)
230	{
231	return tracing_is_on();
232	}
233
234	/*
235	* trace_buf_size is the size in bytes that is allocated
236	* for a buffer. Note, the number of bytes is always rounded
237	* to page size.
238	*
239	* This number is purposely set to a low number of 16384.
240	* If the dump on oops happens, it will be much appreciated
241	* to not have to wait for all that output. Anyway this can be
242	* boot time and run time configurable.
243	*/
244	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
245
246	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
247
248	/* trace_types holds a link list of available tracers. */
249	static struct tracer *trace_types __read_mostly;
250
251	/* current_trace points to the tracer that is currently active */
252	static struct tracer *current_trace __read_mostly;
253
254	/*
255	* trace_types_lock is used to protect the trace_types list.
256	*/
257	static DEFINE_MUTEX(trace_types_lock);
258
259	/*
260	* serialize the access of the ring buffer
261	*
262	* ring buffer serializes readers, but it is low level protection.
263	* The validity of the events (which returns by ring_buffer_peek() ..etc)
264	* are not protected by ring buffer.
265	*
266	* The content of events may become garbage if we allow other process consumes
267	* these events concurrently:
268	* A) the page of the consumed events may become a normal page
269	* (not reader page) in ring buffer, and this page will be rewrited
270	* by events producer.
271	* B) The page of the consumed events may become a page for splice_read,
272	* and this page will be returned to system.
273	*
274	* These primitives allow multi process access to different cpu ring buffer
275	* concurrently.
276	*
277	* These primitives don't distinguish read-only and read-consume access.
278	* Multi read-only access are also serialized.
279	*/
280
281	#ifdef CONFIG_SMP
282	static DECLARE_RWSEM(all_cpu_access_lock);
283	static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
284
285	static inline void trace_access_lock(int cpu)
286	{
287	if (cpu == TRACE_PIPE_ALL_CPU) {
288	/* gain it for accessing the whole ring buffer. */
289	down_write(&all_cpu_access_lock);
290	} else {
291	/* gain it for accessing a cpu ring buffer. */
292
293	/* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
294	down_read(&all_cpu_access_lock);
295
296	/* Secondly block other access to this @cpu ring buffer. */
297	mutex_lock(&per_cpu(cpu_access_lock, cpu));
298	}
299	}
300
301	static inline void trace_access_unlock(int cpu)
302	{
303	if (cpu == TRACE_PIPE_ALL_CPU) {
304	up_write(&all_cpu_access_lock);
305	} else {
306	mutex_unlock(&per_cpu(cpu_access_lock, cpu));
307	up_read(&all_cpu_access_lock);
308	}
309	}
310
311	static inline void trace_access_lock_init(void)
312	{
313	int cpu;
314
315	for_each_possible_cpu(cpu)
316	mutex_init(&per_cpu(cpu_access_lock, cpu));
317	}
318
319	#else
320
321	static DEFINE_MUTEX(access_lock);
322
323	static inline void trace_access_lock(int cpu)
324	{
325	(void)cpu;
326	mutex_lock(&access_lock);
327	}
328
329	static inline void trace_access_unlock(int cpu)
330	{
331	(void)cpu;
332	mutex_unlock(&access_lock);
333	}
334
335	static inline void trace_access_lock_init(void)
336	{
337	}
338
339	#endif
340
341	/* trace_wait is a waitqueue for tasks blocked on trace_poll */
342	static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
343
344	/* trace_flags holds trace_options default values */
345	unsigned long trace_flags = TRACE_ITER_PRINT_PARENT \| TRACE_ITER_PRINTK \|
346	TRACE_ITER_ANNOTATE \| TRACE_ITER_CONTEXT_INFO \| TRACE_ITER_SLEEP_TIME \|
347	TRACE_ITER_GRAPH_TIME \| TRACE_ITER_RECORD_CMD \| TRACE_ITER_OVERWRITE \|
348	TRACE_ITER_IRQ_INFO \| TRACE_ITER_MARKERS;
349
350	static int trace_stop_count;
351	static DEFINE_RAW_SPINLOCK(tracing_start_lock);
352
353	/**
354	* trace_wake_up - wake up tasks waiting for trace input
355	*
356	* Schedules a delayed work to wake up any task that is blocked on the
357	* trace_wait queue. These is used with trace_poll for tasks polling the
358	* trace.
359	*/
360	static void trace_wake_up(struct irq_work *work)
361	{
362	wake_up_all(&trace_wait);
363
364	}
365
366	/**
367	* tracing_on - enable tracing buffers
368	*
369	* This function enables tracing buffers that may have been
370	* disabled with tracing_off.
371	*/
372	void tracing_on(void)
373	{
374	if (global_trace.buffer)
375	ring_buffer_record_on(global_trace.buffer);
376	/*
377	* This flag is only looked at when buffers haven't been
378	* allocated yet. We don't really care about the race
379	* between setting this flag and actually turning
380	* on the buffer.
381	*/
382	global_trace.buffer_disabled = 0;
383	}
384	EXPORT_SYMBOL_GPL(tracing_on);
385
386	/**
387	* tracing_off - turn off tracing buffers
388	*
389	* This function stops the tracing buffers from recording data.
390	* It does not disable any overhead the tracers themselves may
391	* be causing. This function simply causes all recording to
392	* the ring buffers to fail.
393	*/
394	void tracing_off(void)
395	{
396	if (global_trace.buffer)
397	ring_buffer_record_off(global_trace.buffer);
398	/*
399	* This flag is only looked at when buffers haven't been
400	* allocated yet. We don't really care about the race
401	* between setting this flag and actually turning
402	* on the buffer.
403	*/
404	global_trace.buffer_disabled = 1;
405	}
406	EXPORT_SYMBOL_GPL(tracing_off);
407
408	/**
409	* tracing_is_on - show state of ring buffers enabled
410	*/
411	int tracing_is_on(void)
412	{
413	if (global_trace.buffer)
414	return ring_buffer_record_is_on(global_trace.buffer);
415	return !global_trace.buffer_disabled;
416	}
417	EXPORT_SYMBOL_GPL(tracing_is_on);
418
419	static int __init set_buf_size(char *str)
420	{
421	unsigned long buf_size;
422
423	if (!str)
424	return 0;
425	buf_size = memparse(str, &str);
426	/* nr_entries can not be zero */
427	if (buf_size == 0)
428	return 0;
429	trace_buf_size = buf_size;
430	return 1;
431	}
432	__setup("trace_buf_size=", set_buf_size);
433
434	static int __init set_tracing_thresh(char *str)
435	{
436	unsigned long threshold;
437	int ret;
438
439	if (!str)
440	return 0;
441	ret = kstrtoul(str, 0, &threshold);
442	if (ret < 0)
443	return 0;
444	tracing_thresh = threshold * 1000;
445	return 1;
446	}
447	__setup("tracing_thresh=", set_tracing_thresh);
448
449	unsigned long nsecs_to_usecs(unsigned long nsecs)
450	{
451	return nsecs / 1000;
452	}
453
454	/* These must match the bit postions in trace_iterator_flags */
455	static const char *trace_options[] = {
456	"print-parent",
457	"sym-offset",
458	"sym-addr",
459	"verbose",
460	"raw",
461	"hex",
462	"bin",
463	"block",
464	"stacktrace",
465	"trace_printk",
466	"ftrace_preempt",
467	"branch",
468	"annotate",
469	"userstacktrace",
470	"sym-userobj",
471	"printk-msg-only",
472	"context-info",
473	"latency-format",
474	"sleep-time",
475	"graph-time",
476	"record-cmd",
477	"overwrite",
478	"disable_on_free",
479	"irq-info",
480	"markers",
481	NULL
482	};
483
484	static struct {
485	u64 (*func)(void);
486	const char *name;
487	int in_ns; /* is this clock in nanoseconds? */
488	} trace_clocks[] = {
489	{ trace_clock_local, "local", 1 },
490	{ trace_clock_global, "global", 1 },
491	{ trace_clock_counter, "counter", 0 },
492	ARCH_TRACE_CLOCKS
493	};
494
495	int trace_clock_id;
496
497	/*
498	* trace_parser_get_init - gets the buffer for trace parser
499	*/
500	int trace_parser_get_init(struct trace_parser *parser, int size)
501	{
502	memset(parser, 0, sizeof(*parser));
503
504	parser->buffer = kmalloc(size, GFP_KERNEL);
505	if (!parser->buffer)
506	return 1;
507
508	parser->size = size;
509	return 0;
510	}
511
512	/*
513	* trace_parser_put - frees the buffer for trace parser
514	*/
515	void trace_parser_put(struct trace_parser *parser)
516	{
517	kfree(parser->buffer);
518	}
519
520	/*
521	* trace_get_user - reads the user input string separated by space
522	* (matched by isspace(ch))
523	*
524	* For each string found the 'struct trace_parser' is updated,
525	* and the function returns.
526	*
527	* Returns number of bytes read.
528	*
529	* See kernel/trace/trace.h for 'struct trace_parser' details.
530	*/
531	int trace_get_user(struct trace_parser parser, const char __user ubuf,
532	size_t cnt, loff_t *ppos)
533	{
534	char ch;
535	size_t read = 0;
536	ssize_t ret;
537
538	if (!*ppos)
539	trace_parser_clear(parser);
540
541	ret = get_user(ch, ubuf++);
542	if (ret)
543	goto out;
544
545	read++;
546	cnt--;
547
548	/*
549	* The parser is not finished with the last write,
550	* continue reading the user input without skipping spaces.
551	*/
552	if (!parser->cont) {
553	/* skip white space */
554	while (cnt && isspace(ch)) {
555	ret = get_user(ch, ubuf++);
556	if (ret)
557	goto out;
558	read++;
559	cnt--;
560	}
561
562	/* only spaces were written */
563	if (isspace(ch)) {
564	*ppos += read;
565	ret = read;
566	goto out;
567	}
568
569	parser->idx = 0;
570	}
571
572	/* read the non-space input */
573	while (cnt && !isspace(ch)) {
574	if (parser->idx < parser->size - 1)
575	parser->buffer[parser->idx++] = ch;
576	else {
577	ret = -EINVAL;
578	goto out;
579	}
580	ret = get_user(ch, ubuf++);
581	if (ret)
582	goto out;
583	read++;
584	cnt--;
585	}
586
587	/* We either got finished input or we have to wait for another call. */
588	if (isspace(ch)) {
589	parser->buffer[parser->idx] = 0;
590	parser->cont = false;
591	} else {
592	parser->cont = true;
593	parser->buffer[parser->idx++] = ch;
594	}
595
596	*ppos += read;
597	ret = read;
598
599	out:
600	return ret;
601	}
602
603	ssize_t trace_seq_to_user(struct trace_seq s, char __user ubuf, size_t cnt)
604	{
605	int len;
606	int ret;
607
608	if (!cnt)
609	return 0;
610
611	if (s->len <= s->readpos)
612	return -EBUSY;
613
614	len = s->len - s->readpos;
615	if (cnt > len)
616	cnt = len;
617	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
618	if (ret == cnt)
619	return -EFAULT;
620
621	cnt -= ret;
622
623	s->readpos += cnt;
624	return cnt;
625	}
626
627	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
628	{
629	int len;
630
631	if (s->len <= s->readpos)
632	return -EBUSY;
633
634	len = s->len - s->readpos;
635	if (cnt > len)
636	cnt = len;
637	memcpy(buf, s->buffer + s->readpos, cnt);
638
639	s->readpos += cnt;
640	return cnt;
641	}
642
643	/*
644	* ftrace_max_lock is used to protect the swapping of buffers
645	* when taking a max snapshot. The buffers themselves are
646	* protected by per_cpu spinlocks. But the action of the swap
647	* needs its own lock.
648	*
649	* This is defined as a arch_spinlock_t in order to help
650	* with performance when lockdep debugging is enabled.
651	*
652	* It is also used in other places outside the update_max_tr
653	* so it needs to be defined outside of the
654	* CONFIG_TRACER_MAX_TRACE.
655	*/
656	static arch_spinlock_t ftrace_max_lock =
657	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
658
659	unsigned long __read_mostly tracing_thresh;
660
661	#ifdef CONFIG_TRACER_MAX_TRACE
662	unsigned long __read_mostly tracing_max_latency;
663
664	/*
665	* Copy the new maximum trace into the separate maximum-trace
666	* structure. (this way the maximum trace is permanently saved,
667	* for later retrieval via /sys/kernel/debug/tracing/latency_trace)
668	*/
669	static void
670	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
671	{
672	struct trace_array_cpu *data = tr->data[cpu];
673	struct trace_array_cpu *max_data;
674
675	max_tr.cpu = cpu;
676	max_tr.time_start = data->preempt_timestamp;
677
678	max_data = max_tr.data[cpu];
679	max_data->saved_latency = tracing_max_latency;
680	max_data->critical_start = data->critical_start;
681	max_data->critical_end = data->critical_end;
682
683	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
684	max_data->pid = tsk->pid;
685	max_data->uid = task_uid(tsk);
686	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
687	max_data->policy = tsk->policy;
688	max_data->rt_priority = tsk->rt_priority;
689
690	/* record this tasks comm */
691	tracing_record_cmdline(tsk);
692	}
693
694	/**
695	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
696	* @tr: tracer
697	* @tsk: the task with the latency
698	* @cpu: The cpu that initiated the trace.
699	*
700	* Flip the buffers between the @tr and the max_tr and record information
701	* about which task was the cause of this latency.
702	*/
703	void
704	update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
705	{
706	struct ring_buffer *buf = tr->buffer;
707
708	if (trace_stop_count)
709	return;
710
711	WARN_ON_ONCE(!irqs_disabled());
712	if (!current_trace->use_max_tr) {
713	WARN_ON_ONCE(1);
714	return;
715	}
716	arch_spin_lock(&ftrace_max_lock);
717
718	tr->buffer = max_tr.buffer;
719	max_tr.buffer = buf;
720
721	__update_max_tr(tr, tsk, cpu);
722	arch_spin_unlock(&ftrace_max_lock);
723	}
724
725	/**
726	* update_max_tr_single - only copy one trace over, and reset the rest
727	* @tr - tracer
728	* @tsk - task with the latency
729	* @cpu - the cpu of the buffer to copy.
730	*
731	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
732	*/
733	void
734	update_max_tr_single(struct trace_array tr, struct task_struct tsk, int cpu)
735	{
736	int ret;
737
738	if (trace_stop_count)
739	return;
740
741	WARN_ON_ONCE(!irqs_disabled());
742	if (!current_trace->use_max_tr) {
743	WARN_ON_ONCE(1);
744	return;
745	}
746
747	arch_spin_lock(&ftrace_max_lock);
748
749	ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
750
751	if (ret == -EBUSY) {
752	/*
753	* We failed to swap the buffer due to a commit taking
754	* place on this CPU. We fail to record, but we reset
755	* the max trace buffer (no one writes directly to it)
756	* and flag that it failed.
757	*/
758	trace_array_printk(&max_tr, _THIS_IP_,
759	"Failed to swap buffers due to commit in progress\n");
760	}
761
762	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
763
764	__update_max_tr(tr, tsk, cpu);
765	arch_spin_unlock(&ftrace_max_lock);
766	}
767	#endif /* CONFIG_TRACER_MAX_TRACE */
768
769	static void default_wait_pipe(struct trace_iterator *iter)
770	{
771	DEFINE_WAIT(wait);
772
773	prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
774
775	/*
776	* The events can happen in critical sections where
777	* checking a work queue can cause deadlocks.
778	* After adding a task to the queue, this flag is set
779	* only to notify events to try to wake up the queue
780	* using irq_work.
781	*
782	* We don't clear it even if the buffer is no longer
783	* empty. The flag only causes the next event to run
784	* irq_work to do the work queue wake up. The worse
785	* that can happen if we race with !trace_empty() is that
786	* an event will cause an irq_work to try to wake up
787	* an empty queue.
788	*
789	* There's no reason to protect this flag either, as
790	* the work queue and irq_work logic will do the necessary
791	* synchronization for the wake ups. The only thing
792	* that is necessary is that the wake up happens after
793	* a task has been queued. It's OK for spurious wake ups.
794	*/
795	trace_wakeup_needed = true;
796
797	if (trace_empty(iter))
798	schedule();
799
800	finish_wait(&trace_wait, &wait);
801	}
802
803	/**
804	* register_tracer - register a tracer with the ftrace system.
805	* @type - the plugin for the tracer
806	*
807	* Register a new plugin tracer.
808	*/
809	int register_tracer(struct tracer *type)
810	{
811	struct tracer *t;
812	int ret = 0;
813
814	if (!type->name) {
815	pr_info("Tracer must have a name\n");
816	return -1;
817	}
818
819	if (strlen(type->name) >= MAX_TRACER_SIZE) {
820	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
821	return -1;
822	}
823
824	mutex_lock(&trace_types_lock);
825
826	tracing_selftest_running = true;
827
828	for (t = trace_types; t; t = t->next) {
829	if (strcmp(type->name, t->name) == 0) {
830	/* already found */
831	pr_info("Tracer %s already registered\n",
832	type->name);
833	ret = -1;
834	goto out;
835	}
836	}
837
838	if (!type->set_flag)
839	type->set_flag = &dummy_set_flag;
840	if (!type->flags)
841	type->flags = &dummy_tracer_flags;
842	else
843	if (!type->flags->opts)
844	type->flags->opts = dummy_tracer_opt;
845	if (!type->wait_pipe)
846	type->wait_pipe = default_wait_pipe;
847
848
849	#ifdef CONFIG_FTRACE_STARTUP_TEST
850	if (type->selftest && !tracing_selftest_disabled) {
851	struct tracer *saved_tracer = current_trace;
852	struct trace_array *tr = &global_trace;
853
854	/*
855	* Run a selftest on this tracer.
856	* Here we reset the trace buffer, and set the current
857	* tracer to be this tracer. The tracer can then run some
858	* internal tracing to verify that everything is in order.
859	* If we fail, we do not register this tracer.
860	*/
861	tracing_reset_online_cpus(tr);
862
863	current_trace = type;
864
865	/* If we expanded the buffers, make sure the max is expanded too */
866	if (ring_buffer_expanded && type->use_max_tr)
867	ring_buffer_resize(max_tr.buffer, trace_buf_size,
868	RING_BUFFER_ALL_CPUS);
869
870	/* the test is responsible for initializing and enabling */
871	pr_info("Testing tracer %s: ", type->name);
872	ret = type->selftest(type, tr);
873	/* the test is responsible for resetting too */
874	current_trace = saved_tracer;
875	if (ret) {
876	printk(KERN_CONT "FAILED!\n");
877	/* Add the warning after printing 'FAILED' */
878	WARN_ON(1);
879	goto out;
880	}
881	/* Only reset on passing, to avoid touching corrupted buffers */
882	tracing_reset_online_cpus(tr);
883
884	/* Shrink the max buffer again */
885	if (ring_buffer_expanded && type->use_max_tr)
886	ring_buffer_resize(max_tr.buffer, 1,
887	RING_BUFFER_ALL_CPUS);
888
889	printk(KERN_CONT "PASSED\n");
890	}
891	#endif
892
893	type->next = trace_types;
894	trace_types = type;
895
896	out:
897	tracing_selftest_running = false;
898	mutex_unlock(&trace_types_lock);
899
900	if (ret \|\| !default_bootup_tracer)
901	goto out_unlock;
902
903	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
904	goto out_unlock;
905
906	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
907	/* Do we want this tracer to start on bootup? */
908	tracing_set_tracer(type->name);
909	default_bootup_tracer = NULL;
910	/* disable other selftests, since this will break it. */
911	tracing_selftest_disabled = 1;
912	#ifdef CONFIG_FTRACE_STARTUP_TEST
913	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
914	type->name);
915	#endif
916
917	out_unlock:
918	return ret;
919	}
920
921	void tracing_reset(struct trace_array *tr, int cpu)
922	{
923	struct ring_buffer *buffer = tr->buffer;
924
925	ring_buffer_record_disable(buffer);
926
927	/* Make sure all commits have finished */
928	synchronize_sched();
929	ring_buffer_reset_cpu(buffer, cpu);
930
931	ring_buffer_record_enable(buffer);
932	}
933
934	void tracing_reset_online_cpus(struct trace_array *tr)
935	{
936	struct ring_buffer *buffer = tr->buffer;
937	int cpu;
938
939	ring_buffer_record_disable(buffer);
940
941	/* Make sure all commits have finished */
942	synchronize_sched();
943
944	tr->time_start = ftrace_now(tr->cpu);
945
946	for_each_online_cpu(cpu)
947	ring_buffer_reset_cpu(buffer, cpu);
948
949	ring_buffer_record_enable(buffer);
950	}
951
952	void tracing_reset_current(int cpu)
953	{
954	tracing_reset(&global_trace, cpu);
955	}
956
957	void tracing_reset_current_online_cpus(void)
958	{
959	tracing_reset_online_cpus(&global_trace);
960	}
961
962	#define SAVED_CMDLINES 128
963	#define NO_CMDLINE_MAP UINT_MAX
964	static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
965	static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
966	static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
967	static int cmdline_idx;
968	static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
969
970	/* temporary disable recording */
971	static atomic_t trace_record_cmdline_disabled __read_mostly;
972
973	static void trace_init_cmdlines(void)
974	{
975	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
976	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
977	cmdline_idx = 0;
978	}
979
980	int is_tracing_stopped(void)
981	{
982	return trace_stop_count;
983	}
984
985	/**
986	* ftrace_off_permanent - disable all ftrace code permanently
987	*
988	* This should only be called when a serious anomally has
989	* been detected. This will turn off the function tracing,
990	* ring buffers, and other tracing utilites. It takes no
991	* locks and can be called from any context.
992	*/
993	void ftrace_off_permanent(void)
994	{
995	tracing_disabled = 1;
996	ftrace_stop();
997	tracing_off_permanent();
998	}
999
1000	/**
1001	* tracing_start - quick start of the tracer
1002	*
1003	* If tracing is enabled but was stopped by tracing_stop,
1004	* this will start the tracer back up.
1005	*/
1006	void tracing_start(void)
1007	{
1008	struct ring_buffer *buffer;
1009	unsigned long flags;
1010
1011	if (tracing_disabled)
1012	return;
1013
1014	raw_spin_lock_irqsave(&tracing_start_lock, flags);
1015	if (--trace_stop_count) {
1016	if (trace_stop_count < 0) {
1017	/* Someone screwed up their debugging */
1018	WARN_ON_ONCE(1);
1019	trace_stop_count = 0;
1020	}
1021	goto out;
1022	}
1023
1024	/* Prevent the buffers from switching */
1025	arch_spin_lock(&ftrace_max_lock);
1026
1027	buffer = global_trace.buffer;
1028	if (buffer)
1029	ring_buffer_record_enable(buffer);
1030
1031	buffer = max_tr.buffer;
1032	if (buffer)
1033	ring_buffer_record_enable(buffer);
1034
1035	arch_spin_unlock(&ftrace_max_lock);
1036
1037	ftrace_start();
1038	out:
1039	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1040	}
1041
1042	/**
1043	* tracing_stop - quick stop of the tracer
1044	*
1045	* Light weight way to stop tracing. Use in conjunction with
1046	* tracing_start.
1047	*/
1048	void tracing_stop(void)
1049	{
1050	struct ring_buffer *buffer;
1051	unsigned long flags;
1052
1053	ftrace_stop();
1054	raw_spin_lock_irqsave(&tracing_start_lock, flags);
1055	if (trace_stop_count++)
1056	goto out;
1057
1058	/* Prevent the buffers from switching */
1059	arch_spin_lock(&ftrace_max_lock);
1060
1061	buffer = global_trace.buffer;
1062	if (buffer)
1063	ring_buffer_record_disable(buffer);
1064
1065	buffer = max_tr.buffer;
1066	if (buffer)
1067	ring_buffer_record_disable(buffer);
1068
1069	arch_spin_unlock(&ftrace_max_lock);
1070
1071	out:
1072	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1073	}
1074
1075	void trace_stop_cmdline_recording(void);
1076
1077	static void trace_save_cmdline(struct task_struct *tsk)
1078	{
1079	unsigned pid, idx;
1080
1081	if (!tsk->pid \|\| unlikely(tsk->pid > PID_MAX_DEFAULT))
1082	return;
1083
1084	/*
1085	* It's not the end of the world if we don't get
1086	* the lock, but we also don't want to spin
1087	* nor do we want to disable interrupts,
1088	* so if we miss here, then better luck next time.
1089	*/
1090	if (!arch_spin_trylock(&trace_cmdline_lock))
1091	return;
1092
1093	idx = map_pid_to_cmdline[tsk->pid];
1094	if (idx == NO_CMDLINE_MAP) {
1095	idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1096
1097	/*
1098	* Check whether the cmdline buffer at idx has a pid
1099	* mapped. We are going to overwrite that entry so we
1100	* need to clear the map_pid_to_cmdline. Otherwise we
1101	* would read the new comm for the old pid.
1102	*/
1103	pid = map_cmdline_to_pid[idx];
1104	if (pid != NO_CMDLINE_MAP)
1105	map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1106
1107	map_cmdline_to_pid[idx] = tsk->pid;
1108	map_pid_to_cmdline[tsk->pid] = idx;
1109
1110	cmdline_idx = idx;
1111	}
1112
1113	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1114
1115	arch_spin_unlock(&trace_cmdline_lock);
1116	}
1117
1118	void trace_find_cmdline(int pid, char comm[])
1119	{
1120	unsigned map;
1121
1122	if (!pid) {
1123	strcpy(comm, "<idle>");
1124	return;
1125	}
1126
1127	if (WARN_ON_ONCE(pid < 0)) {
1128	strcpy(comm, "<XXX>");
1129	return;
1130	}
1131
1132	if (pid > PID_MAX_DEFAULT) {
1133	strcpy(comm, "<...>");
1134	return;
1135	}
1136
1137	preempt_disable();
1138	arch_spin_lock(&trace_cmdline_lock);
1139	map = map_pid_to_cmdline[pid];
1140	if (map != NO_CMDLINE_MAP)
1141	strcpy(comm, saved_cmdlines[map]);
1142	else
1143	strcpy(comm, "<...>");
1144
1145	arch_spin_unlock(&trace_cmdline_lock);
1146	preempt_enable();
1147	}
1148
1149	void tracing_record_cmdline(struct task_struct *tsk)
1150	{
1151	if (atomic_read(&trace_record_cmdline_disabled) \|\| !tracing_is_on())
1152	return;
1153
1154	if (!__this_cpu_read(trace_cmdline_save))
1155	return;
1156
1157	__this_cpu_write(trace_cmdline_save, false);
1158
1159	trace_save_cmdline(tsk);
1160	}
1161
1162	void
1163	tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1164	int pc)
1165	{
1166	struct task_struct *tsk = current;
1167
1168	entry->preempt_count = pc & 0xff;
1169	entry->pid = (tsk) ? tsk->pid : 0;
1170	entry->padding = 0;
1171	entry->flags =
1172	#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1173	(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) \|
1174	#else
1175	TRACE_FLAG_IRQS_NOSUPPORT \|
1176	#endif
1177	((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) \|
1178	((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) \|
1179	(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1180	}
1181	EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1182
1183	struct ring_buffer_event *
1184	trace_buffer_lock_reserve(struct ring_buffer *buffer,
1185	int type,
1186	unsigned long len,
1187	unsigned long flags, int pc)
1188	{
1189	struct ring_buffer_event *event;
1190
1191	event = ring_buffer_lock_reserve(buffer, len);
1192	if (event != NULL) {
1193	struct trace_entry *ent = ring_buffer_event_data(event);
1194
1195	tracing_generic_entry_update(ent, flags, pc);
1196	ent->type = type;
1197	}
1198
1199	return event;
1200	}
1201
1202	void
1203	__buffer_unlock_commit(struct ring_buffer buffer, struct ring_buffer_event event)
1204	{
1205	__this_cpu_write(trace_cmdline_save, true);
1206	if (trace_wakeup_needed) {
1207	trace_wakeup_needed = false;
1208	/* irq_work_queue() supplies it's own memory barriers */
1209	irq_work_queue(&trace_work_wakeup);
1210	}
1211	ring_buffer_unlock_commit(buffer, event);
1212	}
1213
1214	static inline void
1215	__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1216	struct ring_buffer_event *event,
1217	unsigned long flags, int pc)
1218	{
1219	__buffer_unlock_commit(buffer, event);
1220
1221	ftrace_trace_stack(buffer, flags, 6, pc);
1222	ftrace_trace_userstack(buffer, flags, pc);
1223	}
1224
1225	void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1226	struct ring_buffer_event *event,
1227	unsigned long flags, int pc)
1228	{
1229	__trace_buffer_unlock_commit(buffer, event, flags, pc);
1230	}
1231	EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1232
1233	struct ring_buffer_event *
1234	trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1235	int type, unsigned long len,
1236	unsigned long flags, int pc)
1237	{
1238	*current_rb = global_trace.buffer;
1239	return trace_buffer_lock_reserve(*current_rb,
1240	type, len, flags, pc);
1241	}
1242	EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1243
1244	void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1245	struct ring_buffer_event *event,
1246	unsigned long flags, int pc)
1247	{
1248	__trace_buffer_unlock_commit(buffer, event, flags, pc);
1249	}
1250	EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1251
1252	void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1253	struct ring_buffer_event *event,
1254	unsigned long flags, int pc,
1255	struct pt_regs *regs)
1256	{
1257	__buffer_unlock_commit(buffer, event);
1258
1259	ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1260	ftrace_trace_userstack(buffer, flags, pc);
1261	}
1262	EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1263
1264	void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1265	struct ring_buffer_event *event)
1266	{
1267	ring_buffer_discard_commit(buffer, event);
1268	}
1269	EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1270
1271	void
1272	trace_function(struct trace_array *tr,
1273	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1274	int pc)
1275	{
1276	struct ftrace_event_call *call = &event_function;
1277	struct ring_buffer *buffer = tr->buffer;
1278	struct ring_buffer_event *event;
1279	struct ftrace_entry *entry;
1280
1281	/* If we are reading the ring buffer, don't trace */
1282	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1283	return;
1284
1285	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1286	flags, pc);
1287	if (!event)
1288	return;
1289	entry = ring_buffer_event_data(event);
1290	entry->ip = ip;
1291	entry->parent_ip = parent_ip;
1292
1293	if (!filter_check_discard(call, entry, buffer, event))
1294	__buffer_unlock_commit(buffer, event);
1295	}
1296
1297	void
1298	ftrace(struct trace_array tr, struct trace_array_cpu data,
1299	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1300	int pc)
1301	{
1302	if (likely(!atomic_read(&data->disabled)))
1303	trace_function(tr, ip, parent_ip, flags, pc);
1304	}
1305
1306	#ifdef CONFIG_STACKTRACE
1307
1308	#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1309	struct ftrace_stack {
1310	unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1311	};
1312
1313	static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1314	static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1315
1316	static void __ftrace_trace_stack(struct ring_buffer *buffer,
1317	unsigned long flags,
1318	int skip, int pc, struct pt_regs *regs)
1319	{
1320	struct ftrace_event_call *call = &event_kernel_stack;
1321	struct ring_buffer_event *event;
1322	struct stack_entry *entry;
1323	struct stack_trace trace;
1324	int use_stack;
1325	int size = FTRACE_STACK_ENTRIES;
1326
1327	trace.nr_entries = 0;
1328	trace.skip = skip;
1329
1330	/*
1331	* Since events can happen in NMIs there's no safe way to
1332	* use the per cpu ftrace_stacks. We reserve it and if an interrupt
1333	* or NMI comes in, it will just have to use the default
1334	* FTRACE_STACK_SIZE.
1335	*/
1336	preempt_disable_notrace();
1337
1338	use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1339	/*
1340	* We don't need any atomic variables, just a barrier.
1341	* If an interrupt comes in, we don't care, because it would
1342	* have exited and put the counter back to what we want.
1343	* We just need a barrier to keep gcc from moving things
1344	* around.
1345	*/
1346	barrier();
1347	if (use_stack == 1) {
1348	trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1349	trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1350
1351	if (regs)
1352	save_stack_trace_regs(regs, &trace);
1353	else
1354	save_stack_trace(&trace);
1355
1356	if (trace.nr_entries > size)
1357	size = trace.nr_entries;
1358	} else
1359	/* From now on, use_stack is a boolean */
1360	use_stack = 0;
1361
1362	size *= sizeof(unsigned long);
1363
1364	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1365	sizeof(*entry) + size, flags, pc);
1366	if (!event)
1367	goto out;
1368	entry = ring_buffer_event_data(event);
1369
1370	memset(&entry->caller, 0, size);
1371
1372	if (use_stack)
1373	memcpy(&entry->caller, trace.entries,
1374	trace.nr_entries * sizeof(unsigned long));
1375	else {
1376	trace.max_entries = FTRACE_STACK_ENTRIES;
1377	trace.entries = entry->caller;
1378	if (regs)
1379	save_stack_trace_regs(regs, &trace);
1380	else
1381	save_stack_trace(&trace);
1382	}
1383
1384	entry->size = trace.nr_entries;
1385
1386	if (!filter_check_discard(call, entry, buffer, event))
1387	__buffer_unlock_commit(buffer, event);
1388
1389	out:
1390	/* Again, don't let gcc optimize things here */
1391	barrier();
1392	__get_cpu_var(ftrace_stack_reserve)--;
1393	preempt_enable_notrace();
1394
1395	}
1396
1397	void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1398	int skip, int pc, struct pt_regs *regs)
1399	{
1400	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1401	return;
1402
1403	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
1404	}
1405
1406	void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1407	int skip, int pc)
1408	{
1409	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1410	return;
1411
1412	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1413	}
1414
1415	void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1416	int pc)
1417	{
1418	__ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1419	}
1420
1421	/**
1422	* trace_dump_stack - record a stack back trace in the trace buffer
1423	*/
1424	void trace_dump_stack(void)
1425	{
1426	unsigned long flags;
1427
1428	if (tracing_disabled \|\| tracing_selftest_running)
1429	return;
1430
1431	local_save_flags(flags);
1432
1433	/* skipping 3 traces, seems to get us at the caller of this function */
1434	__ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1435	}
1436
1437	static DEFINE_PER_CPU(int, user_stack_count);
1438
1439	void
1440	ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1441	{
1442	struct ftrace_event_call *call = &event_user_stack;
1443	struct ring_buffer_event *event;
1444	struct userstack_entry *entry;
1445	struct stack_trace trace;
1446
1447	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1448	return;
1449
1450	/*
1451	* NMIs can not handle page faults, even with fix ups.
1452	* The save user stack can (and often does) fault.
1453	*/
1454	if (unlikely(in_nmi()))
1455	return;
1456
1457	/*
1458	* prevent recursion, since the user stack tracing may
1459	* trigger other kernel events.
1460	*/
1461	preempt_disable();
1462	if (__this_cpu_read(user_stack_count))
1463	goto out;
1464
1465	__this_cpu_inc(user_stack_count);
1466
1467	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1468	sizeof(*entry), flags, pc);
1469	if (!event)
1470	goto out_drop_count;
1471	entry = ring_buffer_event_data(event);
1472
1473	entry->tgid = current->tgid;
1474	memset(&entry->caller, 0, sizeof(entry->caller));
1475
1476	trace.nr_entries = 0;
1477	trace.max_entries = FTRACE_STACK_ENTRIES;
1478	trace.skip = 0;
1479	trace.entries = entry->caller;
1480
1481	save_stack_trace_user(&trace);
1482	if (!filter_check_discard(call, entry, buffer, event))
1483	__buffer_unlock_commit(buffer, event);
1484
1485	out_drop_count:
1486	__this_cpu_dec(user_stack_count);
1487	out:
1488	preempt_enable();
1489	}
1490
1491	#ifdef UNUSED
1492	static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1493	{
1494	ftrace_trace_userstack(tr, flags, preempt_count());
1495	}
1496	#endif /* UNUSED */
1497
1498	#endif /* CONFIG_STACKTRACE */
1499
1500	/* created for use with alloc_percpu */
1501	struct trace_buffer_struct {
1502	char buffer[TRACE_BUF_SIZE];
1503	};
1504
1505	static struct trace_buffer_struct *trace_percpu_buffer;
1506	static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1507	static struct trace_buffer_struct *trace_percpu_irq_buffer;
1508	static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1509
1510	/*
1511	* The buffer used is dependent on the context. There is a per cpu
1512	* buffer for normal context, softirq contex, hard irq context and
1513	* for NMI context. Thise allows for lockless recording.
1514	*
1515	* Note, if the buffers failed to be allocated, then this returns NULL
1516	*/
1517	static char *get_trace_buf(void)
1518	{
1519	struct trace_buffer_struct *percpu_buffer;
1520	struct trace_buffer_struct *buffer;
1521
1522	/*
1523	* If we have allocated per cpu buffers, then we do not
1524	* need to do any locking.
1525	*/
1526	if (in_nmi())
1527	percpu_buffer = trace_percpu_nmi_buffer;
1528	else if (in_irq())
1529	percpu_buffer = trace_percpu_irq_buffer;
1530	else if (in_softirq())
1531	percpu_buffer = trace_percpu_sirq_buffer;
1532	else
1533	percpu_buffer = trace_percpu_buffer;
1534
1535	if (!percpu_buffer)
1536	return NULL;
1537
1538	buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1539
1540	return buffer->buffer;
1541	}
1542
1543	static int alloc_percpu_trace_buffer(void)
1544	{
1545	struct trace_buffer_struct *buffers;
1546	struct trace_buffer_struct *sirq_buffers;
1547	struct trace_buffer_struct *irq_buffers;
1548	struct trace_buffer_struct *nmi_buffers;
1549
1550	buffers = alloc_percpu(struct trace_buffer_struct);
1551	if (!buffers)
1552	goto err_warn;
1553
1554	sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1555	if (!sirq_buffers)
1556	goto err_sirq;
1557
1558	irq_buffers = alloc_percpu(struct trace_buffer_struct);
1559	if (!irq_buffers)
1560	goto err_irq;
1561
1562	nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1563	if (!nmi_buffers)
1564	goto err_nmi;
1565
1566	trace_percpu_buffer = buffers;
1567	trace_percpu_sirq_buffer = sirq_buffers;
1568	trace_percpu_irq_buffer = irq_buffers;
1569	trace_percpu_nmi_buffer = nmi_buffers;
1570
1571	return 0;
1572
1573	err_nmi:
1574	free_percpu(irq_buffers);
1575	err_irq:
1576	free_percpu(sirq_buffers);
1577	err_sirq:
1578	free_percpu(buffers);
1579	err_warn:
1580	WARN(1, "Could not allocate percpu trace_printk buffer");
1581	return -ENOMEM;
1582	}
1583
1584	static int buffers_allocated;
1585
1586	void trace_printk_init_buffers(void)
1587	{
1588	if (buffers_allocated)
1589	return;
1590
1591	if (alloc_percpu_trace_buffer())
1592	return;
1593
1594	pr_info("ftrace: Allocated trace_printk buffers\n");
1595
1596	/* Expand the buffers to set size */
1597	tracing_update_buffers();
1598
1599	buffers_allocated = 1;
1600
1601	/*
1602	* trace_printk_init_buffers() can be called by modules.
1603	* If that happens, then we need to start cmdline recording
1604	* directly here. If the global_trace.buffer is already
1605	* allocated here, then this was called by module code.
1606	*/
1607	if (global_trace.buffer)
1608	tracing_start_cmdline_record();
1609	}
1610
1611	void trace_printk_start_comm(void)
1612	{
1613	/* Start tracing comms if trace printk is set */
1614	if (!buffers_allocated)
1615	return;
1616	tracing_start_cmdline_record();
1617	}
1618
1619	static void trace_printk_start_stop_comm(int enabled)
1620	{
1621	if (!buffers_allocated)
1622	return;
1623
1624	if (enabled)
1625	tracing_start_cmdline_record();
1626	else
1627	tracing_stop_cmdline_record();
1628	}
1629
1630	/**
1631	* trace_vbprintk - write binary msg to tracing buffer
1632	*
1633	*/
1634	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1635	{
1636	struct ftrace_event_call *call = &event_bprint;
1637	struct ring_buffer_event *event;
1638	struct ring_buffer *buffer;
1639	struct trace_array *tr = &global_trace;
1640	struct bprint_entry *entry;
1641	unsigned long flags;
1642	char *tbuffer;
1643	int len = 0, size, pc;
1644
1645	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
1646	return 0;
1647
1648	/* Don't pollute graph traces with trace_vprintk internals */
1649	pause_graph_tracing();
1650
1651	pc = preempt_count();
1652	preempt_disable_notrace();
1653
1654	tbuffer = get_trace_buf();
1655	if (!tbuffer) {
1656	len = 0;
1657	goto out;
1658	}
1659
1660	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1661
1662	if (len > TRACE_BUF_SIZE/sizeof(int) \|\| len < 0)
1663	goto out;
1664
1665	local_save_flags(flags);
1666	size = sizeof(entry) + sizeof(u32) len;
1667	buffer = tr->buffer;
1668	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1669	flags, pc);
1670	if (!event)
1671	goto out;
1672	entry = ring_buffer_event_data(event);
1673	entry->ip = ip;
1674	entry->fmt = fmt;
1675
1676	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1677	if (!filter_check_discard(call, entry, buffer, event)) {
1678	__buffer_unlock_commit(buffer, event);
1679	ftrace_trace_stack(buffer, flags, 6, pc);
1680	}
1681
1682	out:
1683	preempt_enable_notrace();
1684	unpause_graph_tracing();
1685
1686	return len;
1687	}
1688	EXPORT_SYMBOL_GPL(trace_vbprintk);
1689
1690	int trace_array_printk(struct trace_array *tr,
1691	unsigned long ip, const char *fmt, ...)
1692	{
1693	int ret;
1694	va_list ap;
1695
1696	if (!(trace_flags & TRACE_ITER_PRINTK))
1697	return 0;
1698
1699	va_start(ap, fmt);
1700	ret = trace_array_vprintk(tr, ip, fmt, ap);
1701	va_end(ap);
1702	return ret;
1703	}
1704
1705	int trace_array_vprintk(struct trace_array *tr,
1706	unsigned long ip, const char *fmt, va_list args)
1707	{
1708	struct ftrace_event_call *call = &event_print;
1709	struct ring_buffer_event *event;
1710	struct ring_buffer *buffer;
1711	int len = 0, size, pc;
1712	struct print_entry *entry;
1713	unsigned long flags;
1714	char *tbuffer;
1715
1716	if (tracing_disabled \|\| tracing_selftest_running)
1717	return 0;
1718
1719	/* Don't pollute graph traces with trace_vprintk internals */
1720	pause_graph_tracing();
1721
1722	pc = preempt_count();
1723	preempt_disable_notrace();
1724
1725
1726	tbuffer = get_trace_buf();
1727	if (!tbuffer) {
1728	len = 0;
1729	goto out;
1730	}
1731
1732	len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1733	if (len > TRACE_BUF_SIZE)
1734	goto out;
1735
1736	local_save_flags(flags);
1737	size = sizeof(*entry) + len + 1;
1738	buffer = tr->buffer;
1739	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1740	flags, pc);
1741	if (!event)
1742	goto out;
1743	entry = ring_buffer_event_data(event);
1744	entry->ip = ip;
1745
1746	memcpy(&entry->buf, tbuffer, len);
1747	entry->buf[len] = '\0';
1748	if (!filter_check_discard(call, entry, buffer, event)) {
1749	__buffer_unlock_commit(buffer, event);
1750	ftrace_trace_stack(buffer, flags, 6, pc);
1751	}
1752	out:
1753	preempt_enable_notrace();
1754	unpause_graph_tracing();
1755
1756	return len;
1757	}
1758
1759	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1760	{
1761	return trace_array_vprintk(&global_trace, ip, fmt, args);
1762	}
1763	EXPORT_SYMBOL_GPL(trace_vprintk);
1764
1765	static void trace_iterator_increment(struct trace_iterator *iter)
1766	{
1767	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1768
1769	iter->idx++;
1770	if (buf_iter)
1771	ring_buffer_read(buf_iter, NULL);
1772	}
1773
1774	static struct trace_entry *
1775	peek_next_entry(struct trace_iterator iter, int cpu, u64 ts,
1776	unsigned long *lost_events)
1777	{
1778	struct ring_buffer_event *event;
1779	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
1780
1781	if (buf_iter)
1782	event = ring_buffer_iter_peek(buf_iter, ts);
1783	else
1784	event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1785	lost_events);
1786
1787	if (event) {
1788	iter->ent_size = ring_buffer_event_length(event);
1789	return ring_buffer_event_data(event);
1790	}
1791	iter->ent_size = 0;
1792	return NULL;
1793	}
1794
1795	static struct trace_entry *
1796	__find_next_entry(struct trace_iterator iter, int ent_cpu,
1797	unsigned long missing_events, u64 ent_ts)
1798	{
1799	struct ring_buffer *buffer = iter->tr->buffer;
1800	struct trace_entry ent, next = NULL;
1801	unsigned long lost_events = 0, next_lost = 0;
1802	int cpu_file = iter->cpu_file;
1803	u64 next_ts = 0, ts;
1804	int next_cpu = -1;
1805	int next_size = 0;
1806	int cpu;
1807
1808	/*
1809	* If we are in a per_cpu trace file, don't bother by iterating over
1810	* all cpu and peek directly.
1811	*/
1812	if (cpu_file > TRACE_PIPE_ALL_CPU) {
1813	if (ring_buffer_empty_cpu(buffer, cpu_file))
1814	return NULL;
1815	ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1816	if (ent_cpu)
1817	*ent_cpu = cpu_file;
1818
1819	return ent;
1820	}
1821
1822	for_each_tracing_cpu(cpu) {
1823
1824	if (ring_buffer_empty_cpu(buffer, cpu))
1825	continue;
1826
1827	ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1828
1829	/*
1830	* Pick the entry with the smallest timestamp:
1831	*/
1832	if (ent && (!next \|\| ts < next_ts)) {
1833	next = ent;
1834	next_cpu = cpu;
1835	next_ts = ts;
1836	next_lost = lost_events;
1837	next_size = iter->ent_size;
1838	}
1839	}
1840
1841	iter->ent_size = next_size;
1842
1843	if (ent_cpu)
1844	*ent_cpu = next_cpu;
1845
1846	if (ent_ts)
1847	*ent_ts = next_ts;
1848
1849	if (missing_events)
1850	*missing_events = next_lost;
1851
1852	return next;
1853	}
1854
1855	/* Find the next real entry, without updating the iterator itself */
1856	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
1857	int ent_cpu, u64 ent_ts)
1858	{
1859	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1860	}
1861
1862	/* Find the next real entry, and increment the iterator to the next entry */
1863	void trace_find_next_entry_inc(struct trace_iterator iter)
1864	{
1865	iter->ent = __find_next_entry(iter, &iter->cpu,
1866	&iter->lost_events, &iter->ts);
1867
1868	if (iter->ent)
1869	trace_iterator_increment(iter);
1870
1871	return iter->ent ? iter : NULL;
1872	}
1873
1874	static void trace_consume(struct trace_iterator *iter)
1875	{
1876	ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1877	&iter->lost_events);
1878	}
1879
1880	static void s_next(struct seq_file m, void v, loff_t pos)
1881	{
1882	struct trace_iterator *iter = m->private;
1883	int i = (int)*pos;
1884	void *ent;
1885
1886	WARN_ON_ONCE(iter->leftover);
1887
1888	(*pos)++;
1889
1890	/* can't go backwards */
1891	if (iter->idx > i)
1892	return NULL;
1893
1894	if (iter->idx < 0)
1895	ent = trace_find_next_entry_inc(iter);
1896	else
1897	ent = iter;
1898
1899	while (ent && iter->idx < i)
1900	ent = trace_find_next_entry_inc(iter);
1901
1902	iter->pos = *pos;
1903
1904	return ent;
1905	}
1906
1907	void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1908	{
1909	struct trace_array *tr = iter->tr;
1910	struct ring_buffer_event *event;
1911	struct ring_buffer_iter *buf_iter;
1912	unsigned long entries = 0;
1913	u64 ts;
1914
1915	tr->data[cpu]->skipped_entries = 0;
1916
1917	buf_iter = trace_buffer_iter(iter, cpu);
1918	if (!buf_iter)
1919	return;
1920
1921	ring_buffer_iter_reset(buf_iter);
1922
1923	/*
1924	* We could have the case with the max latency tracers
1925	* that a reset never took place on a cpu. This is evident
1926	* by the timestamp being before the start of the buffer.
1927	*/
1928	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1929	if (ts >= iter->tr->time_start)
1930	break;
1931	entries++;
1932	ring_buffer_read(buf_iter, NULL);
1933	}
1934
1935	tr->data[cpu]->skipped_entries = entries;
1936	}
1937
1938	/*
1939	* The current tracer is copied to avoid a global locking
1940	* all around.
1941	*/
1942	static void s_start(struct seq_file m, loff_t *pos)
1943	{
1944	struct trace_iterator *iter = m->private;
1945	static struct tracer *old_tracer;
1946	int cpu_file = iter->cpu_file;
1947	void *p = NULL;
1948	loff_t l = 0;
1949	int cpu;
1950
1951	/* copy the tracer to avoid using a global lock all around */
1952	mutex_lock(&trace_types_lock);
1953	if (unlikely(old_tracer != current_trace && current_trace)) {
1954	old_tracer = current_trace;
1955	iter->trace = current_trace;
1956	}
1957	mutex_unlock(&trace_types_lock);
1958
1959	atomic_inc(&trace_record_cmdline_disabled);
1960
1961	if (*pos != iter->pos) {
1962	iter->ent = NULL;
1963	iter->cpu = 0;
1964	iter->idx = -1;
1965
1966	if (cpu_file == TRACE_PIPE_ALL_CPU) {
1967	for_each_tracing_cpu(cpu)
1968	tracing_iter_reset(iter, cpu);
1969	} else
1970	tracing_iter_reset(iter, cpu_file);
1971
1972	iter->leftover = 0;
1973	for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1974	;
1975
1976	} else {
1977	/*
1978	* If we overflowed the seq_file before, then we want
1979	* to just reuse the trace_seq buffer again.
1980	*/
1981	if (iter->leftover)
1982	p = iter;
1983	else {
1984	l = *pos - 1;
1985	p = s_next(m, p, &l);
1986	}
1987	}
1988
1989	trace_event_read_lock();
1990	trace_access_lock(cpu_file);
1991	return p;
1992	}
1993
1994	static void s_stop(struct seq_file m, void p)
1995	{
1996	struct trace_iterator *iter = m->private;
1997
1998	atomic_dec(&trace_record_cmdline_disabled);
1999	trace_access_unlock(iter->cpu_file);
2000	trace_event_read_unlock();
2001	}
2002
2003	static void
2004	get_total_entries(struct trace_array tr, unsigned long total, unsigned long *entries)
2005	{
2006	unsigned long count;
2007	int cpu;
2008
2009	*total = 0;
2010	*entries = 0;
2011
2012	for_each_tracing_cpu(cpu) {
2013	count = ring_buffer_entries_cpu(tr->buffer, cpu);
2014	/*
2015	* If this buffer has skipped entries, then we hold all
2016	* entries for the trace and we need to ignore the
2017	* ones before the time stamp.
2018	*/
2019	if (tr->data[cpu]->skipped_entries) {
2020	count -= tr->data[cpu]->skipped_entries;
2021	/* total is the same as the entries */
2022	*total += count;
2023	} else
2024	*total += count +
2025	ring_buffer_overrun_cpu(tr->buffer, cpu);
2026	*entries += count;
2027	}
2028	}
2029
2030	static void print_lat_help_header(struct seq_file *m)
2031	{
2032	seq_puts(m, "# _------=> CPU# \n");
2033	seq_puts(m, "# / _-----=> irqs-off \n");
2034	seq_puts(m, "# \| / _----=> need-resched \n");
2035	seq_puts(m, "# \|\| / _---=> hardirq/softirq \n");
2036	seq_puts(m, "# \|\|\| / _--=> preempt-depth \n");
2037	seq_puts(m, "# \|\|\|\| / delay \n");
2038	seq_puts(m, "# cmd pid \|\|\|\|\| time \| caller \n");
2039	seq_puts(m, "# \\ / \|\|\|\|\| \\ \| / \n");
2040	}
2041
2042	static void print_event_info(struct trace_array tr, struct seq_file m)
2043	{
2044	unsigned long total;
2045	unsigned long entries;
2046
2047	get_total_entries(tr, &total, &entries);
2048	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2049	entries, total, num_online_cpus());
2050	seq_puts(m, "#\n");
2051	}
2052
2053	static void print_func_help_header(struct trace_array tr, struct seq_file m)
2054	{
2055	print_event_info(tr, m);
2056	seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2057	seq_puts(m, "# \| \| \| \| \|\n");
2058	}
2059
2060	static void print_func_help_header_irq(struct trace_array tr, struct seq_file m)
2061	{
2062	print_event_info(tr, m);
2063	seq_puts(m, "# _-----=> irqs-off\n");
2064	seq_puts(m, "# / _----=> need-resched\n");
2065	seq_puts(m, "# \| / _---=> hardirq/softirq\n");
2066	seq_puts(m, "# \|\| / _--=> preempt-depth\n");
2067	seq_puts(m, "# \|\|\| / delay\n");
2068	seq_puts(m, "# TASK-PID CPU# \|\|\|\| TIMESTAMP FUNCTION\n");
2069	seq_puts(m, "# \| \| \| \|\|\|\| \| \|\n");
2070	}
2071
2072	void
2073	print_trace_header(struct seq_file m, struct trace_iterator iter)
2074	{
2075	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2076	struct trace_array *tr = iter->tr;
2077	struct trace_array_cpu *data = tr->data[tr->cpu];
2078	struct tracer *type = current_trace;
2079	unsigned long entries;
2080	unsigned long total;
2081	const char *name = "preemption";
2082
2083	if (type)
2084	name = type->name;
2085
2086	get_total_entries(tr, &total, &entries);
2087
2088	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2089	name, UTS_RELEASE);
2090	seq_puts(m, "# -----------------------------------"
2091	"---------------------------------\n");
2092	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d \|"
2093	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2094	nsecs_to_usecs(data->saved_latency),
2095	entries,
2096	total,
2097	tr->cpu,
2098	#if defined(CONFIG_PREEMPT_NONE)
2099	"server",
2100	#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2101	"desktop",
2102	#elif defined(CONFIG_PREEMPT)
2103	"preempt",
2104	#else
2105	"unknown",
2106	#endif
2107	/* These are reserved for later use */
2108	0, 0, 0, 0);
2109	#ifdef CONFIG_SMP
2110	seq_printf(m, " #P:%d)\n", num_online_cpus());
2111	#else
2112	seq_puts(m, ")\n");
2113	#endif
2114	seq_puts(m, "# -----------------\n");
2115	seq_printf(m, "# \| task: %.16s-%d "
2116	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2117	data->comm, data->pid,
2118	from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2119	data->policy, data->rt_priority);
2120	seq_puts(m, "# -----------------\n");
2121
2122	if (data->critical_start) {
2123	seq_puts(m, "# => started at: ");
2124	seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2125	trace_print_seq(m, &iter->seq);
2126	seq_puts(m, "\n# => ended at: ");
2127	seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2128	trace_print_seq(m, &iter->seq);
2129	seq_puts(m, "\n#\n");
2130	}
2131
2132	seq_puts(m, "#\n");
2133	}
2134
2135	static void test_cpu_buff_start(struct trace_iterator *iter)
2136	{
2137	struct trace_seq *s = &iter->seq;
2138
2139	if (!(trace_flags & TRACE_ITER_ANNOTATE))
2140	return;
2141
2142	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2143	return;
2144
2145	if (cpumask_test_cpu(iter->cpu, iter->started))
2146	return;
2147
2148	if (iter->tr->data[iter->cpu]->skipped_entries)
2149	return;
2150
2151	cpumask_set_cpu(iter->cpu, iter->started);
2152
2153	/* Don't print started cpu buffer for the first entry of the trace */
2154	if (iter->idx > 1)
2155	trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2156	iter->cpu);
2157	}
2158
2159	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2160	{
2161	struct trace_seq *s = &iter->seq;
2162	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2163	struct trace_entry *entry;
2164	struct trace_event *event;
2165
2166	entry = iter->ent;
2167
2168	test_cpu_buff_start(iter);
2169
2170	event = ftrace_find_event(entry->type);
2171
2172	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2173	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2174	if (!trace_print_lat_context(iter))
2175	goto partial;
2176	} else {
2177	if (!trace_print_context(iter))
2178	goto partial;
2179	}
2180	}
2181
2182	if (event)
2183	return event->funcs->trace(iter, sym_flags, event);
2184
2185	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2186	goto partial;
2187
2188	return TRACE_TYPE_HANDLED;
2189	partial:
2190	return TRACE_TYPE_PARTIAL_LINE;
2191	}
2192
2193	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2194	{
2195	struct trace_seq *s = &iter->seq;
2196	struct trace_entry *entry;
2197	struct trace_event *event;
2198
2199	entry = iter->ent;
2200
2201	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2202	if (!trace_seq_printf(s, "%d %d %llu ",
2203	entry->pid, iter->cpu, iter->ts))
2204	goto partial;
2205	}
2206
2207	event = ftrace_find_event(entry->type);
2208	if (event)
2209	return event->funcs->raw(iter, 0, event);
2210
2211	if (!trace_seq_printf(s, "%d ?\n", entry->type))
2212	goto partial;
2213
2214	return TRACE_TYPE_HANDLED;
2215	partial:
2216	return TRACE_TYPE_PARTIAL_LINE;
2217	}
2218
2219	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2220	{
2221	struct trace_seq *s = &iter->seq;
2222	unsigned char newline = '\n';
2223	struct trace_entry *entry;
2224	struct trace_event *event;
2225
2226	entry = iter->ent;
2227
2228	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2229	SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2230	SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2231	SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2232	}
2233
2234	event = ftrace_find_event(entry->type);
2235	if (event) {
2236	enum print_line_t ret = event->funcs->hex(iter, 0, event);
2237	if (ret != TRACE_TYPE_HANDLED)
2238	return ret;
2239	}
2240
2241	SEQ_PUT_FIELD_RET(s, newline);
2242
2243	return TRACE_TYPE_HANDLED;
2244	}
2245
2246	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2247	{
2248	struct trace_seq *s = &iter->seq;
2249	struct trace_entry *entry;
2250	struct trace_event *event;
2251
2252	entry = iter->ent;
2253
2254	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2255	SEQ_PUT_FIELD_RET(s, entry->pid);
2256	SEQ_PUT_FIELD_RET(s, iter->cpu);
2257	SEQ_PUT_FIELD_RET(s, iter->ts);
2258	}
2259
2260	event = ftrace_find_event(entry->type);
2261	return event ? event->funcs->binary(iter, 0, event) :
2262	TRACE_TYPE_HANDLED;
2263	}
2264
2265	int trace_empty(struct trace_iterator *iter)
2266	{
2267	struct ring_buffer_iter *buf_iter;
2268	int cpu;
2269
2270	/* If we are looking at one CPU buffer, only check that one */
2271	if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2272	cpu = iter->cpu_file;
2273	buf_iter = trace_buffer_iter(iter, cpu);
2274	if (buf_iter) {
2275	if (!ring_buffer_iter_empty(buf_iter))
2276	return 0;
2277	} else {
2278	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2279	return 0;
2280	}
2281	return 1;
2282	}
2283
2284	for_each_tracing_cpu(cpu) {
2285	buf_iter = trace_buffer_iter(iter, cpu);
2286	if (buf_iter) {
2287	if (!ring_buffer_iter_empty(buf_iter))
2288	return 0;
2289	} else {
2290	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2291	return 0;
2292	}
2293	}
2294
2295	return 1;
2296	}
2297
2298	/* Called with trace_event_read_lock() held. */
2299	enum print_line_t print_trace_line(struct trace_iterator *iter)
2300	{
2301	enum print_line_t ret;
2302
2303	if (iter->lost_events &&
2304	!trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2305	iter->cpu, iter->lost_events))
2306	return TRACE_TYPE_PARTIAL_LINE;
2307
2308	if (iter->trace && iter->trace->print_line) {
2309	ret = iter->trace->print_line(iter);
2310	if (ret != TRACE_TYPE_UNHANDLED)
2311	return ret;
2312	}
2313
2314	if (iter->ent->type == TRACE_BPRINT &&
2315	trace_flags & TRACE_ITER_PRINTK &&
2316	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2317	return trace_print_bprintk_msg_only(iter);
2318
2319	if (iter->ent->type == TRACE_PRINT &&
2320	trace_flags & TRACE_ITER_PRINTK &&
2321	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2322	return trace_print_printk_msg_only(iter);
2323
2324	if (trace_flags & TRACE_ITER_BIN)
2325	return print_bin_fmt(iter);
2326
2327	if (trace_flags & TRACE_ITER_HEX)
2328	return print_hex_fmt(iter);
2329
2330	if (trace_flags & TRACE_ITER_RAW)
2331	return print_raw_fmt(iter);
2332
2333	return print_trace_fmt(iter);
2334	}
2335
2336	void trace_latency_header(struct seq_file *m)
2337	{
2338	struct trace_iterator *iter = m->private;
2339
2340	/* print nothing if the buffers are empty */
2341	if (trace_empty(iter))
2342	return;
2343
2344	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2345	print_trace_header(m, iter);
2346
2347	if (!(trace_flags & TRACE_ITER_VERBOSE))
2348	print_lat_help_header(m);
2349	}
2350
2351	void trace_default_header(struct seq_file *m)
2352	{
2353	struct trace_iterator *iter = m->private;
2354
2355	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2356	return;
2357
2358	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2359	/* print nothing if the buffers are empty */
2360	if (trace_empty(iter))
2361	return;
2362	print_trace_header(m, iter);
2363	if (!(trace_flags & TRACE_ITER_VERBOSE))
2364	print_lat_help_header(m);
2365	} else {
2366	if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2367	if (trace_flags & TRACE_ITER_IRQ_INFO)
2368	print_func_help_header_irq(iter->tr, m);
2369	else
2370	print_func_help_header(iter->tr, m);
2371	}
2372	}
2373	}
2374
2375	static void test_ftrace_alive(struct seq_file *m)
2376	{
2377	if (!ftrace_is_dead())
2378	return;
2379	seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2380	seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2381	}
2382
2383	static int s_show(struct seq_file m, void v)
2384	{
2385	struct trace_iterator *iter = v;
2386	int ret;
2387
2388	if (iter->ent == NULL) {
2389	if (iter->tr) {
2390	seq_printf(m, "# tracer: %s\n", iter->trace->name);
2391	seq_puts(m, "#\n");
2392	test_ftrace_alive(m);
2393	}
2394	if (iter->trace && iter->trace->print_header)
2395	iter->trace->print_header(m);
2396	else
2397	trace_default_header(m);
2398
2399	} else if (iter->leftover) {
2400	/*
2401	* If we filled the seq_file buffer earlier, we
2402	* want to just show it now.
2403	*/
2404	ret = trace_print_seq(m, &iter->seq);
2405
2406	/* ret should this time be zero, but you never know */
2407	iter->leftover = ret;
2408
2409	} else {
2410	print_trace_line(iter);
2411	ret = trace_print_seq(m, &iter->seq);
2412	/*
2413	* If we overflow the seq_file buffer, then it will
2414	* ask us for this data again at start up.
2415	* Use that instead.
2416	* ret is 0 if seq_file write succeeded.
2417	* -1 otherwise.
2418	*/
2419	iter->leftover = ret;
2420	}
2421
2422	return 0;
2423	}
2424
2425	static const struct seq_operations tracer_seq_ops = {
2426	.start = s_start,
2427	.next = s_next,
2428	.stop = s_stop,
2429	.show = s_show,
2430	};
2431
2432	static struct trace_iterator *
2433	__tracing_open(struct inode inode, struct file file)
2434	{
2435	long cpu_file = (long) inode->i_private;
2436	struct trace_iterator *iter;
2437	int cpu;
2438
2439	if (tracing_disabled)
2440	return ERR_PTR(-ENODEV);
2441
2442	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2443	if (!iter)
2444	return ERR_PTR(-ENOMEM);
2445
2446	iter->buffer_iter = kzalloc(sizeof(iter->buffer_iter) num_possible_cpus(),
2447	GFP_KERNEL);
2448	if (!iter->buffer_iter)
2449	goto release;
2450
2451	/*
2452	* We make a copy of the current tracer to avoid concurrent
2453	* changes on it while we are reading.
2454	*/
2455	mutex_lock(&trace_types_lock);
2456	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2457	if (!iter->trace)
2458	goto fail;
2459
2460	if (current_trace)
2461	iter->trace = current_trace;
2462
2463	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2464	goto fail;
2465
2466	if (current_trace && current_trace->print_max)
2467	iter->tr = &max_tr;
2468	else
2469	iter->tr = &global_trace;
2470	iter->pos = -1;
2471	mutex_init(&iter->mutex);
2472	iter->cpu_file = cpu_file;
2473
2474	/* Notify the tracer early; before we stop tracing. */
2475	if (iter->trace && iter->trace->open)
2476	iter->trace->open(iter);
2477
2478	/* Annotate start of buffers if we had overruns */
2479	if (ring_buffer_overruns(iter->tr->buffer))
2480	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
2481
2482	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
2483	if (trace_clocks[trace_clock_id].in_ns)
2484	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
2485
2486	/* stop the trace while dumping */
2487	tracing_stop();
2488
2489	if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2490	for_each_tracing_cpu(cpu) {
2491	iter->buffer_iter[cpu] =
2492	ring_buffer_read_prepare(iter->tr->buffer, cpu);
2493	}
2494	ring_buffer_read_prepare_sync();
2495	for_each_tracing_cpu(cpu) {
2496	ring_buffer_read_start(iter->buffer_iter[cpu]);
2497	tracing_iter_reset(iter, cpu);
2498	}
2499	} else {
2500	cpu = iter->cpu_file;
2501	iter->buffer_iter[cpu] =
2502	ring_buffer_read_prepare(iter->tr->buffer, cpu);
2503	ring_buffer_read_prepare_sync();
2504	ring_buffer_read_start(iter->buffer_iter[cpu]);
2505	tracing_iter_reset(iter, cpu);
2506	}
2507
2508	mutex_unlock(&trace_types_lock);
2509
2510	return iter;
2511
2512	fail:
2513	mutex_unlock(&trace_types_lock);
2514	kfree(iter->trace);
2515	kfree(iter->buffer_iter);
2516	release:
2517	seq_release_private(inode, file);
2518	return ERR_PTR(-ENOMEM);
2519	}
2520
2521	int tracing_open_generic(struct inode inode, struct file filp)
2522	{
2523	if (tracing_disabled)
2524	return -ENODEV;
2525
2526	filp->private_data = inode->i_private;
2527	return 0;
2528	}
2529
2530	static int tracing_release(struct inode inode, struct file file)
2531	{
2532	struct seq_file *m = file->private_data;
2533	struct trace_iterator *iter;
2534	int cpu;
2535
2536	if (!(file->f_mode & FMODE_READ))
2537	return 0;
2538
2539	iter = m->private;
2540
2541	mutex_lock(&trace_types_lock);
2542	for_each_tracing_cpu(cpu) {
2543	if (iter->buffer_iter[cpu])
2544	ring_buffer_read_finish(iter->buffer_iter[cpu]);
2545	}
2546
2547	if (iter->trace && iter->trace->close)
2548	iter->trace->close(iter);
2549
2550	/* reenable tracing if it was previously enabled */
2551	tracing_start();
2552	mutex_unlock(&trace_types_lock);
2553
2554	mutex_destroy(&iter->mutex);
2555	free_cpumask_var(iter->started);
2556	kfree(iter->trace);
2557	kfree(iter->buffer_iter);
2558	seq_release_private(inode, file);
2559	return 0;
2560	}
2561
2562	static int tracing_open(struct inode inode, struct file file)
2563	{
2564	struct trace_iterator *iter;
2565	int ret = 0;
2566
2567	/* If this file was open for write, then erase contents */
2568	if ((file->f_mode & FMODE_WRITE) &&
2569	(file->f_flags & O_TRUNC)) {
2570	long cpu = (long) inode->i_private;
2571
2572	if (cpu == TRACE_PIPE_ALL_CPU)
2573	tracing_reset_online_cpus(&global_trace);
2574	else
2575	tracing_reset(&global_trace, cpu);
2576	}
2577
2578	if (file->f_mode & FMODE_READ) {
2579	iter = __tracing_open(inode, file);
2580	if (IS_ERR(iter))
2581	ret = PTR_ERR(iter);
2582	else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2583	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
2584	}
2585	return ret;
2586	}
2587
2588	static void *
2589	t_next(struct seq_file m, void v, loff_t *pos)
2590	{
2591	struct tracer *t = v;
2592
2593	(*pos)++;
2594
2595	if (t)
2596	t = t->next;
2597
2598	return t;
2599	}
2600
2601	static void t_start(struct seq_file m, loff_t *pos)
2602	{
2603	struct tracer *t;
2604	loff_t l = 0;
2605
2606	mutex_lock(&trace_types_lock);
2607	for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2608	;
2609
2610	return t;
2611	}
2612
2613	static void t_stop(struct seq_file m, void p)
2614	{
2615	mutex_unlock(&trace_types_lock);
2616	}
2617
2618	static int t_show(struct seq_file m, void v)
2619	{
2620	struct tracer *t = v;
2621
2622	if (!t)
2623	return 0;
2624
2625	seq_printf(m, "%s", t->name);
2626	if (t->next)
2627	seq_putc(m, ' ');
2628	else
2629	seq_putc(m, '\n');
2630
2631	return 0;
2632	}
2633
2634	static const struct seq_operations show_traces_seq_ops = {
2635	.start = t_start,
2636	.next = t_next,
2637	.stop = t_stop,
2638	.show = t_show,
2639	};
2640
2641	static int show_traces_open(struct inode inode, struct file file)
2642	{
2643	if (tracing_disabled)
2644	return -ENODEV;
2645
2646	return seq_open(file, &show_traces_seq_ops);
2647	}
2648
2649	static ssize_t
2650	tracing_write_stub(struct file filp, const char __user ubuf,
2651	size_t count, loff_t *ppos)
2652	{
2653	return count;
2654	}
2655
2656	static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2657	{
2658	if (file->f_mode & FMODE_READ)
2659	return seq_lseek(file, offset, origin);
2660	else
2661	return 0;
2662	}
2663
2664	static const struct file_operations tracing_fops = {
2665	.open = tracing_open,
2666	.read = seq_read,
2667	.write = tracing_write_stub,
2668	.llseek = tracing_seek,
2669	.release = tracing_release,
2670	};
2671
2672	static const struct file_operations show_traces_fops = {
2673	.open = show_traces_open,
2674	.read = seq_read,
2675	.release = seq_release,
2676	.llseek = seq_lseek,
2677	};
2678
2679	/*
2680	* Only trace on a CPU if the bitmask is set:
2681	*/
2682	static cpumask_var_t tracing_cpumask;
2683
2684	/*
2685	* The tracer itself will not take this lock, but still we want
2686	* to provide a consistent cpumask to user-space:
2687	*/
2688	static DEFINE_MUTEX(tracing_cpumask_update_lock);
2689
2690	/*
2691	* Temporary storage for the character representation of the
2692	* CPU bitmask (and one more byte for the newline):
2693	*/
2694	static char mask_str[NR_CPUS + 1];
2695
2696	static ssize_t
2697	tracing_cpumask_read(struct file filp, char __user ubuf,
2698	size_t count, loff_t *ppos)
2699	{
2700	int len;
2701
2702	mutex_lock(&tracing_cpumask_update_lock);
2703
2704	len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2705	if (count - len < 2) {
2706	count = -EINVAL;
2707	goto out_err;
2708	}
2709	len += sprintf(mask_str + len, "\n");
2710	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2711
2712	out_err:
2713	mutex_unlock(&tracing_cpumask_update_lock);
2714
2715	return count;
2716	}
2717
2718	static ssize_t
2719	tracing_cpumask_write(struct file filp, const char __user ubuf,
2720	size_t count, loff_t *ppos)
2721	{
2722	int err, cpu;
2723	cpumask_var_t tracing_cpumask_new;
2724
2725	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2726	return -ENOMEM;
2727
2728	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2729	if (err)
2730	goto err_unlock;
2731
2732	mutex_lock(&tracing_cpumask_update_lock);
2733
2734	local_irq_disable();
2735	arch_spin_lock(&ftrace_max_lock);
2736	for_each_tracing_cpu(cpu) {
2737	/*
2738	* Increase/decrease the disabled counter if we are
2739	* about to flip a bit in the cpumask:
2740	*/
2741	if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2742	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2743	atomic_inc(&global_trace.data[cpu]->disabled);
2744	ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2745	}
2746	if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2747	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2748	atomic_dec(&global_trace.data[cpu]->disabled);
2749	ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2750	}
2751	}
2752	arch_spin_unlock(&ftrace_max_lock);
2753	local_irq_enable();
2754
2755	cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2756
2757	mutex_unlock(&tracing_cpumask_update_lock);
2758	free_cpumask_var(tracing_cpumask_new);
2759
2760	return count;
2761
2762	err_unlock:
2763	free_cpumask_var(tracing_cpumask_new);
2764
2765	return err;
2766	}
2767
2768	static const struct file_operations tracing_cpumask_fops = {
2769	.open = tracing_open_generic,
2770	.read = tracing_cpumask_read,
2771	.write = tracing_cpumask_write,
2772	.llseek = generic_file_llseek,
2773	};
2774
2775	static int tracing_trace_options_show(struct seq_file m, void v)
2776	{
2777	struct tracer_opt *trace_opts;
2778	u32 tracer_flags;
2779	int i;
2780
2781	mutex_lock(&trace_types_lock);
2782	tracer_flags = current_trace->flags->val;
2783	trace_opts = current_trace->flags->opts;
2784
2785	for (i = 0; trace_options[i]; i++) {
2786	if (trace_flags & (1 << i))
2787	seq_printf(m, "%s\n", trace_options[i]);
2788	else
2789	seq_printf(m, "no%s\n", trace_options[i]);
2790	}
2791
2792	for (i = 0; trace_opts[i].name; i++) {
2793	if (tracer_flags & trace_opts[i].bit)
2794	seq_printf(m, "%s\n", trace_opts[i].name);
2795	else
2796	seq_printf(m, "no%s\n", trace_opts[i].name);
2797	}
2798	mutex_unlock(&trace_types_lock);
2799
2800	return 0;
2801	}
2802
2803	static int __set_tracer_option(struct tracer *trace,
2804	struct tracer_flags *tracer_flags,
2805	struct tracer_opt *opts, int neg)
2806	{
2807	int ret;
2808
2809	ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2810	if (ret)
2811	return ret;
2812
2813	if (neg)
2814	tracer_flags->val &= ~opts->bit;
2815	else
2816	tracer_flags->val \|= opts->bit;
2817	return 0;
2818	}
2819
2820	/* Try to assign a tracer specific option */
2821	static int set_tracer_option(struct tracer trace, char cmp, int neg)
2822	{
2823	struct tracer_flags *tracer_flags = trace->flags;
2824	struct tracer_opt *opts = NULL;
2825	int i;
2826
2827	for (i = 0; tracer_flags->opts[i].name; i++) {
2828	opts = &tracer_flags->opts[i];
2829
2830	if (strcmp(cmp, opts->name) == 0)
2831	return __set_tracer_option(trace, trace->flags,
2832	opts, neg);
2833	}
2834
2835	return -EINVAL;
2836	}
2837
2838	static void set_tracer_flags(unsigned int mask, int enabled)
2839	{
2840	/* do nothing if flag is already set */
2841	if (!!(trace_flags & mask) == !!enabled)
2842	return;
2843
2844	if (enabled)
2845	trace_flags \|= mask;
2846	else
2847	trace_flags &= ~mask;
2848
2849	if (mask == TRACE_ITER_RECORD_CMD)
2850	trace_event_enable_cmd_record(enabled);
2851
2852	if (mask == TRACE_ITER_OVERWRITE)
2853	ring_buffer_change_overwrite(global_trace.buffer, enabled);
2854
2855	if (mask == TRACE_ITER_PRINTK)
2856	trace_printk_start_stop_comm(enabled);
2857	}
2858
2859	static int trace_set_options(char *option)
2860	{
2861	char *cmp;
2862	int neg = 0;
2863	int ret = 0;
2864	int i;
2865
2866	cmp = strstrip(option);
2867
2868	if (strncmp(cmp, "no", 2) == 0) {
2869	neg = 1;
2870	cmp += 2;
2871	}
2872
2873	for (i = 0; trace_options[i]; i++) {
2874	if (strcmp(cmp, trace_options[i]) == 0) {
2875	set_tracer_flags(1 << i, !neg);
2876	break;
2877	}
2878	}
2879
2880	/* If no option could be set, test the specific tracer options */
2881	if (!trace_options[i]) {
2882	mutex_lock(&trace_types_lock);
2883	ret = set_tracer_option(current_trace, cmp, neg);
2884	mutex_unlock(&trace_types_lock);
2885	}
2886
2887	return ret;
2888	}
2889
2890	static ssize_t
2891	tracing_trace_options_write(struct file filp, const char __user ubuf,
2892	size_t cnt, loff_t *ppos)
2893	{
2894	char buf[64];
2895
2896	if (cnt >= sizeof(buf))
2897	return -EINVAL;
2898
2899	if (copy_from_user(&buf, ubuf, cnt))
2900	return -EFAULT;
2901
2902	buf[cnt] = 0;
2903
2904	trace_set_options(buf);
2905
2906	*ppos += cnt;
2907
2908	return cnt;
2909	}
2910
2911	static int tracing_trace_options_open(struct inode inode, struct file file)
2912	{
2913	if (tracing_disabled)
2914	return -ENODEV;
2915	return single_open(file, tracing_trace_options_show, NULL);
2916	}
2917
2918	static const struct file_operations tracing_iter_fops = {
2919	.open = tracing_trace_options_open,
2920	.read = seq_read,
2921	.llseek = seq_lseek,
2922	.release = single_release,
2923	.write = tracing_trace_options_write,
2924	};
2925
2926	static const char readme_msg[] =
2927	"tracing mini-HOWTO:\n\n"
2928	"# mount -t debugfs nodev /sys/kernel/debug\n\n"
2929	"# cat /sys/kernel/debug/tracing/available_tracers\n"
2930	"wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
2931	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2932	"nop\n"
2933	"# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
2934	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2935	"wakeup\n"
2936	"# cat /sys/kernel/debug/tracing/trace_options\n"
2937	"noprint-parent nosym-offset nosym-addr noverbose\n"
2938	"# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2939	"# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2940	"# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2941	"# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
2942	;
2943
2944	static ssize_t
2945	tracing_readme_read(struct file filp, char __user ubuf,
2946	size_t cnt, loff_t *ppos)
2947	{
2948	return simple_read_from_buffer(ubuf, cnt, ppos,
2949	readme_msg, strlen(readme_msg));
2950	}
2951
2952	static const struct file_operations tracing_readme_fops = {
2953	.open = tracing_open_generic,
2954	.read = tracing_readme_read,
2955	.llseek = generic_file_llseek,
2956	};
2957
2958	static ssize_t
2959	tracing_saved_cmdlines_read(struct file file, char __user ubuf,
2960	size_t cnt, loff_t *ppos)
2961	{
2962	char *buf_comm;
2963	char *file_buf;
2964	char *buf;
2965	int len = 0;
2966	int pid;
2967	int i;
2968
2969	file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2970	if (!file_buf)
2971	return -ENOMEM;
2972
2973	buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2974	if (!buf_comm) {
2975	kfree(file_buf);
2976	return -ENOMEM;
2977	}
2978
2979	buf = file_buf;
2980
2981	for (i = 0; i < SAVED_CMDLINES; i++) {
2982	int r;
2983
2984	pid = map_cmdline_to_pid[i];
2985	if (pid == -1 \|\| pid == NO_CMDLINE_MAP)
2986	continue;
2987
2988	trace_find_cmdline(pid, buf_comm);
2989	r = sprintf(buf, "%d %s\n", pid, buf_comm);
2990	buf += r;
2991	len += r;
2992	}
2993
2994	len = simple_read_from_buffer(ubuf, cnt, ppos,
2995	file_buf, len);
2996
2997	kfree(file_buf);
2998	kfree(buf_comm);
2999
3000	return len;
3001	}
3002
3003	static const struct file_operations tracing_saved_cmdlines_fops = {
3004	.open = tracing_open_generic,
3005	.read = tracing_saved_cmdlines_read,
3006	.llseek = generic_file_llseek,
3007	};
3008
3009	static ssize_t
3010	tracing_set_trace_read(struct file filp, char __user ubuf,
3011	size_t cnt, loff_t *ppos)
3012	{
3013	char buf[MAX_TRACER_SIZE+2];
3014	int r;
3015
3016	mutex_lock(&trace_types_lock);
3017	if (current_trace)
3018	r = sprintf(buf, "%s\n", current_trace->name);
3019	else
3020	r = sprintf(buf, "\n");
3021	mutex_unlock(&trace_types_lock);
3022
3023	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3024	}
3025
3026	int tracer_init(struct tracer t, struct trace_array tr)
3027	{
3028	tracing_reset_online_cpus(tr);
3029	return t->init(tr);
3030	}
3031
3032	static void set_buffer_entries(struct trace_array *tr, unsigned long val)
3033	{
3034	int cpu;
3035	for_each_tracing_cpu(cpu)
3036	tr->data[cpu]->entries = val;
3037	}
3038
3039	/* resize @tr's buffer to the size of @size_tr's entries */
3040	static int resize_buffer_duplicate_size(struct trace_array *tr,
3041	struct trace_array *size_tr, int cpu_id)
3042	{
3043	int cpu, ret = 0;
3044
3045	if (cpu_id == RING_BUFFER_ALL_CPUS) {
3046	for_each_tracing_cpu(cpu) {
3047	ret = ring_buffer_resize(tr->buffer,
3048	size_tr->data[cpu]->entries, cpu);
3049	if (ret < 0)
3050	break;
3051	tr->data[cpu]->entries = size_tr->data[cpu]->entries;
3052	}
3053	} else {
3054	ret = ring_buffer_resize(tr->buffer,
3055	size_tr->data[cpu_id]->entries, cpu_id);
3056	if (ret == 0)
3057	tr->data[cpu_id]->entries =
3058	size_tr->data[cpu_id]->entries;
3059	}
3060
3061	return ret;
3062	}
3063
3064	static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
3065	{
3066	int ret;
3067
3068	/*
3069	* If kernel or user changes the size of the ring buffer
3070	* we use the size that was given, and we can forget about
3071	* expanding it later.
3072	*/
3073	ring_buffer_expanded = 1;
3074
3075	/* May be called before buffers are initialized */
3076	if (!global_trace.buffer)
3077	return 0;
3078
3079	ret = ring_buffer_resize(global_trace.buffer, size, cpu);
3080	if (ret < 0)
3081	return ret;
3082
3083	if (!current_trace->use_max_tr)
3084	goto out;
3085
3086	ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3087	if (ret < 0) {
3088	int r = resize_buffer_duplicate_size(&global_trace,
3089	&global_trace, cpu);
3090	if (r < 0) {
3091	/*
3092	* AARGH! We are left with different
3093	* size max buffer!!!!
3094	* The max buffer is our "snapshot" buffer.
3095	* When a tracer needs a snapshot (one of the
3096	* latency tracers), it swaps the max buffer
3097	* with the saved snap shot. We succeeded to
3098	* update the size of the main buffer, but failed to
3099	* update the size of the max buffer. But when we tried
3100	* to reset the main buffer to the original size, we
3101	* failed there too. This is very unlikely to
3102	* happen, but if it does, warn and kill all
3103	* tracing.
3104	*/
3105	WARN_ON(1);
3106	tracing_disabled = 1;
3107	}
3108	return ret;
3109	}
3110
3111	if (cpu == RING_BUFFER_ALL_CPUS)
3112	set_buffer_entries(&max_tr, size);
3113	else
3114	max_tr.data[cpu]->entries = size;
3115
3116	out:
3117	if (cpu == RING_BUFFER_ALL_CPUS)
3118	set_buffer_entries(&global_trace, size);
3119	else
3120	global_trace.data[cpu]->entries = size;
3121
3122	return ret;
3123	}
3124
3125	static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3126	{
3127	int ret = size;
3128
3129	mutex_lock(&trace_types_lock);
3130
3131	if (cpu_id != RING_BUFFER_ALL_CPUS) {
3132	/* make sure, this cpu is enabled in the mask */
3133	if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3134	ret = -EINVAL;
3135	goto out;
3136	}
3137	}
3138
3139	ret = __tracing_resize_ring_buffer(size, cpu_id);
3140	if (ret < 0)
3141	ret = -ENOMEM;
3142
3143	out:
3144	mutex_unlock(&trace_types_lock);
3145
3146	return ret;
3147	}
3148
3149
3150	/**
3151	* tracing_update_buffers - used by tracing facility to expand ring buffers
3152	*
3153	* To save on memory when the tracing is never used on a system with it
3154	* configured in. The ring buffers are set to a minimum size. But once
3155	* a user starts to use the tracing facility, then they need to grow
3156	* to their default size.
3157	*
3158	* This function is to be called when a tracer is about to be used.
3159	*/
3160	int tracing_update_buffers(void)
3161	{
3162	int ret = 0;
3163
3164	mutex_lock(&trace_types_lock);
3165	if (!ring_buffer_expanded)
3166	ret = __tracing_resize_ring_buffer(trace_buf_size,
3167	RING_BUFFER_ALL_CPUS);
3168	mutex_unlock(&trace_types_lock);
3169
3170	return ret;
3171	}
3172
3173	struct trace_option_dentry;
3174
3175	static struct trace_option_dentry *
3176	create_trace_option_files(struct tracer *tracer);
3177
3178	static void
3179	destroy_trace_option_files(struct trace_option_dentry *topts);
3180
3181	static int tracing_set_tracer(const char *buf)
3182	{
3183	static struct trace_option_dentry *topts;
3184	struct trace_array *tr = &global_trace;
3185	struct tracer *t;
3186	int ret = 0;
3187
3188	mutex_lock(&trace_types_lock);
3189
3190	if (!ring_buffer_expanded) {
3191	ret = __tracing_resize_ring_buffer(trace_buf_size,
3192	RING_BUFFER_ALL_CPUS);
3193	if (ret < 0)
3194	goto out;
3195	ret = 0;
3196	}
3197
3198	for (t = trace_types; t; t = t->next) {
3199	if (strcmp(t->name, buf) == 0)
3200	break;
3201	}
3202	if (!t) {
3203	ret = -EINVAL;
3204	goto out;
3205	}
3206	if (t == current_trace)
3207	goto out;
3208
3209	trace_branch_disable();
3210	if (current_trace && current_trace->reset)
3211	current_trace->reset(tr);
3212	if (current_trace && current_trace->use_max_tr) {
3213	/*
3214	* We don't free the ring buffer. instead, resize it because
3215	* The max_tr ring buffer has some state (e.g. ring->clock) and
3216	* we want preserve it.
3217	*/
3218	ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3219	set_buffer_entries(&max_tr, 1);
3220	}
3221	destroy_trace_option_files(topts);
3222
3223	current_trace = &nop_trace;
3224
3225	topts = create_trace_option_files(t);
3226	if (t->use_max_tr) {
3227	/* we need to make per cpu buffer sizes equivalent */
3228	ret = resize_buffer_duplicate_size(&max_tr, &global_trace,
3229	RING_BUFFER_ALL_CPUS);
3230	if (ret < 0)
3231	goto out;
3232	}
3233
3234	if (t->init) {
3235	ret = tracer_init(t, tr);
3236	if (ret)
3237	goto out;
3238	}
3239
3240	current_trace = t;
3241	trace_branch_enable(tr);
3242	out:
3243	mutex_unlock(&trace_types_lock);
3244
3245	return ret;
3246	}
3247
3248	static ssize_t
3249	tracing_set_trace_write(struct file filp, const char __user ubuf,
3250	size_t cnt, loff_t *ppos)
3251	{
3252	char buf[MAX_TRACER_SIZE+1];
3253	int i;
3254	size_t ret;
3255	int err;
3256
3257	ret = cnt;
3258
3259	if (cnt > MAX_TRACER_SIZE)
3260	cnt = MAX_TRACER_SIZE;
3261
3262	if (copy_from_user(&buf, ubuf, cnt))
3263	return -EFAULT;
3264
3265	buf[cnt] = 0;
3266
3267	/* strip ending whitespace. */
3268	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3269	buf[i] = 0;
3270
3271	err = tracing_set_tracer(buf);
3272	if (err)
3273	return err;
3274
3275	*ppos += ret;
3276
3277	return ret;
3278	}
3279
3280	static ssize_t
3281	tracing_max_lat_read(struct file filp, char __user ubuf,
3282	size_t cnt, loff_t *ppos)
3283	{
3284	unsigned long *ptr = filp->private_data;
3285	char buf[64];
3286	int r;
3287
3288	r = snprintf(buf, sizeof(buf), "%ld\n",
3289	ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(ptr));
3290	if (r > sizeof(buf))
3291	r = sizeof(buf);
3292	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3293	}
3294
3295	static ssize_t
3296	tracing_max_lat_write(struct file filp, const char __user ubuf,
3297	size_t cnt, loff_t *ppos)
3298	{
3299	unsigned long *ptr = filp->private_data;
3300	unsigned long val;
3301	int ret;
3302
3303	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3304	if (ret)
3305	return ret;
3306
3307	ptr = val 1000;
3308
3309	return cnt;
3310	}
3311
3312	static int tracing_open_pipe(struct inode inode, struct file filp)
3313	{
3314	long cpu_file = (long) inode->i_private;
3315	struct trace_iterator *iter;
3316	int ret = 0;
3317
3318	if (tracing_disabled)
3319	return -ENODEV;
3320
3321	mutex_lock(&trace_types_lock);
3322
3323	/* create a buffer to store the information to pass to userspace */
3324	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3325	if (!iter) {
3326	ret = -ENOMEM;
3327	goto out;
3328	}
3329
3330	/*
3331	* We make a copy of the current tracer to avoid concurrent
3332	* changes on it while we are reading.
3333	*/
3334	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3335	if (!iter->trace) {
3336	ret = -ENOMEM;
3337	goto fail;
3338	}
3339	if (current_trace)
3340	iter->trace = current_trace;
3341
3342	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3343	ret = -ENOMEM;
3344	goto fail;
3345	}
3346
3347	/* trace pipe does not show start of buffer */
3348	cpumask_setall(iter->started);
3349
3350	if (trace_flags & TRACE_ITER_LATENCY_FMT)
3351	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
3352
3353	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
3354	if (trace_clocks[trace_clock_id].in_ns)
3355	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
3356
3357	iter->cpu_file = cpu_file;
3358	iter->tr = &global_trace;
3359	mutex_init(&iter->mutex);
3360	filp->private_data = iter;
3361
3362	if (iter->trace->pipe_open)
3363	iter->trace->pipe_open(iter);
3364
3365	nonseekable_open(inode, filp);
3366	out:
3367	mutex_unlock(&trace_types_lock);
3368	return ret;
3369
3370	fail:
3371	kfree(iter->trace);
3372	kfree(iter);
3373	mutex_unlock(&trace_types_lock);
3374	return ret;
3375	}
3376
3377	static int tracing_release_pipe(struct inode inode, struct file file)
3378	{
3379	struct trace_iterator *iter = file->private_data;
3380
3381	mutex_lock(&trace_types_lock);
3382
3383	if (iter->trace->pipe_close)
3384	iter->trace->pipe_close(iter);
3385
3386	mutex_unlock(&trace_types_lock);
3387
3388	free_cpumask_var(iter->started);
3389	mutex_destroy(&iter->mutex);
3390	kfree(iter->trace);
3391	kfree(iter);
3392
3393	return 0;
3394	}
3395
3396	static unsigned int
3397	tracing_poll_pipe(struct file filp, poll_table poll_table)
3398	{
3399	struct trace_iterator *iter = filp->private_data;
3400
3401	if (trace_flags & TRACE_ITER_BLOCK) {
3402	/*
3403	* Always select as readable when in blocking mode
3404	*/
3405	return POLLIN \| POLLRDNORM;
3406	} else {
3407	if (!trace_empty(iter))
3408	return POLLIN \| POLLRDNORM;
3409	poll_wait(filp, &trace_wait, poll_table);
3410	if (!trace_empty(iter))
3411	return POLLIN \| POLLRDNORM;
3412
3413	return 0;
3414	}
3415	}
3416
3417	/*
3418	* This is a make-shift waitqueue.
3419	* A tracer might use this callback on some rare cases:
3420	*
3421	* 1) the current tracer might hold the runqueue lock when it wakes up
3422	* a reader, hence a deadlock (sched, function, and function graph tracers)
3423	* 2) the function tracers, trace all functions, we don't want
3424	* the overhead of calling wake_up and friends
3425	* (and tracing them too)
3426	*
3427	* Anyway, this is really very primitive wakeup.
3428	*/
3429	void poll_wait_pipe(struct trace_iterator *iter)
3430	{
3431	set_current_state(TASK_INTERRUPTIBLE);
3432	/* sleep for 100 msecs, and try again. */
3433	schedule_timeout(HZ / 10);
3434	}
3435
3436	/* Must be called with trace_types_lock mutex held. */
3437	static int tracing_wait_pipe(struct file *filp)
3438	{
3439	struct trace_iterator *iter = filp->private_data;
3440
3441	while (trace_empty(iter)) {
3442
3443	if ((filp->f_flags & O_NONBLOCK)) {
3444	return -EAGAIN;
3445	}
3446
3447	mutex_unlock(&iter->mutex);
3448
3449	iter->trace->wait_pipe(iter);
3450
3451	mutex_lock(&iter->mutex);
3452
3453	if (signal_pending(current))
3454	return -EINTR;
3455
3456	/*
3457	* We block until we read something and tracing is disabled.
3458	* We still block if tracing is disabled, but we have never
3459	* read anything. This allows a user to cat this file, and
3460	* then enable tracing. But after we have read something,
3461	* we give an EOF when tracing is again disabled.
3462	*
3463	* iter->pos will be 0 if we haven't read anything.
3464	*/
3465	if (!tracing_is_enabled() && iter->pos)
3466	break;
3467	}
3468
3469	return 1;
3470	}
3471
3472	/*
3473	* Consumer reader.
3474	*/
3475	static ssize_t
3476	tracing_read_pipe(struct file filp, char __user ubuf,
3477	size_t cnt, loff_t *ppos)
3478	{
3479	struct trace_iterator *iter = filp->private_data;
3480	static struct tracer *old_tracer;
3481	ssize_t sret;
3482
3483	/* return any leftover data */
3484	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3485	if (sret != -EBUSY)
3486	return sret;
3487
3488	trace_seq_init(&iter->seq);
3489
3490	/* copy the tracer to avoid using a global lock all around */
3491	mutex_lock(&trace_types_lock);
3492	if (unlikely(old_tracer != current_trace && current_trace)) {
3493	old_tracer = current_trace;
3494	iter->trace = current_trace;
3495	}
3496	mutex_unlock(&trace_types_lock);
3497
3498	/*
3499	* Avoid more than one consumer on a single file descriptor
3500	* This is just a matter of traces coherency, the ring buffer itself
3501	* is protected.
3502	*/
3503	mutex_lock(&iter->mutex);
3504	if (iter->trace->read) {
3505	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3506	if (sret)
3507	goto out;
3508	}
3509
3510	waitagain:
3511	sret = tracing_wait_pipe(filp);
3512	if (sret <= 0)
3513	goto out;
3514
3515	/* stop when tracing is finished */
3516	if (trace_empty(iter)) {
3517	sret = 0;
3518	goto out;
3519	}
3520
3521	if (cnt >= PAGE_SIZE)
3522	cnt = PAGE_SIZE - 1;
3523
3524	/* reset all but tr, trace, and overruns */
3525	memset(&iter->seq, 0,
3526	sizeof(struct trace_iterator) -
3527	offsetof(struct trace_iterator, seq));
3528	iter->pos = -1;
3529
3530	trace_event_read_lock();
3531	trace_access_lock(iter->cpu_file);
3532	while (trace_find_next_entry_inc(iter) != NULL) {
3533	enum print_line_t ret;
3534	int len = iter->seq.len;
3535
3536	ret = print_trace_line(iter);
3537	if (ret == TRACE_TYPE_PARTIAL_LINE) {
3538	/* don't print partial lines */
3539	iter->seq.len = len;
3540	break;
3541	}
3542	if (ret != TRACE_TYPE_NO_CONSUME)
3543	trace_consume(iter);
3544
3545	if (iter->seq.len >= cnt)
3546	break;
3547
3548	/*
3549	* Setting the full flag means we reached the trace_seq buffer
3550	* size and we should leave by partial output condition above.
3551	* One of the trace_seq_* functions is not used properly.
3552	*/
3553	WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3554	iter->ent->type);
3555	}
3556	trace_access_unlock(iter->cpu_file);
3557	trace_event_read_unlock();
3558
3559	/* Now copy what we have to the user */
3560	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3561	if (iter->seq.readpos >= iter->seq.len)
3562	trace_seq_init(&iter->seq);
3563
3564	/*
3565	* If there was nothing to send to user, in spite of consuming trace
3566	* entries, go back to wait for more entries.
3567	*/
3568	if (sret == -EBUSY)
3569	goto waitagain;
3570
3571	out:
3572	mutex_unlock(&iter->mutex);
3573
3574	return sret;
3575	}
3576
3577	static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3578	struct pipe_buffer *buf)
3579	{
3580	__free_page(buf->page);
3581	}
3582
3583	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3584	unsigned int idx)
3585	{
3586	__free_page(spd->pages[idx]);
3587	}
3588
3589	static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3590	.can_merge = 0,
3591	.map = generic_pipe_buf_map,
3592	.unmap = generic_pipe_buf_unmap,
3593	.confirm = generic_pipe_buf_confirm,
3594	.release = tracing_pipe_buf_release,
3595	.steal = generic_pipe_buf_steal,
3596	.get = generic_pipe_buf_get,
3597	};
3598
3599	static size_t
3600	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3601	{
3602	size_t count;
3603	int ret;
3604
3605	/* Seq buffer is page-sized, exactly what we need. */
3606	for (;;) {
3607	count = iter->seq.len;
3608	ret = print_trace_line(iter);
3609	count = iter->seq.len - count;
3610	if (rem < count) {
3611	rem = 0;
3612	iter->seq.len -= count;
3613	break;
3614	}
3615	if (ret == TRACE_TYPE_PARTIAL_LINE) {
3616	iter->seq.len -= count;
3617	break;
3618	}
3619
3620	if (ret != TRACE_TYPE_NO_CONSUME)
3621	trace_consume(iter);
3622	rem -= count;
3623	if (!trace_find_next_entry_inc(iter)) {
3624	rem = 0;
3625	iter->ent = NULL;
3626	break;
3627	}
3628	}
3629
3630	return rem;
3631	}
3632
3633	static ssize_t tracing_splice_read_pipe(struct file *filp,
3634	loff_t *ppos,
3635	struct pipe_inode_info *pipe,
3636	size_t len,
3637	unsigned int flags)
3638	{
3639	struct page *pages_def[PIPE_DEF_BUFFERS];
3640	struct partial_page partial_def[PIPE_DEF_BUFFERS];
3641	struct trace_iterator *iter = filp->private_data;
3642	struct splice_pipe_desc spd = {
3643	.pages = pages_def,
3644	.partial = partial_def,
3645	.nr_pages = 0, /* This gets updated below. */
3646	.nr_pages_max = PIPE_DEF_BUFFERS,
3647	.flags = flags,
3648	.ops = &tracing_pipe_buf_ops,
3649	.spd_release = tracing_spd_release_pipe,
3650	};
3651	static struct tracer *old_tracer;
3652	ssize_t ret;
3653	size_t rem;
3654	unsigned int i;
3655
3656	if (splice_grow_spd(pipe, &spd))
3657	return -ENOMEM;
3658
3659	/* copy the tracer to avoid using a global lock all around */
3660	mutex_lock(&trace_types_lock);
3661	if (unlikely(old_tracer != current_trace && current_trace)) {
3662	old_tracer = current_trace;
3663	iter->trace = current_trace;
3664	}
3665	mutex_unlock(&trace_types_lock);
3666
3667	mutex_lock(&iter->mutex);
3668
3669	if (iter->trace->splice_read) {
3670	ret = iter->trace->splice_read(iter, filp,
3671	ppos, pipe, len, flags);
3672	if (ret)
3673	goto out_err;
3674	}
3675
3676	ret = tracing_wait_pipe(filp);
3677	if (ret <= 0)
3678	goto out_err;
3679
3680	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3681	ret = -EFAULT;
3682	goto out_err;
3683	}
3684
3685	trace_event_read_lock();
3686	trace_access_lock(iter->cpu_file);
3687
3688	/* Fill as many pages as possible. */
3689	for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3690	spd.pages[i] = alloc_page(GFP_KERNEL);
3691	if (!spd.pages[i])
3692	break;
3693
3694	rem = tracing_fill_pipe_page(rem, iter);
3695
3696	/* Copy the data into the page, so we can start over. */
3697	ret = trace_seq_to_buffer(&iter->seq,
3698	page_address(spd.pages[i]),
3699	iter->seq.len);
3700	if (ret < 0) {
3701	__free_page(spd.pages[i]);
3702	break;
3703	}
3704	spd.partial[i].offset = 0;
3705	spd.partial[i].len = iter->seq.len;
3706
3707	trace_seq_init(&iter->seq);
3708	}
3709
3710	trace_access_unlock(iter->cpu_file);
3711	trace_event_read_unlock();
3712	mutex_unlock(&iter->mutex);
3713
3714	spd.nr_pages = i;
3715
3716	ret = splice_to_pipe(pipe, &spd);
3717	out:
3718	splice_shrink_spd(&spd);
3719	return ret;
3720
3721	out_err:
3722	mutex_unlock(&iter->mutex);
3723	goto out;
3724	}
3725
3726	struct ftrace_entries_info {
3727	struct trace_array *tr;
3728	int cpu;
3729	};
3730
3731	static int tracing_entries_open(struct inode inode, struct file filp)
3732	{
3733	struct ftrace_entries_info *info;
3734
3735	if (tracing_disabled)
3736	return -ENODEV;
3737
3738	info = kzalloc(sizeof(*info), GFP_KERNEL);
3739	if (!info)
3740	return -ENOMEM;
3741
3742	info->tr = &global_trace;
3743	info->cpu = (unsigned long)inode->i_private;
3744
3745	filp->private_data = info;
3746
3747	return 0;
3748	}
3749
3750	static ssize_t
3751	tracing_entries_read(struct file filp, char __user ubuf,
3752	size_t cnt, loff_t *ppos)
3753	{
3754	struct ftrace_entries_info *info = filp->private_data;
3755	struct trace_array *tr = info->tr;
3756	char buf[64];
3757	int r = 0;
3758	ssize_t ret;
3759
3760	mutex_lock(&trace_types_lock);
3761
3762	if (info->cpu == RING_BUFFER_ALL_CPUS) {
3763	int cpu, buf_size_same;
3764	unsigned long size;
3765
3766	size = 0;
3767	buf_size_same = 1;
3768	/* check if all cpu sizes are same */
3769	for_each_tracing_cpu(cpu) {
3770	/* fill in the size from first enabled cpu */
3771	if (size == 0)
3772	size = tr->data[cpu]->entries;
3773	if (size != tr->data[cpu]->entries) {
3774	buf_size_same = 0;
3775	break;
3776	}
3777	}
3778
3779	if (buf_size_same) {
3780	if (!ring_buffer_expanded)
3781	r = sprintf(buf, "%lu (expanded: %lu)\n",
3782	size >> 10,
3783	trace_buf_size >> 10);
3784	else
3785	r = sprintf(buf, "%lu\n", size >> 10);
3786	} else
3787	r = sprintf(buf, "X\n");
3788	} else
3789	r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3790
3791	mutex_unlock(&trace_types_lock);
3792
3793	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3794	return ret;
3795	}
3796
3797	static ssize_t
3798	tracing_entries_write(struct file filp, const char __user ubuf,
3799	size_t cnt, loff_t *ppos)
3800	{
3801	struct ftrace_entries_info *info = filp->private_data;
3802	unsigned long val;
3803	int ret;
3804
3805	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3806	if (ret)
3807	return ret;
3808
3809	/* must have at least 1 entry */
3810	if (!val)
3811	return -EINVAL;
3812
3813	/* value is in KB */
3814	val <<= 10;
3815
3816	ret = tracing_resize_ring_buffer(val, info->cpu);
3817	if (ret < 0)
3818	return ret;
3819
3820	*ppos += cnt;
3821
3822	return cnt;
3823	}
3824
3825	static int
3826	tracing_entries_release(struct inode inode, struct file filp)
3827	{
3828	struct ftrace_entries_info *info = filp->private_data;
3829
3830	kfree(info);
3831
3832	return 0;
3833	}
3834
3835	static ssize_t
3836	tracing_total_entries_read(struct file filp, char __user ubuf,
3837	size_t cnt, loff_t *ppos)
3838	{
3839	struct trace_array *tr = filp->private_data;
3840	char buf[64];
3841	int r, cpu;
3842	unsigned long size = 0, expanded_size = 0;
3843
3844	mutex_lock(&trace_types_lock);
3845	for_each_tracing_cpu(cpu) {
3846	size += tr->data[cpu]->entries >> 10;
3847	if (!ring_buffer_expanded)
3848	expanded_size += trace_buf_size >> 10;
3849	}
3850	if (ring_buffer_expanded)
3851	r = sprintf(buf, "%lu\n", size);
3852	else
3853	r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3854	mutex_unlock(&trace_types_lock);
3855
3856	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3857	}
3858
3859	static ssize_t
3860	tracing_free_buffer_write(struct file filp, const char __user ubuf,
3861	size_t cnt, loff_t *ppos)
3862	{
3863	/*
3864	* There is no need to read what the user has written, this function
3865	* is just to make sure that there is no error when "echo" is used
3866	*/
3867
3868	*ppos += cnt;
3869
3870	return cnt;
3871	}
3872
3873	static int
3874	tracing_free_buffer_release(struct inode inode, struct file filp)
3875	{
3876	/* disable tracing ? */
3877	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3878	tracing_off();
3879	/* resize the ring buffer to 0 */
3880	tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
3881
3882	return 0;
3883	}
3884
3885	static ssize_t
3886	tracing_mark_write(struct file filp, const char __user ubuf,
3887	size_t cnt, loff_t *fpos)
3888	{
3889	unsigned long addr = (unsigned long)ubuf;
3890	struct ring_buffer_event *event;
3891	struct ring_buffer *buffer;
3892	struct print_entry *entry;
3893	unsigned long irq_flags;
3894	struct page *pages[2];
3895	void *map_page[2];
3896	int nr_pages = 1;
3897	ssize_t written;
3898	int offset;
3899	int size;
3900	int len;
3901	int ret;
3902	int i;
3903
3904	if (tracing_disabled)
3905	return -EINVAL;
3906
3907	if (!(trace_flags & TRACE_ITER_MARKERS))
3908	return -EINVAL;
3909
3910	if (cnt > TRACE_BUF_SIZE)
3911	cnt = TRACE_BUF_SIZE;
3912
3913	/*
3914	* Userspace is injecting traces into the kernel trace buffer.
3915	* We want to be as non intrusive as possible.
3916	* To do so, we do not want to allocate any special buffers
3917	* or take any locks, but instead write the userspace data
3918	* straight into the ring buffer.
3919	*
3920	* First we need to pin the userspace buffer into memory,
3921	* which, most likely it is, because it just referenced it.
3922	* But there's no guarantee that it is. By using get_user_pages_fast()
3923	* and kmap_atomic/kunmap_atomic() we can get access to the
3924	* pages directly. We then write the data directly into the
3925	* ring buffer.
3926	*/
3927	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
3928
3929	/* check if we cross pages */
3930	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
3931	nr_pages = 2;
3932
3933	offset = addr & (PAGE_SIZE - 1);
3934	addr &= PAGE_MASK;
3935
3936	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
3937	if (ret < nr_pages) {
3938	while (--ret >= 0)
3939	put_page(pages[ret]);
3940	written = -EFAULT;
3941	goto out;
3942	}
3943
3944	for (i = 0; i < nr_pages; i++)
3945	map_page[i] = kmap_atomic(pages[i]);
3946
3947	local_save_flags(irq_flags);
3948	size = sizeof(entry) + cnt + 2; / possible \n added */
3949	buffer = global_trace.buffer;
3950	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3951	irq_flags, preempt_count());
3952	if (!event) {
3953	/* Ring buffer disabled, return as if not open for write */
3954	written = -EBADF;
3955	goto out_unlock;
3956	}
3957
3958	entry = ring_buffer_event_data(event);
3959	entry->ip = _THIS_IP_;
3960
3961	if (nr_pages == 2) {
3962	len = PAGE_SIZE - offset;
3963	memcpy(&entry->buf, map_page[0] + offset, len);
3964	memcpy(&entry->buf[len], map_page[1], cnt - len);
3965	} else
3966	memcpy(&entry->buf, map_page[0] + offset, cnt);
3967
3968	if (entry->buf[cnt - 1] != '\n') {
3969	entry->buf[cnt] = '\n';
3970	entry->buf[cnt + 1] = '\0';
3971	} else
3972	entry->buf[cnt] = '\0';
3973
3974	__buffer_unlock_commit(buffer, event);
3975
3976	written = cnt;
3977
3978	*fpos += written;
3979
3980	out_unlock:
3981	for (i = 0; i < nr_pages; i++){
3982	kunmap_atomic(map_page[i]);
3983	put_page(pages[i]);
3984	}
3985	out:
3986	return written;
3987	}
3988
3989	static int tracing_clock_show(struct seq_file m, void v)
3990	{
3991	int i;
3992
3993	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3994	seq_printf(m,
3995	"%s%s%s%s", i ? " " : "",
3996	i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3997	i == trace_clock_id ? "]" : "");
3998	seq_putc(m, '\n');
3999
4000	return 0;
4001	}
4002
4003	static ssize_t tracing_clock_write(struct file filp, const char __user ubuf,
4004	size_t cnt, loff_t *fpos)
4005	{
4006	char buf[64];
4007	const char *clockstr;
4008	int i;
4009
4010	if (cnt >= sizeof(buf))
4011	return -EINVAL;
4012
4013	if (copy_from_user(&buf, ubuf, cnt))
4014	return -EFAULT;
4015
4016	buf[cnt] = 0;
4017
4018	clockstr = strstrip(buf);
4019
4020	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4021	if (strcmp(trace_clocks[i].name, clockstr) == 0)
4022	break;
4023	}
4024	if (i == ARRAY_SIZE(trace_clocks))
4025	return -EINVAL;
4026
4027	trace_clock_id = i;
4028
4029	mutex_lock(&trace_types_lock);
4030
4031	ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
4032	if (max_tr.buffer)
4033	ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
4034
4035	/*
4036	* New clock may not be consistent with the previous clock.
4037	* Reset the buffer so that it doesn't have incomparable timestamps.
4038	*/
4039	tracing_reset_online_cpus(&global_trace);
4040	if (max_tr.buffer)
4041	tracing_reset_online_cpus(&max_tr);
4042
4043	mutex_unlock(&trace_types_lock);
4044
4045	*fpos += cnt;
4046
4047	return cnt;
4048	}
4049
4050	static int tracing_clock_open(struct inode inode, struct file file)
4051	{
4052	if (tracing_disabled)
4053	return -ENODEV;
4054	return single_open(file, tracing_clock_show, NULL);
4055	}
4056
4057	static const struct file_operations tracing_max_lat_fops = {
4058	.open = tracing_open_generic,
4059	.read = tracing_max_lat_read,
4060	.write = tracing_max_lat_write,
4061	.llseek = generic_file_llseek,
4062	};
4063
4064	static const struct file_operations set_tracer_fops = {
4065	.open = tracing_open_generic,
4066	.read = tracing_set_trace_read,
4067	.write = tracing_set_trace_write,
4068	.llseek = generic_file_llseek,
4069	};
4070
4071	static const struct file_operations tracing_pipe_fops = {
4072	.open = tracing_open_pipe,
4073	.poll = tracing_poll_pipe,
4074	.read = tracing_read_pipe,
4075	.splice_read = tracing_splice_read_pipe,
4076	.release = tracing_release_pipe,
4077	.llseek = no_llseek,
4078	};
4079
4080	static const struct file_operations tracing_entries_fops = {
4081	.open = tracing_entries_open,
4082	.read = tracing_entries_read,
4083	.write = tracing_entries_write,
4084	.release = tracing_entries_release,
4085	.llseek = generic_file_llseek,
4086	};
4087
4088	static const struct file_operations tracing_total_entries_fops = {
4089	.open = tracing_open_generic,
4090	.read = tracing_total_entries_read,
4091	.llseek = generic_file_llseek,
4092	};
4093
4094	static const struct file_operations tracing_free_buffer_fops = {
4095	.write = tracing_free_buffer_write,
4096	.release = tracing_free_buffer_release,
4097	};
4098
4099	static const struct file_operations tracing_mark_fops = {
4100	.open = tracing_open_generic,
4101	.write = tracing_mark_write,
4102	.llseek = generic_file_llseek,
4103	};
4104
4105	static const struct file_operations trace_clock_fops = {
4106	.open = tracing_clock_open,
4107	.read = seq_read,
4108	.llseek = seq_lseek,
4109	.release = single_release,
4110	.write = tracing_clock_write,
4111	};
4112
4113	struct ftrace_buffer_info {
4114	struct trace_array *tr;
4115	void *spare;
4116	int cpu;
4117	unsigned int read;
4118	};
4119
4120	static int tracing_buffers_open(struct inode inode, struct file filp)
4121	{
4122	int cpu = (int)(long)inode->i_private;
4123	struct ftrace_buffer_info *info;
4124
4125	if (tracing_disabled)
4126	return -ENODEV;
4127
4128	info = kzalloc(sizeof(*info), GFP_KERNEL);
4129	if (!info)
4130	return -ENOMEM;
4131
4132	info->tr = &global_trace;
4133	info->cpu = cpu;
4134	info->spare = NULL;
4135	/* Force reading ring buffer for first read */
4136	info->read = (unsigned int)-1;
4137
4138	filp->private_data = info;
4139
4140	return nonseekable_open(inode, filp);
4141	}
4142
4143	static ssize_t
4144	tracing_buffers_read(struct file filp, char __user ubuf,
4145	size_t count, loff_t *ppos)
4146	{
4147	struct ftrace_buffer_info *info = filp->private_data;
4148	ssize_t ret;
4149	size_t size;
4150
4151	if (!count)
4152	return 0;
4153
4154	if (!info->spare)
4155	info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4156	if (!info->spare)
4157	return -ENOMEM;
4158
4159	/* Do we have previous read data to read? */
4160	if (info->read < PAGE_SIZE)
4161	goto read;
4162
4163	trace_access_lock(info->cpu);
4164	ret = ring_buffer_read_page(info->tr->buffer,
4165	&info->spare,
4166	count,
4167	info->cpu, 0);
4168	trace_access_unlock(info->cpu);
4169	if (ret < 0)
4170	return 0;
4171
4172	info->read = 0;
4173
4174	read:
4175	size = PAGE_SIZE - info->read;
4176	if (size > count)
4177	size = count;
4178
4179	ret = copy_to_user(ubuf, info->spare + info->read, size);
4180	if (ret == size)
4181	return -EFAULT;
4182	size -= ret;
4183
4184	*ppos += size;
4185	info->read += size;
4186
4187	return size;
4188	}
4189
4190	static int tracing_buffers_release(struct inode inode, struct file file)
4191	{
4192	struct ftrace_buffer_info *info = file->private_data;
4193
4194	if (info->spare)
4195	ring_buffer_free_read_page(info->tr->buffer, info->spare);
4196	kfree(info);
4197
4198	return 0;
4199	}
4200
4201	struct buffer_ref {
4202	struct ring_buffer *buffer;
4203	void *page;
4204	int ref;
4205	};
4206
4207	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4208	struct pipe_buffer *buf)
4209	{
4210	struct buffer_ref ref = (struct buffer_ref )buf->private;
4211
4212	if (--ref->ref)
4213	return;
4214
4215	ring_buffer_free_read_page(ref->buffer, ref->page);
4216	kfree(ref);
4217	buf->private = 0;
4218	}
4219
4220	static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4221	struct pipe_buffer *buf)
4222	{
4223	struct buffer_ref ref = (struct buffer_ref )buf->private;
4224
4225	ref->ref++;
4226	}
4227
4228	/* Pipe buffer operations for a buffer. */
4229	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4230	.can_merge = 0,
4231	.map = generic_pipe_buf_map,
4232	.unmap = generic_pipe_buf_unmap,
4233	.confirm = generic_pipe_buf_confirm,
4234	.release = buffer_pipe_buf_release,
4235	.steal = generic_pipe_buf_steal,
4236	.get = buffer_pipe_buf_get,
4237	};
4238
4239	/*
4240	* Callback from splice_to_pipe(), if we need to release some pages
4241	* at the end of the spd in case we error'ed out in filling the pipe.
4242	*/
4243	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4244	{
4245	struct buffer_ref *ref =
4246	(struct buffer_ref *)spd->partial[i].private;
4247
4248	if (--ref->ref)
4249	return;
4250
4251	ring_buffer_free_read_page(ref->buffer, ref->page);
4252	kfree(ref);
4253	spd->partial[i].private = 0;
4254	}
4255
4256	static ssize_t
4257	tracing_buffers_splice_read(struct file file, loff_t ppos,
4258	struct pipe_inode_info *pipe, size_t len,
4259	unsigned int flags)
4260	{
4261	struct ftrace_buffer_info *info = file->private_data;
4262	struct partial_page partial_def[PIPE_DEF_BUFFERS];
4263	struct page *pages_def[PIPE_DEF_BUFFERS];
4264	struct splice_pipe_desc spd = {
4265	.pages = pages_def,
4266	.partial = partial_def,
4267	.nr_pages_max = PIPE_DEF_BUFFERS,
4268	.flags = flags,
4269	.ops = &buffer_pipe_buf_ops,
4270	.spd_release = buffer_spd_release,
4271	};
4272	struct buffer_ref *ref;
4273	int entries, size, i;
4274	size_t ret;
4275
4276	if (splice_grow_spd(pipe, &spd))
4277	return -ENOMEM;
4278
4279	if (*ppos & (PAGE_SIZE - 1)) {
4280	ret = -EINVAL;
4281	goto out;
4282	}
4283
4284	if (len & (PAGE_SIZE - 1)) {
4285	if (len < PAGE_SIZE) {
4286	ret = -EINVAL;
4287	goto out;
4288	}
4289	len &= PAGE_MASK;
4290	}
4291
4292	trace_access_lock(info->cpu);
4293	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4294
4295	for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4296	struct page *page;
4297	int r;
4298
4299	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4300	if (!ref)
4301	break;
4302
4303	ref->ref = 1;
4304	ref->buffer = info->tr->buffer;
4305	ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4306	if (!ref->page) {
4307	kfree(ref);
4308	break;
4309	}
4310
4311	r = ring_buffer_read_page(ref->buffer, &ref->page,
4312	len, info->cpu, 1);
4313	if (r < 0) {
4314	ring_buffer_free_read_page(ref->buffer, ref->page);
4315	kfree(ref);
4316	break;
4317	}
4318
4319	/*
4320	* zero out any left over data, this is going to
4321	* user land.
4322	*/
4323	size = ring_buffer_page_len(ref->page);
4324	if (size < PAGE_SIZE)
4325	memset(ref->page + size, 0, PAGE_SIZE - size);
4326
4327	page = virt_to_page(ref->page);
4328
4329	spd.pages[i] = page;
4330	spd.partial[i].len = PAGE_SIZE;
4331	spd.partial[i].offset = 0;
4332	spd.partial[i].private = (unsigned long)ref;
4333	spd.nr_pages++;
4334	*ppos += PAGE_SIZE;
4335
4336	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4337	}
4338
4339	trace_access_unlock(info->cpu);
4340	spd.nr_pages = i;
4341
4342	/* did we read anything? */
4343	if (!spd.nr_pages) {
4344	if (flags & SPLICE_F_NONBLOCK)
4345	ret = -EAGAIN;
4346	else
4347	ret = 0;
4348	/* TODO: block */
4349	goto out;
4350	}
4351
4352	ret = splice_to_pipe(pipe, &spd);
4353	splice_shrink_spd(&spd);
4354	out:
4355	return ret;
4356	}
4357
4358	static const struct file_operations tracing_buffers_fops = {
4359	.open = tracing_buffers_open,
4360	.read = tracing_buffers_read,
4361	.release = tracing_buffers_release,
4362	.splice_read = tracing_buffers_splice_read,
4363	.llseek = no_llseek,
4364	};
4365
4366	static ssize_t
4367	tracing_stats_read(struct file filp, char __user ubuf,
4368	size_t count, loff_t *ppos)
4369	{
4370	unsigned long cpu = (unsigned long)filp->private_data;
4371	struct trace_array *tr = &global_trace;
4372	struct trace_seq *s;
4373	unsigned long cnt;
4374	unsigned long long t;
4375	unsigned long usec_rem;
4376
4377	s = kmalloc(sizeof(*s), GFP_KERNEL);
4378	if (!s)
4379	return -ENOMEM;
4380
4381	trace_seq_init(s);
4382
4383	cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4384	trace_seq_printf(s, "entries: %ld\n", cnt);
4385
4386	cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4387	trace_seq_printf(s, "overrun: %ld\n", cnt);
4388
4389	cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4390	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4391
4392	cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4393	trace_seq_printf(s, "bytes: %ld\n", cnt);
4394
4395	if (trace_clocks[trace_clock_id].in_ns) {
4396	/* local or global for trace_clock */
4397	t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4398	usec_rem = do_div(t, USEC_PER_SEC);
4399	trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
4400	t, usec_rem);
4401
4402	t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4403	usec_rem = do_div(t, USEC_PER_SEC);
4404	trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4405	} else {
4406	/* counter or tsc mode for trace_clock */
4407	trace_seq_printf(s, "oldest event ts: %llu\n",
4408	ring_buffer_oldest_event_ts(tr->buffer, cpu));
4409
4410	trace_seq_printf(s, "now ts: %llu\n",
4411	ring_buffer_time_stamp(tr->buffer, cpu));
4412	}
4413
4414	cnt = ring_buffer_dropped_events_cpu(tr->buffer, cpu);
4415	trace_seq_printf(s, "dropped events: %ld\n", cnt);
4416
4417	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4418
4419	kfree(s);
4420
4421	return count;
4422	}
4423
4424	static const struct file_operations tracing_stats_fops = {
4425	.open = tracing_open_generic,
4426	.read = tracing_stats_read,
4427	.llseek = generic_file_llseek,
4428	};
4429
4430	#ifdef CONFIG_DYNAMIC_FTRACE
4431
4432	int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4433	{
4434	return 0;
4435	}
4436
4437	static ssize_t
4438	tracing_read_dyn_info(struct file filp, char __user ubuf,
4439	size_t cnt, loff_t *ppos)
4440	{
4441	static char ftrace_dyn_info_buffer[1024];
4442	static DEFINE_MUTEX(dyn_info_mutex);
4443	unsigned long *p = filp->private_data;
4444	char *buf = ftrace_dyn_info_buffer;
4445	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4446	int r;
4447
4448	mutex_lock(&dyn_info_mutex);
4449	r = sprintf(buf, "%ld ", *p);
4450
4451	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
4452	buf[r++] = '\n';
4453
4454	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4455
4456	mutex_unlock(&dyn_info_mutex);
4457
4458	return r;
4459	}
4460
4461	static const struct file_operations tracing_dyn_info_fops = {
4462	.open = tracing_open_generic,
4463	.read = tracing_read_dyn_info,
4464	.llseek = generic_file_llseek,
4465	};
4466	#endif
4467
4468	static struct dentry *d_tracer;
4469
4470	struct dentry *tracing_init_dentry(void)
4471	{
4472	static int once;
4473
4474	if (d_tracer)
4475	return d_tracer;
4476
4477	if (!debugfs_initialized())
4478	return NULL;
4479
4480	d_tracer = debugfs_create_dir("tracing", NULL);
4481
4482	if (!d_tracer && !once) {
4483	once = 1;
4484	pr_warning("Could not create debugfs directory 'tracing'\n");
4485	return NULL;
4486	}
4487
4488	return d_tracer;
4489	}
4490
4491	static struct dentry *d_percpu;
4492
4493	struct dentry *tracing_dentry_percpu(void)
4494	{
4495	static int once;
4496	struct dentry *d_tracer;
4497
4498	if (d_percpu)
4499	return d_percpu;
4500
4501	d_tracer = tracing_init_dentry();
4502
4503	if (!d_tracer)
4504	return NULL;
4505
4506	d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4507
4508	if (!d_percpu && !once) {
4509	once = 1;
4510	pr_warning("Could not create debugfs directory 'per_cpu'\n");
4511	return NULL;
4512	}
4513
4514	return d_percpu;
4515	}
4516
4517	static void tracing_init_debugfs_percpu(long cpu)
4518	{
4519	struct dentry *d_percpu = tracing_dentry_percpu();
4520	struct dentry *d_cpu;
4521	char cpu_dir[30]; /* 30 characters should be more than enough */
4522
4523	if (!d_percpu)
4524	return;
4525
4526	snprintf(cpu_dir, 30, "cpu%ld", cpu);
4527	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4528	if (!d_cpu) {
4529	pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4530	return;
4531	}
4532
4533	/* per cpu trace_pipe */
4534	trace_create_file("trace_pipe", 0444, d_cpu,
4535	(void *) cpu, &tracing_pipe_fops);
4536
4537	/* per cpu trace */
4538	trace_create_file("trace", 0644, d_cpu,
4539	(void *) cpu, &tracing_fops);
4540
4541	trace_create_file("trace_pipe_raw", 0444, d_cpu,
4542	(void *) cpu, &tracing_buffers_fops);
4543
4544	trace_create_file("stats", 0444, d_cpu,
4545	(void *) cpu, &tracing_stats_fops);
4546
4547	trace_create_file("buffer_size_kb", 0444, d_cpu,
4548	(void *) cpu, &tracing_entries_fops);
4549	}
4550
4551	#ifdef CONFIG_FTRACE_SELFTEST
4552	/* Let selftest have access to static functions in this file */
4553	#include "trace_selftest.c"
4554	#endif
4555
4556	struct trace_option_dentry {
4557	struct tracer_opt *opt;
4558	struct tracer_flags *flags;
4559	struct dentry *entry;
4560	};
4561
4562	static ssize_t
4563	trace_options_read(struct file filp, char __user ubuf, size_t cnt,
4564	loff_t *ppos)
4565	{
4566	struct trace_option_dentry *topt = filp->private_data;
4567	char *buf;
4568
4569	if (topt->flags->val & topt->opt->bit)
4570	buf = "1\n";
4571	else
4572	buf = "0\n";
4573
4574	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4575	}
4576
4577	static ssize_t
4578	trace_options_write(struct file filp, const char __user ubuf, size_t cnt,
4579	loff_t *ppos)
4580	{
4581	struct trace_option_dentry *topt = filp->private_data;
4582	unsigned long val;
4583	int ret;
4584
4585	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4586	if (ret)
4587	return ret;
4588
4589	if (val != 0 && val != 1)
4590	return -EINVAL;
4591
4592	if (!!(topt->flags->val & topt->opt->bit) != val) {
4593	mutex_lock(&trace_types_lock);
4594	ret = __set_tracer_option(current_trace, topt->flags,
4595	topt->opt, !val);
4596	mutex_unlock(&trace_types_lock);
4597	if (ret)
4598	return ret;
4599	}
4600
4601	*ppos += cnt;
4602
4603	return cnt;
4604	}
4605
4606
4607	static const struct file_operations trace_options_fops = {
4608	.open = tracing_open_generic,
4609	.read = trace_options_read,
4610	.write = trace_options_write,
4611	.llseek = generic_file_llseek,
4612	};
4613
4614	static ssize_t
4615	trace_options_core_read(struct file filp, char __user ubuf, size_t cnt,
4616	loff_t *ppos)
4617	{
4618	long index = (long)filp->private_data;
4619	char *buf;
4620
4621	if (trace_flags & (1 << index))
4622	buf = "1\n";
4623	else
4624	buf = "0\n";
4625
4626	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4627	}
4628
4629	static ssize_t
4630	trace_options_core_write(struct file filp, const char __user ubuf, size_t cnt,
4631	loff_t *ppos)
4632	{
4633	long index = (long)filp->private_data;
4634	unsigned long val;
4635	int ret;
4636
4637	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4638	if (ret)
4639	return ret;
4640
4641	if (val != 0 && val != 1)
4642	return -EINVAL;
4643	set_tracer_flags(1 << index, val);
4644
4645	*ppos += cnt;
4646
4647	return cnt;
4648	}
4649
4650	static const struct file_operations trace_options_core_fops = {
4651	.open = tracing_open_generic,
4652	.read = trace_options_core_read,
4653	.write = trace_options_core_write,
4654	.llseek = generic_file_llseek,
4655	};
4656
4657	struct dentry trace_create_file(const char name,
4658	umode_t mode,
4659	struct dentry *parent,
4660	void *data,
4661	const struct file_operations *fops)
4662	{
4663	struct dentry *ret;
4664
4665	ret = debugfs_create_file(name, mode, parent, data, fops);
4666	if (!ret)
4667	pr_warning("Could not create debugfs '%s' entry\n", name);
4668
4669	return ret;
4670	}
4671
4672
4673	static struct dentry *trace_options_init_dentry(void)
4674	{
4675	struct dentry *d_tracer;
4676	static struct dentry *t_options;
4677
4678	if (t_options)
4679	return t_options;
4680
4681	d_tracer = tracing_init_dentry();
4682	if (!d_tracer)
4683	return NULL;
4684
4685	t_options = debugfs_create_dir("options", d_tracer);
4686	if (!t_options) {
4687	pr_warning("Could not create debugfs directory 'options'\n");
4688	return NULL;
4689	}
4690
4691	return t_options;
4692	}
4693
4694	static void
4695	create_trace_option_file(struct trace_option_dentry *topt,
4696	struct tracer_flags *flags,
4697	struct tracer_opt *opt)
4698	{
4699	struct dentry *t_options;
4700
4701	t_options = trace_options_init_dentry();
4702	if (!t_options)
4703	return;
4704
4705	topt->flags = flags;
4706	topt->opt = opt;
4707
4708	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4709	&trace_options_fops);
4710
4711	}
4712
4713	static struct trace_option_dentry *
4714	create_trace_option_files(struct tracer *tracer)
4715	{
4716	struct trace_option_dentry *topts;
4717	struct tracer_flags *flags;
4718	struct tracer_opt *opts;
4719	int cnt;
4720
4721	if (!tracer)
4722	return NULL;
4723
4724	flags = tracer->flags;
4725
4726	if (!flags \|\| !flags->opts)
4727	return NULL;
4728
4729	opts = flags->opts;
4730
4731	for (cnt = 0; opts[cnt].name; cnt++)
4732	;
4733
4734	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4735	if (!topts)
4736	return NULL;
4737
4738	for (cnt = 0; opts[cnt].name; cnt++)
4739	create_trace_option_file(&topts[cnt], flags,
4740	&opts[cnt]);
4741
4742	return topts;
4743	}
4744
4745	static void
4746	destroy_trace_option_files(struct trace_option_dentry *topts)
4747	{
4748	int cnt;
4749
4750	if (!topts)
4751	return;
4752
4753	for (cnt = 0; topts[cnt].opt; cnt++) {
4754	if (topts[cnt].entry)
4755	debugfs_remove(topts[cnt].entry);
4756	}
4757
4758	kfree(topts);
4759	}
4760
4761	static struct dentry *
4762	create_trace_option_core_file(const char *option, long index)
4763	{
4764	struct dentry *t_options;
4765
4766	t_options = trace_options_init_dentry();
4767	if (!t_options)
4768	return NULL;
4769
4770	return trace_create_file(option, 0644, t_options, (void *)index,
4771	&trace_options_core_fops);
4772	}
4773
4774	static __init void create_trace_options_dir(void)
4775	{
4776	struct dentry *t_options;
4777	int i;
4778
4779	t_options = trace_options_init_dentry();
4780	if (!t_options)
4781	return;
4782
4783	for (i = 0; trace_options[i]; i++)
4784	create_trace_option_core_file(trace_options[i], i);
4785	}
4786
4787	static ssize_t
4788	rb_simple_read(struct file filp, char __user ubuf,
4789	size_t cnt, loff_t *ppos)
4790	{
4791	struct trace_array *tr = filp->private_data;
4792	struct ring_buffer *buffer = tr->buffer;
4793	char buf[64];
4794	int r;
4795
4796	if (buffer)
4797	r = ring_buffer_record_is_on(buffer);
4798	else
4799	r = 0;
4800
4801	r = sprintf(buf, "%d\n", r);
4802
4803	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4804	}
4805
4806	static ssize_t
4807	rb_simple_write(struct file filp, const char __user ubuf,
4808	size_t cnt, loff_t *ppos)
4809	{
4810	struct trace_array *tr = filp->private_data;
4811	struct ring_buffer *buffer = tr->buffer;
4812	unsigned long val;
4813	int ret;
4814
4815	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4816	if (ret)
4817	return ret;
4818
4819	if (buffer) {
4820	mutex_lock(&trace_types_lock);
4821	if (val) {
4822	ring_buffer_record_on(buffer);
4823	if (current_trace->start)
4824	current_trace->start(tr);
4825	} else {
4826	ring_buffer_record_off(buffer);
4827	if (current_trace->stop)
4828	current_trace->stop(tr);
4829	}
4830	mutex_unlock(&trace_types_lock);
4831	}
4832
4833	(*ppos)++;
4834
4835	return cnt;
4836	}
4837
4838	static const struct file_operations rb_simple_fops = {
4839	.open = tracing_open_generic,
4840	.read = rb_simple_read,
4841	.write = rb_simple_write,
4842	.llseek = default_llseek,
4843	};
4844
4845	static __init int tracer_init_debugfs(void)
4846	{
4847	struct dentry *d_tracer;
4848	int cpu;
4849
4850	trace_access_lock_init();
4851
4852	d_tracer = tracing_init_dentry();
4853
4854	trace_create_file("trace_options", 0644, d_tracer,
4855	NULL, &tracing_iter_fops);
4856
4857	trace_create_file("tracing_cpumask", 0644, d_tracer,
4858	NULL, &tracing_cpumask_fops);
4859
4860	trace_create_file("trace", 0644, d_tracer,
4861	(void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4862
4863	trace_create_file("available_tracers", 0444, d_tracer,
4864	&global_trace, &show_traces_fops);
4865
4866	trace_create_file("current_tracer", 0644, d_tracer,
4867	&global_trace, &set_tracer_fops);
4868
4869	#ifdef CONFIG_TRACER_MAX_TRACE
4870	trace_create_file("tracing_max_latency", 0644, d_tracer,
4871	&tracing_max_latency, &tracing_max_lat_fops);
4872	#endif
4873
4874	trace_create_file("tracing_thresh", 0644, d_tracer,
4875	&tracing_thresh, &tracing_max_lat_fops);
4876
4877	trace_create_file("README", 0444, d_tracer,
4878	NULL, &tracing_readme_fops);
4879
4880	trace_create_file("trace_pipe", 0444, d_tracer,
4881	(void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4882
4883	trace_create_file("buffer_size_kb", 0644, d_tracer,
4884	(void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
4885
4886	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4887	&global_trace, &tracing_total_entries_fops);
4888
4889	trace_create_file("free_buffer", 0644, d_tracer,
4890	&global_trace, &tracing_free_buffer_fops);
4891
4892	trace_create_file("trace_marker", 0220, d_tracer,
4893	NULL, &tracing_mark_fops);
4894
4895	trace_create_file("saved_cmdlines", 0444, d_tracer,
4896	NULL, &tracing_saved_cmdlines_fops);
4897
4898	trace_create_file("trace_clock", 0644, d_tracer, NULL,
4899	&trace_clock_fops);
4900
4901	trace_create_file("tracing_on", 0644, d_tracer,
4902	&global_trace, &rb_simple_fops);
4903
4904	#ifdef CONFIG_DYNAMIC_FTRACE
4905	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4906	&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4907	#endif
4908
4909	create_trace_options_dir();
4910
4911	for_each_tracing_cpu(cpu)
4912	tracing_init_debugfs_percpu(cpu);
4913
4914	return 0;
4915	}
4916
4917	static int trace_panic_handler(struct notifier_block *this,
4918	unsigned long event, void *unused)
4919	{
4920	if (ftrace_dump_on_oops)
4921	ftrace_dump(ftrace_dump_on_oops);
4922	return NOTIFY_OK;
4923	}
4924
4925	static struct notifier_block trace_panic_notifier = {
4926	.notifier_call = trace_panic_handler,
4927	.next = NULL,
4928	.priority = 150 /* priority: INT_MAX >= x >= 0 */
4929	};
4930
4931	static int trace_die_handler(struct notifier_block *self,
4932	unsigned long val,
4933	void *data)
4934	{
4935	switch (val) {
4936	case DIE_OOPS:
4937	if (ftrace_dump_on_oops)
4938	ftrace_dump(ftrace_dump_on_oops);
4939	break;
4940	default:
4941	break;
4942	}
4943	return NOTIFY_OK;
4944	}
4945
4946	static struct notifier_block trace_die_notifier = {
4947	.notifier_call = trace_die_handler,
4948	.priority = 200
4949	};
4950
4951	/*
4952	* printk is set to max of 1024, we really don't need it that big.
4953	* Nothing should be printing 1000 characters anyway.
4954	*/
4955	#define TRACE_MAX_PRINT 1000
4956
4957	/*
4958	* Define here KERN_TRACE so that we have one place to modify
4959	* it if we decide to change what log level the ftrace dump
4960	* should be at.
4961	*/
4962	#define KERN_TRACE KERN_EMERG
4963
4964	void
4965	trace_printk_seq(struct trace_seq *s)
4966	{
4967	/* Probably should print a warning here. */
4968	if (s->len >= 1000)
4969	s->len = 1000;
4970
4971	/* should be zero ended, but we are paranoid. */
4972	s->buffer[s->len] = 0;
4973
4974	printk(KERN_TRACE "%s", s->buffer);
4975
4976	trace_seq_init(s);
4977	}
4978
4979	void trace_init_global_iter(struct trace_iterator *iter)
4980	{
4981	iter->tr = &global_trace;
4982	iter->trace = current_trace;
4983	iter->cpu_file = TRACE_PIPE_ALL_CPU;
4984	}
4985
4986	static void
4987	__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4988	{
4989	static arch_spinlock_t ftrace_dump_lock =
4990	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4991	/* use static because iter can be a bit big for the stack */
4992	static struct trace_iterator iter;
4993	unsigned int old_userobj;
4994	static int dump_ran;
4995	unsigned long flags;
4996	int cnt = 0, cpu;
4997
4998	/* only one dump */
4999	local_irq_save(flags);
5000	arch_spin_lock(&ftrace_dump_lock);
5001	if (dump_ran)
5002	goto out;
5003
5004	dump_ran = 1;
5005
5006	tracing_off();
5007
5008	/* Did function tracer already get disabled? */
5009	if (ftrace_is_dead()) {
5010	printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
5011	printk("# MAY BE MISSING FUNCTION EVENTS\n");
5012	}
5013
5014	if (disable_tracing)
5015	ftrace_kill();
5016
5017	trace_init_global_iter(&iter);
5018
5019	for_each_tracing_cpu(cpu) {
5020	atomic_inc(&iter.tr->data[cpu]->disabled);
5021	}
5022
5023	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
5024
5025	/* don't look at user memory in panic mode */
5026	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
5027
5028	/* Simulate the iterator */
5029	iter.tr = &global_trace;
5030	iter.trace = current_trace;
5031
5032	switch (oops_dump_mode) {
5033	case DUMP_ALL:
5034	iter.cpu_file = TRACE_PIPE_ALL_CPU;
5035	break;
5036	case DUMP_ORIG:
5037	iter.cpu_file = raw_smp_processor_id();
5038	break;
5039	case DUMP_NONE:
5040	goto out_enable;
5041	default:
5042	printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
5043	iter.cpu_file = TRACE_PIPE_ALL_CPU;
5044	}
5045
5046	printk(KERN_TRACE "Dumping ftrace buffer:\n");
5047
5048	/*
5049	* We need to stop all tracing on all CPUS to read the
5050	* the next buffer. This is a bit expensive, but is
5051	* not done often. We fill all what we can read,
5052	* and then release the locks again.
5053	*/
5054
5055	while (!trace_empty(&iter)) {
5056
5057	if (!cnt)
5058	printk(KERN_TRACE "---------------------------------\n");
5059
5060	cnt++;
5061
5062	/* reset all but tr, trace, and overruns */
5063	memset(&iter.seq, 0,
5064	sizeof(struct trace_iterator) -
5065	offsetof(struct trace_iterator, seq));
5066	iter.iter_flags \|= TRACE_FILE_LAT_FMT;
5067	iter.pos = -1;
5068
5069	if (trace_find_next_entry_inc(&iter) != NULL) {
5070	int ret;
5071
5072	ret = print_trace_line(&iter);
5073	if (ret != TRACE_TYPE_NO_CONSUME)
5074	trace_consume(&iter);
5075	}
5076	touch_nmi_watchdog();
5077
5078	trace_printk_seq(&iter.seq);
5079	}
5080
5081	if (!cnt)
5082	printk(KERN_TRACE " (ftrace buffer empty)\n");
5083	else
5084	printk(KERN_TRACE "---------------------------------\n");
5085
5086	out_enable:
5087	/* Re-enable tracing if requested */
5088	if (!disable_tracing) {
5089	trace_flags \|= old_userobj;
5090
5091	for_each_tracing_cpu(cpu) {
5092	atomic_dec(&iter.tr->data[cpu]->disabled);
5093	}
5094	tracing_on();
5095	}
5096
5097	out:
5098	arch_spin_unlock(&ftrace_dump_lock);
5099	local_irq_restore(flags);
5100	}
5101
5102	/* By default: disable tracing after the dump */
5103	void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
5104	{
5105	__ftrace_dump(true, oops_dump_mode);
5106	}
5107	EXPORT_SYMBOL_GPL(ftrace_dump);
5108
5109	__init static int tracer_alloc_buffers(void)
5110	{
5111	int ring_buf_size;
5112	enum ring_buffer_flags rb_flags;
5113	int i;
5114	int ret = -ENOMEM;
5115
5116
5117	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5118	goto out;
5119
5120	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5121	goto out_free_buffer_mask;
5122
5123	/* Only allocate trace_printk buffers if a trace_printk exists */
5124	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5125	/* Must be called before global_trace.buffer is allocated */
5126	trace_printk_init_buffers();
5127
5128	/* To save memory, keep the ring buffer size to its minimum */
5129	if (ring_buffer_expanded)
5130	ring_buf_size = trace_buf_size;
5131	else
5132	ring_buf_size = 1;
5133
5134	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5135
5136	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5137	cpumask_copy(tracing_cpumask, cpu_all_mask);
5138
5139	/* TODO: make the number of buffers hot pluggable with CPUS */
5140	global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5141	if (!global_trace.buffer) {
5142	printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5143	WARN_ON(1);
5144	goto out_free_cpumask;
5145	}
5146	if (global_trace.buffer_disabled)
5147	tracing_off();
5148
5149
5150	#ifdef CONFIG_TRACER_MAX_TRACE
5151	max_tr.buffer = ring_buffer_alloc(1, rb_flags);
5152	if (!max_tr.buffer) {
5153	printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5154	WARN_ON(1);
5155	ring_buffer_free(global_trace.buffer);
5156	goto out_free_cpumask;
5157	}
5158	#endif
5159
5160	/* Allocate the first page for all buffers */
5161	for_each_tracing_cpu(i) {
5162	global_trace.data[i] = &per_cpu(global_trace_cpu, i);
5163	max_tr.data[i] = &per_cpu(max_tr_data, i);
5164	}
5165
5166	set_buffer_entries(&global_trace,
5167	ring_buffer_size(global_trace.buffer, 0));
5168	#ifdef CONFIG_TRACER_MAX_TRACE
5169	set_buffer_entries(&max_tr, 1);
5170	#endif
5171
5172	trace_init_cmdlines();
5173	init_irq_work(&trace_work_wakeup, trace_wake_up);
5174
5175	register_tracer(&nop_trace);
5176	current_trace = &nop_trace;
5177	/* All seems OK, enable tracing */
5178	tracing_disabled = 0;
5179
5180	atomic_notifier_chain_register(&panic_notifier_list,
5181	&trace_panic_notifier);
5182
5183	register_die_notifier(&trace_die_notifier);
5184
5185	while (trace_boot_options) {
5186	char *option;
5187
5188	option = strsep(&trace_boot_options, ",");
5189	trace_set_options(option);
5190	}
5191
5192	return 0;
5193
5194	out_free_cpumask:
5195	free_cpumask_var(tracing_cpumask);
5196	out_free_buffer_mask:
5197	free_cpumask_var(tracing_buffer_mask);
5198	out:
5199	return ret;
5200	}
5201
5202	__init static int clear_boot_tracer(void)
5203	{
5204	/*
5205	* The default tracer at boot buffer is an init section.
5206	* This function is called in lateinit. If we did not
5207	* find the boot tracer, then clear it out, to prevent
5208	* later registration from accessing the buffer that is
5209	* about to be freed.
5210	*/
5211	if (!default_bootup_tracer)
5212	return 0;
5213
5214	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5215	default_bootup_tracer);
5216	default_bootup_tracer = NULL;
5217
5218	return 0;
5219	}
5220
5221	early_initcall(tracer_alloc_buffers);
5222	fs_initcall(tracer_init_debugfs);
5223	late_initcall(clear_boot_tracer);