linux - kernel/trace/trace.c

4580 lines · 101.6 KiB

1	/*
2	* ring buffer based function tracer
3	*
4	* Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
5	* Copyright (C) 2008 Ingo Molnar <[email protected]>
6	*
7	* Originally taken from the RT patch by:
8	* Arnaldo Carvalho de Melo <[email protected]>
9	*
10	* Based on code from the latency_tracer, that is:
11	* Copyright (C) 2004-2006 Ingo Molnar
12	* Copyright (C) 2004 William Lee Irwin III
13	*/
14	#include <linux/ring_buffer.h>
15	#include <generated/utsrelease.h>
16	#include <linux/stacktrace.h>
17	#include <linux/writeback.h>
18	#include <linux/kallsyms.h>
19	#include <linux/seq_file.h>
20	#include <linux/smp_lock.h>
21	#include <linux/notifier.h>
22	#include <linux/irqflags.h>
23	#include <linux/debugfs.h>
24	#include <linux/pagemap.h>
25	#include <linux/hardirq.h>
26	#include <linux/linkage.h>
27	#include <linux/uaccess.h>
28	#include <linux/kprobes.h>
29	#include <linux/ftrace.h>
30	#include <linux/module.h>
31	#include <linux/percpu.h>
32	#include <linux/splice.h>
33	#include <linux/kdebug.h>
34	#include <linux/string.h>
35	#include <linux/rwsem.h>
36	#include <linux/slab.h>
37	#include <linux/ctype.h>
38	#include <linux/init.h>
39	#include <linux/poll.h>
40	#include <linux/fs.h>
41
42	#include "trace.h"
43	#include "trace_output.h"
44
45	#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
46
47	/*
48	* On boot up, the ring buffer is set to the minimum size, so that
49	* we do not waste memory on systems that are not using tracing.
50	*/
51	int ring_buffer_expanded;
52
53	/*
54	* We need to change this state when a selftest is running.
55	* A selftest will lurk into the ring-buffer to count the
56	* entries inserted during the selftest although some concurrent
57	* insertions into the ring-buffer such as trace_printk could occurred
58	* at the same time, giving false positive or negative results.
59	*/
60	static bool __read_mostly tracing_selftest_running;
61
62	/*
63	* If a tracer is running, we do not want to run SELFTEST.
64	*/
65	bool __read_mostly tracing_selftest_disabled;
66
67	/* For tracers that don't implement custom flags */
68	static struct tracer_opt dummy_tracer_opt[] = {
69	{ }
70	};
71
72	static struct tracer_flags dummy_tracer_flags = {
73	.val = 0,
74	.opts = dummy_tracer_opt
75	};
76
77	static int dummy_set_flag(u32 old_flags, u32 bit, int set)
78	{
79	return 0;
80	}
81
82	/*
83	* Kill all tracing for good (never come back).
84	* It is initialized to 1 but will turn to zero if the initialization
85	* of the tracer is successful. But that is the only place that sets
86	* this back to zero.
87	*/
88	static int tracing_disabled = 1;
89
90	DEFINE_PER_CPU(int, ftrace_cpu_disabled);
91
92	static inline void ftrace_disable_cpu(void)
93	{
94	preempt_disable();
95	__this_cpu_inc(ftrace_cpu_disabled);
96	}
97
98	static inline void ftrace_enable_cpu(void)
99	{
100	__this_cpu_dec(ftrace_cpu_disabled);
101	preempt_enable();
102	}
103
104	static cpumask_var_t __read_mostly tracing_buffer_mask;
105
106	#define for_each_tracing_cpu(cpu) \
107	for_each_cpu(cpu, tracing_buffer_mask)
108
109	/*
110	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111	*
112	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
113	* is set, then ftrace_dump is called. This will output the contents
114	* of the ftrace buffers to the console. This is very useful for
115	* capturing traces that lead to crashes and outputing it to a
116	* serial console.
117	*
118	* It is default off, but you can enable it with either specifying
119	* "ftrace_dump_on_oops" in the kernel command line, or setting
120	* /proc/sys/kernel/ftrace_dump_on_oops to true.
121	*/
122	int ftrace_dump_on_oops;
123
124	static int tracing_set_tracer(const char *buf);
125
126	#define MAX_TRACER_SIZE 100
127	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
128	static char *default_bootup_tracer;
129
130	static int __init set_cmdline_ftrace(char *str)
131	{
132	strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133	default_bootup_tracer = bootup_tracer_buf;
134	/* We are using ftrace early, expand it */
135	ring_buffer_expanded = 1;
136	return 1;
137	}
138	__setup("ftrace=", set_cmdline_ftrace);
139
140	static int __init set_ftrace_dump_on_oops(char *str)
141	{
142	ftrace_dump_on_oops = 1;
143	return 1;
144	}
145	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
146
147	unsigned long long ns2usecs(cycle_t nsec)
148	{
149	nsec += 500;
150	do_div(nsec, 1000);
151	return nsec;
152	}
153
154	/*
155	* The global_trace is the descriptor that holds the tracing
156	* buffers for the live tracing. For each CPU, it contains
157	* a link list of pages that will store trace entries. The
158	* page descriptor of the pages in the memory is used to hold
159	* the link list by linking the lru item in the page descriptor
160	* to each of the pages in the buffer per CPU.
161	*
162	* For each active CPU there is a data field that holds the
163	* pages for the buffer for that CPU. Each CPU has the same number
164	* of pages allocated for its buffer.
165	*/
166	static struct trace_array global_trace;
167
168	static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
169
170	int filter_current_check_discard(struct ring_buffer *buffer,
171	struct ftrace_event_call call, void rec,
172	struct ring_buffer_event *event)
173	{
174	return filter_check_discard(call, rec, buffer, event);
175	}
176	EXPORT_SYMBOL_GPL(filter_current_check_discard);
177
178	cycle_t ftrace_now(int cpu)
179	{
180	u64 ts;
181
182	/* Early boot up does not have a buffer yet */
183	if (!global_trace.buffer)
184	return trace_clock_local();
185
186	ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
187	ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
188
189	return ts;
190	}
191
192	/*
193	* The max_tr is used to snapshot the global_trace when a maximum
194	* latency is reached. Some tracers will use this to store a maximum
195	* trace while it continues examining live traces.
196	*
197	* The buffers for the max_tr are set up the same as the global_trace.
198	* When a snapshot is taken, the link list of the max_tr is swapped
199	* with the link list of the global_trace and the buffers are reset for
200	* the global_trace so the tracing can continue.
201	*/
202	static struct trace_array max_tr;
203
204	static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
205
206	/* tracer_enabled is used to toggle activation of a tracer */
207	static int tracer_enabled = 1;
208
209	/**
210	* tracing_is_enabled - return tracer_enabled status
211	*
212	* This function is used by other tracers to know the status
213	* of the tracer_enabled flag. Tracers may use this function
214	* to know if it should enable their features when starting
215	* up. See irqsoff tracer for an example (start_irqsoff_tracer).
216	*/
217	int tracing_is_enabled(void)
218	{
219	return tracer_enabled;
220	}
221
222	/*
223	* trace_buf_size is the size in bytes that is allocated
224	* for a buffer. Note, the number of bytes is always rounded
225	* to page size.
226	*
227	* This number is purposely set to a low number of 16384.
228	* If the dump on oops happens, it will be much appreciated
229	* to not have to wait for all that output. Anyway this can be
230	* boot time and run time configurable.
231	*/
232	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
233
234	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
235
236	/* trace_types holds a link list of available tracers. */
237	static struct tracer *trace_types __read_mostly;
238
239	/* current_trace points to the tracer that is currently active */
240	static struct tracer *current_trace __read_mostly;
241
242	/*
243	* trace_types_lock is used to protect the trace_types list.
244	*/
245	static DEFINE_MUTEX(trace_types_lock);
246
247	/*
248	* serialize the access of the ring buffer
249	*
250	* ring buffer serializes readers, but it is low level protection.
251	* The validity of the events (which returns by ring_buffer_peek() ..etc)
252	* are not protected by ring buffer.
253	*
254	* The content of events may become garbage if we allow other process consumes
255	* these events concurrently:
256	* A) the page of the consumed events may become a normal page
257	* (not reader page) in ring buffer, and this page will be rewrited
258	* by events producer.
259	* B) The page of the consumed events may become a page for splice_read,
260	* and this page will be returned to system.
261	*
262	* These primitives allow multi process access to different cpu ring buffer
263	* concurrently.
264	*
265	* These primitives don't distinguish read-only and read-consume access.
266	* Multi read-only access are also serialized.
267	*/
268
269	#ifdef CONFIG_SMP
270	static DECLARE_RWSEM(all_cpu_access_lock);
271	static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
272
273	static inline void trace_access_lock(int cpu)
274	{
275	if (cpu == TRACE_PIPE_ALL_CPU) {
276	/* gain it for accessing the whole ring buffer. */
277	down_write(&all_cpu_access_lock);
278	} else {
279	/* gain it for accessing a cpu ring buffer. */
280
281	/* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
282	down_read(&all_cpu_access_lock);
283
284	/* Secondly block other access to this @cpu ring buffer. */
285	mutex_lock(&per_cpu(cpu_access_lock, cpu));
286	}
287	}
288
289	static inline void trace_access_unlock(int cpu)
290	{
291	if (cpu == TRACE_PIPE_ALL_CPU) {
292	up_write(&all_cpu_access_lock);
293	} else {
294	mutex_unlock(&per_cpu(cpu_access_lock, cpu));
295	up_read(&all_cpu_access_lock);
296	}
297	}
298
299	static inline void trace_access_lock_init(void)
300	{
301	int cpu;
302
303	for_each_possible_cpu(cpu)
304	mutex_init(&per_cpu(cpu_access_lock, cpu));
305	}
306
307	#else
308
309	static DEFINE_MUTEX(access_lock);
310
311	static inline void trace_access_lock(int cpu)
312	{
313	(void)cpu;
314	mutex_lock(&access_lock);
315	}
316
317	static inline void trace_access_unlock(int cpu)
318	{
319	(void)cpu;
320	mutex_unlock(&access_lock);
321	}
322
323	static inline void trace_access_lock_init(void)
324	{
325	}
326
327	#endif
328
329	/* trace_wait is a waitqueue for tasks blocked on trace_poll */
330	static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
331
332	/* trace_flags holds trace_options default values */
333	unsigned long trace_flags = TRACE_ITER_PRINT_PARENT \| TRACE_ITER_PRINTK \|
334	TRACE_ITER_ANNOTATE \| TRACE_ITER_CONTEXT_INFO \| TRACE_ITER_SLEEP_TIME \|
335	TRACE_ITER_GRAPH_TIME;
336
337	static int trace_stop_count;
338	static DEFINE_SPINLOCK(tracing_start_lock);
339
340	/**
341	* trace_wake_up - wake up tasks waiting for trace input
342	*
343	* Simply wakes up any task that is blocked on the trace_wait
344	* queue. These is used with trace_poll for tasks polling the trace.
345	*/
346	void trace_wake_up(void)
347	{
348	int cpu;
349
350	if (trace_flags & TRACE_ITER_BLOCK)
351	return;
352	/*
353	* The runqueue_is_locked() can fail, but this is the best we
354	* have for now:
355	*/
356	cpu = get_cpu();
357	if (!runqueue_is_locked(cpu))
358	wake_up(&trace_wait);
359	put_cpu();
360	}
361
362	static int __init set_buf_size(char *str)
363	{
364	unsigned long buf_size;
365
366	if (!str)
367	return 0;
368	buf_size = memparse(str, &str);
369	/* nr_entries can not be zero */
370	if (buf_size == 0)
371	return 0;
372	trace_buf_size = buf_size;
373	return 1;
374	}
375	__setup("trace_buf_size=", set_buf_size);
376
377	static int __init set_tracing_thresh(char *str)
378	{
379	unsigned long threshhold;
380	int ret;
381
382	if (!str)
383	return 0;
384	ret = strict_strtoul(str, 0, &threshhold);
385	if (ret < 0)
386	return 0;
387	tracing_thresh = threshhold * 1000;
388	return 1;
389	}
390	__setup("tracing_thresh=", set_tracing_thresh);
391
392	unsigned long nsecs_to_usecs(unsigned long nsecs)
393	{
394	return nsecs / 1000;
395	}
396
397	/* These must match the bit postions in trace_iterator_flags */
398	static const char *trace_options[] = {
399	"print-parent",
400	"sym-offset",
401	"sym-addr",
402	"verbose",
403	"raw",
404	"hex",
405	"bin",
406	"block",
407	"stacktrace",
408	"trace_printk",
409	"ftrace_preempt",
410	"branch",
411	"annotate",
412	"userstacktrace",
413	"sym-userobj",
414	"printk-msg-only",
415	"context-info",
416	"latency-format",
417	"sleep-time",
418	"graph-time",
419	NULL
420	};
421
422	static struct {
423	u64 (*func)(void);
424	const char *name;
425	} trace_clocks[] = {
426	{ trace_clock_local, "local" },
427	{ trace_clock_global, "global" },
428	};
429
430	int trace_clock_id;
431
432	/*
433	* trace_parser_get_init - gets the buffer for trace parser
434	*/
435	int trace_parser_get_init(struct trace_parser *parser, int size)
436	{
437	memset(parser, 0, sizeof(*parser));
438
439	parser->buffer = kmalloc(size, GFP_KERNEL);
440	if (!parser->buffer)
441	return 1;
442
443	parser->size = size;
444	return 0;
445	}
446
447	/*
448	* trace_parser_put - frees the buffer for trace parser
449	*/
450	void trace_parser_put(struct trace_parser *parser)
451	{
452	kfree(parser->buffer);
453	}
454
455	/*
456	* trace_get_user - reads the user input string separated by space
457	* (matched by isspace(ch))
458	*
459	* For each string found the 'struct trace_parser' is updated,
460	* and the function returns.
461	*
462	* Returns number of bytes read.
463	*
464	* See kernel/trace/trace.h for 'struct trace_parser' details.
465	*/
466	int trace_get_user(struct trace_parser parser, const char __user ubuf,
467	size_t cnt, loff_t *ppos)
468	{
469	char ch;
470	size_t read = 0;
471	ssize_t ret;
472
473	if (!*ppos)
474	trace_parser_clear(parser);
475
476	ret = get_user(ch, ubuf++);
477	if (ret)
478	goto out;
479
480	read++;
481	cnt--;
482
483	/*
484	* The parser is not finished with the last write,
485	* continue reading the user input without skipping spaces.
486	*/
487	if (!parser->cont) {
488	/* skip white space */
489	while (cnt && isspace(ch)) {
490	ret = get_user(ch, ubuf++);
491	if (ret)
492	goto out;
493	read++;
494	cnt--;
495	}
496
497	/* only spaces were written */
498	if (isspace(ch)) {
499	*ppos += read;
500	ret = read;
501	goto out;
502	}
503
504	parser->idx = 0;
505	}
506
507	/* read the non-space input */
508	while (cnt && !isspace(ch)) {
509	if (parser->idx < parser->size - 1)
510	parser->buffer[parser->idx++] = ch;
511	else {
512	ret = -EINVAL;
513	goto out;
514	}
515	ret = get_user(ch, ubuf++);
516	if (ret)
517	goto out;
518	read++;
519	cnt--;
520	}
521
522	/* We either got finished input or we have to wait for another call. */
523	if (isspace(ch)) {
524	parser->buffer[parser->idx] = 0;
525	parser->cont = false;
526	} else {
527	parser->cont = true;
528	parser->buffer[parser->idx++] = ch;
529	}
530
531	*ppos += read;
532	ret = read;
533
534	out:
535	return ret;
536	}
537
538	ssize_t trace_seq_to_user(struct trace_seq s, char __user ubuf, size_t cnt)
539	{
540	int len;
541	int ret;
542
543	if (!cnt)
544	return 0;
545
546	if (s->len <= s->readpos)
547	return -EBUSY;
548
549	len = s->len - s->readpos;
550	if (cnt > len)
551	cnt = len;
552	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
553	if (ret == cnt)
554	return -EFAULT;
555
556	cnt -= ret;
557
558	s->readpos += cnt;
559	return cnt;
560	}
561
562	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
563	{
564	int len;
565	void *ret;
566
567	if (s->len <= s->readpos)
568	return -EBUSY;
569
570	len = s->len - s->readpos;
571	if (cnt > len)
572	cnt = len;
573	ret = memcpy(buf, s->buffer + s->readpos, cnt);
574	if (!ret)
575	return -EFAULT;
576
577	s->readpos += cnt;
578	return cnt;
579	}
580
581	/*
582	* ftrace_max_lock is used to protect the swapping of buffers
583	* when taking a max snapshot. The buffers themselves are
584	* protected by per_cpu spinlocks. But the action of the swap
585	* needs its own lock.
586	*
587	* This is defined as a arch_spinlock_t in order to help
588	* with performance when lockdep debugging is enabled.
589	*
590	* It is also used in other places outside the update_max_tr
591	* so it needs to be defined outside of the
592	* CONFIG_TRACER_MAX_TRACE.
593	*/
594	static arch_spinlock_t ftrace_max_lock =
595	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
596
597	unsigned long __read_mostly tracing_thresh;
598
599	#ifdef CONFIG_TRACER_MAX_TRACE
600	unsigned long __read_mostly tracing_max_latency;
601
602	/*
603	* Copy the new maximum trace into the separate maximum-trace
604	* structure. (this way the maximum trace is permanently saved,
605	* for later retrieval via /sys/kernel/debug/tracing/latency_trace)
606	*/
607	static void
608	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
609	{
610	struct trace_array_cpu *data = tr->data[cpu];
611	struct trace_array_cpu *max_data;
612
613	max_tr.cpu = cpu;
614	max_tr.time_start = data->preempt_timestamp;
615
616	max_data = max_tr.data[cpu];
617	max_data->saved_latency = tracing_max_latency;
618	max_data->critical_start = data->critical_start;
619	max_data->critical_end = data->critical_end;
620
621	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
622	max_data->pid = tsk->pid;
623	max_data->uid = task_uid(tsk);
624	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
625	max_data->policy = tsk->policy;
626	max_data->rt_priority = tsk->rt_priority;
627
628	/* record this tasks comm */
629	tracing_record_cmdline(tsk);
630	}
631
632	/**
633	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
634	* @tr: tracer
635	* @tsk: the task with the latency
636	* @cpu: The cpu that initiated the trace.
637	*
638	* Flip the buffers between the @tr and the max_tr and record information
639	* about which task was the cause of this latency.
640	*/
641	void
642	update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
643	{
644	struct ring_buffer *buf = tr->buffer;
645
646	if (trace_stop_count)
647	return;
648
649	WARN_ON_ONCE(!irqs_disabled());
650	arch_spin_lock(&ftrace_max_lock);
651
652	tr->buffer = max_tr.buffer;
653	max_tr.buffer = buf;
654
655	__update_max_tr(tr, tsk, cpu);
656	arch_spin_unlock(&ftrace_max_lock);
657	}
658
659	/**
660	* update_max_tr_single - only copy one trace over, and reset the rest
661	* @tr - tracer
662	* @tsk - task with the latency
663	* @cpu - the cpu of the buffer to copy.
664	*
665	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
666	*/
667	void
668	update_max_tr_single(struct trace_array tr, struct task_struct tsk, int cpu)
669	{
670	int ret;
671
672	if (trace_stop_count)
673	return;
674
675	WARN_ON_ONCE(!irqs_disabled());
676	arch_spin_lock(&ftrace_max_lock);
677
678	ftrace_disable_cpu();
679
680	ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
681
682	if (ret == -EBUSY) {
683	/*
684	* We failed to swap the buffer due to a commit taking
685	* place on this CPU. We fail to record, but we reset
686	* the max trace buffer (no one writes directly to it)
687	* and flag that it failed.
688	*/
689	trace_array_printk(&max_tr, _THIS_IP_,
690	"Failed to swap buffers due to commit in progress\n");
691	}
692
693	ftrace_enable_cpu();
694
695	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
696
697	__update_max_tr(tr, tsk, cpu);
698	arch_spin_unlock(&ftrace_max_lock);
699	}
700	#endif /* CONFIG_TRACER_MAX_TRACE */
701
702	/**
703	* register_tracer - register a tracer with the ftrace system.
704	* @type - the plugin for the tracer
705	*
706	* Register a new plugin tracer.
707	*/
708	int register_tracer(struct tracer *type)
709	__releases(kernel_lock)
710	__acquires(kernel_lock)
711	{
712	struct tracer *t;
713	int ret = 0;
714
715	if (!type->name) {
716	pr_info("Tracer must have a name\n");
717	return -1;
718	}
719
720	if (strlen(type->name) > MAX_TRACER_SIZE) {
721	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
722	return -1;
723	}
724
725	/*
726	* When this gets called we hold the BKL which means that
727	* preemption is disabled. Various trace selftests however
728	* need to disable and enable preemption for successful tests.
729	* So we drop the BKL here and grab it after the tests again.
730	*/
731	unlock_kernel();
732	mutex_lock(&trace_types_lock);
733
734	tracing_selftest_running = true;
735
736	for (t = trace_types; t; t = t->next) {
737	if (strcmp(type->name, t->name) == 0) {
738	/* already found */
739	pr_info("Tracer %s already registered\n",
740	type->name);
741	ret = -1;
742	goto out;
743	}
744	}
745
746	if (!type->set_flag)
747	type->set_flag = &dummy_set_flag;
748	if (!type->flags)
749	type->flags = &dummy_tracer_flags;
750	else
751	if (!type->flags->opts)
752	type->flags->opts = dummy_tracer_opt;
753	if (!type->wait_pipe)
754	type->wait_pipe = default_wait_pipe;
755
756
757	#ifdef CONFIG_FTRACE_STARTUP_TEST
758	if (type->selftest && !tracing_selftest_disabled) {
759	struct tracer *saved_tracer = current_trace;
760	struct trace_array *tr = &global_trace;
761
762	/*
763	* Run a selftest on this tracer.
764	* Here we reset the trace buffer, and set the current
765	* tracer to be this tracer. The tracer can then run some
766	* internal tracing to verify that everything is in order.
767	* If we fail, we do not register this tracer.
768	*/
769	tracing_reset_online_cpus(tr);
770
771	current_trace = type;
772	/* the test is responsible for initializing and enabling */
773	pr_info("Testing tracer %s: ", type->name);
774	ret = type->selftest(type, tr);
775	/* the test is responsible for resetting too */
776	current_trace = saved_tracer;
777	if (ret) {
778	printk(KERN_CONT "FAILED!\n");
779	goto out;
780	}
781	/* Only reset on passing, to avoid touching corrupted buffers */
782	tracing_reset_online_cpus(tr);
783
784	printk(KERN_CONT "PASSED\n");
785	}
786	#endif
787
788	type->next = trace_types;
789	trace_types = type;
790
791	out:
792	tracing_selftest_running = false;
793	mutex_unlock(&trace_types_lock);
794
795	if (ret \|\| !default_bootup_tracer)
796	goto out_unlock;
797
798	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
799	goto out_unlock;
800
801	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
802	/* Do we want this tracer to start on bootup? */
803	tracing_set_tracer(type->name);
804	default_bootup_tracer = NULL;
805	/* disable other selftests, since this will break it. */
806	tracing_selftest_disabled = 1;
807	#ifdef CONFIG_FTRACE_STARTUP_TEST
808	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
809	type->name);
810	#endif
811
812	out_unlock:
813	lock_kernel();
814	return ret;
815	}
816
817	void unregister_tracer(struct tracer *type)
818	{
819	struct tracer **t;
820
821	mutex_lock(&trace_types_lock);
822	for (t = &trace_types; t; t = &(t)->next) {
823	if (*t == type)
824	goto found;
825	}
826	pr_info("Tracer %s not registered\n", type->name);
827	goto out;
828
829	found:
830	t = (t)->next;
831
832	if (type == current_trace && tracer_enabled) {
833	tracer_enabled = 0;
834	tracing_stop();
835	if (current_trace->stop)
836	current_trace->stop(&global_trace);
837	current_trace = &nop_trace;
838	}
839	out:
840	mutex_unlock(&trace_types_lock);
841	}
842
843	static void __tracing_reset(struct ring_buffer *buffer, int cpu)
844	{
845	ftrace_disable_cpu();
846	ring_buffer_reset_cpu(buffer, cpu);
847	ftrace_enable_cpu();
848	}
849
850	void tracing_reset(struct trace_array *tr, int cpu)
851	{
852	struct ring_buffer *buffer = tr->buffer;
853
854	ring_buffer_record_disable(buffer);
855
856	/* Make sure all commits have finished */
857	synchronize_sched();
858	__tracing_reset(buffer, cpu);
859
860	ring_buffer_record_enable(buffer);
861	}
862
863	void tracing_reset_online_cpus(struct trace_array *tr)
864	{
865	struct ring_buffer *buffer = tr->buffer;
866	int cpu;
867
868	ring_buffer_record_disable(buffer);
869
870	/* Make sure all commits have finished */
871	synchronize_sched();
872
873	tr->time_start = ftrace_now(tr->cpu);
874
875	for_each_online_cpu(cpu)
876	__tracing_reset(buffer, cpu);
877
878	ring_buffer_record_enable(buffer);
879	}
880
881	void tracing_reset_current(int cpu)
882	{
883	tracing_reset(&global_trace, cpu);
884	}
885
886	void tracing_reset_current_online_cpus(void)
887	{
888	tracing_reset_online_cpus(&global_trace);
889	}
890
891	#define SAVED_CMDLINES 128
892	#define NO_CMDLINE_MAP UINT_MAX
893	static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
894	static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
895	static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
896	static int cmdline_idx;
897	static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
898
899	/* temporary disable recording */
900	static atomic_t trace_record_cmdline_disabled __read_mostly;
901
902	static void trace_init_cmdlines(void)
903	{
904	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
905	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
906	cmdline_idx = 0;
907	}
908
909	int is_tracing_stopped(void)
910	{
911	return trace_stop_count;
912	}
913
914	/**
915	* ftrace_off_permanent - disable all ftrace code permanently
916	*
917	* This should only be called when a serious anomally has
918	* been detected. This will turn off the function tracing,
919	* ring buffers, and other tracing utilites. It takes no
920	* locks and can be called from any context.
921	*/
922	void ftrace_off_permanent(void)
923	{
924	tracing_disabled = 1;
925	ftrace_stop();
926	tracing_off_permanent();
927	}
928
929	/**
930	* tracing_start - quick start of the tracer
931	*
932	* If tracing is enabled but was stopped by tracing_stop,
933	* this will start the tracer back up.
934	*/
935	void tracing_start(void)
936	{
937	struct ring_buffer *buffer;
938	unsigned long flags;
939
940	if (tracing_disabled)
941	return;
942
943	spin_lock_irqsave(&tracing_start_lock, flags);
944	if (--trace_stop_count) {
945	if (trace_stop_count < 0) {
946	/* Someone screwed up their debugging */
947	WARN_ON_ONCE(1);
948	trace_stop_count = 0;
949	}
950	goto out;
951	}
952
953	/* Prevent the buffers from switching */
954	arch_spin_lock(&ftrace_max_lock);
955
956	buffer = global_trace.buffer;
957	if (buffer)
958	ring_buffer_record_enable(buffer);
959
960	buffer = max_tr.buffer;
961	if (buffer)
962	ring_buffer_record_enable(buffer);
963
964	arch_spin_unlock(&ftrace_max_lock);
965
966	ftrace_start();
967	out:
968	spin_unlock_irqrestore(&tracing_start_lock, flags);
969	}
970
971	/**
972	* tracing_stop - quick stop of the tracer
973	*
974	* Light weight way to stop tracing. Use in conjunction with
975	* tracing_start.
976	*/
977	void tracing_stop(void)
978	{
979	struct ring_buffer *buffer;
980	unsigned long flags;
981
982	ftrace_stop();
983	spin_lock_irqsave(&tracing_start_lock, flags);
984	if (trace_stop_count++)
985	goto out;
986
987	/* Prevent the buffers from switching */
988	arch_spin_lock(&ftrace_max_lock);
989
990	buffer = global_trace.buffer;
991	if (buffer)
992	ring_buffer_record_disable(buffer);
993
994	buffer = max_tr.buffer;
995	if (buffer)
996	ring_buffer_record_disable(buffer);
997
998	arch_spin_unlock(&ftrace_max_lock);
999
1000	out:
1001	spin_unlock_irqrestore(&tracing_start_lock, flags);
1002	}
1003
1004	void trace_stop_cmdline_recording(void);
1005
1006	static void trace_save_cmdline(struct task_struct *tsk)
1007	{
1008	unsigned pid, idx;
1009
1010	if (!tsk->pid \|\| unlikely(tsk->pid > PID_MAX_DEFAULT))
1011	return;
1012
1013	/*
1014	* It's not the end of the world if we don't get
1015	* the lock, but we also don't want to spin
1016	* nor do we want to disable interrupts,
1017	* so if we miss here, then better luck next time.
1018	*/
1019	if (!arch_spin_trylock(&trace_cmdline_lock))
1020	return;
1021
1022	idx = map_pid_to_cmdline[tsk->pid];
1023	if (idx == NO_CMDLINE_MAP) {
1024	idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1025
1026	/*
1027	* Check whether the cmdline buffer at idx has a pid
1028	* mapped. We are going to overwrite that entry so we
1029	* need to clear the map_pid_to_cmdline. Otherwise we
1030	* would read the new comm for the old pid.
1031	*/
1032	pid = map_cmdline_to_pid[idx];
1033	if (pid != NO_CMDLINE_MAP)
1034	map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1035
1036	map_cmdline_to_pid[idx] = tsk->pid;
1037	map_pid_to_cmdline[tsk->pid] = idx;
1038
1039	cmdline_idx = idx;
1040	}
1041
1042	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1043
1044	arch_spin_unlock(&trace_cmdline_lock);
1045	}
1046
1047	void trace_find_cmdline(int pid, char comm[])
1048	{
1049	unsigned map;
1050
1051	if (!pid) {
1052	strcpy(comm, "<idle>");
1053	return;
1054	}
1055
1056	if (WARN_ON_ONCE(pid < 0)) {
1057	strcpy(comm, "<XXX>");
1058	return;
1059	}
1060
1061	if (pid > PID_MAX_DEFAULT) {
1062	strcpy(comm, "<...>");
1063	return;
1064	}
1065
1066	preempt_disable();
1067	arch_spin_lock(&trace_cmdline_lock);
1068	map = map_pid_to_cmdline[pid];
1069	if (map != NO_CMDLINE_MAP)
1070	strcpy(comm, saved_cmdlines[map]);
1071	else
1072	strcpy(comm, "<...>");
1073
1074	arch_spin_unlock(&trace_cmdline_lock);
1075	preempt_enable();
1076	}
1077
1078	void tracing_record_cmdline(struct task_struct *tsk)
1079	{
1080	if (atomic_read(&trace_record_cmdline_disabled) \|\| !tracer_enabled \|\|
1081	!tracing_is_on())
1082	return;
1083
1084	trace_save_cmdline(tsk);
1085	}
1086
1087	void
1088	tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1089	int pc)
1090	{
1091	struct task_struct *tsk = current;
1092
1093	entry->preempt_count = pc & 0xff;
1094	entry->pid = (tsk) ? tsk->pid : 0;
1095	entry->lock_depth = (tsk) ? tsk->lock_depth : 0;
1096	entry->flags =
1097	#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1098	(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) \|
1099	#else
1100	TRACE_FLAG_IRQS_NOSUPPORT \|
1101	#endif
1102	((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) \|
1103	((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) \|
1104	(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1105	}
1106	EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1107
1108	struct ring_buffer_event *
1109	trace_buffer_lock_reserve(struct ring_buffer *buffer,
1110	int type,
1111	unsigned long len,
1112	unsigned long flags, int pc)
1113	{
1114	struct ring_buffer_event *event;
1115
1116	event = ring_buffer_lock_reserve(buffer, len);
1117	if (event != NULL) {
1118	struct trace_entry *ent = ring_buffer_event_data(event);
1119
1120	tracing_generic_entry_update(ent, flags, pc);
1121	ent->type = type;
1122	}
1123
1124	return event;
1125	}
1126
1127	static inline void
1128	__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1129	struct ring_buffer_event *event,
1130	unsigned long flags, int pc,
1131	int wake)
1132	{
1133	ring_buffer_unlock_commit(buffer, event);
1134
1135	ftrace_trace_stack(buffer, flags, 6, pc);
1136	ftrace_trace_userstack(buffer, flags, pc);
1137
1138	if (wake)
1139	trace_wake_up();
1140	}
1141
1142	void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1143	struct ring_buffer_event *event,
1144	unsigned long flags, int pc)
1145	{
1146	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1147	}
1148
1149	struct ring_buffer_event *
1150	trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1151	int type, unsigned long len,
1152	unsigned long flags, int pc)
1153	{
1154	*current_rb = global_trace.buffer;
1155	return trace_buffer_lock_reserve(*current_rb,
1156	type, len, flags, pc);
1157	}
1158	EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1159
1160	void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1161	struct ring_buffer_event *event,
1162	unsigned long flags, int pc)
1163	{
1164	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1165	}
1166	EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1167
1168	void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1169	struct ring_buffer_event *event,
1170	unsigned long flags, int pc)
1171	{
1172	__trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1173	}
1174	EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1175
1176	void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1177	struct ring_buffer_event *event)
1178	{
1179	ring_buffer_discard_commit(buffer, event);
1180	}
1181	EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1182
1183	void
1184	trace_function(struct trace_array *tr,
1185	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1186	int pc)
1187	{
1188	struct ftrace_event_call *call = &event_function;
1189	struct ring_buffer *buffer = tr->buffer;
1190	struct ring_buffer_event *event;
1191	struct ftrace_entry *entry;
1192
1193	/* If we are reading the ring buffer, don't trace */
1194	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1195	return;
1196
1197	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1198	flags, pc);
1199	if (!event)
1200	return;
1201	entry = ring_buffer_event_data(event);
1202	entry->ip = ip;
1203	entry->parent_ip = parent_ip;
1204
1205	if (!filter_check_discard(call, entry, buffer, event))
1206	ring_buffer_unlock_commit(buffer, event);
1207	}
1208
1209	void
1210	ftrace(struct trace_array tr, struct trace_array_cpu data,
1211	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1212	int pc)
1213	{
1214	if (likely(!atomic_read(&data->disabled)))
1215	trace_function(tr, ip, parent_ip, flags, pc);
1216	}
1217
1218	#ifdef CONFIG_STACKTRACE
1219	static void __ftrace_trace_stack(struct ring_buffer *buffer,
1220	unsigned long flags,
1221	int skip, int pc)
1222	{
1223	struct ftrace_event_call *call = &event_kernel_stack;
1224	struct ring_buffer_event *event;
1225	struct stack_entry *entry;
1226	struct stack_trace trace;
1227
1228	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1229	sizeof(*entry), flags, pc);
1230	if (!event)
1231	return;
1232	entry = ring_buffer_event_data(event);
1233	memset(&entry->caller, 0, sizeof(entry->caller));
1234
1235	trace.nr_entries = 0;
1236	trace.max_entries = FTRACE_STACK_ENTRIES;
1237	trace.skip = skip;
1238	trace.entries = entry->caller;
1239
1240	save_stack_trace(&trace);
1241	if (!filter_check_discard(call, entry, buffer, event))
1242	ring_buffer_unlock_commit(buffer, event);
1243	}
1244
1245	void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1246	int skip, int pc)
1247	{
1248	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1249	return;
1250
1251	__ftrace_trace_stack(buffer, flags, skip, pc);
1252	}
1253
1254	void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1255	int pc)
1256	{
1257	__ftrace_trace_stack(tr->buffer, flags, skip, pc);
1258	}
1259
1260	/**
1261	* trace_dump_stack - record a stack back trace in the trace buffer
1262	*/
1263	void trace_dump_stack(void)
1264	{
1265	unsigned long flags;
1266
1267	if (tracing_disabled \|\| tracing_selftest_running)
1268	return;
1269
1270	local_save_flags(flags);
1271
1272	/* skipping 3 traces, seems to get us at the caller of this function */
1273	__ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count());
1274	}
1275
1276	void
1277	ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1278	{
1279	struct ftrace_event_call *call = &event_user_stack;
1280	struct ring_buffer_event *event;
1281	struct userstack_entry *entry;
1282	struct stack_trace trace;
1283
1284	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1285	return;
1286
1287	/*
1288	* NMIs can not handle page faults, even with fix ups.
1289	* The save user stack can (and often does) fault.
1290	*/
1291	if (unlikely(in_nmi()))
1292	return;
1293
1294	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1295	sizeof(*entry), flags, pc);
1296	if (!event)
1297	return;
1298	entry = ring_buffer_event_data(event);
1299
1300	entry->tgid = current->tgid;
1301	memset(&entry->caller, 0, sizeof(entry->caller));
1302
1303	trace.nr_entries = 0;
1304	trace.max_entries = FTRACE_STACK_ENTRIES;
1305	trace.skip = 0;
1306	trace.entries = entry->caller;
1307
1308	save_stack_trace_user(&trace);
1309	if (!filter_check_discard(call, entry, buffer, event))
1310	ring_buffer_unlock_commit(buffer, event);
1311	}
1312
1313	#ifdef UNUSED
1314	static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1315	{
1316	ftrace_trace_userstack(tr, flags, preempt_count());
1317	}
1318	#endif /* UNUSED */
1319
1320	#endif /* CONFIG_STACKTRACE */
1321
1322	static void
1323	ftrace_trace_special(void *__tr,
1324	unsigned long arg1, unsigned long arg2, unsigned long arg3,
1325	int pc)
1326	{
1327	struct ftrace_event_call *call = &event_special;
1328	struct ring_buffer_event *event;
1329	struct trace_array *tr = __tr;
1330	struct ring_buffer *buffer = tr->buffer;
1331	struct special_entry *entry;
1332
1333	event = trace_buffer_lock_reserve(buffer, TRACE_SPECIAL,
1334	sizeof(*entry), 0, pc);
1335	if (!event)
1336	return;
1337	entry = ring_buffer_event_data(event);
1338	entry->arg1 = arg1;
1339	entry->arg2 = arg2;
1340	entry->arg3 = arg3;
1341
1342	if (!filter_check_discard(call, entry, buffer, event))
1343	trace_buffer_unlock_commit(buffer, event, 0, pc);
1344	}
1345
1346	void
1347	__trace_special(void __tr, void __data,
1348	unsigned long arg1, unsigned long arg2, unsigned long arg3)
1349	{
1350	ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
1351	}
1352
1353	void
1354	ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1355	{
1356	struct trace_array *tr = &global_trace;
1357	struct trace_array_cpu *data;
1358	unsigned long flags;
1359	int cpu;
1360	int pc;
1361
1362	if (tracing_disabled)
1363	return;
1364
1365	pc = preempt_count();
1366	local_irq_save(flags);
1367	cpu = raw_smp_processor_id();
1368	data = tr->data[cpu];
1369
1370	if (likely(atomic_inc_return(&data->disabled) == 1))
1371	ftrace_trace_special(tr, arg1, arg2, arg3, pc);
1372
1373	atomic_dec(&data->disabled);
1374	local_irq_restore(flags);
1375	}
1376
1377	/**
1378	* trace_vbprintk - write binary msg to tracing buffer
1379	*
1380	*/
1381	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1382	{
1383	static arch_spinlock_t trace_buf_lock =
1384	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1385	static u32 trace_buf[TRACE_BUF_SIZE];
1386
1387	struct ftrace_event_call *call = &event_bprint;
1388	struct ring_buffer_event *event;
1389	struct ring_buffer *buffer;
1390	struct trace_array *tr = &global_trace;
1391	struct trace_array_cpu *data;
1392	struct bprint_entry *entry;
1393	unsigned long flags;
1394	int disable;
1395	int resched;
1396	int cpu, len = 0, size, pc;
1397
1398	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
1399	return 0;
1400
1401	/* Don't pollute graph traces with trace_vprintk internals */
1402	pause_graph_tracing();
1403
1404	pc = preempt_count();
1405	resched = ftrace_preempt_disable();
1406	cpu = raw_smp_processor_id();
1407	data = tr->data[cpu];
1408
1409	disable = atomic_inc_return(&data->disabled);
1410	if (unlikely(disable != 1))
1411	goto out;
1412
1413	/* Lockdep uses trace_printk for lock tracing */
1414	local_irq_save(flags);
1415	arch_spin_lock(&trace_buf_lock);
1416	len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1417
1418	if (len > TRACE_BUF_SIZE \|\| len < 0)
1419	goto out_unlock;
1420
1421	size = sizeof(entry) + sizeof(u32) len;
1422	buffer = tr->buffer;
1423	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1424	flags, pc);
1425	if (!event)
1426	goto out_unlock;
1427	entry = ring_buffer_event_data(event);
1428	entry->ip = ip;
1429	entry->fmt = fmt;
1430
1431	memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1432	if (!filter_check_discard(call, entry, buffer, event)) {
1433	ring_buffer_unlock_commit(buffer, event);
1434	ftrace_trace_stack(buffer, flags, 6, pc);
1435	}
1436
1437	out_unlock:
1438	arch_spin_unlock(&trace_buf_lock);
1439	local_irq_restore(flags);
1440
1441	out:
1442	atomic_dec_return(&data->disabled);
1443	ftrace_preempt_enable(resched);
1444	unpause_graph_tracing();
1445
1446	return len;
1447	}
1448	EXPORT_SYMBOL_GPL(trace_vbprintk);
1449
1450	int trace_array_printk(struct trace_array *tr,
1451	unsigned long ip, const char *fmt, ...)
1452	{
1453	int ret;
1454	va_list ap;
1455
1456	if (!(trace_flags & TRACE_ITER_PRINTK))
1457	return 0;
1458
1459	va_start(ap, fmt);
1460	ret = trace_array_vprintk(tr, ip, fmt, ap);
1461	va_end(ap);
1462	return ret;
1463	}
1464
1465	int trace_array_vprintk(struct trace_array *tr,
1466	unsigned long ip, const char *fmt, va_list args)
1467	{
1468	static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1469	static char trace_buf[TRACE_BUF_SIZE];
1470
1471	struct ftrace_event_call *call = &event_print;
1472	struct ring_buffer_event *event;
1473	struct ring_buffer *buffer;
1474	struct trace_array_cpu *data;
1475	int cpu, len = 0, size, pc;
1476	struct print_entry *entry;
1477	unsigned long irq_flags;
1478	int disable;
1479
1480	if (tracing_disabled \|\| tracing_selftest_running)
1481	return 0;
1482
1483	pc = preempt_count();
1484	preempt_disable_notrace();
1485	cpu = raw_smp_processor_id();
1486	data = tr->data[cpu];
1487
1488	disable = atomic_inc_return(&data->disabled);
1489	if (unlikely(disable != 1))
1490	goto out;
1491
1492	pause_graph_tracing();
1493	raw_local_irq_save(irq_flags);
1494	arch_spin_lock(&trace_buf_lock);
1495	len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1496
1497	size = sizeof(*entry) + len + 1;
1498	buffer = tr->buffer;
1499	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1500	irq_flags, pc);
1501	if (!event)
1502	goto out_unlock;
1503	entry = ring_buffer_event_data(event);
1504	entry->ip = ip;
1505
1506	memcpy(&entry->buf, trace_buf, len);
1507	entry->buf[len] = '\0';
1508	if (!filter_check_discard(call, entry, buffer, event)) {
1509	ring_buffer_unlock_commit(buffer, event);
1510	ftrace_trace_stack(buffer, irq_flags, 6, pc);
1511	}
1512
1513	out_unlock:
1514	arch_spin_unlock(&trace_buf_lock);
1515	raw_local_irq_restore(irq_flags);
1516	unpause_graph_tracing();
1517	out:
1518	atomic_dec_return(&data->disabled);
1519	preempt_enable_notrace();
1520
1521	return len;
1522	}
1523
1524	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1525	{
1526	return trace_array_vprintk(&global_trace, ip, fmt, args);
1527	}
1528	EXPORT_SYMBOL_GPL(trace_vprintk);
1529
1530	enum trace_file_type {
1531	TRACE_FILE_LAT_FMT = 1,
1532	TRACE_FILE_ANNOTATE = 2,
1533	};
1534
1535	static void trace_iterator_increment(struct trace_iterator *iter)
1536	{
1537	/* Don't allow ftrace to trace into the ring buffers */
1538	ftrace_disable_cpu();
1539
1540	iter->idx++;
1541	if (iter->buffer_iter[iter->cpu])
1542	ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1543
1544	ftrace_enable_cpu();
1545	}
1546
1547	static struct trace_entry *
1548	peek_next_entry(struct trace_iterator iter, int cpu, u64 ts)
1549	{
1550	struct ring_buffer_event *event;
1551	struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1552
1553	/* Don't allow ftrace to trace into the ring buffers */
1554	ftrace_disable_cpu();
1555
1556	if (buf_iter)
1557	event = ring_buffer_iter_peek(buf_iter, ts);
1558	else
1559	event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1560
1561	ftrace_enable_cpu();
1562
1563	return event ? ring_buffer_event_data(event) : NULL;
1564	}
1565
1566	static struct trace_entry *
1567	__find_next_entry(struct trace_iterator iter, int ent_cpu, u64 *ent_ts)
1568	{
1569	struct ring_buffer *buffer = iter->tr->buffer;
1570	struct trace_entry ent, next = NULL;
1571	int cpu_file = iter->cpu_file;
1572	u64 next_ts = 0, ts;
1573	int next_cpu = -1;
1574	int cpu;
1575
1576	/*
1577	* If we are in a per_cpu trace file, don't bother by iterating over
1578	* all cpu and peek directly.
1579	*/
1580	if (cpu_file > TRACE_PIPE_ALL_CPU) {
1581	if (ring_buffer_empty_cpu(buffer, cpu_file))
1582	return NULL;
1583	ent = peek_next_entry(iter, cpu_file, ent_ts);
1584	if (ent_cpu)
1585	*ent_cpu = cpu_file;
1586
1587	return ent;
1588	}
1589
1590	for_each_tracing_cpu(cpu) {
1591
1592	if (ring_buffer_empty_cpu(buffer, cpu))
1593	continue;
1594
1595	ent = peek_next_entry(iter, cpu, &ts);
1596
1597	/*
1598	* Pick the entry with the smallest timestamp:
1599	*/
1600	if (ent && (!next \|\| ts < next_ts)) {
1601	next = ent;
1602	next_cpu = cpu;
1603	next_ts = ts;
1604	}
1605	}
1606
1607	if (ent_cpu)
1608	*ent_cpu = next_cpu;
1609
1610	if (ent_ts)
1611	*ent_ts = next_ts;
1612
1613	return next;
1614	}
1615
1616	/* Find the next real entry, without updating the iterator itself */
1617	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
1618	int ent_cpu, u64 ent_ts)
1619	{
1620	return __find_next_entry(iter, ent_cpu, ent_ts);
1621	}
1622
1623	/* Find the next real entry, and increment the iterator to the next entry */
1624	static void find_next_entry_inc(struct trace_iterator iter)
1625	{
1626	iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1627
1628	if (iter->ent)
1629	trace_iterator_increment(iter);
1630
1631	return iter->ent ? iter : NULL;
1632	}
1633
1634	static void trace_consume(struct trace_iterator *iter)
1635	{
1636	/* Don't allow ftrace to trace into the ring buffers */
1637	ftrace_disable_cpu();
1638	ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1639	ftrace_enable_cpu();
1640	}
1641
1642	static void s_next(struct seq_file m, void v, loff_t pos)
1643	{
1644	struct trace_iterator *iter = m->private;
1645	int i = (int)*pos;
1646	void *ent;
1647
1648	WARN_ON_ONCE(iter->leftover);
1649
1650	(*pos)++;
1651
1652	/* can't go backwards */
1653	if (iter->idx > i)
1654	return NULL;
1655
1656	if (iter->idx < 0)
1657	ent = find_next_entry_inc(iter);
1658	else
1659	ent = iter;
1660
1661	while (ent && iter->idx < i)
1662	ent = find_next_entry_inc(iter);
1663
1664	iter->pos = *pos;
1665
1666	return ent;
1667	}
1668
1669	static void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1670	{
1671	struct trace_array *tr = iter->tr;
1672	struct ring_buffer_event *event;
1673	struct ring_buffer_iter *buf_iter;
1674	unsigned long entries = 0;
1675	u64 ts;
1676
1677	tr->data[cpu]->skipped_entries = 0;
1678
1679	if (!iter->buffer_iter[cpu])
1680	return;
1681
1682	buf_iter = iter->buffer_iter[cpu];
1683	ring_buffer_iter_reset(buf_iter);
1684
1685	/*
1686	* We could have the case with the max latency tracers
1687	* that a reset never took place on a cpu. This is evident
1688	* by the timestamp being before the start of the buffer.
1689	*/
1690	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1691	if (ts >= iter->tr->time_start)
1692	break;
1693	entries++;
1694	ring_buffer_read(buf_iter, NULL);
1695	}
1696
1697	tr->data[cpu]->skipped_entries = entries;
1698	}
1699
1700	/*
1701	* The current tracer is copied to avoid a global locking
1702	* all around.
1703	*/
1704	static void s_start(struct seq_file m, loff_t *pos)
1705	{
1706	struct trace_iterator *iter = m->private;
1707	static struct tracer *old_tracer;
1708	int cpu_file = iter->cpu_file;
1709	void *p = NULL;
1710	loff_t l = 0;
1711	int cpu;
1712
1713	/* copy the tracer to avoid using a global lock all around */
1714	mutex_lock(&trace_types_lock);
1715	if (unlikely(old_tracer != current_trace && current_trace)) {
1716	old_tracer = current_trace;
1717	iter->trace = current_trace;
1718	}
1719	mutex_unlock(&trace_types_lock);
1720
1721	atomic_inc(&trace_record_cmdline_disabled);
1722
1723	if (*pos != iter->pos) {
1724	iter->ent = NULL;
1725	iter->cpu = 0;
1726	iter->idx = -1;
1727
1728	ftrace_disable_cpu();
1729
1730	if (cpu_file == TRACE_PIPE_ALL_CPU) {
1731	for_each_tracing_cpu(cpu)
1732	tracing_iter_reset(iter, cpu);
1733	} else
1734	tracing_iter_reset(iter, cpu_file);
1735
1736	ftrace_enable_cpu();
1737
1738	iter->leftover = 0;
1739	for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1740	;
1741
1742	} else {
1743	/*
1744	* If we overflowed the seq_file before, then we want
1745	* to just reuse the trace_seq buffer again.
1746	*/
1747	if (iter->leftover)
1748	p = iter;
1749	else {
1750	l = *pos - 1;
1751	p = s_next(m, p, &l);
1752	}
1753	}
1754
1755	trace_event_read_lock();
1756	trace_access_lock(cpu_file);
1757	return p;
1758	}
1759
1760	static void s_stop(struct seq_file m, void p)
1761	{
1762	struct trace_iterator *iter = m->private;
1763
1764	atomic_dec(&trace_record_cmdline_disabled);
1765	trace_access_unlock(iter->cpu_file);
1766	trace_event_read_unlock();
1767	}
1768
1769	static void print_lat_help_header(struct seq_file *m)
1770	{
1771	seq_puts(m, "# _------=> CPU# \n");
1772	seq_puts(m, "# / _-----=> irqs-off \n");
1773	seq_puts(m, "# \| / _----=> need-resched \n");
1774	seq_puts(m, "# \|\| / _---=> hardirq/softirq \n");
1775	seq_puts(m, "# \|\|\| / _--=> preempt-depth \n");
1776	seq_puts(m, "# \|\|\|\| /_--=> lock-depth \n");
1777	seq_puts(m, "# \|\|\|\|\|/ delay \n");
1778	seq_puts(m, "# cmd pid \|\|\|\|\|\| time \| caller \n");
1779	seq_puts(m, "# \\ / \|\|\|\|\|\| \\ \| / \n");
1780	}
1781
1782	static void print_func_help_header(struct seq_file *m)
1783	{
1784	seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1785	seq_puts(m, "# \| \| \| \| \|\n");
1786	}
1787
1788
1789	static void
1790	print_trace_header(struct seq_file m, struct trace_iterator iter)
1791	{
1792	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1793	struct trace_array *tr = iter->tr;
1794	struct trace_array_cpu *data = tr->data[tr->cpu];
1795	struct tracer *type = current_trace;
1796	unsigned long entries = 0;
1797	unsigned long total = 0;
1798	unsigned long count;
1799	const char *name = "preemption";
1800	int cpu;
1801
1802	if (type)
1803	name = type->name;
1804
1805
1806	for_each_tracing_cpu(cpu) {
1807	count = ring_buffer_entries_cpu(tr->buffer, cpu);
1808	/*
1809	* If this buffer has skipped entries, then we hold all
1810	* entries for the trace and we need to ignore the
1811	* ones before the time stamp.
1812	*/
1813	if (tr->data[cpu]->skipped_entries) {
1814	count -= tr->data[cpu]->skipped_entries;
1815	/* total is the same as the entries */
1816	total += count;
1817	} else
1818	total += count +
1819	ring_buffer_overrun_cpu(tr->buffer, cpu);
1820	entries += count;
1821	}
1822
1823	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1824	name, UTS_RELEASE);
1825	seq_puts(m, "# -----------------------------------"
1826	"---------------------------------\n");
1827	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d \|"
1828	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1829	nsecs_to_usecs(data->saved_latency),
1830	entries,
1831	total,
1832	tr->cpu,
1833	#if defined(CONFIG_PREEMPT_NONE)
1834	"server",
1835	#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1836	"desktop",
1837	#elif defined(CONFIG_PREEMPT)
1838	"preempt",
1839	#else
1840	"unknown",
1841	#endif
1842	/* These are reserved for later use */
1843	0, 0, 0, 0);
1844	#ifdef CONFIG_SMP
1845	seq_printf(m, " #P:%d)\n", num_online_cpus());
1846	#else
1847	seq_puts(m, ")\n");
1848	#endif
1849	seq_puts(m, "# -----------------\n");
1850	seq_printf(m, "# \| task: %.16s-%d "
1851	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1852	data->comm, data->pid, data->uid, data->nice,
1853	data->policy, data->rt_priority);
1854	seq_puts(m, "# -----------------\n");
1855
1856	if (data->critical_start) {
1857	seq_puts(m, "# => started at: ");
1858	seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1859	trace_print_seq(m, &iter->seq);
1860	seq_puts(m, "\n# => ended at: ");
1861	seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1862	trace_print_seq(m, &iter->seq);
1863	seq_puts(m, "\n#\n");
1864	}
1865
1866	seq_puts(m, "#\n");
1867	}
1868
1869	static void test_cpu_buff_start(struct trace_iterator *iter)
1870	{
1871	struct trace_seq *s = &iter->seq;
1872
1873	if (!(trace_flags & TRACE_ITER_ANNOTATE))
1874	return;
1875
1876	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1877	return;
1878
1879	if (cpumask_test_cpu(iter->cpu, iter->started))
1880	return;
1881
1882	if (iter->tr->data[iter->cpu]->skipped_entries)
1883	return;
1884
1885	cpumask_set_cpu(iter->cpu, iter->started);
1886
1887	/* Don't print started cpu buffer for the first entry of the trace */
1888	if (iter->idx > 1)
1889	trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1890	iter->cpu);
1891	}
1892
1893	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1894	{
1895	struct trace_seq *s = &iter->seq;
1896	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1897	struct trace_entry *entry;
1898	struct trace_event *event;
1899
1900	entry = iter->ent;
1901
1902	test_cpu_buff_start(iter);
1903
1904	event = ftrace_find_event(entry->type);
1905
1906	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1907	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1908	if (!trace_print_lat_context(iter))
1909	goto partial;
1910	} else {
1911	if (!trace_print_context(iter))
1912	goto partial;
1913	}
1914	}
1915
1916	if (event)
1917	return event->trace(iter, sym_flags);
1918
1919	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1920	goto partial;
1921
1922	return TRACE_TYPE_HANDLED;
1923	partial:
1924	return TRACE_TYPE_PARTIAL_LINE;
1925	}
1926
1927	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1928	{
1929	struct trace_seq *s = &iter->seq;
1930	struct trace_entry *entry;
1931	struct trace_event *event;
1932
1933	entry = iter->ent;
1934
1935	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1936	if (!trace_seq_printf(s, "%d %d %llu ",
1937	entry->pid, iter->cpu, iter->ts))
1938	goto partial;
1939	}
1940
1941	event = ftrace_find_event(entry->type);
1942	if (event)
1943	return event->raw(iter, 0);
1944
1945	if (!trace_seq_printf(s, "%d ?\n", entry->type))
1946	goto partial;
1947
1948	return TRACE_TYPE_HANDLED;
1949	partial:
1950	return TRACE_TYPE_PARTIAL_LINE;
1951	}
1952
1953	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1954	{
1955	struct trace_seq *s = &iter->seq;
1956	unsigned char newline = '\n';
1957	struct trace_entry *entry;
1958	struct trace_event *event;
1959
1960	entry = iter->ent;
1961
1962	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1963	SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1964	SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1965	SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1966	}
1967
1968	event = ftrace_find_event(entry->type);
1969	if (event) {
1970	enum print_line_t ret = event->hex(iter, 0);
1971	if (ret != TRACE_TYPE_HANDLED)
1972	return ret;
1973	}
1974
1975	SEQ_PUT_FIELD_RET(s, newline);
1976
1977	return TRACE_TYPE_HANDLED;
1978	}
1979
1980	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1981	{
1982	struct trace_seq *s = &iter->seq;
1983	struct trace_entry *entry;
1984	struct trace_event *event;
1985
1986	entry = iter->ent;
1987
1988	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1989	SEQ_PUT_FIELD_RET(s, entry->pid);
1990	SEQ_PUT_FIELD_RET(s, iter->cpu);
1991	SEQ_PUT_FIELD_RET(s, iter->ts);
1992	}
1993
1994	event = ftrace_find_event(entry->type);
1995	return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
1996	}
1997
1998	static int trace_empty(struct trace_iterator *iter)
1999	{
2000	int cpu;
2001
2002	/* If we are looking at one CPU buffer, only check that one */
2003	if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2004	cpu = iter->cpu_file;
2005	if (iter->buffer_iter[cpu]) {
2006	if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2007	return 0;
2008	} else {
2009	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2010	return 0;
2011	}
2012	return 1;
2013	}
2014
2015	for_each_tracing_cpu(cpu) {
2016	if (iter->buffer_iter[cpu]) {
2017	if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2018	return 0;
2019	} else {
2020	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2021	return 0;
2022	}
2023	}
2024
2025	return 1;
2026	}
2027
2028	/* Called with trace_event_read_lock() held. */
2029	static enum print_line_t print_trace_line(struct trace_iterator *iter)
2030	{
2031	enum print_line_t ret;
2032
2033	if (iter->trace && iter->trace->print_line) {
2034	ret = iter->trace->print_line(iter);
2035	if (ret != TRACE_TYPE_UNHANDLED)
2036	return ret;
2037	}
2038
2039	if (iter->ent->type == TRACE_BPRINT &&
2040	trace_flags & TRACE_ITER_PRINTK &&
2041	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2042	return trace_print_bprintk_msg_only(iter);
2043
2044	if (iter->ent->type == TRACE_PRINT &&
2045	trace_flags & TRACE_ITER_PRINTK &&
2046	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2047	return trace_print_printk_msg_only(iter);
2048
2049	if (trace_flags & TRACE_ITER_BIN)
2050	return print_bin_fmt(iter);
2051
2052	if (trace_flags & TRACE_ITER_HEX)
2053	return print_hex_fmt(iter);
2054
2055	if (trace_flags & TRACE_ITER_RAW)
2056	return print_raw_fmt(iter);
2057
2058	return print_trace_fmt(iter);
2059	}
2060
2061	static int s_show(struct seq_file m, void v)
2062	{
2063	struct trace_iterator *iter = v;
2064	int ret;
2065
2066	if (iter->ent == NULL) {
2067	if (iter->tr) {
2068	seq_printf(m, "# tracer: %s\n", iter->trace->name);
2069	seq_puts(m, "#\n");
2070	}
2071	if (iter->trace && iter->trace->print_header)
2072	iter->trace->print_header(m);
2073	else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2074	/* print nothing if the buffers are empty */
2075	if (trace_empty(iter))
2076	return 0;
2077	print_trace_header(m, iter);
2078	if (!(trace_flags & TRACE_ITER_VERBOSE))
2079	print_lat_help_header(m);
2080	} else {
2081	if (!(trace_flags & TRACE_ITER_VERBOSE))
2082	print_func_help_header(m);
2083	}
2084	} else if (iter->leftover) {
2085	/*
2086	* If we filled the seq_file buffer earlier, we
2087	* want to just show it now.
2088	*/
2089	ret = trace_print_seq(m, &iter->seq);
2090
2091	/* ret should this time be zero, but you never know */
2092	iter->leftover = ret;
2093
2094	} else {
2095	print_trace_line(iter);
2096	ret = trace_print_seq(m, &iter->seq);
2097	/*
2098	* If we overflow the seq_file buffer, then it will
2099	* ask us for this data again at start up.
2100	* Use that instead.
2101	* ret is 0 if seq_file write succeeded.
2102	* -1 otherwise.
2103	*/
2104	iter->leftover = ret;
2105	}
2106
2107	return 0;
2108	}
2109
2110	static const struct seq_operations tracer_seq_ops = {
2111	.start = s_start,
2112	.next = s_next,
2113	.stop = s_stop,
2114	.show = s_show,
2115	};
2116
2117	static struct trace_iterator *
2118	__tracing_open(struct inode inode, struct file file)
2119	{
2120	long cpu_file = (long) inode->i_private;
2121	void *fail_ret = ERR_PTR(-ENOMEM);
2122	struct trace_iterator *iter;
2123	struct seq_file *m;
2124	int cpu, ret;
2125
2126	if (tracing_disabled)
2127	return ERR_PTR(-ENODEV);
2128
2129	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2130	if (!iter)
2131	return ERR_PTR(-ENOMEM);
2132
2133	/*
2134	* We make a copy of the current tracer to avoid concurrent
2135	* changes on it while we are reading.
2136	*/
2137	mutex_lock(&trace_types_lock);
2138	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2139	if (!iter->trace)
2140	goto fail;
2141
2142	if (current_trace)
2143	iter->trace = current_trace;
2144
2145	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2146	goto fail;
2147
2148	if (current_trace && current_trace->print_max)
2149	iter->tr = &max_tr;
2150	else
2151	iter->tr = &global_trace;
2152	iter->pos = -1;
2153	mutex_init(&iter->mutex);
2154	iter->cpu_file = cpu_file;
2155
2156	/* Notify the tracer early; before we stop tracing. */
2157	if (iter->trace && iter->trace->open)
2158	iter->trace->open(iter);
2159
2160	/* Annotate start of buffers if we had overruns */
2161	if (ring_buffer_overruns(iter->tr->buffer))
2162	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
2163
2164	/* stop the trace while dumping */
2165	tracing_stop();
2166
2167	if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2168	for_each_tracing_cpu(cpu) {
2169
2170	iter->buffer_iter[cpu] =
2171	ring_buffer_read_start(iter->tr->buffer, cpu);
2172	tracing_iter_reset(iter, cpu);
2173	}
2174	} else {
2175	cpu = iter->cpu_file;
2176	iter->buffer_iter[cpu] =
2177	ring_buffer_read_start(iter->tr->buffer, cpu);
2178	tracing_iter_reset(iter, cpu);
2179	}
2180
2181	ret = seq_open(file, &tracer_seq_ops);
2182	if (ret < 0) {
2183	fail_ret = ERR_PTR(ret);
2184	goto fail_buffer;
2185	}
2186
2187	m = file->private_data;
2188	m->private = iter;
2189
2190	mutex_unlock(&trace_types_lock);
2191
2192	return iter;
2193
2194	fail_buffer:
2195	for_each_tracing_cpu(cpu) {
2196	if (iter->buffer_iter[cpu])
2197	ring_buffer_read_finish(iter->buffer_iter[cpu]);
2198	}
2199	free_cpumask_var(iter->started);
2200	tracing_start();
2201	fail:
2202	mutex_unlock(&trace_types_lock);
2203	kfree(iter->trace);
2204	kfree(iter);
2205
2206	return fail_ret;
2207	}
2208
2209	int tracing_open_generic(struct inode inode, struct file filp)
2210	{
2211	if (tracing_disabled)
2212	return -ENODEV;
2213
2214	filp->private_data = inode->i_private;
2215	return 0;
2216	}
2217
2218	static int tracing_release(struct inode inode, struct file file)
2219	{
2220	struct seq_file m = (struct seq_file )file->private_data;
2221	struct trace_iterator *iter;
2222	int cpu;
2223
2224	if (!(file->f_mode & FMODE_READ))
2225	return 0;
2226
2227	iter = m->private;
2228
2229	mutex_lock(&trace_types_lock);
2230	for_each_tracing_cpu(cpu) {
2231	if (iter->buffer_iter[cpu])
2232	ring_buffer_read_finish(iter->buffer_iter[cpu]);
2233	}
2234
2235	if (iter->trace && iter->trace->close)
2236	iter->trace->close(iter);
2237
2238	/* reenable tracing if it was previously enabled */
2239	tracing_start();
2240	mutex_unlock(&trace_types_lock);
2241
2242	seq_release(inode, file);
2243	mutex_destroy(&iter->mutex);
2244	free_cpumask_var(iter->started);
2245	kfree(iter->trace);
2246	kfree(iter);
2247	return 0;
2248	}
2249
2250	static int tracing_open(struct inode inode, struct file file)
2251	{
2252	struct trace_iterator *iter;
2253	int ret = 0;
2254
2255	/* If this file was open for write, then erase contents */
2256	if ((file->f_mode & FMODE_WRITE) &&
2257	(file->f_flags & O_TRUNC)) {
2258	long cpu = (long) inode->i_private;
2259
2260	if (cpu == TRACE_PIPE_ALL_CPU)
2261	tracing_reset_online_cpus(&global_trace);
2262	else
2263	tracing_reset(&global_trace, cpu);
2264	}
2265
2266	if (file->f_mode & FMODE_READ) {
2267	iter = __tracing_open(inode, file);
2268	if (IS_ERR(iter))
2269	ret = PTR_ERR(iter);
2270	else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2271	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
2272	}
2273	return ret;
2274	}
2275
2276	static void *
2277	t_next(struct seq_file m, void v, loff_t *pos)
2278	{
2279	struct tracer *t = v;
2280
2281	(*pos)++;
2282
2283	if (t)
2284	t = t->next;
2285
2286	return t;
2287	}
2288
2289	static void t_start(struct seq_file m, loff_t *pos)
2290	{
2291	struct tracer *t;
2292	loff_t l = 0;
2293
2294	mutex_lock(&trace_types_lock);
2295	for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2296	;
2297
2298	return t;
2299	}
2300
2301	static void t_stop(struct seq_file m, void p)
2302	{
2303	mutex_unlock(&trace_types_lock);
2304	}
2305
2306	static int t_show(struct seq_file m, void v)
2307	{
2308	struct tracer *t = v;
2309
2310	if (!t)
2311	return 0;
2312
2313	seq_printf(m, "%s", t->name);
2314	if (t->next)
2315	seq_putc(m, ' ');
2316	else
2317	seq_putc(m, '\n');
2318
2319	return 0;
2320	}
2321
2322	static const struct seq_operations show_traces_seq_ops = {
2323	.start = t_start,
2324	.next = t_next,
2325	.stop = t_stop,
2326	.show = t_show,
2327	};
2328
2329	static int show_traces_open(struct inode inode, struct file file)
2330	{
2331	if (tracing_disabled)
2332	return -ENODEV;
2333
2334	return seq_open(file, &show_traces_seq_ops);
2335	}
2336
2337	static ssize_t
2338	tracing_write_stub(struct file filp, const char __user ubuf,
2339	size_t count, loff_t *ppos)
2340	{
2341	return count;
2342	}
2343
2344	static const struct file_operations tracing_fops = {
2345	.open = tracing_open,
2346	.read = seq_read,
2347	.write = tracing_write_stub,
2348	.llseek = seq_lseek,
2349	.release = tracing_release,
2350	};
2351
2352	static const struct file_operations show_traces_fops = {
2353	.open = show_traces_open,
2354	.read = seq_read,
2355	.release = seq_release,
2356	};
2357
2358	/*
2359	* Only trace on a CPU if the bitmask is set:
2360	*/
2361	static cpumask_var_t tracing_cpumask;
2362
2363	/*
2364	* The tracer itself will not take this lock, but still we want
2365	* to provide a consistent cpumask to user-space:
2366	*/
2367	static DEFINE_MUTEX(tracing_cpumask_update_lock);
2368
2369	/*
2370	* Temporary storage for the character representation of the
2371	* CPU bitmask (and one more byte for the newline):
2372	*/
2373	static char mask_str[NR_CPUS + 1];
2374
2375	static ssize_t
2376	tracing_cpumask_read(struct file filp, char __user ubuf,
2377	size_t count, loff_t *ppos)
2378	{
2379	int len;
2380
2381	mutex_lock(&tracing_cpumask_update_lock);
2382
2383	len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2384	if (count - len < 2) {
2385	count = -EINVAL;
2386	goto out_err;
2387	}
2388	len += sprintf(mask_str + len, "\n");
2389	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2390
2391	out_err:
2392	mutex_unlock(&tracing_cpumask_update_lock);
2393
2394	return count;
2395	}
2396
2397	static ssize_t
2398	tracing_cpumask_write(struct file filp, const char __user ubuf,
2399	size_t count, loff_t *ppos)
2400	{
2401	int err, cpu;
2402	cpumask_var_t tracing_cpumask_new;
2403
2404	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2405	return -ENOMEM;
2406
2407	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2408	if (err)
2409	goto err_unlock;
2410
2411	mutex_lock(&tracing_cpumask_update_lock);
2412
2413	local_irq_disable();
2414	arch_spin_lock(&ftrace_max_lock);
2415	for_each_tracing_cpu(cpu) {
2416	/*
2417	* Increase/decrease the disabled counter if we are
2418	* about to flip a bit in the cpumask:
2419	*/
2420	if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2421	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2422	atomic_inc(&global_trace.data[cpu]->disabled);
2423	}
2424	if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2425	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2426	atomic_dec(&global_trace.data[cpu]->disabled);
2427	}
2428	}
2429	arch_spin_unlock(&ftrace_max_lock);
2430	local_irq_enable();
2431
2432	cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2433
2434	mutex_unlock(&tracing_cpumask_update_lock);
2435	free_cpumask_var(tracing_cpumask_new);
2436
2437	return count;
2438
2439	err_unlock:
2440	free_cpumask_var(tracing_cpumask_new);
2441
2442	return err;
2443	}
2444
2445	static const struct file_operations tracing_cpumask_fops = {
2446	.open = tracing_open_generic,
2447	.read = tracing_cpumask_read,
2448	.write = tracing_cpumask_write,
2449	};
2450
2451	static int tracing_trace_options_show(struct seq_file m, void v)
2452	{
2453	struct tracer_opt *trace_opts;
2454	u32 tracer_flags;
2455	int i;
2456
2457	mutex_lock(&trace_types_lock);
2458	tracer_flags = current_trace->flags->val;
2459	trace_opts = current_trace->flags->opts;
2460
2461	for (i = 0; trace_options[i]; i++) {
2462	if (trace_flags & (1 << i))
2463	seq_printf(m, "%s\n", trace_options[i]);
2464	else
2465	seq_printf(m, "no%s\n", trace_options[i]);
2466	}
2467
2468	for (i = 0; trace_opts[i].name; i++) {
2469	if (tracer_flags & trace_opts[i].bit)
2470	seq_printf(m, "%s\n", trace_opts[i].name);
2471	else
2472	seq_printf(m, "no%s\n", trace_opts[i].name);
2473	}
2474	mutex_unlock(&trace_types_lock);
2475
2476	return 0;
2477	}
2478
2479	static int __set_tracer_option(struct tracer *trace,
2480	struct tracer_flags *tracer_flags,
2481	struct tracer_opt *opts, int neg)
2482	{
2483	int ret;
2484
2485	ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2486	if (ret)
2487	return ret;
2488
2489	if (neg)
2490	tracer_flags->val &= ~opts->bit;
2491	else
2492	tracer_flags->val \|= opts->bit;
2493	return 0;
2494	}
2495
2496	/* Try to assign a tracer specific option */
2497	static int set_tracer_option(struct tracer trace, char cmp, int neg)
2498	{
2499	struct tracer_flags *tracer_flags = trace->flags;
2500	struct tracer_opt *opts = NULL;
2501	int i;
2502
2503	for (i = 0; tracer_flags->opts[i].name; i++) {
2504	opts = &tracer_flags->opts[i];
2505
2506	if (strcmp(cmp, opts->name) == 0)
2507	return __set_tracer_option(trace, trace->flags,
2508	opts, neg);
2509	}
2510
2511	return -EINVAL;
2512	}
2513
2514	static void set_tracer_flags(unsigned int mask, int enabled)
2515	{
2516	/* do nothing if flag is already set */
2517	if (!!(trace_flags & mask) == !!enabled)
2518	return;
2519
2520	if (enabled)
2521	trace_flags \|= mask;
2522	else
2523	trace_flags &= ~mask;
2524	}
2525
2526	static ssize_t
2527	tracing_trace_options_write(struct file filp, const char __user ubuf,
2528	size_t cnt, loff_t *ppos)
2529	{
2530	char buf[64];
2531	char *cmp;
2532	int neg = 0;
2533	int ret;
2534	int i;
2535
2536	if (cnt >= sizeof(buf))
2537	return -EINVAL;
2538
2539	if (copy_from_user(&buf, ubuf, cnt))
2540	return -EFAULT;
2541
2542	buf[cnt] = 0;
2543	cmp = strstrip(buf);
2544
2545	if (strncmp(cmp, "no", 2) == 0) {
2546	neg = 1;
2547	cmp += 2;
2548	}
2549
2550	for (i = 0; trace_options[i]; i++) {
2551	if (strcmp(cmp, trace_options[i]) == 0) {
2552	set_tracer_flags(1 << i, !neg);
2553	break;
2554	}
2555	}
2556
2557	/* If no option could be set, test the specific tracer options */
2558	if (!trace_options[i]) {
2559	mutex_lock(&trace_types_lock);
2560	ret = set_tracer_option(current_trace, cmp, neg);
2561	mutex_unlock(&trace_types_lock);
2562	if (ret)
2563	return ret;
2564	}
2565
2566	*ppos += cnt;
2567
2568	return cnt;
2569	}
2570
2571	static int tracing_trace_options_open(struct inode inode, struct file file)
2572	{
2573	if (tracing_disabled)
2574	return -ENODEV;
2575	return single_open(file, tracing_trace_options_show, NULL);
2576	}
2577
2578	static const struct file_operations tracing_iter_fops = {
2579	.open = tracing_trace_options_open,
2580	.read = seq_read,
2581	.llseek = seq_lseek,
2582	.release = single_release,
2583	.write = tracing_trace_options_write,
2584	};
2585
2586	static const char readme_msg[] =
2587	"tracing mini-HOWTO:\n\n"
2588	"# mount -t debugfs nodev /sys/kernel/debug\n\n"
2589	"# cat /sys/kernel/debug/tracing/available_tracers\n"
2590	"wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
2591	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2592	"nop\n"
2593	"# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2594	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2595	"sched_switch\n"
2596	"# cat /sys/kernel/debug/tracing/trace_options\n"
2597	"noprint-parent nosym-offset nosym-addr noverbose\n"
2598	"# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2599	"# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
2600	"# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2601	"# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
2602	;
2603
2604	static ssize_t
2605	tracing_readme_read(struct file filp, char __user ubuf,
2606	size_t cnt, loff_t *ppos)
2607	{
2608	return simple_read_from_buffer(ubuf, cnt, ppos,
2609	readme_msg, strlen(readme_msg));
2610	}
2611
2612	static const struct file_operations tracing_readme_fops = {
2613	.open = tracing_open_generic,
2614	.read = tracing_readme_read,
2615	};
2616
2617	static ssize_t
2618	tracing_saved_cmdlines_read(struct file file, char __user ubuf,
2619	size_t cnt, loff_t *ppos)
2620	{
2621	char *buf_comm;
2622	char *file_buf;
2623	char *buf;
2624	int len = 0;
2625	int pid;
2626	int i;
2627
2628	file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2629	if (!file_buf)
2630	return -ENOMEM;
2631
2632	buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2633	if (!buf_comm) {
2634	kfree(file_buf);
2635	return -ENOMEM;
2636	}
2637
2638	buf = file_buf;
2639
2640	for (i = 0; i < SAVED_CMDLINES; i++) {
2641	int r;
2642
2643	pid = map_cmdline_to_pid[i];
2644	if (pid == -1 \|\| pid == NO_CMDLINE_MAP)
2645	continue;
2646
2647	trace_find_cmdline(pid, buf_comm);
2648	r = sprintf(buf, "%d %s\n", pid, buf_comm);
2649	buf += r;
2650	len += r;
2651	}
2652
2653	len = simple_read_from_buffer(ubuf, cnt, ppos,
2654	file_buf, len);
2655
2656	kfree(file_buf);
2657	kfree(buf_comm);
2658
2659	return len;
2660	}
2661
2662	static const struct file_operations tracing_saved_cmdlines_fops = {
2663	.open = tracing_open_generic,
2664	.read = tracing_saved_cmdlines_read,
2665	};
2666
2667	static ssize_t
2668	tracing_ctrl_read(struct file filp, char __user ubuf,
2669	size_t cnt, loff_t *ppos)
2670	{
2671	char buf[64];
2672	int r;
2673
2674	r = sprintf(buf, "%u\n", tracer_enabled);
2675	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2676	}
2677
2678	static ssize_t
2679	tracing_ctrl_write(struct file filp, const char __user ubuf,
2680	size_t cnt, loff_t *ppos)
2681	{
2682	struct trace_array *tr = filp->private_data;
2683	char buf[64];
2684	unsigned long val;
2685	int ret;
2686
2687	if (cnt >= sizeof(buf))
2688	return -EINVAL;
2689
2690	if (copy_from_user(&buf, ubuf, cnt))
2691	return -EFAULT;
2692
2693	buf[cnt] = 0;
2694
2695	ret = strict_strtoul(buf, 10, &val);
2696	if (ret < 0)
2697	return ret;
2698
2699	val = !!val;
2700
2701	mutex_lock(&trace_types_lock);
2702	if (tracer_enabled ^ val) {
2703	if (val) {
2704	tracer_enabled = 1;
2705	if (current_trace->start)
2706	current_trace->start(tr);
2707	tracing_start();
2708	} else {
2709	tracer_enabled = 0;
2710	tracing_stop();
2711	if (current_trace->stop)
2712	current_trace->stop(tr);
2713	}
2714	}
2715	mutex_unlock(&trace_types_lock);
2716
2717	*ppos += cnt;
2718
2719	return cnt;
2720	}
2721
2722	static ssize_t
2723	tracing_set_trace_read(struct file filp, char __user ubuf,
2724	size_t cnt, loff_t *ppos)
2725	{
2726	char buf[MAX_TRACER_SIZE+2];
2727	int r;
2728
2729	mutex_lock(&trace_types_lock);
2730	if (current_trace)
2731	r = sprintf(buf, "%s\n", current_trace->name);
2732	else
2733	r = sprintf(buf, "\n");
2734	mutex_unlock(&trace_types_lock);
2735
2736	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2737	}
2738
2739	int tracer_init(struct tracer t, struct trace_array tr)
2740	{
2741	tracing_reset_online_cpus(tr);
2742	return t->init(tr);
2743	}
2744
2745	static int tracing_resize_ring_buffer(unsigned long size)
2746	{
2747	int ret;
2748
2749	/*
2750	* If kernel or user changes the size of the ring buffer
2751	* we use the size that was given, and we can forget about
2752	* expanding it later.
2753	*/
2754	ring_buffer_expanded = 1;
2755
2756	ret = ring_buffer_resize(global_trace.buffer, size);
2757	if (ret < 0)
2758	return ret;
2759
2760	ret = ring_buffer_resize(max_tr.buffer, size);
2761	if (ret < 0) {
2762	int r;
2763
2764	r = ring_buffer_resize(global_trace.buffer,
2765	global_trace.entries);
2766	if (r < 0) {
2767	/*
2768	* AARGH! We are left with different
2769	* size max buffer!!!!
2770	* The max buffer is our "snapshot" buffer.
2771	* When a tracer needs a snapshot (one of the
2772	* latency tracers), it swaps the max buffer
2773	* with the saved snap shot. We succeeded to
2774	* update the size of the main buffer, but failed to
2775	* update the size of the max buffer. But when we tried
2776	* to reset the main buffer to the original size, we
2777	* failed there too. This is very unlikely to
2778	* happen, but if it does, warn and kill all
2779	* tracing.
2780	*/
2781	WARN_ON(1);
2782	tracing_disabled = 1;
2783	}
2784	return ret;
2785	}
2786
2787	global_trace.entries = size;
2788
2789	return ret;
2790	}
2791
2792	/**
2793	* tracing_update_buffers - used by tracing facility to expand ring buffers
2794	*
2795	* To save on memory when the tracing is never used on a system with it
2796	* configured in. The ring buffers are set to a minimum size. But once
2797	* a user starts to use the tracing facility, then they need to grow
2798	* to their default size.
2799	*
2800	* This function is to be called when a tracer is about to be used.
2801	*/
2802	int tracing_update_buffers(void)
2803	{
2804	int ret = 0;
2805
2806	mutex_lock(&trace_types_lock);
2807	if (!ring_buffer_expanded)
2808	ret = tracing_resize_ring_buffer(trace_buf_size);
2809	mutex_unlock(&trace_types_lock);
2810
2811	return ret;
2812	}
2813
2814	struct trace_option_dentry;
2815
2816	static struct trace_option_dentry *
2817	create_trace_option_files(struct tracer *tracer);
2818
2819	static void
2820	destroy_trace_option_files(struct trace_option_dentry *topts);
2821
2822	static int tracing_set_tracer(const char *buf)
2823	{
2824	static struct trace_option_dentry *topts;
2825	struct trace_array *tr = &global_trace;
2826	struct tracer *t;
2827	int ret = 0;
2828
2829	mutex_lock(&trace_types_lock);
2830
2831	if (!ring_buffer_expanded) {
2832	ret = tracing_resize_ring_buffer(trace_buf_size);
2833	if (ret < 0)
2834	goto out;
2835	ret = 0;
2836	}
2837
2838	for (t = trace_types; t; t = t->next) {
2839	if (strcmp(t->name, buf) == 0)
2840	break;
2841	}
2842	if (!t) {
2843	ret = -EINVAL;
2844	goto out;
2845	}
2846	if (t == current_trace)
2847	goto out;
2848
2849	trace_branch_disable();
2850	if (current_trace && current_trace->reset)
2851	current_trace->reset(tr);
2852
2853	destroy_trace_option_files(topts);
2854
2855	current_trace = t;
2856
2857	topts = create_trace_option_files(current_trace);
2858
2859	if (t->init) {
2860	ret = tracer_init(t, tr);
2861	if (ret)
2862	goto out;
2863	}
2864
2865	trace_branch_enable(tr);
2866	out:
2867	mutex_unlock(&trace_types_lock);
2868
2869	return ret;
2870	}
2871
2872	static ssize_t
2873	tracing_set_trace_write(struct file filp, const char __user ubuf,
2874	size_t cnt, loff_t *ppos)
2875	{
2876	char buf[MAX_TRACER_SIZE+1];
2877	int i;
2878	size_t ret;
2879	int err;
2880
2881	ret = cnt;
2882
2883	if (cnt > MAX_TRACER_SIZE)
2884	cnt = MAX_TRACER_SIZE;
2885
2886	if (copy_from_user(&buf, ubuf, cnt))
2887	return -EFAULT;
2888
2889	buf[cnt] = 0;
2890
2891	/* strip ending whitespace. */
2892	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2893	buf[i] = 0;
2894
2895	err = tracing_set_tracer(buf);
2896	if (err)
2897	return err;
2898
2899	*ppos += ret;
2900
2901	return ret;
2902	}
2903
2904	static ssize_t
2905	tracing_max_lat_read(struct file filp, char __user ubuf,
2906	size_t cnt, loff_t *ppos)
2907	{
2908	unsigned long *ptr = filp->private_data;
2909	char buf[64];
2910	int r;
2911
2912	r = snprintf(buf, sizeof(buf), "%ld\n",
2913	ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(ptr));
2914	if (r > sizeof(buf))
2915	r = sizeof(buf);
2916	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2917	}
2918
2919	static ssize_t
2920	tracing_max_lat_write(struct file filp, const char __user ubuf,
2921	size_t cnt, loff_t *ppos)
2922	{
2923	unsigned long *ptr = filp->private_data;
2924	char buf[64];
2925	unsigned long val;
2926	int ret;
2927
2928	if (cnt >= sizeof(buf))
2929	return -EINVAL;
2930
2931	if (copy_from_user(&buf, ubuf, cnt))
2932	return -EFAULT;
2933
2934	buf[cnt] = 0;
2935
2936	ret = strict_strtoul(buf, 10, &val);
2937	if (ret < 0)
2938	return ret;
2939
2940	ptr = val 1000;
2941
2942	return cnt;
2943	}
2944
2945	static int tracing_open_pipe(struct inode inode, struct file filp)
2946	{
2947	long cpu_file = (long) inode->i_private;
2948	struct trace_iterator *iter;
2949	int ret = 0;
2950
2951	if (tracing_disabled)
2952	return -ENODEV;
2953
2954	mutex_lock(&trace_types_lock);
2955
2956	/* create a buffer to store the information to pass to userspace */
2957	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2958	if (!iter) {
2959	ret = -ENOMEM;
2960	goto out;
2961	}
2962
2963	/*
2964	* We make a copy of the current tracer to avoid concurrent
2965	* changes on it while we are reading.
2966	*/
2967	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
2968	if (!iter->trace) {
2969	ret = -ENOMEM;
2970	goto fail;
2971	}
2972	if (current_trace)
2973	iter->trace = current_trace;
2974
2975	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
2976	ret = -ENOMEM;
2977	goto fail;
2978	}
2979
2980	/* trace pipe does not show start of buffer */
2981	cpumask_setall(iter->started);
2982
2983	if (trace_flags & TRACE_ITER_LATENCY_FMT)
2984	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
2985
2986	iter->cpu_file = cpu_file;
2987	iter->tr = &global_trace;
2988	mutex_init(&iter->mutex);
2989	filp->private_data = iter;
2990
2991	if (iter->trace->pipe_open)
2992	iter->trace->pipe_open(iter);
2993
2994	out:
2995	mutex_unlock(&trace_types_lock);
2996	return ret;
2997
2998	fail:
2999	kfree(iter->trace);
3000	kfree(iter);
3001	mutex_unlock(&trace_types_lock);
3002	return ret;
3003	}
3004
3005	static int tracing_release_pipe(struct inode inode, struct file file)
3006	{
3007	struct trace_iterator *iter = file->private_data;
3008
3009	mutex_lock(&trace_types_lock);
3010
3011	if (iter->trace->pipe_close)
3012	iter->trace->pipe_close(iter);
3013
3014	mutex_unlock(&trace_types_lock);
3015
3016	free_cpumask_var(iter->started);
3017	mutex_destroy(&iter->mutex);
3018	kfree(iter->trace);
3019	kfree(iter);
3020
3021	return 0;
3022	}
3023
3024	static unsigned int
3025	tracing_poll_pipe(struct file filp, poll_table poll_table)
3026	{
3027	struct trace_iterator *iter = filp->private_data;
3028
3029	if (trace_flags & TRACE_ITER_BLOCK) {
3030	/*
3031	* Always select as readable when in blocking mode
3032	*/
3033	return POLLIN \| POLLRDNORM;
3034	} else {
3035	if (!trace_empty(iter))
3036	return POLLIN \| POLLRDNORM;
3037	poll_wait(filp, &trace_wait, poll_table);
3038	if (!trace_empty(iter))
3039	return POLLIN \| POLLRDNORM;
3040
3041	return 0;
3042	}
3043	}
3044
3045
3046	void default_wait_pipe(struct trace_iterator *iter)
3047	{
3048	DEFINE_WAIT(wait);
3049
3050	prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3051
3052	if (trace_empty(iter))
3053	schedule();
3054
3055	finish_wait(&trace_wait, &wait);
3056	}
3057
3058	/*
3059	* This is a make-shift waitqueue.
3060	* A tracer might use this callback on some rare cases:
3061	*
3062	* 1) the current tracer might hold the runqueue lock when it wakes up
3063	* a reader, hence a deadlock (sched, function, and function graph tracers)
3064	* 2) the function tracers, trace all functions, we don't want
3065	* the overhead of calling wake_up and friends
3066	* (and tracing them too)
3067	*
3068	* Anyway, this is really very primitive wakeup.
3069	*/
3070	void poll_wait_pipe(struct trace_iterator *iter)
3071	{
3072	set_current_state(TASK_INTERRUPTIBLE);
3073	/* sleep for 100 msecs, and try again. */
3074	schedule_timeout(HZ / 10);
3075	}
3076
3077	/* Must be called with trace_types_lock mutex held. */
3078	static int tracing_wait_pipe(struct file *filp)
3079	{
3080	struct trace_iterator *iter = filp->private_data;
3081
3082	while (trace_empty(iter)) {
3083
3084	if ((filp->f_flags & O_NONBLOCK)) {
3085	return -EAGAIN;
3086	}
3087
3088	mutex_unlock(&iter->mutex);
3089
3090	iter->trace->wait_pipe(iter);
3091
3092	mutex_lock(&iter->mutex);
3093
3094	if (signal_pending(current))
3095	return -EINTR;
3096
3097	/*
3098	* We block until we read something and tracing is disabled.
3099	* We still block if tracing is disabled, but we have never
3100	* read anything. This allows a user to cat this file, and
3101	* then enable tracing. But after we have read something,
3102	* we give an EOF when tracing is again disabled.
3103	*
3104	* iter->pos will be 0 if we haven't read anything.
3105	*/
3106	if (!tracer_enabled && iter->pos)
3107	break;
3108	}
3109
3110	return 1;
3111	}
3112
3113	/*
3114	* Consumer reader.
3115	*/
3116	static ssize_t
3117	tracing_read_pipe(struct file filp, char __user ubuf,
3118	size_t cnt, loff_t *ppos)
3119	{
3120	struct trace_iterator *iter = filp->private_data;
3121	static struct tracer *old_tracer;
3122	ssize_t sret;
3123
3124	/* return any leftover data */
3125	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3126	if (sret != -EBUSY)
3127	return sret;
3128
3129	trace_seq_init(&iter->seq);
3130
3131	/* copy the tracer to avoid using a global lock all around */
3132	mutex_lock(&trace_types_lock);
3133	if (unlikely(old_tracer != current_trace && current_trace)) {
3134	old_tracer = current_trace;
3135	iter->trace = current_trace;
3136	}
3137	mutex_unlock(&trace_types_lock);
3138
3139	/*
3140	* Avoid more than one consumer on a single file descriptor
3141	* This is just a matter of traces coherency, the ring buffer itself
3142	* is protected.
3143	*/
3144	mutex_lock(&iter->mutex);
3145	if (iter->trace->read) {
3146	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3147	if (sret)
3148	goto out;
3149	}
3150
3151	waitagain:
3152	sret = tracing_wait_pipe(filp);
3153	if (sret <= 0)
3154	goto out;
3155
3156	/* stop when tracing is finished */
3157	if (trace_empty(iter)) {
3158	sret = 0;
3159	goto out;
3160	}
3161
3162	if (cnt >= PAGE_SIZE)
3163	cnt = PAGE_SIZE - 1;
3164
3165	/* reset all but tr, trace, and overruns */
3166	memset(&iter->seq, 0,
3167	sizeof(struct trace_iterator) -
3168	offsetof(struct trace_iterator, seq));
3169	iter->pos = -1;
3170
3171	trace_event_read_lock();
3172	trace_access_lock(iter->cpu_file);
3173	while (find_next_entry_inc(iter) != NULL) {
3174	enum print_line_t ret;
3175	int len = iter->seq.len;
3176
3177	ret = print_trace_line(iter);
3178	if (ret == TRACE_TYPE_PARTIAL_LINE) {
3179	/* don't print partial lines */
3180	iter->seq.len = len;
3181	break;
3182	}
3183	if (ret != TRACE_TYPE_NO_CONSUME)
3184	trace_consume(iter);
3185
3186	if (iter->seq.len >= cnt)
3187	break;
3188	}
3189	trace_access_unlock(iter->cpu_file);
3190	trace_event_read_unlock();
3191
3192	/* Now copy what we have to the user */
3193	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3194	if (iter->seq.readpos >= iter->seq.len)
3195	trace_seq_init(&iter->seq);
3196
3197	/*
3198	* If there was nothing to send to user, inspite of consuming trace
3199	* entries, go back to wait for more entries.
3200	*/
3201	if (sret == -EBUSY)
3202	goto waitagain;
3203
3204	out:
3205	mutex_unlock(&iter->mutex);
3206
3207	return sret;
3208	}
3209
3210	static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3211	struct pipe_buffer *buf)
3212	{
3213	__free_page(buf->page);
3214	}
3215
3216	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3217	unsigned int idx)
3218	{
3219	__free_page(spd->pages[idx]);
3220	}
3221
3222	static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3223	.can_merge = 0,
3224	.map = generic_pipe_buf_map,
3225	.unmap = generic_pipe_buf_unmap,
3226	.confirm = generic_pipe_buf_confirm,
3227	.release = tracing_pipe_buf_release,
3228	.steal = generic_pipe_buf_steal,
3229	.get = generic_pipe_buf_get,
3230	};
3231
3232	static size_t
3233	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3234	{
3235	size_t count;
3236	int ret;
3237
3238	/* Seq buffer is page-sized, exactly what we need. */
3239	for (;;) {
3240	count = iter->seq.len;
3241	ret = print_trace_line(iter);
3242	count = iter->seq.len - count;
3243	if (rem < count) {
3244	rem = 0;
3245	iter->seq.len -= count;
3246	break;
3247	}
3248	if (ret == TRACE_TYPE_PARTIAL_LINE) {
3249	iter->seq.len -= count;
3250	break;
3251	}
3252
3253	if (ret != TRACE_TYPE_NO_CONSUME)
3254	trace_consume(iter);
3255	rem -= count;
3256	if (!find_next_entry_inc(iter)) {
3257	rem = 0;
3258	iter->ent = NULL;
3259	break;
3260	}
3261	}
3262
3263	return rem;
3264	}
3265
3266	static ssize_t tracing_splice_read_pipe(struct file *filp,
3267	loff_t *ppos,
3268	struct pipe_inode_info *pipe,
3269	size_t len,
3270	unsigned int flags)
3271	{
3272	struct page *pages[PIPE_BUFFERS];
3273	struct partial_page partial[PIPE_BUFFERS];
3274	struct trace_iterator *iter = filp->private_data;
3275	struct splice_pipe_desc spd = {
3276	.pages = pages,
3277	.partial = partial,
3278	.nr_pages = 0, /* This gets updated below. */
3279	.flags = flags,
3280	.ops = &tracing_pipe_buf_ops,
3281	.spd_release = tracing_spd_release_pipe,
3282	};
3283	static struct tracer *old_tracer;
3284	ssize_t ret;
3285	size_t rem;
3286	unsigned int i;
3287
3288	/* copy the tracer to avoid using a global lock all around */
3289	mutex_lock(&trace_types_lock);
3290	if (unlikely(old_tracer != current_trace && current_trace)) {
3291	old_tracer = current_trace;
3292	iter->trace = current_trace;
3293	}
3294	mutex_unlock(&trace_types_lock);
3295
3296	mutex_lock(&iter->mutex);
3297
3298	if (iter->trace->splice_read) {
3299	ret = iter->trace->splice_read(iter, filp,
3300	ppos, pipe, len, flags);
3301	if (ret)
3302	goto out_err;
3303	}
3304
3305	ret = tracing_wait_pipe(filp);
3306	if (ret <= 0)
3307	goto out_err;
3308
3309	if (!iter->ent && !find_next_entry_inc(iter)) {
3310	ret = -EFAULT;
3311	goto out_err;
3312	}
3313
3314	trace_event_read_lock();
3315	trace_access_lock(iter->cpu_file);
3316
3317	/* Fill as many pages as possible. */
3318	for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
3319	pages[i] = alloc_page(GFP_KERNEL);
3320	if (!pages[i])
3321	break;
3322
3323	rem = tracing_fill_pipe_page(rem, iter);
3324
3325	/* Copy the data into the page, so we can start over. */
3326	ret = trace_seq_to_buffer(&iter->seq,
3327	page_address(pages[i]),
3328	iter->seq.len);
3329	if (ret < 0) {
3330	__free_page(pages[i]);
3331	break;
3332	}
3333	partial[i].offset = 0;
3334	partial[i].len = iter->seq.len;
3335
3336	trace_seq_init(&iter->seq);
3337	}
3338
3339	trace_access_unlock(iter->cpu_file);
3340	trace_event_read_unlock();
3341	mutex_unlock(&iter->mutex);
3342
3343	spd.nr_pages = i;
3344
3345	return splice_to_pipe(pipe, &spd);
3346
3347	out_err:
3348	mutex_unlock(&iter->mutex);
3349
3350	return ret;
3351	}
3352
3353	static ssize_t
3354	tracing_entries_read(struct file filp, char __user ubuf,
3355	size_t cnt, loff_t *ppos)
3356	{
3357	struct trace_array *tr = filp->private_data;
3358	char buf[96];
3359	int r;
3360
3361	mutex_lock(&trace_types_lock);
3362	if (!ring_buffer_expanded)
3363	r = sprintf(buf, "%lu (expanded: %lu)\n",
3364	tr->entries >> 10,
3365	trace_buf_size >> 10);
3366	else
3367	r = sprintf(buf, "%lu\n", tr->entries >> 10);
3368	mutex_unlock(&trace_types_lock);
3369
3370	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3371	}
3372
3373	static ssize_t
3374	tracing_entries_write(struct file filp, const char __user ubuf,
3375	size_t cnt, loff_t *ppos)
3376	{
3377	unsigned long val;
3378	char buf[64];
3379	int ret, cpu;
3380
3381	if (cnt >= sizeof(buf))
3382	return -EINVAL;
3383
3384	if (copy_from_user(&buf, ubuf, cnt))
3385	return -EFAULT;
3386
3387	buf[cnt] = 0;
3388
3389	ret = strict_strtoul(buf, 10, &val);
3390	if (ret < 0)
3391	return ret;
3392
3393	/* must have at least 1 entry */
3394	if (!val)
3395	return -EINVAL;
3396
3397	mutex_lock(&trace_types_lock);
3398
3399	tracing_stop();
3400
3401	/* disable all cpu buffers */
3402	for_each_tracing_cpu(cpu) {
3403	if (global_trace.data[cpu])
3404	atomic_inc(&global_trace.data[cpu]->disabled);
3405	if (max_tr.data[cpu])
3406	atomic_inc(&max_tr.data[cpu]->disabled);
3407	}
3408
3409	/* value is in KB */
3410	val <<= 10;
3411
3412	if (val != global_trace.entries) {
3413	ret = tracing_resize_ring_buffer(val);
3414	if (ret < 0) {
3415	cnt = ret;
3416	goto out;
3417	}
3418	}
3419
3420	*ppos += cnt;
3421
3422	/* If check pages failed, return ENOMEM */
3423	if (tracing_disabled)
3424	cnt = -ENOMEM;
3425	out:
3426	for_each_tracing_cpu(cpu) {
3427	if (global_trace.data[cpu])
3428	atomic_dec(&global_trace.data[cpu]->disabled);
3429	if (max_tr.data[cpu])
3430	atomic_dec(&max_tr.data[cpu]->disabled);
3431	}
3432
3433	tracing_start();
3434	max_tr.entries = global_trace.entries;
3435	mutex_unlock(&trace_types_lock);
3436
3437	return cnt;
3438	}
3439
3440	static int mark_printk(const char *fmt, ...)
3441	{
3442	int ret;
3443	va_list args;
3444	va_start(args, fmt);
3445	ret = trace_vprintk(0, fmt, args);
3446	va_end(args);
3447	return ret;
3448	}
3449
3450	static ssize_t
3451	tracing_mark_write(struct file filp, const char __user ubuf,
3452	size_t cnt, loff_t *fpos)
3453	{
3454	char *buf;
3455
3456	if (tracing_disabled)
3457	return -EINVAL;
3458
3459	if (cnt > TRACE_BUF_SIZE)
3460	cnt = TRACE_BUF_SIZE;
3461
3462	buf = kmalloc(cnt + 2, GFP_KERNEL);
3463	if (buf == NULL)
3464	return -ENOMEM;
3465
3466	if (copy_from_user(buf, ubuf, cnt)) {
3467	kfree(buf);
3468	return -EFAULT;
3469	}
3470	if (buf[cnt-1] != '\n') {
3471	buf[cnt] = '\n';
3472	buf[cnt+1] = '\0';
3473	} else
3474	buf[cnt] = '\0';
3475
3476	cnt = mark_printk("%s", buf);
3477	kfree(buf);
3478	*fpos += cnt;
3479
3480	return cnt;
3481	}
3482
3483	static int tracing_clock_show(struct seq_file m, void v)
3484	{
3485	int i;
3486
3487	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3488	seq_printf(m,
3489	"%s%s%s%s", i ? " " : "",
3490	i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3491	i == trace_clock_id ? "]" : "");
3492	seq_putc(m, '\n');
3493
3494	return 0;
3495	}
3496
3497	static ssize_t tracing_clock_write(struct file filp, const char __user ubuf,
3498	size_t cnt, loff_t *fpos)
3499	{
3500	char buf[64];
3501	const char *clockstr;
3502	int i;
3503
3504	if (cnt >= sizeof(buf))
3505	return -EINVAL;
3506
3507	if (copy_from_user(&buf, ubuf, cnt))
3508	return -EFAULT;
3509
3510	buf[cnt] = 0;
3511
3512	clockstr = strstrip(buf);
3513
3514	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3515	if (strcmp(trace_clocks[i].name, clockstr) == 0)
3516	break;
3517	}
3518	if (i == ARRAY_SIZE(trace_clocks))
3519	return -EINVAL;
3520
3521	trace_clock_id = i;
3522
3523	mutex_lock(&trace_types_lock);
3524
3525	ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3526	if (max_tr.buffer)
3527	ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3528
3529	mutex_unlock(&trace_types_lock);
3530
3531	*fpos += cnt;
3532
3533	return cnt;
3534	}
3535
3536	static int tracing_clock_open(struct inode inode, struct file file)
3537	{
3538	if (tracing_disabled)
3539	return -ENODEV;
3540	return single_open(file, tracing_clock_show, NULL);
3541	}
3542
3543	static const struct file_operations tracing_max_lat_fops = {
3544	.open = tracing_open_generic,
3545	.read = tracing_max_lat_read,
3546	.write = tracing_max_lat_write,
3547	};
3548
3549	static const struct file_operations tracing_ctrl_fops = {
3550	.open = tracing_open_generic,
3551	.read = tracing_ctrl_read,
3552	.write = tracing_ctrl_write,
3553	};
3554
3555	static const struct file_operations set_tracer_fops = {
3556	.open = tracing_open_generic,
3557	.read = tracing_set_trace_read,
3558	.write = tracing_set_trace_write,
3559	};
3560
3561	static const struct file_operations tracing_pipe_fops = {
3562	.open = tracing_open_pipe,
3563	.poll = tracing_poll_pipe,
3564	.read = tracing_read_pipe,
3565	.splice_read = tracing_splice_read_pipe,
3566	.release = tracing_release_pipe,
3567	};
3568
3569	static const struct file_operations tracing_entries_fops = {
3570	.open = tracing_open_generic,
3571	.read = tracing_entries_read,
3572	.write = tracing_entries_write,
3573	};
3574
3575	static const struct file_operations tracing_mark_fops = {
3576	.open = tracing_open_generic,
3577	.write = tracing_mark_write,
3578	};
3579
3580	static const struct file_operations trace_clock_fops = {
3581	.open = tracing_clock_open,
3582	.read = seq_read,
3583	.llseek = seq_lseek,
3584	.release = single_release,
3585	.write = tracing_clock_write,
3586	};
3587
3588	struct ftrace_buffer_info {
3589	struct trace_array *tr;
3590	void *spare;
3591	int cpu;
3592	unsigned int read;
3593	};
3594
3595	static int tracing_buffers_open(struct inode inode, struct file filp)
3596	{
3597	int cpu = (int)(long)inode->i_private;
3598	struct ftrace_buffer_info *info;
3599
3600	if (tracing_disabled)
3601	return -ENODEV;
3602
3603	info = kzalloc(sizeof(*info), GFP_KERNEL);
3604	if (!info)
3605	return -ENOMEM;
3606
3607	info->tr = &global_trace;
3608	info->cpu = cpu;
3609	info->spare = NULL;
3610	/* Force reading ring buffer for first read */
3611	info->read = (unsigned int)-1;
3612
3613	filp->private_data = info;
3614
3615	return nonseekable_open(inode, filp);
3616	}
3617
3618	static ssize_t
3619	tracing_buffers_read(struct file filp, char __user ubuf,
3620	size_t count, loff_t *ppos)
3621	{
3622	struct ftrace_buffer_info *info = filp->private_data;
3623	unsigned int pos;
3624	ssize_t ret;
3625	size_t size;
3626
3627	if (!count)
3628	return 0;
3629
3630	if (!info->spare)
3631	info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3632	if (!info->spare)
3633	return -ENOMEM;
3634
3635	/* Do we have previous read data to read? */
3636	if (info->read < PAGE_SIZE)
3637	goto read;
3638
3639	info->read = 0;
3640
3641	trace_access_lock(info->cpu);
3642	ret = ring_buffer_read_page(info->tr->buffer,
3643	&info->spare,
3644	count,
3645	info->cpu, 0);
3646	trace_access_unlock(info->cpu);
3647	if (ret < 0)
3648	return 0;
3649
3650	pos = ring_buffer_page_len(info->spare);
3651
3652	if (pos < PAGE_SIZE)
3653	memset(info->spare + pos, 0, PAGE_SIZE - pos);
3654
3655	read:
3656	size = PAGE_SIZE - info->read;
3657	if (size > count)
3658	size = count;
3659
3660	ret = copy_to_user(ubuf, info->spare + info->read, size);
3661	if (ret == size)
3662	return -EFAULT;
3663	size -= ret;
3664
3665	*ppos += size;
3666	info->read += size;
3667
3668	return size;
3669	}
3670
3671	static int tracing_buffers_release(struct inode inode, struct file file)
3672	{
3673	struct ftrace_buffer_info *info = file->private_data;
3674
3675	if (info->spare)
3676	ring_buffer_free_read_page(info->tr->buffer, info->spare);
3677	kfree(info);
3678
3679	return 0;
3680	}
3681
3682	struct buffer_ref {
3683	struct ring_buffer *buffer;
3684	void *page;
3685	int ref;
3686	};
3687
3688	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3689	struct pipe_buffer *buf)
3690	{
3691	struct buffer_ref ref = (struct buffer_ref )buf->private;
3692
3693	if (--ref->ref)
3694	return;
3695
3696	ring_buffer_free_read_page(ref->buffer, ref->page);
3697	kfree(ref);
3698	buf->private = 0;
3699	}
3700
3701	static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3702	struct pipe_buffer *buf)
3703	{
3704	return 1;
3705	}
3706
3707	static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3708	struct pipe_buffer *buf)
3709	{
3710	struct buffer_ref ref = (struct buffer_ref )buf->private;
3711
3712	ref->ref++;
3713	}
3714
3715	/* Pipe buffer operations for a buffer. */
3716	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
3717	.can_merge = 0,
3718	.map = generic_pipe_buf_map,
3719	.unmap = generic_pipe_buf_unmap,
3720	.confirm = generic_pipe_buf_confirm,
3721	.release = buffer_pipe_buf_release,
3722	.steal = buffer_pipe_buf_steal,
3723	.get = buffer_pipe_buf_get,
3724	};
3725
3726	/*
3727	* Callback from splice_to_pipe(), if we need to release some pages
3728	* at the end of the spd in case we error'ed out in filling the pipe.
3729	*/
3730	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3731	{
3732	struct buffer_ref *ref =
3733	(struct buffer_ref *)spd->partial[i].private;
3734
3735	if (--ref->ref)
3736	return;
3737
3738	ring_buffer_free_read_page(ref->buffer, ref->page);
3739	kfree(ref);
3740	spd->partial[i].private = 0;
3741	}
3742
3743	static ssize_t
3744	tracing_buffers_splice_read(struct file file, loff_t ppos,
3745	struct pipe_inode_info *pipe, size_t len,
3746	unsigned int flags)
3747	{
3748	struct ftrace_buffer_info *info = file->private_data;
3749	struct partial_page partial[PIPE_BUFFERS];
3750	struct page *pages[PIPE_BUFFERS];
3751	struct splice_pipe_desc spd = {
3752	.pages = pages,
3753	.partial = partial,
3754	.flags = flags,
3755	.ops = &buffer_pipe_buf_ops,
3756	.spd_release = buffer_spd_release,
3757	};
3758	struct buffer_ref *ref;
3759	int entries, size, i;
3760	size_t ret;
3761
3762	if (*ppos & (PAGE_SIZE - 1)) {
3763	WARN_ONCE(1, "Ftrace: previous read must page-align\n");
3764	return -EINVAL;
3765	}
3766
3767	if (len & (PAGE_SIZE - 1)) {
3768	WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
3769	if (len < PAGE_SIZE)
3770	return -EINVAL;
3771	len &= PAGE_MASK;
3772	}
3773
3774	trace_access_lock(info->cpu);
3775	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3776
3777	for (i = 0; i < PIPE_BUFFERS && len && entries; i++, len -= PAGE_SIZE) {
3778	struct page *page;
3779	int r;
3780
3781	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3782	if (!ref)
3783	break;
3784
3785	ref->ref = 1;
3786	ref->buffer = info->tr->buffer;
3787	ref->page = ring_buffer_alloc_read_page(ref->buffer);
3788	if (!ref->page) {
3789	kfree(ref);
3790	break;
3791	}
3792
3793	r = ring_buffer_read_page(ref->buffer, &ref->page,
3794	len, info->cpu, 1);
3795	if (r < 0) {
3796	ring_buffer_free_read_page(ref->buffer,
3797	ref->page);
3798	kfree(ref);
3799	break;
3800	}
3801
3802	/*
3803	* zero out any left over data, this is going to
3804	* user land.
3805	*/
3806	size = ring_buffer_page_len(ref->page);
3807	if (size < PAGE_SIZE)
3808	memset(ref->page + size, 0, PAGE_SIZE - size);
3809
3810	page = virt_to_page(ref->page);
3811
3812	spd.pages[i] = page;
3813	spd.partial[i].len = PAGE_SIZE;
3814	spd.partial[i].offset = 0;
3815	spd.partial[i].private = (unsigned long)ref;
3816	spd.nr_pages++;
3817	*ppos += PAGE_SIZE;
3818
3819	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3820	}
3821
3822	trace_access_unlock(info->cpu);
3823	spd.nr_pages = i;
3824
3825	/* did we read anything? */
3826	if (!spd.nr_pages) {
3827	if (flags & SPLICE_F_NONBLOCK)
3828	ret = -EAGAIN;
3829	else
3830	ret = 0;
3831	/* TODO: block */
3832	return ret;
3833	}
3834
3835	ret = splice_to_pipe(pipe, &spd);
3836
3837	return ret;
3838	}
3839
3840	static const struct file_operations tracing_buffers_fops = {
3841	.open = tracing_buffers_open,
3842	.read = tracing_buffers_read,
3843	.release = tracing_buffers_release,
3844	.splice_read = tracing_buffers_splice_read,
3845	.llseek = no_llseek,
3846	};
3847
3848	static ssize_t
3849	tracing_stats_read(struct file filp, char __user ubuf,
3850	size_t count, loff_t *ppos)
3851	{
3852	unsigned long cpu = (unsigned long)filp->private_data;
3853	struct trace_array *tr = &global_trace;
3854	struct trace_seq *s;
3855	unsigned long cnt;
3856
3857	s = kmalloc(sizeof(*s), GFP_KERNEL);
3858	if (!s)
3859	return -ENOMEM;
3860
3861	trace_seq_init(s);
3862
3863	cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
3864	trace_seq_printf(s, "entries: %ld\n", cnt);
3865
3866	cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
3867	trace_seq_printf(s, "overrun: %ld\n", cnt);
3868
3869	cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
3870	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
3871
3872	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
3873
3874	kfree(s);
3875
3876	return count;
3877	}
3878
3879	static const struct file_operations tracing_stats_fops = {
3880	.open = tracing_open_generic,
3881	.read = tracing_stats_read,
3882	};
3883
3884	#ifdef CONFIG_DYNAMIC_FTRACE
3885
3886	int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3887	{
3888	return 0;
3889	}
3890
3891	static ssize_t
3892	tracing_read_dyn_info(struct file filp, char __user ubuf,
3893	size_t cnt, loff_t *ppos)
3894	{
3895	static char ftrace_dyn_info_buffer[1024];
3896	static DEFINE_MUTEX(dyn_info_mutex);
3897	unsigned long *p = filp->private_data;
3898	char *buf = ftrace_dyn_info_buffer;
3899	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
3900	int r;
3901
3902	mutex_lock(&dyn_info_mutex);
3903	r = sprintf(buf, "%ld ", *p);
3904
3905	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
3906	buf[r++] = '\n';
3907
3908	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3909
3910	mutex_unlock(&dyn_info_mutex);
3911
3912	return r;
3913	}
3914
3915	static const struct file_operations tracing_dyn_info_fops = {
3916	.open = tracing_open_generic,
3917	.read = tracing_read_dyn_info,
3918	};
3919	#endif
3920
3921	static struct dentry *d_tracer;
3922
3923	struct dentry *tracing_init_dentry(void)
3924	{
3925	static int once;
3926
3927	if (d_tracer)
3928	return d_tracer;
3929
3930	if (!debugfs_initialized())
3931	return NULL;
3932
3933	d_tracer = debugfs_create_dir("tracing", NULL);
3934
3935	if (!d_tracer && !once) {
3936	once = 1;
3937	pr_warning("Could not create debugfs directory 'tracing'\n");
3938	return NULL;
3939	}
3940
3941	return d_tracer;
3942	}
3943
3944	static struct dentry *d_percpu;
3945
3946	struct dentry *tracing_dentry_percpu(void)
3947	{
3948	static int once;
3949	struct dentry *d_tracer;
3950
3951	if (d_percpu)
3952	return d_percpu;
3953
3954	d_tracer = tracing_init_dentry();
3955
3956	if (!d_tracer)
3957	return NULL;
3958
3959	d_percpu = debugfs_create_dir("per_cpu", d_tracer);
3960
3961	if (!d_percpu && !once) {
3962	once = 1;
3963	pr_warning("Could not create debugfs directory 'per_cpu'\n");
3964	return NULL;
3965	}
3966
3967	return d_percpu;
3968	}
3969
3970	static void tracing_init_debugfs_percpu(long cpu)
3971	{
3972	struct dentry *d_percpu = tracing_dentry_percpu();
3973	struct dentry *d_cpu;
3974	/* strlen(cpu) + MAX(log10(cpu)) + '\0' */
3975	char cpu_dir[7];
3976
3977	if (cpu > 999 \|\| cpu < 0)
3978	return;
3979
3980	sprintf(cpu_dir, "cpu%ld", cpu);
3981	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
3982	if (!d_cpu) {
3983	pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
3984	return;
3985	}
3986
3987	/* per cpu trace_pipe */
3988	trace_create_file("trace_pipe", 0444, d_cpu,
3989	(void *) cpu, &tracing_pipe_fops);
3990
3991	/* per cpu trace */
3992	trace_create_file("trace", 0644, d_cpu,
3993	(void *) cpu, &tracing_fops);
3994
3995	trace_create_file("trace_pipe_raw", 0444, d_cpu,
3996	(void *) cpu, &tracing_buffers_fops);
3997
3998	trace_create_file("stats", 0444, d_cpu,
3999	(void *) cpu, &tracing_stats_fops);
4000	}
4001
4002	#ifdef CONFIG_FTRACE_SELFTEST
4003	/* Let selftest have access to static functions in this file */
4004	#include "trace_selftest.c"
4005	#endif
4006
4007	struct trace_option_dentry {
4008	struct tracer_opt *opt;
4009	struct tracer_flags *flags;
4010	struct dentry *entry;
4011	};
4012
4013	static ssize_t
4014	trace_options_read(struct file filp, char __user ubuf, size_t cnt,
4015	loff_t *ppos)
4016	{
4017	struct trace_option_dentry *topt = filp->private_data;
4018	char *buf;
4019
4020	if (topt->flags->val & topt->opt->bit)
4021	buf = "1\n";
4022	else
4023	buf = "0\n";
4024
4025	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4026	}
4027
4028	static ssize_t
4029	trace_options_write(struct file filp, const char __user ubuf, size_t cnt,
4030	loff_t *ppos)
4031	{
4032	struct trace_option_dentry *topt = filp->private_data;
4033	unsigned long val;
4034	char buf[64];
4035	int ret;
4036
4037	if (cnt >= sizeof(buf))
4038	return -EINVAL;
4039
4040	if (copy_from_user(&buf, ubuf, cnt))
4041	return -EFAULT;
4042
4043	buf[cnt] = 0;
4044
4045	ret = strict_strtoul(buf, 10, &val);
4046	if (ret < 0)
4047	return ret;
4048
4049	if (val != 0 && val != 1)
4050	return -EINVAL;
4051
4052	if (!!(topt->flags->val & topt->opt->bit) != val) {
4053	mutex_lock(&trace_types_lock);
4054	ret = __set_tracer_option(current_trace, topt->flags,
4055	topt->opt, !val);
4056	mutex_unlock(&trace_types_lock);
4057	if (ret)
4058	return ret;
4059	}
4060
4061	*ppos += cnt;
4062
4063	return cnt;
4064	}
4065
4066
4067	static const struct file_operations trace_options_fops = {
4068	.open = tracing_open_generic,
4069	.read = trace_options_read,
4070	.write = trace_options_write,
4071	};
4072
4073	static ssize_t
4074	trace_options_core_read(struct file filp, char __user ubuf, size_t cnt,
4075	loff_t *ppos)
4076	{
4077	long index = (long)filp->private_data;
4078	char *buf;
4079
4080	if (trace_flags & (1 << index))
4081	buf = "1\n";
4082	else
4083	buf = "0\n";
4084
4085	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4086	}
4087
4088	static ssize_t
4089	trace_options_core_write(struct file filp, const char __user ubuf, size_t cnt,
4090	loff_t *ppos)
4091	{
4092	long index = (long)filp->private_data;
4093	char buf[64];
4094	unsigned long val;
4095	int ret;
4096
4097	if (cnt >= sizeof(buf))
4098	return -EINVAL;
4099
4100	if (copy_from_user(&buf, ubuf, cnt))
4101	return -EFAULT;
4102
4103	buf[cnt] = 0;
4104
4105	ret = strict_strtoul(buf, 10, &val);
4106	if (ret < 0)
4107	return ret;
4108
4109	if (val != 0 && val != 1)
4110	return -EINVAL;
4111	set_tracer_flags(1 << index, val);
4112
4113	*ppos += cnt;
4114
4115	return cnt;
4116	}
4117
4118	static const struct file_operations trace_options_core_fops = {
4119	.open = tracing_open_generic,
4120	.read = trace_options_core_read,
4121	.write = trace_options_core_write,
4122	};
4123
4124	struct dentry trace_create_file(const char name,
4125	mode_t mode,
4126	struct dentry *parent,
4127	void *data,
4128	const struct file_operations *fops)
4129	{
4130	struct dentry *ret;
4131
4132	ret = debugfs_create_file(name, mode, parent, data, fops);
4133	if (!ret)
4134	pr_warning("Could not create debugfs '%s' entry\n", name);
4135
4136	return ret;
4137	}
4138
4139
4140	static struct dentry *trace_options_init_dentry(void)
4141	{
4142	struct dentry *d_tracer;
4143	static struct dentry *t_options;
4144
4145	if (t_options)
4146	return t_options;
4147
4148	d_tracer = tracing_init_dentry();
4149	if (!d_tracer)
4150	return NULL;
4151
4152	t_options = debugfs_create_dir("options", d_tracer);
4153	if (!t_options) {
4154	pr_warning("Could not create debugfs directory 'options'\n");
4155	return NULL;
4156	}
4157
4158	return t_options;
4159	}
4160
4161	static void
4162	create_trace_option_file(struct trace_option_dentry *topt,
4163	struct tracer_flags *flags,
4164	struct tracer_opt *opt)
4165	{
4166	struct dentry *t_options;
4167
4168	t_options = trace_options_init_dentry();
4169	if (!t_options)
4170	return;
4171
4172	topt->flags = flags;
4173	topt->opt = opt;
4174
4175	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4176	&trace_options_fops);
4177
4178	}
4179
4180	static struct trace_option_dentry *
4181	create_trace_option_files(struct tracer *tracer)
4182	{
4183	struct trace_option_dentry *topts;
4184	struct tracer_flags *flags;
4185	struct tracer_opt *opts;
4186	int cnt;
4187
4188	if (!tracer)
4189	return NULL;
4190
4191	flags = tracer->flags;
4192
4193	if (!flags \|\| !flags->opts)
4194	return NULL;
4195
4196	opts = flags->opts;
4197
4198	for (cnt = 0; opts[cnt].name; cnt++)
4199	;
4200
4201	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4202	if (!topts)
4203	return NULL;
4204
4205	for (cnt = 0; opts[cnt].name; cnt++)
4206	create_trace_option_file(&topts[cnt], flags,
4207	&opts[cnt]);
4208
4209	return topts;
4210	}
4211
4212	static void
4213	destroy_trace_option_files(struct trace_option_dentry *topts)
4214	{
4215	int cnt;
4216
4217	if (!topts)
4218	return;
4219
4220	for (cnt = 0; topts[cnt].opt; cnt++) {
4221	if (topts[cnt].entry)
4222	debugfs_remove(topts[cnt].entry);
4223	}
4224
4225	kfree(topts);
4226	}
4227
4228	static struct dentry *
4229	create_trace_option_core_file(const char *option, long index)
4230	{
4231	struct dentry *t_options;
4232
4233	t_options = trace_options_init_dentry();
4234	if (!t_options)
4235	return NULL;
4236
4237	return trace_create_file(option, 0644, t_options, (void *)index,
4238	&trace_options_core_fops);
4239	}
4240
4241	static __init void create_trace_options_dir(void)
4242	{
4243	struct dentry *t_options;
4244	int i;
4245
4246	t_options = trace_options_init_dentry();
4247	if (!t_options)
4248	return;
4249
4250	for (i = 0; trace_options[i]; i++)
4251	create_trace_option_core_file(trace_options[i], i);
4252	}
4253
4254	static __init int tracer_init_debugfs(void)
4255	{
4256	struct dentry *d_tracer;
4257	int cpu;
4258
4259	trace_access_lock_init();
4260
4261	d_tracer = tracing_init_dentry();
4262
4263	trace_create_file("tracing_enabled", 0644, d_tracer,
4264	&global_trace, &tracing_ctrl_fops);
4265
4266	trace_create_file("trace_options", 0644, d_tracer,
4267	NULL, &tracing_iter_fops);
4268
4269	trace_create_file("tracing_cpumask", 0644, d_tracer,
4270	NULL, &tracing_cpumask_fops);
4271
4272	trace_create_file("trace", 0644, d_tracer,
4273	(void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4274
4275	trace_create_file("available_tracers", 0444, d_tracer,
4276	&global_trace, &show_traces_fops);
4277
4278	trace_create_file("current_tracer", 0644, d_tracer,
4279	&global_trace, &set_tracer_fops);
4280
4281	#ifdef CONFIG_TRACER_MAX_TRACE
4282	trace_create_file("tracing_max_latency", 0644, d_tracer,
4283	&tracing_max_latency, &tracing_max_lat_fops);
4284	#endif
4285
4286	trace_create_file("tracing_thresh", 0644, d_tracer,
4287	&tracing_thresh, &tracing_max_lat_fops);
4288
4289	trace_create_file("README", 0444, d_tracer,
4290	NULL, &tracing_readme_fops);
4291
4292	trace_create_file("trace_pipe", 0444, d_tracer,
4293	(void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4294
4295	trace_create_file("buffer_size_kb", 0644, d_tracer,
4296	&global_trace, &tracing_entries_fops);
4297
4298	trace_create_file("trace_marker", 0220, d_tracer,
4299	NULL, &tracing_mark_fops);
4300
4301	trace_create_file("saved_cmdlines", 0444, d_tracer,
4302	NULL, &tracing_saved_cmdlines_fops);
4303
4304	trace_create_file("trace_clock", 0644, d_tracer, NULL,
4305	&trace_clock_fops);
4306
4307	#ifdef CONFIG_DYNAMIC_FTRACE
4308	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4309	&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4310	#endif
4311	#ifdef CONFIG_SYSPROF_TRACER
4312	init_tracer_sysprof_debugfs(d_tracer);
4313	#endif
4314
4315	create_trace_options_dir();
4316
4317	for_each_tracing_cpu(cpu)
4318	tracing_init_debugfs_percpu(cpu);
4319
4320	return 0;
4321	}
4322
4323	static int trace_panic_handler(struct notifier_block *this,
4324	unsigned long event, void *unused)
4325	{
4326	if (ftrace_dump_on_oops)
4327	ftrace_dump();
4328	return NOTIFY_OK;
4329	}
4330
4331	static struct notifier_block trace_panic_notifier = {
4332	.notifier_call = trace_panic_handler,
4333	.next = NULL,
4334	.priority = 150 /* priority: INT_MAX >= x >= 0 */
4335	};
4336
4337	static int trace_die_handler(struct notifier_block *self,
4338	unsigned long val,
4339	void *data)
4340	{
4341	switch (val) {
4342	case DIE_OOPS:
4343	if (ftrace_dump_on_oops)
4344	ftrace_dump();
4345	break;
4346	default:
4347	break;
4348	}
4349	return NOTIFY_OK;
4350	}
4351
4352	static struct notifier_block trace_die_notifier = {
4353	.notifier_call = trace_die_handler,
4354	.priority = 200
4355	};
4356
4357	/*
4358	* printk is set to max of 1024, we really don't need it that big.
4359	* Nothing should be printing 1000 characters anyway.
4360	*/
4361	#define TRACE_MAX_PRINT 1000
4362
4363	/*
4364	* Define here KERN_TRACE so that we have one place to modify
4365	* it if we decide to change what log level the ftrace dump
4366	* should be at.
4367	*/
4368	#define KERN_TRACE KERN_EMERG
4369
4370	static void
4371	trace_printk_seq(struct trace_seq *s)
4372	{
4373	/* Probably should print a warning here. */
4374	if (s->len >= 1000)
4375	s->len = 1000;
4376
4377	/* should be zero ended, but we are paranoid. */
4378	s->buffer[s->len] = 0;
4379
4380	printk(KERN_TRACE "%s", s->buffer);
4381
4382	trace_seq_init(s);
4383	}
4384
4385	static void __ftrace_dump(bool disable_tracing)
4386	{
4387	static arch_spinlock_t ftrace_dump_lock =
4388	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4389	/* use static because iter can be a bit big for the stack */
4390	static struct trace_iterator iter;
4391	unsigned int old_userobj;
4392	static int dump_ran;
4393	unsigned long flags;
4394	int cnt = 0, cpu;
4395
4396	/* only one dump */
4397	local_irq_save(flags);
4398	arch_spin_lock(&ftrace_dump_lock);
4399	if (dump_ran)
4400	goto out;
4401
4402	dump_ran = 1;
4403
4404	tracing_off();
4405
4406	if (disable_tracing)
4407	ftrace_kill();
4408
4409	for_each_tracing_cpu(cpu) {
4410	atomic_inc(&global_trace.data[cpu]->disabled);
4411	}
4412
4413	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4414
4415	/* don't look at user memory in panic mode */
4416	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4417
4418	printk(KERN_TRACE "Dumping ftrace buffer:\n");
4419
4420	/* Simulate the iterator */
4421	iter.tr = &global_trace;
4422	iter.trace = current_trace;
4423	iter.cpu_file = TRACE_PIPE_ALL_CPU;
4424
4425	/*
4426	* We need to stop all tracing on all CPUS to read the
4427	* the next buffer. This is a bit expensive, but is
4428	* not done often. We fill all what we can read,
4429	* and then release the locks again.
4430	*/
4431
4432	while (!trace_empty(&iter)) {
4433
4434	if (!cnt)
4435	printk(KERN_TRACE "---------------------------------\n");
4436
4437	cnt++;
4438
4439	/* reset all but tr, trace, and overruns */
4440	memset(&iter.seq, 0,
4441	sizeof(struct trace_iterator) -
4442	offsetof(struct trace_iterator, seq));
4443	iter.iter_flags \|= TRACE_FILE_LAT_FMT;
4444	iter.pos = -1;
4445
4446	if (find_next_entry_inc(&iter) != NULL) {
4447	int ret;
4448
4449	ret = print_trace_line(&iter);
4450	if (ret != TRACE_TYPE_NO_CONSUME)
4451	trace_consume(&iter);
4452	}
4453
4454	trace_printk_seq(&iter.seq);
4455	}
4456
4457	if (!cnt)
4458	printk(KERN_TRACE " (ftrace buffer empty)\n");
4459	else
4460	printk(KERN_TRACE "---------------------------------\n");
4461
4462	/* Re-enable tracing if requested */
4463	if (!disable_tracing) {
4464	trace_flags \|= old_userobj;
4465
4466	for_each_tracing_cpu(cpu) {
4467	atomic_dec(&global_trace.data[cpu]->disabled);
4468	}
4469	tracing_on();
4470	}
4471
4472	out:
4473	arch_spin_unlock(&ftrace_dump_lock);
4474	local_irq_restore(flags);
4475	}
4476
4477	/* By default: disable tracing after the dump */
4478	void ftrace_dump(void)
4479	{
4480	__ftrace_dump(true);
4481	}
4482
4483	__init static int tracer_alloc_buffers(void)
4484	{
4485	int ring_buf_size;
4486	int i;
4487	int ret = -ENOMEM;
4488
4489	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4490	goto out;
4491
4492	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4493	goto out_free_buffer_mask;
4494
4495	/* To save memory, keep the ring buffer size to its minimum */
4496	if (ring_buffer_expanded)
4497	ring_buf_size = trace_buf_size;
4498	else
4499	ring_buf_size = 1;
4500
4501	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4502	cpumask_copy(tracing_cpumask, cpu_all_mask);
4503
4504	/* TODO: make the number of buffers hot pluggable with CPUS */
4505	global_trace.buffer = ring_buffer_alloc(ring_buf_size,
4506	TRACE_BUFFER_FLAGS);
4507	if (!global_trace.buffer) {
4508	printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4509	WARN_ON(1);
4510	goto out_free_cpumask;
4511	}
4512	global_trace.entries = ring_buffer_size(global_trace.buffer);
4513
4514
4515	#ifdef CONFIG_TRACER_MAX_TRACE
4516	max_tr.buffer = ring_buffer_alloc(ring_buf_size,
4517	TRACE_BUFFER_FLAGS);
4518	if (!max_tr.buffer) {
4519	printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4520	WARN_ON(1);
4521	ring_buffer_free(global_trace.buffer);
4522	goto out_free_cpumask;
4523	}
4524	max_tr.entries = ring_buffer_size(max_tr.buffer);
4525	WARN_ON(max_tr.entries != global_trace.entries);
4526	#endif
4527
4528	/* Allocate the first page for all buffers */
4529	for_each_tracing_cpu(i) {
4530	global_trace.data[i] = &per_cpu(global_trace_cpu, i);
4531	max_tr.data[i] = &per_cpu(max_tr_data, i);
4532	}
4533
4534	trace_init_cmdlines();
4535
4536	register_tracer(&nop_trace);
4537	current_trace = &nop_trace;
4538	#ifdef CONFIG_BOOT_TRACER
4539	register_tracer(&boot_tracer);
4540	#endif
4541	/* All seems OK, enable tracing */
4542	tracing_disabled = 0;
4543
4544	atomic_notifier_chain_register(&panic_notifier_list,
4545	&trace_panic_notifier);
4546
4547	register_die_notifier(&trace_die_notifier);
4548
4549	return 0;
4550
4551	out_free_cpumask:
4552	free_cpumask_var(tracing_cpumask);
4553	out_free_buffer_mask:
4554	free_cpumask_var(tracing_buffer_mask);
4555	out:
4556	return ret;
4557	}
4558
4559	__init static int clear_boot_tracer(void)
4560	{
4561	/*
4562	* The default tracer at boot buffer is an init section.
4563	* This function is called in lateinit. If we did not
4564	* find the boot tracer, then clear it out, to prevent
4565	* later registration from accessing the buffer that is
4566	* about to be freed.
4567	*/
4568	if (!default_bootup_tracer)
4569	return 0;
4570
4571	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4572	default_bootup_tracer);
4573	default_bootup_tracer = NULL;
4574
4575	return 0;
4576	}
4577
4578	early_initcall(tracer_alloc_buffers);
4579	fs_initcall(tracer_init_debugfs);
4580	late_initcall(clear_boot_tracer);