linux - kernel/trace/trace.c

5041 lines · 114.2 KiB

1	/*
2	* ring buffer based function tracer
3	*
4	* Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
5	* Copyright (C) 2008 Ingo Molnar <[email protected]>
6	*
7	* Originally taken from the RT patch by:
8	* Arnaldo Carvalho de Melo <[email protected]>
9	*
10	* Based on code from the latency_tracer, that is:
11	* Copyright (C) 2004-2006 Ingo Molnar
12	* Copyright (C) 2004 William Lee Irwin III
13	*/
14	#include <linux/ring_buffer.h>
15	#include <generated/utsrelease.h>
16	#include <linux/stacktrace.h>
17	#include <linux/writeback.h>
18	#include <linux/kallsyms.h>
19	#include <linux/seq_file.h>
20	#include <linux/notifier.h>
21	#include <linux/irqflags.h>
22	#include <linux/debugfs.h>
23	#include <linux/pagemap.h>
24	#include <linux/hardirq.h>
25	#include <linux/linkage.h>
26	#include <linux/uaccess.h>
27	#include <linux/kprobes.h>
28	#include <linux/ftrace.h>
29	#include <linux/module.h>
30	#include <linux/percpu.h>
31	#include <linux/splice.h>
32	#include <linux/kdebug.h>
33	#include <linux/string.h>
34	#include <linux/rwsem.h>
35	#include <linux/slab.h>
36	#include <linux/ctype.h>
37	#include <linux/init.h>
38	#include <linux/poll.h>
39	#include <linux/nmi.h>
40	#include <linux/fs.h>
41
42	#include "trace.h"
43	#include "trace_output.h"
44
45	/*
46	* On boot up, the ring buffer is set to the minimum size, so that
47	* we do not waste memory on systems that are not using tracing.
48	*/
49	int ring_buffer_expanded;
50
51	/*
52	* We need to change this state when a selftest is running.
53	* A selftest will lurk into the ring-buffer to count the
54	* entries inserted during the selftest although some concurrent
55	* insertions into the ring-buffer such as trace_printk could occurred
56	* at the same time, giving false positive or negative results.
57	*/
58	static bool __read_mostly tracing_selftest_running;
59
60	/*
61	* If a tracer is running, we do not want to run SELFTEST.
62	*/
63	bool __read_mostly tracing_selftest_disabled;
64
65	/* For tracers that don't implement custom flags */
66	static struct tracer_opt dummy_tracer_opt[] = {
67	{ }
68	};
69
70	static struct tracer_flags dummy_tracer_flags = {
71	.val = 0,
72	.opts = dummy_tracer_opt
73	};
74
75	static int dummy_set_flag(u32 old_flags, u32 bit, int set)
76	{
77	return 0;
78	}
79
80	/*
81	* Kill all tracing for good (never come back).
82	* It is initialized to 1 but will turn to zero if the initialization
83	* of the tracer is successful. But that is the only place that sets
84	* this back to zero.
85	*/
86	static int tracing_disabled = 1;
87
88	DEFINE_PER_CPU(int, ftrace_cpu_disabled);
89
90	static inline void ftrace_disable_cpu(void)
91	{
92	preempt_disable();
93	__this_cpu_inc(ftrace_cpu_disabled);
94	}
95
96	static inline void ftrace_enable_cpu(void)
97	{
98	__this_cpu_dec(ftrace_cpu_disabled);
99	preempt_enable();
100	}
101
102	cpumask_var_t __read_mostly tracing_buffer_mask;
103
104	/*
105	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
106	*
107	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
108	* is set, then ftrace_dump is called. This will output the contents
109	* of the ftrace buffers to the console. This is very useful for
110	* capturing traces that lead to crashes and outputing it to a
111	* serial console.
112	*
113	* It is default off, but you can enable it with either specifying
114	* "ftrace_dump_on_oops" in the kernel command line, or setting
115	* /proc/sys/kernel/ftrace_dump_on_oops
116	* Set 1 if you want to dump buffers of all CPUs
117	* Set 2 if you want to dump the buffer of the CPU that triggered oops
118	*/
119
120	enum ftrace_dump_mode ftrace_dump_on_oops;
121
122	static int tracing_set_tracer(const char *buf);
123
124	#define MAX_TRACER_SIZE 100
125	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
126	static char *default_bootup_tracer;
127
128	static int __init set_cmdline_ftrace(char *str)
129	{
130	strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
131	default_bootup_tracer = bootup_tracer_buf;
132	/* We are using ftrace early, expand it */
133	ring_buffer_expanded = 1;
134	return 1;
135	}
136	__setup("ftrace=", set_cmdline_ftrace);
137
138	static int __init set_ftrace_dump_on_oops(char *str)
139	{
140	if (str++ != '=' \|\| !str) {
141	ftrace_dump_on_oops = DUMP_ALL;
142	return 1;
143	}
144
145	if (!strcmp("orig_cpu", str)) {
146	ftrace_dump_on_oops = DUMP_ORIG;
147	return 1;
148	}
149
150	return 0;
151	}
152	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
153
154	unsigned long long ns2usecs(cycle_t nsec)
155	{
156	nsec += 500;
157	do_div(nsec, 1000);
158	return nsec;
159	}
160
161	/*
162	* The global_trace is the descriptor that holds the tracing
163	* buffers for the live tracing. For each CPU, it contains
164	* a link list of pages that will store trace entries. The
165	* page descriptor of the pages in the memory is used to hold
166	* the link list by linking the lru item in the page descriptor
167	* to each of the pages in the buffer per CPU.
168	*
169	* For each active CPU there is a data field that holds the
170	* pages for the buffer for that CPU. Each CPU has the same number
171	* of pages allocated for its buffer.
172	*/
173	static struct trace_array global_trace;
174
175	static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
176
177	int filter_current_check_discard(struct ring_buffer *buffer,
178	struct ftrace_event_call call, void rec,
179	struct ring_buffer_event *event)
180	{
181	return filter_check_discard(call, rec, buffer, event);
182	}
183	EXPORT_SYMBOL_GPL(filter_current_check_discard);
184
185	cycle_t ftrace_now(int cpu)
186	{
187	u64 ts;
188
189	/* Early boot up does not have a buffer yet */
190	if (!global_trace.buffer)
191	return trace_clock_local();
192
193	ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
194	ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
195
196	return ts;
197	}
198
199	/*
200	* The max_tr is used to snapshot the global_trace when a maximum
201	* latency is reached. Some tracers will use this to store a maximum
202	* trace while it continues examining live traces.
203	*
204	* The buffers for the max_tr are set up the same as the global_trace.
205	* When a snapshot is taken, the link list of the max_tr is swapped
206	* with the link list of the global_trace and the buffers are reset for
207	* the global_trace so the tracing can continue.
208	*/
209	static struct trace_array max_tr;
210
211	static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
212
213	/* tracer_enabled is used to toggle activation of a tracer */
214	static int tracer_enabled = 1;
215
216	/**
217	* tracing_is_enabled - return tracer_enabled status
218	*
219	* This function is used by other tracers to know the status
220	* of the tracer_enabled flag. Tracers may use this function
221	* to know if it should enable their features when starting
222	* up. See irqsoff tracer for an example (start_irqsoff_tracer).
223	*/
224	int tracing_is_enabled(void)
225	{
226	return tracer_enabled;
227	}
228
229	/*
230	* trace_buf_size is the size in bytes that is allocated
231	* for a buffer. Note, the number of bytes is always rounded
232	* to page size.
233	*
234	* This number is purposely set to a low number of 16384.
235	* If the dump on oops happens, it will be much appreciated
236	* to not have to wait for all that output. Anyway this can be
237	* boot time and run time configurable.
238	*/
239	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
240
241	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
242
243	/* trace_types holds a link list of available tracers. */
244	static struct tracer *trace_types __read_mostly;
245
246	/* current_trace points to the tracer that is currently active */
247	static struct tracer *current_trace __read_mostly;
248
249	/*
250	* trace_types_lock is used to protect the trace_types list.
251	*/
252	static DEFINE_MUTEX(trace_types_lock);
253
254	/*
255	* serialize the access of the ring buffer
256	*
257	* ring buffer serializes readers, but it is low level protection.
258	* The validity of the events (which returns by ring_buffer_peek() ..etc)
259	* are not protected by ring buffer.
260	*
261	* The content of events may become garbage if we allow other process consumes
262	* these events concurrently:
263	* A) the page of the consumed events may become a normal page
264	* (not reader page) in ring buffer, and this page will be rewrited
265	* by events producer.
266	* B) The page of the consumed events may become a page for splice_read,
267	* and this page will be returned to system.
268	*
269	* These primitives allow multi process access to different cpu ring buffer
270	* concurrently.
271	*
272	* These primitives don't distinguish read-only and read-consume access.
273	* Multi read-only access are also serialized.
274	*/
275
276	#ifdef CONFIG_SMP
277	static DECLARE_RWSEM(all_cpu_access_lock);
278	static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
279
280	static inline void trace_access_lock(int cpu)
281	{
282	if (cpu == TRACE_PIPE_ALL_CPU) {
283	/* gain it for accessing the whole ring buffer. */
284	down_write(&all_cpu_access_lock);
285	} else {
286	/* gain it for accessing a cpu ring buffer. */
287
288	/* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
289	down_read(&all_cpu_access_lock);
290
291	/* Secondly block other access to this @cpu ring buffer. */
292	mutex_lock(&per_cpu(cpu_access_lock, cpu));
293	}
294	}
295
296	static inline void trace_access_unlock(int cpu)
297	{
298	if (cpu == TRACE_PIPE_ALL_CPU) {
299	up_write(&all_cpu_access_lock);
300	} else {
301	mutex_unlock(&per_cpu(cpu_access_lock, cpu));
302	up_read(&all_cpu_access_lock);
303	}
304	}
305
306	static inline void trace_access_lock_init(void)
307	{
308	int cpu;
309
310	for_each_possible_cpu(cpu)
311	mutex_init(&per_cpu(cpu_access_lock, cpu));
312	}
313
314	#else
315
316	static DEFINE_MUTEX(access_lock);
317
318	static inline void trace_access_lock(int cpu)
319	{
320	(void)cpu;
321	mutex_lock(&access_lock);
322	}
323
324	static inline void trace_access_unlock(int cpu)
325	{
326	(void)cpu;
327	mutex_unlock(&access_lock);
328	}
329
330	static inline void trace_access_lock_init(void)
331	{
332	}
333
334	#endif
335
336	/* trace_wait is a waitqueue for tasks blocked on trace_poll */
337	static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
338
339	/* trace_flags holds trace_options default values */
340	unsigned long trace_flags = TRACE_ITER_PRINT_PARENT \| TRACE_ITER_PRINTK \|
341	TRACE_ITER_ANNOTATE \| TRACE_ITER_CONTEXT_INFO \| TRACE_ITER_SLEEP_TIME \|
342	TRACE_ITER_GRAPH_TIME \| TRACE_ITER_RECORD_CMD \| TRACE_ITER_OVERWRITE \|
343	TRACE_ITER_IRQ_INFO;
344
345	static int trace_stop_count;
346	static DEFINE_RAW_SPINLOCK(tracing_start_lock);
347
348	static void wakeup_work_handler(struct work_struct *work)
349	{
350	wake_up(&trace_wait);
351	}
352
353	static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
354
355	/**
356	* tracing_on - enable tracing buffers
357	*
358	* This function enables tracing buffers that may have been
359	* disabled with tracing_off.
360	*/
361	void tracing_on(void)
362	{
363	if (global_trace.buffer)
364	ring_buffer_record_on(global_trace.buffer);
365	/*
366	* This flag is only looked at when buffers haven't been
367	* allocated yet. We don't really care about the race
368	* between setting this flag and actually turning
369	* on the buffer.
370	*/
371	global_trace.buffer_disabled = 0;
372	}
373	EXPORT_SYMBOL_GPL(tracing_on);
374
375	/**
376	* tracing_off - turn off tracing buffers
377	*
378	* This function stops the tracing buffers from recording data.
379	* It does not disable any overhead the tracers themselves may
380	* be causing. This function simply causes all recording to
381	* the ring buffers to fail.
382	*/
383	void tracing_off(void)
384	{
385	if (global_trace.buffer)
386	ring_buffer_record_on(global_trace.buffer);
387	/*
388	* This flag is only looked at when buffers haven't been
389	* allocated yet. We don't really care about the race
390	* between setting this flag and actually turning
391	* on the buffer.
392	*/
393	global_trace.buffer_disabled = 1;
394	}
395	EXPORT_SYMBOL_GPL(tracing_off);
396
397	/**
398	* tracing_is_on - show state of ring buffers enabled
399	*/
400	int tracing_is_on(void)
401	{
402	if (global_trace.buffer)
403	return ring_buffer_record_is_on(global_trace.buffer);
404	return !global_trace.buffer_disabled;
405	}
406	EXPORT_SYMBOL_GPL(tracing_is_on);
407
408	/**
409	* trace_wake_up - wake up tasks waiting for trace input
410	*
411	* Schedules a delayed work to wake up any task that is blocked on the
412	* trace_wait queue. These is used with trace_poll for tasks polling the
413	* trace.
414	*/
415	void trace_wake_up(void)
416	{
417	const unsigned long delay = msecs_to_jiffies(2);
418
419	if (trace_flags & TRACE_ITER_BLOCK)
420	return;
421	schedule_delayed_work(&wakeup_work, delay);
422	}
423
424	static int __init set_buf_size(char *str)
425	{
426	unsigned long buf_size;
427
428	if (!str)
429	return 0;
430	buf_size = memparse(str, &str);
431	/* nr_entries can not be zero */
432	if (buf_size == 0)
433	return 0;
434	trace_buf_size = buf_size;
435	return 1;
436	}
437	__setup("trace_buf_size=", set_buf_size);
438
439	static int __init set_tracing_thresh(char *str)
440	{
441	unsigned long threshhold;
442	int ret;
443
444	if (!str)
445	return 0;
446	ret = strict_strtoul(str, 0, &threshhold);
447	if (ret < 0)
448	return 0;
449	tracing_thresh = threshhold * 1000;
450	return 1;
451	}
452	__setup("tracing_thresh=", set_tracing_thresh);
453
454	unsigned long nsecs_to_usecs(unsigned long nsecs)
455	{
456	return nsecs / 1000;
457	}
458
459	/* These must match the bit postions in trace_iterator_flags */
460	static const char *trace_options[] = {
461	"print-parent",
462	"sym-offset",
463	"sym-addr",
464	"verbose",
465	"raw",
466	"hex",
467	"bin",
468	"block",
469	"stacktrace",
470	"trace_printk",
471	"ftrace_preempt",
472	"branch",
473	"annotate",
474	"userstacktrace",
475	"sym-userobj",
476	"printk-msg-only",
477	"context-info",
478	"latency-format",
479	"sleep-time",
480	"graph-time",
481	"record-cmd",
482	"overwrite",
483	"disable_on_free",
484	"irq-info",
485	NULL
486	};
487
488	static struct {
489	u64 (*func)(void);
490	const char *name;
491	} trace_clocks[] = {
492	{ trace_clock_local, "local" },
493	{ trace_clock_global, "global" },
494	{ trace_clock_counter, "counter" },
495	};
496
497	int trace_clock_id;
498
499	/*
500	* trace_parser_get_init - gets the buffer for trace parser
501	*/
502	int trace_parser_get_init(struct trace_parser *parser, int size)
503	{
504	memset(parser, 0, sizeof(*parser));
505
506	parser->buffer = kmalloc(size, GFP_KERNEL);
507	if (!parser->buffer)
508	return 1;
509
510	parser->size = size;
511	return 0;
512	}
513
514	/*
515	* trace_parser_put - frees the buffer for trace parser
516	*/
517	void trace_parser_put(struct trace_parser *parser)
518	{
519	kfree(parser->buffer);
520	}
521
522	/*
523	* trace_get_user - reads the user input string separated by space
524	* (matched by isspace(ch))
525	*
526	* For each string found the 'struct trace_parser' is updated,
527	* and the function returns.
528	*
529	* Returns number of bytes read.
530	*
531	* See kernel/trace/trace.h for 'struct trace_parser' details.
532	*/
533	int trace_get_user(struct trace_parser parser, const char __user ubuf,
534	size_t cnt, loff_t *ppos)
535	{
536	char ch;
537	size_t read = 0;
538	ssize_t ret;
539
540	if (!*ppos)
541	trace_parser_clear(parser);
542
543	ret = get_user(ch, ubuf++);
544	if (ret)
545	goto out;
546
547	read++;
548	cnt--;
549
550	/*
551	* The parser is not finished with the last write,
552	* continue reading the user input without skipping spaces.
553	*/
554	if (!parser->cont) {
555	/* skip white space */
556	while (cnt && isspace(ch)) {
557	ret = get_user(ch, ubuf++);
558	if (ret)
559	goto out;
560	read++;
561	cnt--;
562	}
563
564	/* only spaces were written */
565	if (isspace(ch)) {
566	*ppos += read;
567	ret = read;
568	goto out;
569	}
570
571	parser->idx = 0;
572	}
573
574	/* read the non-space input */
575	while (cnt && !isspace(ch)) {
576	if (parser->idx < parser->size - 1)
577	parser->buffer[parser->idx++] = ch;
578	else {
579	ret = -EINVAL;
580	goto out;
581	}
582	ret = get_user(ch, ubuf++);
583	if (ret)
584	goto out;
585	read++;
586	cnt--;
587	}
588
589	/* We either got finished input or we have to wait for another call. */
590	if (isspace(ch)) {
591	parser->buffer[parser->idx] = 0;
592	parser->cont = false;
593	} else {
594	parser->cont = true;
595	parser->buffer[parser->idx++] = ch;
596	}
597
598	*ppos += read;
599	ret = read;
600
601	out:
602	return ret;
603	}
604
605	ssize_t trace_seq_to_user(struct trace_seq s, char __user ubuf, size_t cnt)
606	{
607	int len;
608	int ret;
609
610	if (!cnt)
611	return 0;
612
613	if (s->len <= s->readpos)
614	return -EBUSY;
615
616	len = s->len - s->readpos;
617	if (cnt > len)
618	cnt = len;
619	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
620	if (ret == cnt)
621	return -EFAULT;
622
623	cnt -= ret;
624
625	s->readpos += cnt;
626	return cnt;
627	}
628
629	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
630	{
631	int len;
632	void *ret;
633
634	if (s->len <= s->readpos)
635	return -EBUSY;
636
637	len = s->len - s->readpos;
638	if (cnt > len)
639	cnt = len;
640	ret = memcpy(buf, s->buffer + s->readpos, cnt);
641	if (!ret)
642	return -EFAULT;
643
644	s->readpos += cnt;
645	return cnt;
646	}
647
648	/*
649	* ftrace_max_lock is used to protect the swapping of buffers
650	* when taking a max snapshot. The buffers themselves are
651	* protected by per_cpu spinlocks. But the action of the swap
652	* needs its own lock.
653	*
654	* This is defined as a arch_spinlock_t in order to help
655	* with performance when lockdep debugging is enabled.
656	*
657	* It is also used in other places outside the update_max_tr
658	* so it needs to be defined outside of the
659	* CONFIG_TRACER_MAX_TRACE.
660	*/
661	static arch_spinlock_t ftrace_max_lock =
662	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
663
664	unsigned long __read_mostly tracing_thresh;
665
666	#ifdef CONFIG_TRACER_MAX_TRACE
667	unsigned long __read_mostly tracing_max_latency;
668
669	/*
670	* Copy the new maximum trace into the separate maximum-trace
671	* structure. (this way the maximum trace is permanently saved,
672	* for later retrieval via /sys/kernel/debug/tracing/latency_trace)
673	*/
674	static void
675	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
676	{
677	struct trace_array_cpu *data = tr->data[cpu];
678	struct trace_array_cpu *max_data;
679
680	max_tr.cpu = cpu;
681	max_tr.time_start = data->preempt_timestamp;
682
683	max_data = max_tr.data[cpu];
684	max_data->saved_latency = tracing_max_latency;
685	max_data->critical_start = data->critical_start;
686	max_data->critical_end = data->critical_end;
687
688	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
689	max_data->pid = tsk->pid;
690	max_data->uid = task_uid(tsk);
691	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
692	max_data->policy = tsk->policy;
693	max_data->rt_priority = tsk->rt_priority;
694
695	/* record this tasks comm */
696	tracing_record_cmdline(tsk);
697	}
698
699	/**
700	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
701	* @tr: tracer
702	* @tsk: the task with the latency
703	* @cpu: The cpu that initiated the trace.
704	*
705	* Flip the buffers between the @tr and the max_tr and record information
706	* about which task was the cause of this latency.
707	*/
708	void
709	update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
710	{
711	struct ring_buffer *buf = tr->buffer;
712
713	if (trace_stop_count)
714	return;
715
716	WARN_ON_ONCE(!irqs_disabled());
717	if (!current_trace->use_max_tr) {
718	WARN_ON_ONCE(1);
719	return;
720	}
721	arch_spin_lock(&ftrace_max_lock);
722
723	tr->buffer = max_tr.buffer;
724	max_tr.buffer = buf;
725
726	__update_max_tr(tr, tsk, cpu);
727	arch_spin_unlock(&ftrace_max_lock);
728	}
729
730	/**
731	* update_max_tr_single - only copy one trace over, and reset the rest
732	* @tr - tracer
733	* @tsk - task with the latency
734	* @cpu - the cpu of the buffer to copy.
735	*
736	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
737	*/
738	void
739	update_max_tr_single(struct trace_array tr, struct task_struct tsk, int cpu)
740	{
741	int ret;
742
743	if (trace_stop_count)
744	return;
745
746	WARN_ON_ONCE(!irqs_disabled());
747	if (!current_trace->use_max_tr) {
748	WARN_ON_ONCE(1);
749	return;
750	}
751
752	arch_spin_lock(&ftrace_max_lock);
753
754	ftrace_disable_cpu();
755
756	ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
757
758	if (ret == -EBUSY) {
759	/*
760	* We failed to swap the buffer due to a commit taking
761	* place on this CPU. We fail to record, but we reset
762	* the max trace buffer (no one writes directly to it)
763	* and flag that it failed.
764	*/
765	trace_array_printk(&max_tr, _THIS_IP_,
766	"Failed to swap buffers due to commit in progress\n");
767	}
768
769	ftrace_enable_cpu();
770
771	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
772
773	__update_max_tr(tr, tsk, cpu);
774	arch_spin_unlock(&ftrace_max_lock);
775	}
776	#endif /* CONFIG_TRACER_MAX_TRACE */
777
778	/**
779	* register_tracer - register a tracer with the ftrace system.
780	* @type - the plugin for the tracer
781	*
782	* Register a new plugin tracer.
783	*/
784	int register_tracer(struct tracer *type)
785	__releases(kernel_lock)
786	__acquires(kernel_lock)
787	{
788	struct tracer *t;
789	int ret = 0;
790
791	if (!type->name) {
792	pr_info("Tracer must have a name\n");
793	return -1;
794	}
795
796	if (strlen(type->name) >= MAX_TRACER_SIZE) {
797	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
798	return -1;
799	}
800
801	mutex_lock(&trace_types_lock);
802
803	tracing_selftest_running = true;
804
805	for (t = trace_types; t; t = t->next) {
806	if (strcmp(type->name, t->name) == 0) {
807	/* already found */
808	pr_info("Tracer %s already registered\n",
809	type->name);
810	ret = -1;
811	goto out;
812	}
813	}
814
815	if (!type->set_flag)
816	type->set_flag = &dummy_set_flag;
817	if (!type->flags)
818	type->flags = &dummy_tracer_flags;
819	else
820	if (!type->flags->opts)
821	type->flags->opts = dummy_tracer_opt;
822	if (!type->wait_pipe)
823	type->wait_pipe = default_wait_pipe;
824
825
826	#ifdef CONFIG_FTRACE_STARTUP_TEST
827	if (type->selftest && !tracing_selftest_disabled) {
828	struct tracer *saved_tracer = current_trace;
829	struct trace_array *tr = &global_trace;
830
831	/*
832	* Run a selftest on this tracer.
833	* Here we reset the trace buffer, and set the current
834	* tracer to be this tracer. The tracer can then run some
835	* internal tracing to verify that everything is in order.
836	* If we fail, we do not register this tracer.
837	*/
838	tracing_reset_online_cpus(tr);
839
840	current_trace = type;
841
842	/* If we expanded the buffers, make sure the max is expanded too */
843	if (ring_buffer_expanded && type->use_max_tr)
844	ring_buffer_resize(max_tr.buffer, trace_buf_size);
845
846	/* the test is responsible for initializing and enabling */
847	pr_info("Testing tracer %s: ", type->name);
848	ret = type->selftest(type, tr);
849	/* the test is responsible for resetting too */
850	current_trace = saved_tracer;
851	if (ret) {
852	printk(KERN_CONT "FAILED!\n");
853	goto out;
854	}
855	/* Only reset on passing, to avoid touching corrupted buffers */
856	tracing_reset_online_cpus(tr);
857
858	/* Shrink the max buffer again */
859	if (ring_buffer_expanded && type->use_max_tr)
860	ring_buffer_resize(max_tr.buffer, 1);
861
862	printk(KERN_CONT "PASSED\n");
863	}
864	#endif
865
866	type->next = trace_types;
867	trace_types = type;
868
869	out:
870	tracing_selftest_running = false;
871	mutex_unlock(&trace_types_lock);
872
873	if (ret \|\| !default_bootup_tracer)
874	goto out_unlock;
875
876	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
877	goto out_unlock;
878
879	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
880	/* Do we want this tracer to start on bootup? */
881	tracing_set_tracer(type->name);
882	default_bootup_tracer = NULL;
883	/* disable other selftests, since this will break it. */
884	tracing_selftest_disabled = 1;
885	#ifdef CONFIG_FTRACE_STARTUP_TEST
886	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
887	type->name);
888	#endif
889
890	out_unlock:
891	return ret;
892	}
893
894	void unregister_tracer(struct tracer *type)
895	{
896	struct tracer **t;
897
898	mutex_lock(&trace_types_lock);
899	for (t = &trace_types; t; t = &(t)->next) {
900	if (*t == type)
901	goto found;
902	}
903	pr_info("Tracer %s not registered\n", type->name);
904	goto out;
905
906	found:
907	t = (t)->next;
908
909	if (type == current_trace && tracer_enabled) {
910	tracer_enabled = 0;
911	tracing_stop();
912	if (current_trace->stop)
913	current_trace->stop(&global_trace);
914	current_trace = &nop_trace;
915	}
916	out:
917	mutex_unlock(&trace_types_lock);
918	}
919
920	static void __tracing_reset(struct ring_buffer *buffer, int cpu)
921	{
922	ftrace_disable_cpu();
923	ring_buffer_reset_cpu(buffer, cpu);
924	ftrace_enable_cpu();
925	}
926
927	void tracing_reset(struct trace_array *tr, int cpu)
928	{
929	struct ring_buffer *buffer = tr->buffer;
930
931	ring_buffer_record_disable(buffer);
932
933	/* Make sure all commits have finished */
934	synchronize_sched();
935	__tracing_reset(buffer, cpu);
936
937	ring_buffer_record_enable(buffer);
938	}
939
940	void tracing_reset_online_cpus(struct trace_array *tr)
941	{
942	struct ring_buffer *buffer = tr->buffer;
943	int cpu;
944
945	ring_buffer_record_disable(buffer);
946
947	/* Make sure all commits have finished */
948	synchronize_sched();
949
950	tr->time_start = ftrace_now(tr->cpu);
951
952	for_each_online_cpu(cpu)
953	__tracing_reset(buffer, cpu);
954
955	ring_buffer_record_enable(buffer);
956	}
957
958	void tracing_reset_current(int cpu)
959	{
960	tracing_reset(&global_trace, cpu);
961	}
962
963	void tracing_reset_current_online_cpus(void)
964	{
965	tracing_reset_online_cpus(&global_trace);
966	}
967
968	#define SAVED_CMDLINES 128
969	#define NO_CMDLINE_MAP UINT_MAX
970	static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
971	static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
972	static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
973	static int cmdline_idx;
974	static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
975
976	/* temporary disable recording */
977	static atomic_t trace_record_cmdline_disabled __read_mostly;
978
979	static void trace_init_cmdlines(void)
980	{
981	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
982	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
983	cmdline_idx = 0;
984	}
985
986	int is_tracing_stopped(void)
987	{
988	return trace_stop_count;
989	}
990
991	/**
992	* ftrace_off_permanent - disable all ftrace code permanently
993	*
994	* This should only be called when a serious anomally has
995	* been detected. This will turn off the function tracing,
996	* ring buffers, and other tracing utilites. It takes no
997	* locks and can be called from any context.
998	*/
999	void ftrace_off_permanent(void)
1000	{
1001	tracing_disabled = 1;
1002	ftrace_stop();
1003	tracing_off_permanent();
1004	}
1005
1006	/**
1007	* tracing_start - quick start of the tracer
1008	*
1009	* If tracing is enabled but was stopped by tracing_stop,
1010	* this will start the tracer back up.
1011	*/
1012	void tracing_start(void)
1013	{
1014	struct ring_buffer *buffer;
1015	unsigned long flags;
1016
1017	if (tracing_disabled)
1018	return;
1019
1020	raw_spin_lock_irqsave(&tracing_start_lock, flags);
1021	if (--trace_stop_count) {
1022	if (trace_stop_count < 0) {
1023	/* Someone screwed up their debugging */
1024	WARN_ON_ONCE(1);
1025	trace_stop_count = 0;
1026	}
1027	goto out;
1028	}
1029
1030	/* Prevent the buffers from switching */
1031	arch_spin_lock(&ftrace_max_lock);
1032
1033	buffer = global_trace.buffer;
1034	if (buffer)
1035	ring_buffer_record_enable(buffer);
1036
1037	buffer = max_tr.buffer;
1038	if (buffer)
1039	ring_buffer_record_enable(buffer);
1040
1041	arch_spin_unlock(&ftrace_max_lock);
1042
1043	ftrace_start();
1044	out:
1045	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1046	}
1047
1048	/**
1049	* tracing_stop - quick stop of the tracer
1050	*
1051	* Light weight way to stop tracing. Use in conjunction with
1052	* tracing_start.
1053	*/
1054	void tracing_stop(void)
1055	{
1056	struct ring_buffer *buffer;
1057	unsigned long flags;
1058
1059	ftrace_stop();
1060	raw_spin_lock_irqsave(&tracing_start_lock, flags);
1061	if (trace_stop_count++)
1062	goto out;
1063
1064	/* Prevent the buffers from switching */
1065	arch_spin_lock(&ftrace_max_lock);
1066
1067	buffer = global_trace.buffer;
1068	if (buffer)
1069	ring_buffer_record_disable(buffer);
1070
1071	buffer = max_tr.buffer;
1072	if (buffer)
1073	ring_buffer_record_disable(buffer);
1074
1075	arch_spin_unlock(&ftrace_max_lock);
1076
1077	out:
1078	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1079	}
1080
1081	void trace_stop_cmdline_recording(void);
1082
1083	static void trace_save_cmdline(struct task_struct *tsk)
1084	{
1085	unsigned pid, idx;
1086
1087	if (!tsk->pid \|\| unlikely(tsk->pid > PID_MAX_DEFAULT))
1088	return;
1089
1090	/*
1091	* It's not the end of the world if we don't get
1092	* the lock, but we also don't want to spin
1093	* nor do we want to disable interrupts,
1094	* so if we miss here, then better luck next time.
1095	*/
1096	if (!arch_spin_trylock(&trace_cmdline_lock))
1097	return;
1098
1099	idx = map_pid_to_cmdline[tsk->pid];
1100	if (idx == NO_CMDLINE_MAP) {
1101	idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1102
1103	/*
1104	* Check whether the cmdline buffer at idx has a pid
1105	* mapped. We are going to overwrite that entry so we
1106	* need to clear the map_pid_to_cmdline. Otherwise we
1107	* would read the new comm for the old pid.
1108	*/
1109	pid = map_cmdline_to_pid[idx];
1110	if (pid != NO_CMDLINE_MAP)
1111	map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1112
1113	map_cmdline_to_pid[idx] = tsk->pid;
1114	map_pid_to_cmdline[tsk->pid] = idx;
1115
1116	cmdline_idx = idx;
1117	}
1118
1119	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1120
1121	arch_spin_unlock(&trace_cmdline_lock);
1122	}
1123
1124	void trace_find_cmdline(int pid, char comm[])
1125	{
1126	unsigned map;
1127
1128	if (!pid) {
1129	strcpy(comm, "<idle>");
1130	return;
1131	}
1132
1133	if (WARN_ON_ONCE(pid < 0)) {
1134	strcpy(comm, "<XXX>");
1135	return;
1136	}
1137
1138	if (pid > PID_MAX_DEFAULT) {
1139	strcpy(comm, "<...>");
1140	return;
1141	}
1142
1143	preempt_disable();
1144	arch_spin_lock(&trace_cmdline_lock);
1145	map = map_pid_to_cmdline[pid];
1146	if (map != NO_CMDLINE_MAP)
1147	strcpy(comm, saved_cmdlines[map]);
1148	else
1149	strcpy(comm, "<...>");
1150
1151	arch_spin_unlock(&trace_cmdline_lock);
1152	preempt_enable();
1153	}
1154
1155	void tracing_record_cmdline(struct task_struct *tsk)
1156	{
1157	if (atomic_read(&trace_record_cmdline_disabled) \|\| !tracer_enabled \|\|
1158	!tracing_is_on())
1159	return;
1160
1161	trace_save_cmdline(tsk);
1162	}
1163
1164	void
1165	tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1166	int pc)
1167	{
1168	struct task_struct *tsk = current;
1169
1170	entry->preempt_count = pc & 0xff;
1171	entry->pid = (tsk) ? tsk->pid : 0;
1172	entry->padding = 0;
1173	entry->flags =
1174	#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1175	(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) \|
1176	#else
1177	TRACE_FLAG_IRQS_NOSUPPORT \|
1178	#endif
1179	((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) \|
1180	((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) \|
1181	(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1182	}
1183	EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1184
1185	struct ring_buffer_event *
1186	trace_buffer_lock_reserve(struct ring_buffer *buffer,
1187	int type,
1188	unsigned long len,
1189	unsigned long flags, int pc)
1190	{
1191	struct ring_buffer_event *event;
1192
1193	event = ring_buffer_lock_reserve(buffer, len);
1194	if (event != NULL) {
1195	struct trace_entry *ent = ring_buffer_event_data(event);
1196
1197	tracing_generic_entry_update(ent, flags, pc);
1198	ent->type = type;
1199	}
1200
1201	return event;
1202	}
1203
1204	static inline void
1205	__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1206	struct ring_buffer_event *event,
1207	unsigned long flags, int pc,
1208	int wake)
1209	{
1210	ring_buffer_unlock_commit(buffer, event);
1211
1212	ftrace_trace_stack(buffer, flags, 6, pc);
1213	ftrace_trace_userstack(buffer, flags, pc);
1214
1215	if (wake)
1216	trace_wake_up();
1217	}
1218
1219	void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1220	struct ring_buffer_event *event,
1221	unsigned long flags, int pc)
1222	{
1223	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1224	}
1225
1226	struct ring_buffer_event *
1227	trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1228	int type, unsigned long len,
1229	unsigned long flags, int pc)
1230	{
1231	*current_rb = global_trace.buffer;
1232	return trace_buffer_lock_reserve(*current_rb,
1233	type, len, flags, pc);
1234	}
1235	EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1236
1237	void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1238	struct ring_buffer_event *event,
1239	unsigned long flags, int pc)
1240	{
1241	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1242	}
1243	EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1244
1245	void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1246	struct ring_buffer_event *event,
1247	unsigned long flags, int pc)
1248	{
1249	__trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1250	}
1251	EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1252
1253	void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1254	struct ring_buffer_event *event,
1255	unsigned long flags, int pc,
1256	struct pt_regs *regs)
1257	{
1258	ring_buffer_unlock_commit(buffer, event);
1259
1260	ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1261	ftrace_trace_userstack(buffer, flags, pc);
1262	}
1263	EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1264
1265	void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1266	struct ring_buffer_event *event)
1267	{
1268	ring_buffer_discard_commit(buffer, event);
1269	}
1270	EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1271
1272	void
1273	trace_function(struct trace_array *tr,
1274	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1275	int pc)
1276	{
1277	struct ftrace_event_call *call = &event_function;
1278	struct ring_buffer *buffer = tr->buffer;
1279	struct ring_buffer_event *event;
1280	struct ftrace_entry *entry;
1281
1282	/* If we are reading the ring buffer, don't trace */
1283	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1284	return;
1285
1286	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1287	flags, pc);
1288	if (!event)
1289	return;
1290	entry = ring_buffer_event_data(event);
1291	entry->ip = ip;
1292	entry->parent_ip = parent_ip;
1293
1294	if (!filter_check_discard(call, entry, buffer, event))
1295	ring_buffer_unlock_commit(buffer, event);
1296	}
1297
1298	void
1299	ftrace(struct trace_array tr, struct trace_array_cpu data,
1300	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1301	int pc)
1302	{
1303	if (likely(!atomic_read(&data->disabled)))
1304	trace_function(tr, ip, parent_ip, flags, pc);
1305	}
1306
1307	#ifdef CONFIG_STACKTRACE
1308
1309	#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1310	struct ftrace_stack {
1311	unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1312	};
1313
1314	static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1315	static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1316
1317	static void __ftrace_trace_stack(struct ring_buffer *buffer,
1318	unsigned long flags,
1319	int skip, int pc, struct pt_regs *regs)
1320	{
1321	struct ftrace_event_call *call = &event_kernel_stack;
1322	struct ring_buffer_event *event;
1323	struct stack_entry *entry;
1324	struct stack_trace trace;
1325	int use_stack;
1326	int size = FTRACE_STACK_ENTRIES;
1327
1328	trace.nr_entries = 0;
1329	trace.skip = skip;
1330
1331	/*
1332	* Since events can happen in NMIs there's no safe way to
1333	* use the per cpu ftrace_stacks. We reserve it and if an interrupt
1334	* or NMI comes in, it will just have to use the default
1335	* FTRACE_STACK_SIZE.
1336	*/
1337	preempt_disable_notrace();
1338
1339	use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1340	/*
1341	* We don't need any atomic variables, just a barrier.
1342	* If an interrupt comes in, we don't care, because it would
1343	* have exited and put the counter back to what we want.
1344	* We just need a barrier to keep gcc from moving things
1345	* around.
1346	*/
1347	barrier();
1348	if (use_stack == 1) {
1349	trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1350	trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1351
1352	if (regs)
1353	save_stack_trace_regs(regs, &trace);
1354	else
1355	save_stack_trace(&trace);
1356
1357	if (trace.nr_entries > size)
1358	size = trace.nr_entries;
1359	} else
1360	/* From now on, use_stack is a boolean */
1361	use_stack = 0;
1362
1363	size *= sizeof(unsigned long);
1364
1365	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1366	sizeof(*entry) + size, flags, pc);
1367	if (!event)
1368	goto out;
1369	entry = ring_buffer_event_data(event);
1370
1371	memset(&entry->caller, 0, size);
1372
1373	if (use_stack)
1374	memcpy(&entry->caller, trace.entries,
1375	trace.nr_entries * sizeof(unsigned long));
1376	else {
1377	trace.max_entries = FTRACE_STACK_ENTRIES;
1378	trace.entries = entry->caller;
1379	if (regs)
1380	save_stack_trace_regs(regs, &trace);
1381	else
1382	save_stack_trace(&trace);
1383	}
1384
1385	entry->size = trace.nr_entries;
1386
1387	if (!filter_check_discard(call, entry, buffer, event))
1388	ring_buffer_unlock_commit(buffer, event);
1389
1390	out:
1391	/* Again, don't let gcc optimize things here */
1392	barrier();
1393	__get_cpu_var(ftrace_stack_reserve)--;
1394	preempt_enable_notrace();
1395
1396	}
1397
1398	void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1399	int skip, int pc, struct pt_regs *regs)
1400	{
1401	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1402	return;
1403
1404	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
1405	}
1406
1407	void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1408	int skip, int pc)
1409	{
1410	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1411	return;
1412
1413	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1414	}
1415
1416	void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1417	int pc)
1418	{
1419	__ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1420	}
1421
1422	/**
1423	* trace_dump_stack - record a stack back trace in the trace buffer
1424	*/
1425	void trace_dump_stack(void)
1426	{
1427	unsigned long flags;
1428
1429	if (tracing_disabled \|\| tracing_selftest_running)
1430	return;
1431
1432	local_save_flags(flags);
1433
1434	/* skipping 3 traces, seems to get us at the caller of this function */
1435	__ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1436	}
1437
1438	static DEFINE_PER_CPU(int, user_stack_count);
1439
1440	void
1441	ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1442	{
1443	struct ftrace_event_call *call = &event_user_stack;
1444	struct ring_buffer_event *event;
1445	struct userstack_entry *entry;
1446	struct stack_trace trace;
1447
1448	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1449	return;
1450
1451	/*
1452	* NMIs can not handle page faults, even with fix ups.
1453	* The save user stack can (and often does) fault.
1454	*/
1455	if (unlikely(in_nmi()))
1456	return;
1457
1458	/*
1459	* prevent recursion, since the user stack tracing may
1460	* trigger other kernel events.
1461	*/
1462	preempt_disable();
1463	if (__this_cpu_read(user_stack_count))
1464	goto out;
1465
1466	__this_cpu_inc(user_stack_count);
1467
1468	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1469	sizeof(*entry), flags, pc);
1470	if (!event)
1471	goto out_drop_count;
1472	entry = ring_buffer_event_data(event);
1473
1474	entry->tgid = current->tgid;
1475	memset(&entry->caller, 0, sizeof(entry->caller));
1476
1477	trace.nr_entries = 0;
1478	trace.max_entries = FTRACE_STACK_ENTRIES;
1479	trace.skip = 0;
1480	trace.entries = entry->caller;
1481
1482	save_stack_trace_user(&trace);
1483	if (!filter_check_discard(call, entry, buffer, event))
1484	ring_buffer_unlock_commit(buffer, event);
1485
1486	out_drop_count:
1487	__this_cpu_dec(user_stack_count);
1488	out:
1489	preempt_enable();
1490	}
1491
1492	#ifdef UNUSED
1493	static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1494	{
1495	ftrace_trace_userstack(tr, flags, preempt_count());
1496	}
1497	#endif /* UNUSED */
1498
1499	#endif /* CONFIG_STACKTRACE */
1500
1501	/**
1502	* trace_vbprintk - write binary msg to tracing buffer
1503	*
1504	*/
1505	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1506	{
1507	static arch_spinlock_t trace_buf_lock =
1508	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1509	static u32 trace_buf[TRACE_BUF_SIZE];
1510
1511	struct ftrace_event_call *call = &event_bprint;
1512	struct ring_buffer_event *event;
1513	struct ring_buffer *buffer;
1514	struct trace_array *tr = &global_trace;
1515	struct trace_array_cpu *data;
1516	struct bprint_entry *entry;
1517	unsigned long flags;
1518	int disable;
1519	int cpu, len = 0, size, pc;
1520
1521	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
1522	return 0;
1523
1524	/* Don't pollute graph traces with trace_vprintk internals */
1525	pause_graph_tracing();
1526
1527	pc = preempt_count();
1528	preempt_disable_notrace();
1529	cpu = raw_smp_processor_id();
1530	data = tr->data[cpu];
1531
1532	disable = atomic_inc_return(&data->disabled);
1533	if (unlikely(disable != 1))
1534	goto out;
1535
1536	/* Lockdep uses trace_printk for lock tracing */
1537	local_irq_save(flags);
1538	arch_spin_lock(&trace_buf_lock);
1539	len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1540
1541	if (len > TRACE_BUF_SIZE \|\| len < 0)
1542	goto out_unlock;
1543
1544	size = sizeof(entry) + sizeof(u32) len;
1545	buffer = tr->buffer;
1546	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1547	flags, pc);
1548	if (!event)
1549	goto out_unlock;
1550	entry = ring_buffer_event_data(event);
1551	entry->ip = ip;
1552	entry->fmt = fmt;
1553
1554	memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1555	if (!filter_check_discard(call, entry, buffer, event)) {
1556	ring_buffer_unlock_commit(buffer, event);
1557	ftrace_trace_stack(buffer, flags, 6, pc);
1558	}
1559
1560	out_unlock:
1561	arch_spin_unlock(&trace_buf_lock);
1562	local_irq_restore(flags);
1563
1564	out:
1565	atomic_dec_return(&data->disabled);
1566	preempt_enable_notrace();
1567	unpause_graph_tracing();
1568
1569	return len;
1570	}
1571	EXPORT_SYMBOL_GPL(trace_vbprintk);
1572
1573	int trace_array_printk(struct trace_array *tr,
1574	unsigned long ip, const char *fmt, ...)
1575	{
1576	int ret;
1577	va_list ap;
1578
1579	if (!(trace_flags & TRACE_ITER_PRINTK))
1580	return 0;
1581
1582	va_start(ap, fmt);
1583	ret = trace_array_vprintk(tr, ip, fmt, ap);
1584	va_end(ap);
1585	return ret;
1586	}
1587
1588	int trace_array_vprintk(struct trace_array *tr,
1589	unsigned long ip, const char *fmt, va_list args)
1590	{
1591	static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1592	static char trace_buf[TRACE_BUF_SIZE];
1593
1594	struct ftrace_event_call *call = &event_print;
1595	struct ring_buffer_event *event;
1596	struct ring_buffer *buffer;
1597	struct trace_array_cpu *data;
1598	int cpu, len = 0, size, pc;
1599	struct print_entry *entry;
1600	unsigned long irq_flags;
1601	int disable;
1602
1603	if (tracing_disabled \|\| tracing_selftest_running)
1604	return 0;
1605
1606	pc = preempt_count();
1607	preempt_disable_notrace();
1608	cpu = raw_smp_processor_id();
1609	data = tr->data[cpu];
1610
1611	disable = atomic_inc_return(&data->disabled);
1612	if (unlikely(disable != 1))
1613	goto out;
1614
1615	pause_graph_tracing();
1616	raw_local_irq_save(irq_flags);
1617	arch_spin_lock(&trace_buf_lock);
1618	len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1619
1620	size = sizeof(*entry) + len + 1;
1621	buffer = tr->buffer;
1622	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1623	irq_flags, pc);
1624	if (!event)
1625	goto out_unlock;
1626	entry = ring_buffer_event_data(event);
1627	entry->ip = ip;
1628
1629	memcpy(&entry->buf, trace_buf, len);
1630	entry->buf[len] = '\0';
1631	if (!filter_check_discard(call, entry, buffer, event)) {
1632	ring_buffer_unlock_commit(buffer, event);
1633	ftrace_trace_stack(buffer, irq_flags, 6, pc);
1634	}
1635
1636	out_unlock:
1637	arch_spin_unlock(&trace_buf_lock);
1638	raw_local_irq_restore(irq_flags);
1639	unpause_graph_tracing();
1640	out:
1641	atomic_dec_return(&data->disabled);
1642	preempt_enable_notrace();
1643
1644	return len;
1645	}
1646
1647	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1648	{
1649	return trace_array_vprintk(&global_trace, ip, fmt, args);
1650	}
1651	EXPORT_SYMBOL_GPL(trace_vprintk);
1652
1653	static void trace_iterator_increment(struct trace_iterator *iter)
1654	{
1655	/* Don't allow ftrace to trace into the ring buffers */
1656	ftrace_disable_cpu();
1657
1658	iter->idx++;
1659	if (iter->buffer_iter[iter->cpu])
1660	ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1661
1662	ftrace_enable_cpu();
1663	}
1664
1665	static struct trace_entry *
1666	peek_next_entry(struct trace_iterator iter, int cpu, u64 ts,
1667	unsigned long *lost_events)
1668	{
1669	struct ring_buffer_event *event;
1670	struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1671
1672	/* Don't allow ftrace to trace into the ring buffers */
1673	ftrace_disable_cpu();
1674
1675	if (buf_iter)
1676	event = ring_buffer_iter_peek(buf_iter, ts);
1677	else
1678	event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1679	lost_events);
1680
1681	ftrace_enable_cpu();
1682
1683	if (event) {
1684	iter->ent_size = ring_buffer_event_length(event);
1685	return ring_buffer_event_data(event);
1686	}
1687	iter->ent_size = 0;
1688	return NULL;
1689	}
1690
1691	static struct trace_entry *
1692	__find_next_entry(struct trace_iterator iter, int ent_cpu,
1693	unsigned long missing_events, u64 ent_ts)
1694	{
1695	struct ring_buffer *buffer = iter->tr->buffer;
1696	struct trace_entry ent, next = NULL;
1697	unsigned long lost_events = 0, next_lost = 0;
1698	int cpu_file = iter->cpu_file;
1699	u64 next_ts = 0, ts;
1700	int next_cpu = -1;
1701	int next_size = 0;
1702	int cpu;
1703
1704	/*
1705	* If we are in a per_cpu trace file, don't bother by iterating over
1706	* all cpu and peek directly.
1707	*/
1708	if (cpu_file > TRACE_PIPE_ALL_CPU) {
1709	if (ring_buffer_empty_cpu(buffer, cpu_file))
1710	return NULL;
1711	ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1712	if (ent_cpu)
1713	*ent_cpu = cpu_file;
1714
1715	return ent;
1716	}
1717
1718	for_each_tracing_cpu(cpu) {
1719
1720	if (ring_buffer_empty_cpu(buffer, cpu))
1721	continue;
1722
1723	ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1724
1725	/*
1726	* Pick the entry with the smallest timestamp:
1727	*/
1728	if (ent && (!next \|\| ts < next_ts)) {
1729	next = ent;
1730	next_cpu = cpu;
1731	next_ts = ts;
1732	next_lost = lost_events;
1733	next_size = iter->ent_size;
1734	}
1735	}
1736
1737	iter->ent_size = next_size;
1738
1739	if (ent_cpu)
1740	*ent_cpu = next_cpu;
1741
1742	if (ent_ts)
1743	*ent_ts = next_ts;
1744
1745	if (missing_events)
1746	*missing_events = next_lost;
1747
1748	return next;
1749	}
1750
1751	/* Find the next real entry, without updating the iterator itself */
1752	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
1753	int ent_cpu, u64 ent_ts)
1754	{
1755	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1756	}
1757
1758	/* Find the next real entry, and increment the iterator to the next entry */
1759	void trace_find_next_entry_inc(struct trace_iterator iter)
1760	{
1761	iter->ent = __find_next_entry(iter, &iter->cpu,
1762	&iter->lost_events, &iter->ts);
1763
1764	if (iter->ent)
1765	trace_iterator_increment(iter);
1766
1767	return iter->ent ? iter : NULL;
1768	}
1769
1770	static void trace_consume(struct trace_iterator *iter)
1771	{
1772	/* Don't allow ftrace to trace into the ring buffers */
1773	ftrace_disable_cpu();
1774	ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1775	&iter->lost_events);
1776	ftrace_enable_cpu();
1777	}
1778
1779	static void s_next(struct seq_file m, void v, loff_t pos)
1780	{
1781	struct trace_iterator *iter = m->private;
1782	int i = (int)*pos;
1783	void *ent;
1784
1785	WARN_ON_ONCE(iter->leftover);
1786
1787	(*pos)++;
1788
1789	/* can't go backwards */
1790	if (iter->idx > i)
1791	return NULL;
1792
1793	if (iter->idx < 0)
1794	ent = trace_find_next_entry_inc(iter);
1795	else
1796	ent = iter;
1797
1798	while (ent && iter->idx < i)
1799	ent = trace_find_next_entry_inc(iter);
1800
1801	iter->pos = *pos;
1802
1803	return ent;
1804	}
1805
1806	void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1807	{
1808	struct trace_array *tr = iter->tr;
1809	struct ring_buffer_event *event;
1810	struct ring_buffer_iter *buf_iter;
1811	unsigned long entries = 0;
1812	u64 ts;
1813
1814	tr->data[cpu]->skipped_entries = 0;
1815
1816	if (!iter->buffer_iter[cpu])
1817	return;
1818
1819	buf_iter = iter->buffer_iter[cpu];
1820	ring_buffer_iter_reset(buf_iter);
1821
1822	/*
1823	* We could have the case with the max latency tracers
1824	* that a reset never took place on a cpu. This is evident
1825	* by the timestamp being before the start of the buffer.
1826	*/
1827	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1828	if (ts >= iter->tr->time_start)
1829	break;
1830	entries++;
1831	ring_buffer_read(buf_iter, NULL);
1832	}
1833
1834	tr->data[cpu]->skipped_entries = entries;
1835	}
1836
1837	/*
1838	* The current tracer is copied to avoid a global locking
1839	* all around.
1840	*/
1841	static void s_start(struct seq_file m, loff_t *pos)
1842	{
1843	struct trace_iterator *iter = m->private;
1844	static struct tracer *old_tracer;
1845	int cpu_file = iter->cpu_file;
1846	void *p = NULL;
1847	loff_t l = 0;
1848	int cpu;
1849
1850	/* copy the tracer to avoid using a global lock all around */
1851	mutex_lock(&trace_types_lock);
1852	if (unlikely(old_tracer != current_trace && current_trace)) {
1853	old_tracer = current_trace;
1854	iter->trace = current_trace;
1855	}
1856	mutex_unlock(&trace_types_lock);
1857
1858	atomic_inc(&trace_record_cmdline_disabled);
1859
1860	if (*pos != iter->pos) {
1861	iter->ent = NULL;
1862	iter->cpu = 0;
1863	iter->idx = -1;
1864
1865	ftrace_disable_cpu();
1866
1867	if (cpu_file == TRACE_PIPE_ALL_CPU) {
1868	for_each_tracing_cpu(cpu)
1869	tracing_iter_reset(iter, cpu);
1870	} else
1871	tracing_iter_reset(iter, cpu_file);
1872
1873	ftrace_enable_cpu();
1874
1875	iter->leftover = 0;
1876	for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1877	;
1878
1879	} else {
1880	/*
1881	* If we overflowed the seq_file before, then we want
1882	* to just reuse the trace_seq buffer again.
1883	*/
1884	if (iter->leftover)
1885	p = iter;
1886	else {
1887	l = *pos - 1;
1888	p = s_next(m, p, &l);
1889	}
1890	}
1891
1892	trace_event_read_lock();
1893	trace_access_lock(cpu_file);
1894	return p;
1895	}
1896
1897	static void s_stop(struct seq_file m, void p)
1898	{
1899	struct trace_iterator *iter = m->private;
1900
1901	atomic_dec(&trace_record_cmdline_disabled);
1902	trace_access_unlock(iter->cpu_file);
1903	trace_event_read_unlock();
1904	}
1905
1906	static void
1907	get_total_entries(struct trace_array tr, unsigned long total, unsigned long *entries)
1908	{
1909	unsigned long count;
1910	int cpu;
1911
1912	*total = 0;
1913	*entries = 0;
1914
1915	for_each_tracing_cpu(cpu) {
1916	count = ring_buffer_entries_cpu(tr->buffer, cpu);
1917	/*
1918	* If this buffer has skipped entries, then we hold all
1919	* entries for the trace and we need to ignore the
1920	* ones before the time stamp.
1921	*/
1922	if (tr->data[cpu]->skipped_entries) {
1923	count -= tr->data[cpu]->skipped_entries;
1924	/* total is the same as the entries */
1925	*total += count;
1926	} else
1927	*total += count +
1928	ring_buffer_overrun_cpu(tr->buffer, cpu);
1929	*entries += count;
1930	}
1931	}
1932
1933	static void print_lat_help_header(struct seq_file *m)
1934	{
1935	seq_puts(m, "# _------=> CPU# \n");
1936	seq_puts(m, "# / _-----=> irqs-off \n");
1937	seq_puts(m, "# \| / _----=> need-resched \n");
1938	seq_puts(m, "# \|\| / _---=> hardirq/softirq \n");
1939	seq_puts(m, "# \|\|\| / _--=> preempt-depth \n");
1940	seq_puts(m, "# \|\|\|\| / delay \n");
1941	seq_puts(m, "# cmd pid \|\|\|\|\| time \| caller \n");
1942	seq_puts(m, "# \\ / \|\|\|\|\| \\ \| / \n");
1943	}
1944
1945	static void print_event_info(struct trace_array tr, struct seq_file m)
1946	{
1947	unsigned long total;
1948	unsigned long entries;
1949
1950	get_total_entries(tr, &total, &entries);
1951	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
1952	entries, total, num_online_cpus());
1953	seq_puts(m, "#\n");
1954	}
1955
1956	static void print_func_help_header(struct trace_array tr, struct seq_file m)
1957	{
1958	print_event_info(tr, m);
1959	seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1960	seq_puts(m, "# \| \| \| \| \|\n");
1961	}
1962
1963	static void print_func_help_header_irq(struct trace_array tr, struct seq_file m)
1964	{
1965	print_event_info(tr, m);
1966	seq_puts(m, "# _-----=> irqs-off\n");
1967	seq_puts(m, "# / _----=> need-resched\n");
1968	seq_puts(m, "# \| / _---=> hardirq/softirq\n");
1969	seq_puts(m, "# \|\| / _--=> preempt-depth\n");
1970	seq_puts(m, "# \|\|\| / delay\n");
1971	seq_puts(m, "# TASK-PID CPU# \|\|\|\| TIMESTAMP FUNCTION\n");
1972	seq_puts(m, "# \| \| \| \|\|\|\| \| \|\n");
1973	}
1974
1975	void
1976	print_trace_header(struct seq_file m, struct trace_iterator iter)
1977	{
1978	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1979	struct trace_array *tr = iter->tr;
1980	struct trace_array_cpu *data = tr->data[tr->cpu];
1981	struct tracer *type = current_trace;
1982	unsigned long entries;
1983	unsigned long total;
1984	const char *name = "preemption";
1985
1986	if (type)
1987	name = type->name;
1988
1989	get_total_entries(tr, &total, &entries);
1990
1991	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1992	name, UTS_RELEASE);
1993	seq_puts(m, "# -----------------------------------"
1994	"---------------------------------\n");
1995	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d \|"
1996	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1997	nsecs_to_usecs(data->saved_latency),
1998	entries,
1999	total,
2000	tr->cpu,
2001	#if defined(CONFIG_PREEMPT_NONE)
2002	"server",
2003	#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2004	"desktop",
2005	#elif defined(CONFIG_PREEMPT)
2006	"preempt",
2007	#else
2008	"unknown",
2009	#endif
2010	/* These are reserved for later use */
2011	0, 0, 0, 0);
2012	#ifdef CONFIG_SMP
2013	seq_printf(m, " #P:%d)\n", num_online_cpus());
2014	#else
2015	seq_puts(m, ")\n");
2016	#endif
2017	seq_puts(m, "# -----------------\n");
2018	seq_printf(m, "# \| task: %.16s-%d "
2019	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2020	data->comm, data->pid, data->uid, data->nice,
2021	data->policy, data->rt_priority);
2022	seq_puts(m, "# -----------------\n");
2023
2024	if (data->critical_start) {
2025	seq_puts(m, "# => started at: ");
2026	seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2027	trace_print_seq(m, &iter->seq);
2028	seq_puts(m, "\n# => ended at: ");
2029	seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2030	trace_print_seq(m, &iter->seq);
2031	seq_puts(m, "\n#\n");
2032	}
2033
2034	seq_puts(m, "#\n");
2035	}
2036
2037	static void test_cpu_buff_start(struct trace_iterator *iter)
2038	{
2039	struct trace_seq *s = &iter->seq;
2040
2041	if (!(trace_flags & TRACE_ITER_ANNOTATE))
2042	return;
2043
2044	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2045	return;
2046
2047	if (cpumask_test_cpu(iter->cpu, iter->started))
2048	return;
2049
2050	if (iter->tr->data[iter->cpu]->skipped_entries)
2051	return;
2052
2053	cpumask_set_cpu(iter->cpu, iter->started);
2054
2055	/* Don't print started cpu buffer for the first entry of the trace */
2056	if (iter->idx > 1)
2057	trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2058	iter->cpu);
2059	}
2060
2061	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2062	{
2063	struct trace_seq *s = &iter->seq;
2064	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2065	struct trace_entry *entry;
2066	struct trace_event *event;
2067
2068	entry = iter->ent;
2069
2070	test_cpu_buff_start(iter);
2071
2072	event = ftrace_find_event(entry->type);
2073
2074	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2075	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2076	if (!trace_print_lat_context(iter))
2077	goto partial;
2078	} else {
2079	if (!trace_print_context(iter))
2080	goto partial;
2081	}
2082	}
2083
2084	if (event)
2085	return event->funcs->trace(iter, sym_flags, event);
2086
2087	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2088	goto partial;
2089
2090	return TRACE_TYPE_HANDLED;
2091	partial:
2092	return TRACE_TYPE_PARTIAL_LINE;
2093	}
2094
2095	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2096	{
2097	struct trace_seq *s = &iter->seq;
2098	struct trace_entry *entry;
2099	struct trace_event *event;
2100
2101	entry = iter->ent;
2102
2103	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2104	if (!trace_seq_printf(s, "%d %d %llu ",
2105	entry->pid, iter->cpu, iter->ts))
2106	goto partial;
2107	}
2108
2109	event = ftrace_find_event(entry->type);
2110	if (event)
2111	return event->funcs->raw(iter, 0, event);
2112
2113	if (!trace_seq_printf(s, "%d ?\n", entry->type))
2114	goto partial;
2115
2116	return TRACE_TYPE_HANDLED;
2117	partial:
2118	return TRACE_TYPE_PARTIAL_LINE;
2119	}
2120
2121	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2122	{
2123	struct trace_seq *s = &iter->seq;
2124	unsigned char newline = '\n';
2125	struct trace_entry *entry;
2126	struct trace_event *event;
2127
2128	entry = iter->ent;
2129
2130	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2131	SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2132	SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2133	SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2134	}
2135
2136	event = ftrace_find_event(entry->type);
2137	if (event) {
2138	enum print_line_t ret = event->funcs->hex(iter, 0, event);
2139	if (ret != TRACE_TYPE_HANDLED)
2140	return ret;
2141	}
2142
2143	SEQ_PUT_FIELD_RET(s, newline);
2144
2145	return TRACE_TYPE_HANDLED;
2146	}
2147
2148	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2149	{
2150	struct trace_seq *s = &iter->seq;
2151	struct trace_entry *entry;
2152	struct trace_event *event;
2153
2154	entry = iter->ent;
2155
2156	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2157	SEQ_PUT_FIELD_RET(s, entry->pid);
2158	SEQ_PUT_FIELD_RET(s, iter->cpu);
2159	SEQ_PUT_FIELD_RET(s, iter->ts);
2160	}
2161
2162	event = ftrace_find_event(entry->type);
2163	return event ? event->funcs->binary(iter, 0, event) :
2164	TRACE_TYPE_HANDLED;
2165	}
2166
2167	int trace_empty(struct trace_iterator *iter)
2168	{
2169	int cpu;
2170
2171	/* If we are looking at one CPU buffer, only check that one */
2172	if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2173	cpu = iter->cpu_file;
2174	if (iter->buffer_iter[cpu]) {
2175	if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2176	return 0;
2177	} else {
2178	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2179	return 0;
2180	}
2181	return 1;
2182	}
2183
2184	for_each_tracing_cpu(cpu) {
2185	if (iter->buffer_iter[cpu]) {
2186	if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2187	return 0;
2188	} else {
2189	if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2190	return 0;
2191	}
2192	}
2193
2194	return 1;
2195	}
2196
2197	/* Called with trace_event_read_lock() held. */
2198	enum print_line_t print_trace_line(struct trace_iterator *iter)
2199	{
2200	enum print_line_t ret;
2201
2202	if (iter->lost_events &&
2203	!trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2204	iter->cpu, iter->lost_events))
2205	return TRACE_TYPE_PARTIAL_LINE;
2206
2207	if (iter->trace && iter->trace->print_line) {
2208	ret = iter->trace->print_line(iter);
2209	if (ret != TRACE_TYPE_UNHANDLED)
2210	return ret;
2211	}
2212
2213	if (iter->ent->type == TRACE_BPRINT &&
2214	trace_flags & TRACE_ITER_PRINTK &&
2215	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2216	return trace_print_bprintk_msg_only(iter);
2217
2218	if (iter->ent->type == TRACE_PRINT &&
2219	trace_flags & TRACE_ITER_PRINTK &&
2220	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2221	return trace_print_printk_msg_only(iter);
2222
2223	if (trace_flags & TRACE_ITER_BIN)
2224	return print_bin_fmt(iter);
2225
2226	if (trace_flags & TRACE_ITER_HEX)
2227	return print_hex_fmt(iter);
2228
2229	if (trace_flags & TRACE_ITER_RAW)
2230	return print_raw_fmt(iter);
2231
2232	return print_trace_fmt(iter);
2233	}
2234
2235	void trace_latency_header(struct seq_file *m)
2236	{
2237	struct trace_iterator *iter = m->private;
2238
2239	/* print nothing if the buffers are empty */
2240	if (trace_empty(iter))
2241	return;
2242
2243	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2244	print_trace_header(m, iter);
2245
2246	if (!(trace_flags & TRACE_ITER_VERBOSE))
2247	print_lat_help_header(m);
2248	}
2249
2250	void trace_default_header(struct seq_file *m)
2251	{
2252	struct trace_iterator *iter = m->private;
2253
2254	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2255	return;
2256
2257	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2258	/* print nothing if the buffers are empty */
2259	if (trace_empty(iter))
2260	return;
2261	print_trace_header(m, iter);
2262	if (!(trace_flags & TRACE_ITER_VERBOSE))
2263	print_lat_help_header(m);
2264	} else {
2265	if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2266	if (trace_flags & TRACE_ITER_IRQ_INFO)
2267	print_func_help_header_irq(iter->tr, m);
2268	else
2269	print_func_help_header(iter->tr, m);
2270	}
2271	}
2272	}
2273
2274	static void test_ftrace_alive(struct seq_file *m)
2275	{
2276	if (!ftrace_is_dead())
2277	return;
2278	seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2279	seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2280	}
2281
2282	static int s_show(struct seq_file m, void v)
2283	{
2284	struct trace_iterator *iter = v;
2285	int ret;
2286
2287	if (iter->ent == NULL) {
2288	if (iter->tr) {
2289	seq_printf(m, "# tracer: %s\n", iter->trace->name);
2290	seq_puts(m, "#\n");
2291	test_ftrace_alive(m);
2292	}
2293	if (iter->trace && iter->trace->print_header)
2294	iter->trace->print_header(m);
2295	else
2296	trace_default_header(m);
2297
2298	} else if (iter->leftover) {
2299	/*
2300	* If we filled the seq_file buffer earlier, we
2301	* want to just show it now.
2302	*/
2303	ret = trace_print_seq(m, &iter->seq);
2304
2305	/* ret should this time be zero, but you never know */
2306	iter->leftover = ret;
2307
2308	} else {
2309	print_trace_line(iter);
2310	ret = trace_print_seq(m, &iter->seq);
2311	/*
2312	* If we overflow the seq_file buffer, then it will
2313	* ask us for this data again at start up.
2314	* Use that instead.
2315	* ret is 0 if seq_file write succeeded.
2316	* -1 otherwise.
2317	*/
2318	iter->leftover = ret;
2319	}
2320
2321	return 0;
2322	}
2323
2324	static const struct seq_operations tracer_seq_ops = {
2325	.start = s_start,
2326	.next = s_next,
2327	.stop = s_stop,
2328	.show = s_show,
2329	};
2330
2331	static struct trace_iterator *
2332	__tracing_open(struct inode inode, struct file file)
2333	{
2334	long cpu_file = (long) inode->i_private;
2335	void *fail_ret = ERR_PTR(-ENOMEM);
2336	struct trace_iterator *iter;
2337	struct seq_file *m;
2338	int cpu, ret;
2339
2340	if (tracing_disabled)
2341	return ERR_PTR(-ENODEV);
2342
2343	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2344	if (!iter)
2345	return ERR_PTR(-ENOMEM);
2346
2347	/*
2348	* We make a copy of the current tracer to avoid concurrent
2349	* changes on it while we are reading.
2350	*/
2351	mutex_lock(&trace_types_lock);
2352	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2353	if (!iter->trace)
2354	goto fail;
2355
2356	if (current_trace)
2357	iter->trace = current_trace;
2358
2359	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2360	goto fail;
2361
2362	if (current_trace && current_trace->print_max)
2363	iter->tr = &max_tr;
2364	else
2365	iter->tr = &global_trace;
2366	iter->pos = -1;
2367	mutex_init(&iter->mutex);
2368	iter->cpu_file = cpu_file;
2369
2370	/* Notify the tracer early; before we stop tracing. */
2371	if (iter->trace && iter->trace->open)
2372	iter->trace->open(iter);
2373
2374	/* Annotate start of buffers if we had overruns */
2375	if (ring_buffer_overruns(iter->tr->buffer))
2376	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
2377
2378	/* stop the trace while dumping */
2379	tracing_stop();
2380
2381	if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2382	for_each_tracing_cpu(cpu) {
2383	iter->buffer_iter[cpu] =
2384	ring_buffer_read_prepare(iter->tr->buffer, cpu);
2385	}
2386	ring_buffer_read_prepare_sync();
2387	for_each_tracing_cpu(cpu) {
2388	ring_buffer_read_start(iter->buffer_iter[cpu]);
2389	tracing_iter_reset(iter, cpu);
2390	}
2391	} else {
2392	cpu = iter->cpu_file;
2393	iter->buffer_iter[cpu] =
2394	ring_buffer_read_prepare(iter->tr->buffer, cpu);
2395	ring_buffer_read_prepare_sync();
2396	ring_buffer_read_start(iter->buffer_iter[cpu]);
2397	tracing_iter_reset(iter, cpu);
2398	}
2399
2400	ret = seq_open(file, &tracer_seq_ops);
2401	if (ret < 0) {
2402	fail_ret = ERR_PTR(ret);
2403	goto fail_buffer;
2404	}
2405
2406	m = file->private_data;
2407	m->private = iter;
2408
2409	mutex_unlock(&trace_types_lock);
2410
2411	return iter;
2412
2413	fail_buffer:
2414	for_each_tracing_cpu(cpu) {
2415	if (iter->buffer_iter[cpu])
2416	ring_buffer_read_finish(iter->buffer_iter[cpu]);
2417	}
2418	free_cpumask_var(iter->started);
2419	tracing_start();
2420	fail:
2421	mutex_unlock(&trace_types_lock);
2422	kfree(iter->trace);
2423	kfree(iter);
2424
2425	return fail_ret;
2426	}
2427
2428	int tracing_open_generic(struct inode inode, struct file filp)
2429	{
2430	if (tracing_disabled)
2431	return -ENODEV;
2432
2433	filp->private_data = inode->i_private;
2434	return 0;
2435	}
2436
2437	static int tracing_release(struct inode inode, struct file file)
2438	{
2439	struct seq_file *m = file->private_data;
2440	struct trace_iterator *iter;
2441	int cpu;
2442
2443	if (!(file->f_mode & FMODE_READ))
2444	return 0;
2445
2446	iter = m->private;
2447
2448	mutex_lock(&trace_types_lock);
2449	for_each_tracing_cpu(cpu) {
2450	if (iter->buffer_iter[cpu])
2451	ring_buffer_read_finish(iter->buffer_iter[cpu]);
2452	}
2453
2454	if (iter->trace && iter->trace->close)
2455	iter->trace->close(iter);
2456
2457	/* reenable tracing if it was previously enabled */
2458	tracing_start();
2459	mutex_unlock(&trace_types_lock);
2460
2461	seq_release(inode, file);
2462	mutex_destroy(&iter->mutex);
2463	free_cpumask_var(iter->started);
2464	kfree(iter->trace);
2465	kfree(iter);
2466	return 0;
2467	}
2468
2469	static int tracing_open(struct inode inode, struct file file)
2470	{
2471	struct trace_iterator *iter;
2472	int ret = 0;
2473
2474	/* If this file was open for write, then erase contents */
2475	if ((file->f_mode & FMODE_WRITE) &&
2476	(file->f_flags & O_TRUNC)) {
2477	long cpu = (long) inode->i_private;
2478
2479	if (cpu == TRACE_PIPE_ALL_CPU)
2480	tracing_reset_online_cpus(&global_trace);
2481	else
2482	tracing_reset(&global_trace, cpu);
2483	}
2484
2485	if (file->f_mode & FMODE_READ) {
2486	iter = __tracing_open(inode, file);
2487	if (IS_ERR(iter))
2488	ret = PTR_ERR(iter);
2489	else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2490	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
2491	}
2492	return ret;
2493	}
2494
2495	static void *
2496	t_next(struct seq_file m, void v, loff_t *pos)
2497	{
2498	struct tracer *t = v;
2499
2500	(*pos)++;
2501
2502	if (t)
2503	t = t->next;
2504
2505	return t;
2506	}
2507
2508	static void t_start(struct seq_file m, loff_t *pos)
2509	{
2510	struct tracer *t;
2511	loff_t l = 0;
2512
2513	mutex_lock(&trace_types_lock);
2514	for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2515	;
2516
2517	return t;
2518	}
2519
2520	static void t_stop(struct seq_file m, void p)
2521	{
2522	mutex_unlock(&trace_types_lock);
2523	}
2524
2525	static int t_show(struct seq_file m, void v)
2526	{
2527	struct tracer *t = v;
2528
2529	if (!t)
2530	return 0;
2531
2532	seq_printf(m, "%s", t->name);
2533	if (t->next)
2534	seq_putc(m, ' ');
2535	else
2536	seq_putc(m, '\n');
2537
2538	return 0;
2539	}
2540
2541	static const struct seq_operations show_traces_seq_ops = {
2542	.start = t_start,
2543	.next = t_next,
2544	.stop = t_stop,
2545	.show = t_show,
2546	};
2547
2548	static int show_traces_open(struct inode inode, struct file file)
2549	{
2550	if (tracing_disabled)
2551	return -ENODEV;
2552
2553	return seq_open(file, &show_traces_seq_ops);
2554	}
2555
2556	static ssize_t
2557	tracing_write_stub(struct file filp, const char __user ubuf,
2558	size_t count, loff_t *ppos)
2559	{
2560	return count;
2561	}
2562
2563	static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2564	{
2565	if (file->f_mode & FMODE_READ)
2566	return seq_lseek(file, offset, origin);
2567	else
2568	return 0;
2569	}
2570
2571	static const struct file_operations tracing_fops = {
2572	.open = tracing_open,
2573	.read = seq_read,
2574	.write = tracing_write_stub,
2575	.llseek = tracing_seek,
2576	.release = tracing_release,
2577	};
2578
2579	static const struct file_operations show_traces_fops = {
2580	.open = show_traces_open,
2581	.read = seq_read,
2582	.release = seq_release,
2583	.llseek = seq_lseek,
2584	};
2585
2586	/*
2587	* Only trace on a CPU if the bitmask is set:
2588	*/
2589	static cpumask_var_t tracing_cpumask;
2590
2591	/*
2592	* The tracer itself will not take this lock, but still we want
2593	* to provide a consistent cpumask to user-space:
2594	*/
2595	static DEFINE_MUTEX(tracing_cpumask_update_lock);
2596
2597	/*
2598	* Temporary storage for the character representation of the
2599	* CPU bitmask (and one more byte for the newline):
2600	*/
2601	static char mask_str[NR_CPUS + 1];
2602
2603	static ssize_t
2604	tracing_cpumask_read(struct file filp, char __user ubuf,
2605	size_t count, loff_t *ppos)
2606	{
2607	int len;
2608
2609	mutex_lock(&tracing_cpumask_update_lock);
2610
2611	len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2612	if (count - len < 2) {
2613	count = -EINVAL;
2614	goto out_err;
2615	}
2616	len += sprintf(mask_str + len, "\n");
2617	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2618
2619	out_err:
2620	mutex_unlock(&tracing_cpumask_update_lock);
2621
2622	return count;
2623	}
2624
2625	static ssize_t
2626	tracing_cpumask_write(struct file filp, const char __user ubuf,
2627	size_t count, loff_t *ppos)
2628	{
2629	int err, cpu;
2630	cpumask_var_t tracing_cpumask_new;
2631
2632	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2633	return -ENOMEM;
2634
2635	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2636	if (err)
2637	goto err_unlock;
2638
2639	mutex_lock(&tracing_cpumask_update_lock);
2640
2641	local_irq_disable();
2642	arch_spin_lock(&ftrace_max_lock);
2643	for_each_tracing_cpu(cpu) {
2644	/*
2645	* Increase/decrease the disabled counter if we are
2646	* about to flip a bit in the cpumask:
2647	*/
2648	if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2649	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2650	atomic_inc(&global_trace.data[cpu]->disabled);
2651	}
2652	if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2653	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2654	atomic_dec(&global_trace.data[cpu]->disabled);
2655	}
2656	}
2657	arch_spin_unlock(&ftrace_max_lock);
2658	local_irq_enable();
2659
2660	cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2661
2662	mutex_unlock(&tracing_cpumask_update_lock);
2663	free_cpumask_var(tracing_cpumask_new);
2664
2665	return count;
2666
2667	err_unlock:
2668	free_cpumask_var(tracing_cpumask_new);
2669
2670	return err;
2671	}
2672
2673	static const struct file_operations tracing_cpumask_fops = {
2674	.open = tracing_open_generic,
2675	.read = tracing_cpumask_read,
2676	.write = tracing_cpumask_write,
2677	.llseek = generic_file_llseek,
2678	};
2679
2680	static int tracing_trace_options_show(struct seq_file m, void v)
2681	{
2682	struct tracer_opt *trace_opts;
2683	u32 tracer_flags;
2684	int i;
2685
2686	mutex_lock(&trace_types_lock);
2687	tracer_flags = current_trace->flags->val;
2688	trace_opts = current_trace->flags->opts;
2689
2690	for (i = 0; trace_options[i]; i++) {
2691	if (trace_flags & (1 << i))
2692	seq_printf(m, "%s\n", trace_options[i]);
2693	else
2694	seq_printf(m, "no%s\n", trace_options[i]);
2695	}
2696
2697	for (i = 0; trace_opts[i].name; i++) {
2698	if (tracer_flags & trace_opts[i].bit)
2699	seq_printf(m, "%s\n", trace_opts[i].name);
2700	else
2701	seq_printf(m, "no%s\n", trace_opts[i].name);
2702	}
2703	mutex_unlock(&trace_types_lock);
2704
2705	return 0;
2706	}
2707
2708	static int __set_tracer_option(struct tracer *trace,
2709	struct tracer_flags *tracer_flags,
2710	struct tracer_opt *opts, int neg)
2711	{
2712	int ret;
2713
2714	ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2715	if (ret)
2716	return ret;
2717
2718	if (neg)
2719	tracer_flags->val &= ~opts->bit;
2720	else
2721	tracer_flags->val \|= opts->bit;
2722	return 0;
2723	}
2724
2725	/* Try to assign a tracer specific option */
2726	static int set_tracer_option(struct tracer trace, char cmp, int neg)
2727	{
2728	struct tracer_flags *tracer_flags = trace->flags;
2729	struct tracer_opt *opts = NULL;
2730	int i;
2731
2732	for (i = 0; tracer_flags->opts[i].name; i++) {
2733	opts = &tracer_flags->opts[i];
2734
2735	if (strcmp(cmp, opts->name) == 0)
2736	return __set_tracer_option(trace, trace->flags,
2737	opts, neg);
2738	}
2739
2740	return -EINVAL;
2741	}
2742
2743	static void set_tracer_flags(unsigned int mask, int enabled)
2744	{
2745	/* do nothing if flag is already set */
2746	if (!!(trace_flags & mask) == !!enabled)
2747	return;
2748
2749	if (enabled)
2750	trace_flags \|= mask;
2751	else
2752	trace_flags &= ~mask;
2753
2754	if (mask == TRACE_ITER_RECORD_CMD)
2755	trace_event_enable_cmd_record(enabled);
2756
2757	if (mask == TRACE_ITER_OVERWRITE)
2758	ring_buffer_change_overwrite(global_trace.buffer, enabled);
2759	}
2760
2761	static ssize_t
2762	tracing_trace_options_write(struct file filp, const char __user ubuf,
2763	size_t cnt, loff_t *ppos)
2764	{
2765	char buf[64];
2766	char *cmp;
2767	int neg = 0;
2768	int ret;
2769	int i;
2770
2771	if (cnt >= sizeof(buf))
2772	return -EINVAL;
2773
2774	if (copy_from_user(&buf, ubuf, cnt))
2775	return -EFAULT;
2776
2777	buf[cnt] = 0;
2778	cmp = strstrip(buf);
2779
2780	if (strncmp(cmp, "no", 2) == 0) {
2781	neg = 1;
2782	cmp += 2;
2783	}
2784
2785	for (i = 0; trace_options[i]; i++) {
2786	if (strcmp(cmp, trace_options[i]) == 0) {
2787	set_tracer_flags(1 << i, !neg);
2788	break;
2789	}
2790	}
2791
2792	/* If no option could be set, test the specific tracer options */
2793	if (!trace_options[i]) {
2794	mutex_lock(&trace_types_lock);
2795	ret = set_tracer_option(current_trace, cmp, neg);
2796	mutex_unlock(&trace_types_lock);
2797	if (ret)
2798	return ret;
2799	}
2800
2801	*ppos += cnt;
2802
2803	return cnt;
2804	}
2805
2806	static int tracing_trace_options_open(struct inode inode, struct file file)
2807	{
2808	if (tracing_disabled)
2809	return -ENODEV;
2810	return single_open(file, tracing_trace_options_show, NULL);
2811	}
2812
2813	static const struct file_operations tracing_iter_fops = {
2814	.open = tracing_trace_options_open,
2815	.read = seq_read,
2816	.llseek = seq_lseek,
2817	.release = single_release,
2818	.write = tracing_trace_options_write,
2819	};
2820
2821	static const char readme_msg[] =
2822	"tracing mini-HOWTO:\n\n"
2823	"# mount -t debugfs nodev /sys/kernel/debug\n\n"
2824	"# cat /sys/kernel/debug/tracing/available_tracers\n"
2825	"wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
2826	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2827	"nop\n"
2828	"# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
2829	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2830	"wakeup\n"
2831	"# cat /sys/kernel/debug/tracing/trace_options\n"
2832	"noprint-parent nosym-offset nosym-addr noverbose\n"
2833	"# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2834	"# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2835	"# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2836	"# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
2837	;
2838
2839	static ssize_t
2840	tracing_readme_read(struct file filp, char __user ubuf,
2841	size_t cnt, loff_t *ppos)
2842	{
2843	return simple_read_from_buffer(ubuf, cnt, ppos,
2844	readme_msg, strlen(readme_msg));
2845	}
2846
2847	static const struct file_operations tracing_readme_fops = {
2848	.open = tracing_open_generic,
2849	.read = tracing_readme_read,
2850	.llseek = generic_file_llseek,
2851	};
2852
2853	static ssize_t
2854	tracing_saved_cmdlines_read(struct file file, char __user ubuf,
2855	size_t cnt, loff_t *ppos)
2856	{
2857	char *buf_comm;
2858	char *file_buf;
2859	char *buf;
2860	int len = 0;
2861	int pid;
2862	int i;
2863
2864	file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2865	if (!file_buf)
2866	return -ENOMEM;
2867
2868	buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2869	if (!buf_comm) {
2870	kfree(file_buf);
2871	return -ENOMEM;
2872	}
2873
2874	buf = file_buf;
2875
2876	for (i = 0; i < SAVED_CMDLINES; i++) {
2877	int r;
2878
2879	pid = map_cmdline_to_pid[i];
2880	if (pid == -1 \|\| pid == NO_CMDLINE_MAP)
2881	continue;
2882
2883	trace_find_cmdline(pid, buf_comm);
2884	r = sprintf(buf, "%d %s\n", pid, buf_comm);
2885	buf += r;
2886	len += r;
2887	}
2888
2889	len = simple_read_from_buffer(ubuf, cnt, ppos,
2890	file_buf, len);
2891
2892	kfree(file_buf);
2893	kfree(buf_comm);
2894
2895	return len;
2896	}
2897
2898	static const struct file_operations tracing_saved_cmdlines_fops = {
2899	.open = tracing_open_generic,
2900	.read = tracing_saved_cmdlines_read,
2901	.llseek = generic_file_llseek,
2902	};
2903
2904	static ssize_t
2905	tracing_ctrl_read(struct file filp, char __user ubuf,
2906	size_t cnt, loff_t *ppos)
2907	{
2908	char buf[64];
2909	int r;
2910
2911	r = sprintf(buf, "%u\n", tracer_enabled);
2912	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2913	}
2914
2915	static ssize_t
2916	tracing_ctrl_write(struct file filp, const char __user ubuf,
2917	size_t cnt, loff_t *ppos)
2918	{
2919	struct trace_array *tr = filp->private_data;
2920	unsigned long val;
2921	int ret;
2922
2923	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2924	if (ret)
2925	return ret;
2926
2927	val = !!val;
2928
2929	mutex_lock(&trace_types_lock);
2930	if (tracer_enabled ^ val) {
2931
2932	/* Only need to warn if this is used to change the state */
2933	WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2934
2935	if (val) {
2936	tracer_enabled = 1;
2937	if (current_trace->start)
2938	current_trace->start(tr);
2939	tracing_start();
2940	} else {
2941	tracer_enabled = 0;
2942	tracing_stop();
2943	if (current_trace->stop)
2944	current_trace->stop(tr);
2945	}
2946	}
2947	mutex_unlock(&trace_types_lock);
2948
2949	*ppos += cnt;
2950
2951	return cnt;
2952	}
2953
2954	static ssize_t
2955	tracing_set_trace_read(struct file filp, char __user ubuf,
2956	size_t cnt, loff_t *ppos)
2957	{
2958	char buf[MAX_TRACER_SIZE+2];
2959	int r;
2960
2961	mutex_lock(&trace_types_lock);
2962	if (current_trace)
2963	r = sprintf(buf, "%s\n", current_trace->name);
2964	else
2965	r = sprintf(buf, "\n");
2966	mutex_unlock(&trace_types_lock);
2967
2968	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2969	}
2970
2971	int tracer_init(struct tracer t, struct trace_array tr)
2972	{
2973	tracing_reset_online_cpus(tr);
2974	return t->init(tr);
2975	}
2976
2977	static int __tracing_resize_ring_buffer(unsigned long size)
2978	{
2979	int ret;
2980
2981	/*
2982	* If kernel or user changes the size of the ring buffer
2983	* we use the size that was given, and we can forget about
2984	* expanding it later.
2985	*/
2986	ring_buffer_expanded = 1;
2987
2988	ret = ring_buffer_resize(global_trace.buffer, size);
2989	if (ret < 0)
2990	return ret;
2991
2992	if (!current_trace->use_max_tr)
2993	goto out;
2994
2995	ret = ring_buffer_resize(max_tr.buffer, size);
2996	if (ret < 0) {
2997	int r;
2998
2999	r = ring_buffer_resize(global_trace.buffer,
3000	global_trace.entries);
3001	if (r < 0) {
3002	/*
3003	* AARGH! We are left with different
3004	* size max buffer!!!!
3005	* The max buffer is our "snapshot" buffer.
3006	* When a tracer needs a snapshot (one of the
3007	* latency tracers), it swaps the max buffer
3008	* with the saved snap shot. We succeeded to
3009	* update the size of the main buffer, but failed to
3010	* update the size of the max buffer. But when we tried
3011	* to reset the main buffer to the original size, we
3012	* failed there too. This is very unlikely to
3013	* happen, but if it does, warn and kill all
3014	* tracing.
3015	*/
3016	WARN_ON(1);
3017	tracing_disabled = 1;
3018	}
3019	return ret;
3020	}
3021
3022	max_tr.entries = size;
3023	out:
3024	global_trace.entries = size;
3025
3026	return ret;
3027	}
3028
3029	static ssize_t tracing_resize_ring_buffer(unsigned long size)
3030	{
3031	int cpu, ret = size;
3032
3033	mutex_lock(&trace_types_lock);
3034
3035	tracing_stop();
3036
3037	/* disable all cpu buffers */
3038	for_each_tracing_cpu(cpu) {
3039	if (global_trace.data[cpu])
3040	atomic_inc(&global_trace.data[cpu]->disabled);
3041	if (max_tr.data[cpu])
3042	atomic_inc(&max_tr.data[cpu]->disabled);
3043	}
3044
3045	if (size != global_trace.entries)
3046	ret = __tracing_resize_ring_buffer(size);
3047
3048	if (ret < 0)
3049	ret = -ENOMEM;
3050
3051	for_each_tracing_cpu(cpu) {
3052	if (global_trace.data[cpu])
3053	atomic_dec(&global_trace.data[cpu]->disabled);
3054	if (max_tr.data[cpu])
3055	atomic_dec(&max_tr.data[cpu]->disabled);
3056	}
3057
3058	tracing_start();
3059	mutex_unlock(&trace_types_lock);
3060
3061	return ret;
3062	}
3063
3064
3065	/**
3066	* tracing_update_buffers - used by tracing facility to expand ring buffers
3067	*
3068	* To save on memory when the tracing is never used on a system with it
3069	* configured in. The ring buffers are set to a minimum size. But once
3070	* a user starts to use the tracing facility, then they need to grow
3071	* to their default size.
3072	*
3073	* This function is to be called when a tracer is about to be used.
3074	*/
3075	int tracing_update_buffers(void)
3076	{
3077	int ret = 0;
3078
3079	mutex_lock(&trace_types_lock);
3080	if (!ring_buffer_expanded)
3081	ret = __tracing_resize_ring_buffer(trace_buf_size);
3082	mutex_unlock(&trace_types_lock);
3083
3084	return ret;
3085	}
3086
3087	struct trace_option_dentry;
3088
3089	static struct trace_option_dentry *
3090	create_trace_option_files(struct tracer *tracer);
3091
3092	static void
3093	destroy_trace_option_files(struct trace_option_dentry *topts);
3094
3095	static int tracing_set_tracer(const char *buf)
3096	{
3097	static struct trace_option_dentry *topts;
3098	struct trace_array *tr = &global_trace;
3099	struct tracer *t;
3100	int ret = 0;
3101
3102	mutex_lock(&trace_types_lock);
3103
3104	if (!ring_buffer_expanded) {
3105	ret = __tracing_resize_ring_buffer(trace_buf_size);
3106	if (ret < 0)
3107	goto out;
3108	ret = 0;
3109	}
3110
3111	for (t = trace_types; t; t = t->next) {
3112	if (strcmp(t->name, buf) == 0)
3113	break;
3114	}
3115	if (!t) {
3116	ret = -EINVAL;
3117	goto out;
3118	}
3119	if (t == current_trace)
3120	goto out;
3121
3122	trace_branch_disable();
3123	if (current_trace && current_trace->reset)
3124	current_trace->reset(tr);
3125	if (current_trace && current_trace->use_max_tr) {
3126	/*
3127	* We don't free the ring buffer. instead, resize it because
3128	* The max_tr ring buffer has some state (e.g. ring->clock) and
3129	* we want preserve it.
3130	*/
3131	ring_buffer_resize(max_tr.buffer, 1);
3132	max_tr.entries = 1;
3133	}
3134	destroy_trace_option_files(topts);
3135
3136	current_trace = t;
3137
3138	topts = create_trace_option_files(current_trace);
3139	if (current_trace->use_max_tr) {
3140	ret = ring_buffer_resize(max_tr.buffer, global_trace.entries);
3141	if (ret < 0)
3142	goto out;
3143	max_tr.entries = global_trace.entries;
3144	}
3145
3146	if (t->init) {
3147	ret = tracer_init(t, tr);
3148	if (ret)
3149	goto out;
3150	}
3151
3152	trace_branch_enable(tr);
3153	out:
3154	mutex_unlock(&trace_types_lock);
3155
3156	return ret;
3157	}
3158
3159	static ssize_t
3160	tracing_set_trace_write(struct file filp, const char __user ubuf,
3161	size_t cnt, loff_t *ppos)
3162	{
3163	char buf[MAX_TRACER_SIZE+1];
3164	int i;
3165	size_t ret;
3166	int err;
3167
3168	ret = cnt;
3169
3170	if (cnt > MAX_TRACER_SIZE)
3171	cnt = MAX_TRACER_SIZE;
3172
3173	if (copy_from_user(&buf, ubuf, cnt))
3174	return -EFAULT;
3175
3176	buf[cnt] = 0;
3177
3178	/* strip ending whitespace. */
3179	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3180	buf[i] = 0;
3181
3182	err = tracing_set_tracer(buf);
3183	if (err)
3184	return err;
3185
3186	*ppos += ret;
3187
3188	return ret;
3189	}
3190
3191	static ssize_t
3192	tracing_max_lat_read(struct file filp, char __user ubuf,
3193	size_t cnt, loff_t *ppos)
3194	{
3195	unsigned long *ptr = filp->private_data;
3196	char buf[64];
3197	int r;
3198
3199	r = snprintf(buf, sizeof(buf), "%ld\n",
3200	ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(ptr));
3201	if (r > sizeof(buf))
3202	r = sizeof(buf);
3203	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3204	}
3205
3206	static ssize_t
3207	tracing_max_lat_write(struct file filp, const char __user ubuf,
3208	size_t cnt, loff_t *ppos)
3209	{
3210	unsigned long *ptr = filp->private_data;
3211	unsigned long val;
3212	int ret;
3213
3214	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3215	if (ret)
3216	return ret;
3217
3218	ptr = val 1000;
3219
3220	return cnt;
3221	}
3222
3223	static int tracing_open_pipe(struct inode inode, struct file filp)
3224	{
3225	long cpu_file = (long) inode->i_private;
3226	struct trace_iterator *iter;
3227	int ret = 0;
3228
3229	if (tracing_disabled)
3230	return -ENODEV;
3231
3232	mutex_lock(&trace_types_lock);
3233
3234	/* create a buffer to store the information to pass to userspace */
3235	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3236	if (!iter) {
3237	ret = -ENOMEM;
3238	goto out;
3239	}
3240
3241	/*
3242	* We make a copy of the current tracer to avoid concurrent
3243	* changes on it while we are reading.
3244	*/
3245	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3246	if (!iter->trace) {
3247	ret = -ENOMEM;
3248	goto fail;
3249	}
3250	if (current_trace)
3251	iter->trace = current_trace;
3252
3253	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3254	ret = -ENOMEM;
3255	goto fail;
3256	}
3257
3258	/* trace pipe does not show start of buffer */
3259	cpumask_setall(iter->started);
3260
3261	if (trace_flags & TRACE_ITER_LATENCY_FMT)
3262	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
3263
3264	iter->cpu_file = cpu_file;
3265	iter->tr = &global_trace;
3266	mutex_init(&iter->mutex);
3267	filp->private_data = iter;
3268
3269	if (iter->trace->pipe_open)
3270	iter->trace->pipe_open(iter);
3271
3272	nonseekable_open(inode, filp);
3273	out:
3274	mutex_unlock(&trace_types_lock);
3275	return ret;
3276
3277	fail:
3278	kfree(iter->trace);
3279	kfree(iter);
3280	mutex_unlock(&trace_types_lock);
3281	return ret;
3282	}
3283
3284	static int tracing_release_pipe(struct inode inode, struct file file)
3285	{
3286	struct trace_iterator *iter = file->private_data;
3287
3288	mutex_lock(&trace_types_lock);
3289
3290	if (iter->trace->pipe_close)
3291	iter->trace->pipe_close(iter);
3292
3293	mutex_unlock(&trace_types_lock);
3294
3295	free_cpumask_var(iter->started);
3296	mutex_destroy(&iter->mutex);
3297	kfree(iter->trace);
3298	kfree(iter);
3299
3300	return 0;
3301	}
3302
3303	static unsigned int
3304	tracing_poll_pipe(struct file filp, poll_table poll_table)
3305	{
3306	struct trace_iterator *iter = filp->private_data;
3307
3308	if (trace_flags & TRACE_ITER_BLOCK) {
3309	/*
3310	* Always select as readable when in blocking mode
3311	*/
3312	return POLLIN \| POLLRDNORM;
3313	} else {
3314	if (!trace_empty(iter))
3315	return POLLIN \| POLLRDNORM;
3316	poll_wait(filp, &trace_wait, poll_table);
3317	if (!trace_empty(iter))
3318	return POLLIN \| POLLRDNORM;
3319
3320	return 0;
3321	}
3322	}
3323
3324
3325	void default_wait_pipe(struct trace_iterator *iter)
3326	{
3327	DEFINE_WAIT(wait);
3328
3329	prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3330
3331	if (trace_empty(iter))
3332	schedule();
3333
3334	finish_wait(&trace_wait, &wait);
3335	}
3336
3337	/*
3338	* This is a make-shift waitqueue.
3339	* A tracer might use this callback on some rare cases:
3340	*
3341	* 1) the current tracer might hold the runqueue lock when it wakes up
3342	* a reader, hence a deadlock (sched, function, and function graph tracers)
3343	* 2) the function tracers, trace all functions, we don't want
3344	* the overhead of calling wake_up and friends
3345	* (and tracing them too)
3346	*
3347	* Anyway, this is really very primitive wakeup.
3348	*/
3349	void poll_wait_pipe(struct trace_iterator *iter)
3350	{
3351	set_current_state(TASK_INTERRUPTIBLE);
3352	/* sleep for 100 msecs, and try again. */
3353	schedule_timeout(HZ / 10);
3354	}
3355
3356	/* Must be called with trace_types_lock mutex held. */
3357	static int tracing_wait_pipe(struct file *filp)
3358	{
3359	struct trace_iterator *iter = filp->private_data;
3360
3361	while (trace_empty(iter)) {
3362
3363	if ((filp->f_flags & O_NONBLOCK)) {
3364	return -EAGAIN;
3365	}
3366
3367	mutex_unlock(&iter->mutex);
3368
3369	iter->trace->wait_pipe(iter);
3370
3371	mutex_lock(&iter->mutex);
3372
3373	if (signal_pending(current))
3374	return -EINTR;
3375
3376	/*
3377	* We block until we read something and tracing is disabled.
3378	* We still block if tracing is disabled, but we have never
3379	* read anything. This allows a user to cat this file, and
3380	* then enable tracing. But after we have read something,
3381	* we give an EOF when tracing is again disabled.
3382	*
3383	* iter->pos will be 0 if we haven't read anything.
3384	*/
3385	if (!tracer_enabled && iter->pos)
3386	break;
3387	}
3388
3389	return 1;
3390	}
3391
3392	/*
3393	* Consumer reader.
3394	*/
3395	static ssize_t
3396	tracing_read_pipe(struct file filp, char __user ubuf,
3397	size_t cnt, loff_t *ppos)
3398	{
3399	struct trace_iterator *iter = filp->private_data;
3400	static struct tracer *old_tracer;
3401	ssize_t sret;
3402
3403	/* return any leftover data */
3404	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3405	if (sret != -EBUSY)
3406	return sret;
3407
3408	trace_seq_init(&iter->seq);
3409
3410	/* copy the tracer to avoid using a global lock all around */
3411	mutex_lock(&trace_types_lock);
3412	if (unlikely(old_tracer != current_trace && current_trace)) {
3413	old_tracer = current_trace;
3414	iter->trace = current_trace;
3415	}
3416	mutex_unlock(&trace_types_lock);
3417
3418	/*
3419	* Avoid more than one consumer on a single file descriptor
3420	* This is just a matter of traces coherency, the ring buffer itself
3421	* is protected.
3422	*/
3423	mutex_lock(&iter->mutex);
3424	if (iter->trace->read) {
3425	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3426	if (sret)
3427	goto out;
3428	}
3429
3430	waitagain:
3431	sret = tracing_wait_pipe(filp);
3432	if (sret <= 0)
3433	goto out;
3434
3435	/* stop when tracing is finished */
3436	if (trace_empty(iter)) {
3437	sret = 0;
3438	goto out;
3439	}
3440
3441	if (cnt >= PAGE_SIZE)
3442	cnt = PAGE_SIZE - 1;
3443
3444	/* reset all but tr, trace, and overruns */
3445	memset(&iter->seq, 0,
3446	sizeof(struct trace_iterator) -
3447	offsetof(struct trace_iterator, seq));
3448	iter->pos = -1;
3449
3450	trace_event_read_lock();
3451	trace_access_lock(iter->cpu_file);
3452	while (trace_find_next_entry_inc(iter) != NULL) {
3453	enum print_line_t ret;
3454	int len = iter->seq.len;
3455
3456	ret = print_trace_line(iter);
3457	if (ret == TRACE_TYPE_PARTIAL_LINE) {
3458	/* don't print partial lines */
3459	iter->seq.len = len;
3460	break;
3461	}
3462	if (ret != TRACE_TYPE_NO_CONSUME)
3463	trace_consume(iter);
3464
3465	if (iter->seq.len >= cnt)
3466	break;
3467
3468	/*
3469	* Setting the full flag means we reached the trace_seq buffer
3470	* size and we should leave by partial output condition above.
3471	* One of the trace_seq_* functions is not used properly.
3472	*/
3473	WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3474	iter->ent->type);
3475	}
3476	trace_access_unlock(iter->cpu_file);
3477	trace_event_read_unlock();
3478
3479	/* Now copy what we have to the user */
3480	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3481	if (iter->seq.readpos >= iter->seq.len)
3482	trace_seq_init(&iter->seq);
3483
3484	/*
3485	* If there was nothing to send to user, in spite of consuming trace
3486	* entries, go back to wait for more entries.
3487	*/
3488	if (sret == -EBUSY)
3489	goto waitagain;
3490
3491	out:
3492	mutex_unlock(&iter->mutex);
3493
3494	return sret;
3495	}
3496
3497	static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3498	struct pipe_buffer *buf)
3499	{
3500	__free_page(buf->page);
3501	}
3502
3503	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3504	unsigned int idx)
3505	{
3506	__free_page(spd->pages[idx]);
3507	}
3508
3509	static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3510	.can_merge = 0,
3511	.map = generic_pipe_buf_map,
3512	.unmap = generic_pipe_buf_unmap,
3513	.confirm = generic_pipe_buf_confirm,
3514	.release = tracing_pipe_buf_release,
3515	.steal = generic_pipe_buf_steal,
3516	.get = generic_pipe_buf_get,
3517	};
3518
3519	static size_t
3520	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3521	{
3522	size_t count;
3523	int ret;
3524
3525	/* Seq buffer is page-sized, exactly what we need. */
3526	for (;;) {
3527	count = iter->seq.len;
3528	ret = print_trace_line(iter);
3529	count = iter->seq.len - count;
3530	if (rem < count) {
3531	rem = 0;
3532	iter->seq.len -= count;
3533	break;
3534	}
3535	if (ret == TRACE_TYPE_PARTIAL_LINE) {
3536	iter->seq.len -= count;
3537	break;
3538	}
3539
3540	if (ret != TRACE_TYPE_NO_CONSUME)
3541	trace_consume(iter);
3542	rem -= count;
3543	if (!trace_find_next_entry_inc(iter)) {
3544	rem = 0;
3545	iter->ent = NULL;
3546	break;
3547	}
3548	}
3549
3550	return rem;
3551	}
3552
3553	static ssize_t tracing_splice_read_pipe(struct file *filp,
3554	loff_t *ppos,
3555	struct pipe_inode_info *pipe,
3556	size_t len,
3557	unsigned int flags)
3558	{
3559	struct page *pages_def[PIPE_DEF_BUFFERS];
3560	struct partial_page partial_def[PIPE_DEF_BUFFERS];
3561	struct trace_iterator *iter = filp->private_data;
3562	struct splice_pipe_desc spd = {
3563	.pages = pages_def,
3564	.partial = partial_def,
3565	.nr_pages = 0, /* This gets updated below. */
3566	.flags = flags,
3567	.ops = &tracing_pipe_buf_ops,
3568	.spd_release = tracing_spd_release_pipe,
3569	};
3570	static struct tracer *old_tracer;
3571	ssize_t ret;
3572	size_t rem;
3573	unsigned int i;
3574
3575	if (splice_grow_spd(pipe, &spd))
3576	return -ENOMEM;
3577
3578	/* copy the tracer to avoid using a global lock all around */
3579	mutex_lock(&trace_types_lock);
3580	if (unlikely(old_tracer != current_trace && current_trace)) {
3581	old_tracer = current_trace;
3582	iter->trace = current_trace;
3583	}
3584	mutex_unlock(&trace_types_lock);
3585
3586	mutex_lock(&iter->mutex);
3587
3588	if (iter->trace->splice_read) {
3589	ret = iter->trace->splice_read(iter, filp,
3590	ppos, pipe, len, flags);
3591	if (ret)
3592	goto out_err;
3593	}
3594
3595	ret = tracing_wait_pipe(filp);
3596	if (ret <= 0)
3597	goto out_err;
3598
3599	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3600	ret = -EFAULT;
3601	goto out_err;
3602	}
3603
3604	trace_event_read_lock();
3605	trace_access_lock(iter->cpu_file);
3606
3607	/* Fill as many pages as possible. */
3608	for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3609	spd.pages[i] = alloc_page(GFP_KERNEL);
3610	if (!spd.pages[i])
3611	break;
3612
3613	rem = tracing_fill_pipe_page(rem, iter);
3614
3615	/* Copy the data into the page, so we can start over. */
3616	ret = trace_seq_to_buffer(&iter->seq,
3617	page_address(spd.pages[i]),
3618	iter->seq.len);
3619	if (ret < 0) {
3620	__free_page(spd.pages[i]);
3621	break;
3622	}
3623	spd.partial[i].offset = 0;
3624	spd.partial[i].len = iter->seq.len;
3625
3626	trace_seq_init(&iter->seq);
3627	}
3628
3629	trace_access_unlock(iter->cpu_file);
3630	trace_event_read_unlock();
3631	mutex_unlock(&iter->mutex);
3632
3633	spd.nr_pages = i;
3634
3635	ret = splice_to_pipe(pipe, &spd);
3636	out:
3637	splice_shrink_spd(pipe, &spd);
3638	return ret;
3639
3640	out_err:
3641	mutex_unlock(&iter->mutex);
3642	goto out;
3643	}
3644
3645	static ssize_t
3646	tracing_entries_read(struct file filp, char __user ubuf,
3647	size_t cnt, loff_t *ppos)
3648	{
3649	struct trace_array *tr = filp->private_data;
3650	char buf[96];
3651	int r;
3652
3653	mutex_lock(&trace_types_lock);
3654	if (!ring_buffer_expanded)
3655	r = sprintf(buf, "%lu (expanded: %lu)\n",
3656	tr->entries >> 10,
3657	trace_buf_size >> 10);
3658	else
3659	r = sprintf(buf, "%lu\n", tr->entries >> 10);
3660	mutex_unlock(&trace_types_lock);
3661
3662	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3663	}
3664
3665	static ssize_t
3666	tracing_entries_write(struct file filp, const char __user ubuf,
3667	size_t cnt, loff_t *ppos)
3668	{
3669	unsigned long val;
3670	int ret;
3671
3672	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3673	if (ret)
3674	return ret;
3675
3676	/* must have at least 1 entry */
3677	if (!val)
3678	return -EINVAL;
3679
3680	/* value is in KB */
3681	val <<= 10;
3682
3683	ret = tracing_resize_ring_buffer(val);
3684	if (ret < 0)
3685	return ret;
3686
3687	*ppos += cnt;
3688
3689	return cnt;
3690	}
3691
3692	static ssize_t
3693	tracing_total_entries_read(struct file filp, char __user ubuf,
3694	size_t cnt, loff_t *ppos)
3695	{
3696	struct trace_array *tr = filp->private_data;
3697	char buf[64];
3698	int r, cpu;
3699	unsigned long size = 0, expanded_size = 0;
3700
3701	mutex_lock(&trace_types_lock);
3702	for_each_tracing_cpu(cpu) {
3703	size += tr->entries >> 10;
3704	if (!ring_buffer_expanded)
3705	expanded_size += trace_buf_size >> 10;
3706	}
3707	if (ring_buffer_expanded)
3708	r = sprintf(buf, "%lu\n", size);
3709	else
3710	r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3711	mutex_unlock(&trace_types_lock);
3712
3713	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3714	}
3715
3716	static ssize_t
3717	tracing_free_buffer_write(struct file filp, const char __user ubuf,
3718	size_t cnt, loff_t *ppos)
3719	{
3720	/*
3721	* There is no need to read what the user has written, this function
3722	* is just to make sure that there is no error when "echo" is used
3723	*/
3724
3725	*ppos += cnt;
3726
3727	return cnt;
3728	}
3729
3730	static int
3731	tracing_free_buffer_release(struct inode inode, struct file filp)
3732	{
3733	/* disable tracing ? */
3734	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3735	tracing_off();
3736	/* resize the ring buffer to 0 */
3737	tracing_resize_ring_buffer(0);
3738
3739	return 0;
3740	}
3741
3742	static ssize_t
3743	tracing_mark_write(struct file filp, const char __user ubuf,
3744	size_t cnt, loff_t *fpos)
3745	{
3746	unsigned long addr = (unsigned long)ubuf;
3747	struct ring_buffer_event *event;
3748	struct ring_buffer *buffer;
3749	struct print_entry *entry;
3750	unsigned long irq_flags;
3751	struct page *pages[2];
3752	int nr_pages = 1;
3753	ssize_t written;
3754	void *page1;
3755	void *page2;
3756	int offset;
3757	int size;
3758	int len;
3759	int ret;
3760
3761	if (tracing_disabled)
3762	return -EINVAL;
3763
3764	if (cnt > TRACE_BUF_SIZE)
3765	cnt = TRACE_BUF_SIZE;
3766
3767	/*
3768	* Userspace is injecting traces into the kernel trace buffer.
3769	* We want to be as non intrusive as possible.
3770	* To do so, we do not want to allocate any special buffers
3771	* or take any locks, but instead write the userspace data
3772	* straight into the ring buffer.
3773	*
3774	* First we need to pin the userspace buffer into memory,
3775	* which, most likely it is, because it just referenced it.
3776	* But there's no guarantee that it is. By using get_user_pages_fast()
3777	* and kmap_atomic/kunmap_atomic() we can get access to the
3778	* pages directly. We then write the data directly into the
3779	* ring buffer.
3780	*/
3781	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
3782
3783	/* check if we cross pages */
3784	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
3785	nr_pages = 2;
3786
3787	offset = addr & (PAGE_SIZE - 1);
3788	addr &= PAGE_MASK;
3789
3790	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
3791	if (ret < nr_pages) {
3792	while (--ret >= 0)
3793	put_page(pages[ret]);
3794	written = -EFAULT;
3795	goto out;
3796	}
3797
3798	page1 = kmap_atomic(pages[0]);
3799	if (nr_pages == 2)
3800	page2 = kmap_atomic(pages[1]);
3801
3802	local_save_flags(irq_flags);
3803	size = sizeof(entry) + cnt + 2; / possible \n added */
3804	buffer = global_trace.buffer;
3805	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3806	irq_flags, preempt_count());
3807	if (!event) {
3808	/* Ring buffer disabled, return as if not open for write */
3809	written = -EBADF;
3810	goto out_unlock;
3811	}
3812
3813	entry = ring_buffer_event_data(event);
3814	entry->ip = _THIS_IP_;
3815
3816	if (nr_pages == 2) {
3817	len = PAGE_SIZE - offset;
3818	memcpy(&entry->buf, page1 + offset, len);
3819	memcpy(&entry->buf[len], page2, cnt - len);
3820	} else
3821	memcpy(&entry->buf, page1 + offset, cnt);
3822
3823	if (entry->buf[cnt - 1] != '\n') {
3824	entry->buf[cnt] = '\n';
3825	entry->buf[cnt + 1] = '\0';
3826	} else
3827	entry->buf[cnt] = '\0';
3828
3829	ring_buffer_unlock_commit(buffer, event);
3830
3831	written = cnt;
3832
3833	*fpos += written;
3834
3835	out_unlock:
3836	if (nr_pages == 2)
3837	kunmap_atomic(page2);
3838	kunmap_atomic(page1);
3839	while (nr_pages > 0)
3840	put_page(pages[--nr_pages]);
3841	out:
3842	return written;
3843	}
3844
3845	static int tracing_clock_show(struct seq_file m, void v)
3846	{
3847	int i;
3848
3849	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3850	seq_printf(m,
3851	"%s%s%s%s", i ? " " : "",
3852	i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3853	i == trace_clock_id ? "]" : "");
3854	seq_putc(m, '\n');
3855
3856	return 0;
3857	}
3858
3859	static ssize_t tracing_clock_write(struct file filp, const char __user ubuf,
3860	size_t cnt, loff_t *fpos)
3861	{
3862	char buf[64];
3863	const char *clockstr;
3864	int i;
3865
3866	if (cnt >= sizeof(buf))
3867	return -EINVAL;
3868
3869	if (copy_from_user(&buf, ubuf, cnt))
3870	return -EFAULT;
3871
3872	buf[cnt] = 0;
3873
3874	clockstr = strstrip(buf);
3875
3876	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3877	if (strcmp(trace_clocks[i].name, clockstr) == 0)
3878	break;
3879	}
3880	if (i == ARRAY_SIZE(trace_clocks))
3881	return -EINVAL;
3882
3883	trace_clock_id = i;
3884
3885	mutex_lock(&trace_types_lock);
3886
3887	ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3888	if (max_tr.buffer)
3889	ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3890
3891	mutex_unlock(&trace_types_lock);
3892
3893	*fpos += cnt;
3894
3895	return cnt;
3896	}
3897
3898	static int tracing_clock_open(struct inode inode, struct file file)
3899	{
3900	if (tracing_disabled)
3901	return -ENODEV;
3902	return single_open(file, tracing_clock_show, NULL);
3903	}
3904
3905	static const struct file_operations tracing_max_lat_fops = {
3906	.open = tracing_open_generic,
3907	.read = tracing_max_lat_read,
3908	.write = tracing_max_lat_write,
3909	.llseek = generic_file_llseek,
3910	};
3911
3912	static const struct file_operations tracing_ctrl_fops = {
3913	.open = tracing_open_generic,
3914	.read = tracing_ctrl_read,
3915	.write = tracing_ctrl_write,
3916	.llseek = generic_file_llseek,
3917	};
3918
3919	static const struct file_operations set_tracer_fops = {
3920	.open = tracing_open_generic,
3921	.read = tracing_set_trace_read,
3922	.write = tracing_set_trace_write,
3923	.llseek = generic_file_llseek,
3924	};
3925
3926	static const struct file_operations tracing_pipe_fops = {
3927	.open = tracing_open_pipe,
3928	.poll = tracing_poll_pipe,
3929	.read = tracing_read_pipe,
3930	.splice_read = tracing_splice_read_pipe,
3931	.release = tracing_release_pipe,
3932	.llseek = no_llseek,
3933	};
3934
3935	static const struct file_operations tracing_entries_fops = {
3936	.open = tracing_open_generic,
3937	.read = tracing_entries_read,
3938	.write = tracing_entries_write,
3939	.llseek = generic_file_llseek,
3940	};
3941
3942	static const struct file_operations tracing_total_entries_fops = {
3943	.open = tracing_open_generic,
3944	.read = tracing_total_entries_read,
3945	.llseek = generic_file_llseek,
3946	};
3947
3948	static const struct file_operations tracing_free_buffer_fops = {
3949	.write = tracing_free_buffer_write,
3950	.release = tracing_free_buffer_release,
3951	};
3952
3953	static const struct file_operations tracing_mark_fops = {
3954	.open = tracing_open_generic,
3955	.write = tracing_mark_write,
3956	.llseek = generic_file_llseek,
3957	};
3958
3959	static const struct file_operations trace_clock_fops = {
3960	.open = tracing_clock_open,
3961	.read = seq_read,
3962	.llseek = seq_lseek,
3963	.release = single_release,
3964	.write = tracing_clock_write,
3965	};
3966
3967	struct ftrace_buffer_info {
3968	struct trace_array *tr;
3969	void *spare;
3970	int cpu;
3971	unsigned int read;
3972	};
3973
3974	static int tracing_buffers_open(struct inode inode, struct file filp)
3975	{
3976	int cpu = (int)(long)inode->i_private;
3977	struct ftrace_buffer_info *info;
3978
3979	if (tracing_disabled)
3980	return -ENODEV;
3981
3982	info = kzalloc(sizeof(*info), GFP_KERNEL);
3983	if (!info)
3984	return -ENOMEM;
3985
3986	info->tr = &global_trace;
3987	info->cpu = cpu;
3988	info->spare = NULL;
3989	/* Force reading ring buffer for first read */
3990	info->read = (unsigned int)-1;
3991
3992	filp->private_data = info;
3993
3994	return nonseekable_open(inode, filp);
3995	}
3996
3997	static ssize_t
3998	tracing_buffers_read(struct file filp, char __user ubuf,
3999	size_t count, loff_t *ppos)
4000	{
4001	struct ftrace_buffer_info *info = filp->private_data;
4002	ssize_t ret;
4003	size_t size;
4004
4005	if (!count)
4006	return 0;
4007
4008	if (!info->spare)
4009	info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4010	if (!info->spare)
4011	return -ENOMEM;
4012
4013	/* Do we have previous read data to read? */
4014	if (info->read < PAGE_SIZE)
4015	goto read;
4016
4017	trace_access_lock(info->cpu);
4018	ret = ring_buffer_read_page(info->tr->buffer,
4019	&info->spare,
4020	count,
4021	info->cpu, 0);
4022	trace_access_unlock(info->cpu);
4023	if (ret < 0)
4024	return 0;
4025
4026	info->read = 0;
4027
4028	read:
4029	size = PAGE_SIZE - info->read;
4030	if (size > count)
4031	size = count;
4032
4033	ret = copy_to_user(ubuf, info->spare + info->read, size);
4034	if (ret == size)
4035	return -EFAULT;
4036	size -= ret;
4037
4038	*ppos += size;
4039	info->read += size;
4040
4041	return size;
4042	}
4043
4044	static int tracing_buffers_release(struct inode inode, struct file file)
4045	{
4046	struct ftrace_buffer_info *info = file->private_data;
4047
4048	if (info->spare)
4049	ring_buffer_free_read_page(info->tr->buffer, info->spare);
4050	kfree(info);
4051
4052	return 0;
4053	}
4054
4055	struct buffer_ref {
4056	struct ring_buffer *buffer;
4057	void *page;
4058	int ref;
4059	};
4060
4061	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4062	struct pipe_buffer *buf)
4063	{
4064	struct buffer_ref ref = (struct buffer_ref )buf->private;
4065
4066	if (--ref->ref)
4067	return;
4068
4069	ring_buffer_free_read_page(ref->buffer, ref->page);
4070	kfree(ref);
4071	buf->private = 0;
4072	}
4073
4074	static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
4075	struct pipe_buffer *buf)
4076	{
4077	return 1;
4078	}
4079
4080	static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4081	struct pipe_buffer *buf)
4082	{
4083	struct buffer_ref ref = (struct buffer_ref )buf->private;
4084
4085	ref->ref++;
4086	}
4087
4088	/* Pipe buffer operations for a buffer. */
4089	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4090	.can_merge = 0,
4091	.map = generic_pipe_buf_map,
4092	.unmap = generic_pipe_buf_unmap,
4093	.confirm = generic_pipe_buf_confirm,
4094	.release = buffer_pipe_buf_release,
4095	.steal = buffer_pipe_buf_steal,
4096	.get = buffer_pipe_buf_get,
4097	};
4098
4099	/*
4100	* Callback from splice_to_pipe(), if we need to release some pages
4101	* at the end of the spd in case we error'ed out in filling the pipe.
4102	*/
4103	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4104	{
4105	struct buffer_ref *ref =
4106	(struct buffer_ref *)spd->partial[i].private;
4107
4108	if (--ref->ref)
4109	return;
4110
4111	ring_buffer_free_read_page(ref->buffer, ref->page);
4112	kfree(ref);
4113	spd->partial[i].private = 0;
4114	}
4115
4116	static ssize_t
4117	tracing_buffers_splice_read(struct file file, loff_t ppos,
4118	struct pipe_inode_info *pipe, size_t len,
4119	unsigned int flags)
4120	{
4121	struct ftrace_buffer_info *info = file->private_data;
4122	struct partial_page partial_def[PIPE_DEF_BUFFERS];
4123	struct page *pages_def[PIPE_DEF_BUFFERS];
4124	struct splice_pipe_desc spd = {
4125	.pages = pages_def,
4126	.partial = partial_def,
4127	.flags = flags,
4128	.ops = &buffer_pipe_buf_ops,
4129	.spd_release = buffer_spd_release,
4130	};
4131	struct buffer_ref *ref;
4132	int entries, size, i;
4133	size_t ret;
4134
4135	if (splice_grow_spd(pipe, &spd))
4136	return -ENOMEM;
4137
4138	if (*ppos & (PAGE_SIZE - 1)) {
4139	WARN_ONCE(1, "Ftrace: previous read must page-align\n");
4140	ret = -EINVAL;
4141	goto out;
4142	}
4143
4144	if (len & (PAGE_SIZE - 1)) {
4145	WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
4146	if (len < PAGE_SIZE) {
4147	ret = -EINVAL;
4148	goto out;
4149	}
4150	len &= PAGE_MASK;
4151	}
4152
4153	trace_access_lock(info->cpu);
4154	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4155
4156	for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4157	struct page *page;
4158	int r;
4159
4160	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4161	if (!ref)
4162	break;
4163
4164	ref->ref = 1;
4165	ref->buffer = info->tr->buffer;
4166	ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4167	if (!ref->page) {
4168	kfree(ref);
4169	break;
4170	}
4171
4172	r = ring_buffer_read_page(ref->buffer, &ref->page,
4173	len, info->cpu, 1);
4174	if (r < 0) {
4175	ring_buffer_free_read_page(ref->buffer, ref->page);
4176	kfree(ref);
4177	break;
4178	}
4179
4180	/*
4181	* zero out any left over data, this is going to
4182	* user land.
4183	*/
4184	size = ring_buffer_page_len(ref->page);
4185	if (size < PAGE_SIZE)
4186	memset(ref->page + size, 0, PAGE_SIZE - size);
4187
4188	page = virt_to_page(ref->page);
4189
4190	spd.pages[i] = page;
4191	spd.partial[i].len = PAGE_SIZE;
4192	spd.partial[i].offset = 0;
4193	spd.partial[i].private = (unsigned long)ref;
4194	spd.nr_pages++;
4195	*ppos += PAGE_SIZE;
4196
4197	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4198	}
4199
4200	trace_access_unlock(info->cpu);
4201	spd.nr_pages = i;
4202
4203	/* did we read anything? */
4204	if (!spd.nr_pages) {
4205	if (flags & SPLICE_F_NONBLOCK)
4206	ret = -EAGAIN;
4207	else
4208	ret = 0;
4209	/* TODO: block */
4210	goto out;
4211	}
4212
4213	ret = splice_to_pipe(pipe, &spd);
4214	splice_shrink_spd(pipe, &spd);
4215	out:
4216	return ret;
4217	}
4218
4219	static const struct file_operations tracing_buffers_fops = {
4220	.open = tracing_buffers_open,
4221	.read = tracing_buffers_read,
4222	.release = tracing_buffers_release,
4223	.splice_read = tracing_buffers_splice_read,
4224	.llseek = no_llseek,
4225	};
4226
4227	static ssize_t
4228	tracing_stats_read(struct file filp, char __user ubuf,
4229	size_t count, loff_t *ppos)
4230	{
4231	unsigned long cpu = (unsigned long)filp->private_data;
4232	struct trace_array *tr = &global_trace;
4233	struct trace_seq *s;
4234	unsigned long cnt;
4235	unsigned long long t;
4236	unsigned long usec_rem;
4237
4238	s = kmalloc(sizeof(*s), GFP_KERNEL);
4239	if (!s)
4240	return -ENOMEM;
4241
4242	trace_seq_init(s);
4243
4244	cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4245	trace_seq_printf(s, "entries: %ld\n", cnt);
4246
4247	cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4248	trace_seq_printf(s, "overrun: %ld\n", cnt);
4249
4250	cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4251	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4252
4253	cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4254	trace_seq_printf(s, "bytes: %ld\n", cnt);
4255
4256	t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4257	usec_rem = do_div(t, USEC_PER_SEC);
4258	trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);
4259
4260	t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4261	usec_rem = do_div(t, USEC_PER_SEC);
4262	trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4263
4264	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4265
4266	kfree(s);
4267
4268	return count;
4269	}
4270
4271	static const struct file_operations tracing_stats_fops = {
4272	.open = tracing_open_generic,
4273	.read = tracing_stats_read,
4274	.llseek = generic_file_llseek,
4275	};
4276
4277	#ifdef CONFIG_DYNAMIC_FTRACE
4278
4279	int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4280	{
4281	return 0;
4282	}
4283
4284	static ssize_t
4285	tracing_read_dyn_info(struct file filp, char __user ubuf,
4286	size_t cnt, loff_t *ppos)
4287	{
4288	static char ftrace_dyn_info_buffer[1024];
4289	static DEFINE_MUTEX(dyn_info_mutex);
4290	unsigned long *p = filp->private_data;
4291	char *buf = ftrace_dyn_info_buffer;
4292	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4293	int r;
4294
4295	mutex_lock(&dyn_info_mutex);
4296	r = sprintf(buf, "%ld ", *p);
4297
4298	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
4299	buf[r++] = '\n';
4300
4301	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4302
4303	mutex_unlock(&dyn_info_mutex);
4304
4305	return r;
4306	}
4307
4308	static const struct file_operations tracing_dyn_info_fops = {
4309	.open = tracing_open_generic,
4310	.read = tracing_read_dyn_info,
4311	.llseek = generic_file_llseek,
4312	};
4313	#endif
4314
4315	static struct dentry *d_tracer;
4316
4317	struct dentry *tracing_init_dentry(void)
4318	{
4319	static int once;
4320
4321	if (d_tracer)
4322	return d_tracer;
4323
4324	if (!debugfs_initialized())
4325	return NULL;
4326
4327	d_tracer = debugfs_create_dir("tracing", NULL);
4328
4329	if (!d_tracer && !once) {
4330	once = 1;
4331	pr_warning("Could not create debugfs directory 'tracing'\n");
4332	return NULL;
4333	}
4334
4335	return d_tracer;
4336	}
4337
4338	static struct dentry *d_percpu;
4339
4340	struct dentry *tracing_dentry_percpu(void)
4341	{
4342	static int once;
4343	struct dentry *d_tracer;
4344
4345	if (d_percpu)
4346	return d_percpu;
4347
4348	d_tracer = tracing_init_dentry();
4349
4350	if (!d_tracer)
4351	return NULL;
4352
4353	d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4354
4355	if (!d_percpu && !once) {
4356	once = 1;
4357	pr_warning("Could not create debugfs directory 'per_cpu'\n");
4358	return NULL;
4359	}
4360
4361	return d_percpu;
4362	}
4363
4364	static void tracing_init_debugfs_percpu(long cpu)
4365	{
4366	struct dentry *d_percpu = tracing_dentry_percpu();
4367	struct dentry *d_cpu;
4368	char cpu_dir[30]; /* 30 characters should be more than enough */
4369
4370	snprintf(cpu_dir, 30, "cpu%ld", cpu);
4371	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4372	if (!d_cpu) {
4373	pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4374	return;
4375	}
4376
4377	/* per cpu trace_pipe */
4378	trace_create_file("trace_pipe", 0444, d_cpu,
4379	(void *) cpu, &tracing_pipe_fops);
4380
4381	/* per cpu trace */
4382	trace_create_file("trace", 0644, d_cpu,
4383	(void *) cpu, &tracing_fops);
4384
4385	trace_create_file("trace_pipe_raw", 0444, d_cpu,
4386	(void *) cpu, &tracing_buffers_fops);
4387
4388	trace_create_file("stats", 0444, d_cpu,
4389	(void *) cpu, &tracing_stats_fops);
4390	}
4391
4392	#ifdef CONFIG_FTRACE_SELFTEST
4393	/* Let selftest have access to static functions in this file */
4394	#include "trace_selftest.c"
4395	#endif
4396
4397	struct trace_option_dentry {
4398	struct tracer_opt *opt;
4399	struct tracer_flags *flags;
4400	struct dentry *entry;
4401	};
4402
4403	static ssize_t
4404	trace_options_read(struct file filp, char __user ubuf, size_t cnt,
4405	loff_t *ppos)
4406	{
4407	struct trace_option_dentry *topt = filp->private_data;
4408	char *buf;
4409
4410	if (topt->flags->val & topt->opt->bit)
4411	buf = "1\n";
4412	else
4413	buf = "0\n";
4414
4415	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4416	}
4417
4418	static ssize_t
4419	trace_options_write(struct file filp, const char __user ubuf, size_t cnt,
4420	loff_t *ppos)
4421	{
4422	struct trace_option_dentry *topt = filp->private_data;
4423	unsigned long val;
4424	int ret;
4425
4426	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4427	if (ret)
4428	return ret;
4429
4430	if (val != 0 && val != 1)
4431	return -EINVAL;
4432
4433	if (!!(topt->flags->val & topt->opt->bit) != val) {
4434	mutex_lock(&trace_types_lock);
4435	ret = __set_tracer_option(current_trace, topt->flags,
4436	topt->opt, !val);
4437	mutex_unlock(&trace_types_lock);
4438	if (ret)
4439	return ret;
4440	}
4441
4442	*ppos += cnt;
4443
4444	return cnt;
4445	}
4446
4447
4448	static const struct file_operations trace_options_fops = {
4449	.open = tracing_open_generic,
4450	.read = trace_options_read,
4451	.write = trace_options_write,
4452	.llseek = generic_file_llseek,
4453	};
4454
4455	static ssize_t
4456	trace_options_core_read(struct file filp, char __user ubuf, size_t cnt,
4457	loff_t *ppos)
4458	{
4459	long index = (long)filp->private_data;
4460	char *buf;
4461
4462	if (trace_flags & (1 << index))
4463	buf = "1\n";
4464	else
4465	buf = "0\n";
4466
4467	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4468	}
4469
4470	static ssize_t
4471	trace_options_core_write(struct file filp, const char __user ubuf, size_t cnt,
4472	loff_t *ppos)
4473	{
4474	long index = (long)filp->private_data;
4475	unsigned long val;
4476	int ret;
4477
4478	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4479	if (ret)
4480	return ret;
4481
4482	if (val != 0 && val != 1)
4483	return -EINVAL;
4484	set_tracer_flags(1 << index, val);
4485
4486	*ppos += cnt;
4487
4488	return cnt;
4489	}
4490
4491	static const struct file_operations trace_options_core_fops = {
4492	.open = tracing_open_generic,
4493	.read = trace_options_core_read,
4494	.write = trace_options_core_write,
4495	.llseek = generic_file_llseek,
4496	};
4497
4498	struct dentry trace_create_file(const char name,
4499	umode_t mode,
4500	struct dentry *parent,
4501	void *data,
4502	const struct file_operations *fops)
4503	{
4504	struct dentry *ret;
4505
4506	ret = debugfs_create_file(name, mode, parent, data, fops);
4507	if (!ret)
4508	pr_warning("Could not create debugfs '%s' entry\n", name);
4509
4510	return ret;
4511	}
4512
4513
4514	static struct dentry *trace_options_init_dentry(void)
4515	{
4516	struct dentry *d_tracer;
4517	static struct dentry *t_options;
4518
4519	if (t_options)
4520	return t_options;
4521
4522	d_tracer = tracing_init_dentry();
4523	if (!d_tracer)
4524	return NULL;
4525
4526	t_options = debugfs_create_dir("options", d_tracer);
4527	if (!t_options) {
4528	pr_warning("Could not create debugfs directory 'options'\n");
4529	return NULL;
4530	}
4531
4532	return t_options;
4533	}
4534
4535	static void
4536	create_trace_option_file(struct trace_option_dentry *topt,
4537	struct tracer_flags *flags,
4538	struct tracer_opt *opt)
4539	{
4540	struct dentry *t_options;
4541
4542	t_options = trace_options_init_dentry();
4543	if (!t_options)
4544	return;
4545
4546	topt->flags = flags;
4547	topt->opt = opt;
4548
4549	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4550	&trace_options_fops);
4551
4552	}
4553
4554	static struct trace_option_dentry *
4555	create_trace_option_files(struct tracer *tracer)
4556	{
4557	struct trace_option_dentry *topts;
4558	struct tracer_flags *flags;
4559	struct tracer_opt *opts;
4560	int cnt;
4561
4562	if (!tracer)
4563	return NULL;
4564
4565	flags = tracer->flags;
4566
4567	if (!flags \|\| !flags->opts)
4568	return NULL;
4569
4570	opts = flags->opts;
4571
4572	for (cnt = 0; opts[cnt].name; cnt++)
4573	;
4574
4575	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4576	if (!topts)
4577	return NULL;
4578
4579	for (cnt = 0; opts[cnt].name; cnt++)
4580	create_trace_option_file(&topts[cnt], flags,
4581	&opts[cnt]);
4582
4583	return topts;
4584	}
4585
4586	static void
4587	destroy_trace_option_files(struct trace_option_dentry *topts)
4588	{
4589	int cnt;
4590
4591	if (!topts)
4592	return;
4593
4594	for (cnt = 0; topts[cnt].opt; cnt++) {
4595	if (topts[cnt].entry)
4596	debugfs_remove(topts[cnt].entry);
4597	}
4598
4599	kfree(topts);
4600	}
4601
4602	static struct dentry *
4603	create_trace_option_core_file(const char *option, long index)
4604	{
4605	struct dentry *t_options;
4606
4607	t_options = trace_options_init_dentry();
4608	if (!t_options)
4609	return NULL;
4610
4611	return trace_create_file(option, 0644, t_options, (void *)index,
4612	&trace_options_core_fops);
4613	}
4614
4615	static __init void create_trace_options_dir(void)
4616	{
4617	struct dentry *t_options;
4618	int i;
4619
4620	t_options = trace_options_init_dentry();
4621	if (!t_options)
4622	return;
4623
4624	for (i = 0; trace_options[i]; i++)
4625	create_trace_option_core_file(trace_options[i], i);
4626	}
4627
4628	static ssize_t
4629	rb_simple_read(struct file filp, char __user ubuf,
4630	size_t cnt, loff_t *ppos)
4631	{
4632	struct ring_buffer *buffer = filp->private_data;
4633	char buf[64];
4634	int r;
4635
4636	if (buffer)
4637	r = ring_buffer_record_is_on(buffer);
4638	else
4639	r = 0;
4640
4641	r = sprintf(buf, "%d\n", r);
4642
4643	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4644	}
4645
4646	static ssize_t
4647	rb_simple_write(struct file filp, const char __user ubuf,
4648	size_t cnt, loff_t *ppos)
4649	{
4650	struct ring_buffer *buffer = filp->private_data;
4651	unsigned long val;
4652	int ret;
4653
4654	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4655	if (ret)
4656	return ret;
4657
4658	if (buffer) {
4659	if (val)
4660	ring_buffer_record_on(buffer);
4661	else
4662	ring_buffer_record_off(buffer);
4663	}
4664
4665	(*ppos)++;
4666
4667	return cnt;
4668	}
4669
4670	static const struct file_operations rb_simple_fops = {
4671	.open = tracing_open_generic,
4672	.read = rb_simple_read,
4673	.write = rb_simple_write,
4674	.llseek = default_llseek,
4675	};
4676
4677	static __init int tracer_init_debugfs(void)
4678	{
4679	struct dentry *d_tracer;
4680	int cpu;
4681
4682	trace_access_lock_init();
4683
4684	d_tracer = tracing_init_dentry();
4685
4686	trace_create_file("tracing_enabled", 0644, d_tracer,
4687	&global_trace, &tracing_ctrl_fops);
4688
4689	trace_create_file("trace_options", 0644, d_tracer,
4690	NULL, &tracing_iter_fops);
4691
4692	trace_create_file("tracing_cpumask", 0644, d_tracer,
4693	NULL, &tracing_cpumask_fops);
4694
4695	trace_create_file("trace", 0644, d_tracer,
4696	(void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4697
4698	trace_create_file("available_tracers", 0444, d_tracer,
4699	&global_trace, &show_traces_fops);
4700
4701	trace_create_file("current_tracer", 0644, d_tracer,
4702	&global_trace, &set_tracer_fops);
4703
4704	#ifdef CONFIG_TRACER_MAX_TRACE
4705	trace_create_file("tracing_max_latency", 0644, d_tracer,
4706	&tracing_max_latency, &tracing_max_lat_fops);
4707	#endif
4708
4709	trace_create_file("tracing_thresh", 0644, d_tracer,
4710	&tracing_thresh, &tracing_max_lat_fops);
4711
4712	trace_create_file("README", 0444, d_tracer,
4713	NULL, &tracing_readme_fops);
4714
4715	trace_create_file("trace_pipe", 0444, d_tracer,
4716	(void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4717
4718	trace_create_file("buffer_size_kb", 0644, d_tracer,
4719	&global_trace, &tracing_entries_fops);
4720
4721	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4722	&global_trace, &tracing_total_entries_fops);
4723
4724	trace_create_file("free_buffer", 0644, d_tracer,
4725	&global_trace, &tracing_free_buffer_fops);
4726
4727	trace_create_file("trace_marker", 0220, d_tracer,
4728	NULL, &tracing_mark_fops);
4729
4730	trace_create_file("saved_cmdlines", 0444, d_tracer,
4731	NULL, &tracing_saved_cmdlines_fops);
4732
4733	trace_create_file("trace_clock", 0644, d_tracer, NULL,
4734	&trace_clock_fops);
4735
4736	trace_create_file("tracing_on", 0644, d_tracer,
4737	global_trace.buffer, &rb_simple_fops);
4738
4739	#ifdef CONFIG_DYNAMIC_FTRACE
4740	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4741	&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4742	#endif
4743
4744	create_trace_options_dir();
4745
4746	for_each_tracing_cpu(cpu)
4747	tracing_init_debugfs_percpu(cpu);
4748
4749	return 0;
4750	}
4751
4752	static int trace_panic_handler(struct notifier_block *this,
4753	unsigned long event, void *unused)
4754	{
4755	if (ftrace_dump_on_oops)
4756	ftrace_dump(ftrace_dump_on_oops);
4757	return NOTIFY_OK;
4758	}
4759
4760	static struct notifier_block trace_panic_notifier = {
4761	.notifier_call = trace_panic_handler,
4762	.next = NULL,
4763	.priority = 150 /* priority: INT_MAX >= x >= 0 */
4764	};
4765
4766	static int trace_die_handler(struct notifier_block *self,
4767	unsigned long val,
4768	void *data)
4769	{
4770	switch (val) {
4771	case DIE_OOPS:
4772	if (ftrace_dump_on_oops)
4773	ftrace_dump(ftrace_dump_on_oops);
4774	break;
4775	default:
4776	break;
4777	}
4778	return NOTIFY_OK;
4779	}
4780
4781	static struct notifier_block trace_die_notifier = {
4782	.notifier_call = trace_die_handler,
4783	.priority = 200
4784	};
4785
4786	/*
4787	* printk is set to max of 1024, we really don't need it that big.
4788	* Nothing should be printing 1000 characters anyway.
4789	*/
4790	#define TRACE_MAX_PRINT 1000
4791
4792	/*
4793	* Define here KERN_TRACE so that we have one place to modify
4794	* it if we decide to change what log level the ftrace dump
4795	* should be at.
4796	*/
4797	#define KERN_TRACE KERN_EMERG
4798
4799	void
4800	trace_printk_seq(struct trace_seq *s)
4801	{
4802	/* Probably should print a warning here. */
4803	if (s->len >= 1000)
4804	s->len = 1000;
4805
4806	/* should be zero ended, but we are paranoid. */
4807	s->buffer[s->len] = 0;
4808
4809	printk(KERN_TRACE "%s", s->buffer);
4810
4811	trace_seq_init(s);
4812	}
4813
4814	void trace_init_global_iter(struct trace_iterator *iter)
4815	{
4816	iter->tr = &global_trace;
4817	iter->trace = current_trace;
4818	iter->cpu_file = TRACE_PIPE_ALL_CPU;
4819	}
4820
4821	static void
4822	__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4823	{
4824	static arch_spinlock_t ftrace_dump_lock =
4825	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4826	/* use static because iter can be a bit big for the stack */
4827	static struct trace_iterator iter;
4828	unsigned int old_userobj;
4829	static int dump_ran;
4830	unsigned long flags;
4831	int cnt = 0, cpu;
4832
4833	/* only one dump */
4834	local_irq_save(flags);
4835	arch_spin_lock(&ftrace_dump_lock);
4836	if (dump_ran)
4837	goto out;
4838
4839	dump_ran = 1;
4840
4841	tracing_off();
4842
4843	/* Did function tracer already get disabled? */
4844	if (ftrace_is_dead()) {
4845	printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
4846	printk("# MAY BE MISSING FUNCTION EVENTS\n");
4847	}
4848
4849	if (disable_tracing)
4850	ftrace_kill();
4851
4852	trace_init_global_iter(&iter);
4853
4854	for_each_tracing_cpu(cpu) {
4855	atomic_inc(&iter.tr->data[cpu]->disabled);
4856	}
4857
4858	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4859
4860	/* don't look at user memory in panic mode */
4861	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4862
4863	/* Simulate the iterator */
4864	iter.tr = &global_trace;
4865	iter.trace = current_trace;
4866
4867	switch (oops_dump_mode) {
4868	case DUMP_ALL:
4869	iter.cpu_file = TRACE_PIPE_ALL_CPU;
4870	break;
4871	case DUMP_ORIG:
4872	iter.cpu_file = raw_smp_processor_id();
4873	break;
4874	case DUMP_NONE:
4875	goto out_enable;
4876	default:
4877	printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
4878	iter.cpu_file = TRACE_PIPE_ALL_CPU;
4879	}
4880
4881	printk(KERN_TRACE "Dumping ftrace buffer:\n");
4882
4883	/*
4884	* We need to stop all tracing on all CPUS to read the
4885	* the next buffer. This is a bit expensive, but is
4886	* not done often. We fill all what we can read,
4887	* and then release the locks again.
4888	*/
4889
4890	while (!trace_empty(&iter)) {
4891
4892	if (!cnt)
4893	printk(KERN_TRACE "---------------------------------\n");
4894
4895	cnt++;
4896
4897	/* reset all but tr, trace, and overruns */
4898	memset(&iter.seq, 0,
4899	sizeof(struct trace_iterator) -
4900	offsetof(struct trace_iterator, seq));
4901	iter.iter_flags \|= TRACE_FILE_LAT_FMT;
4902	iter.pos = -1;
4903
4904	if (trace_find_next_entry_inc(&iter) != NULL) {
4905	int ret;
4906
4907	ret = print_trace_line(&iter);
4908	if (ret != TRACE_TYPE_NO_CONSUME)
4909	trace_consume(&iter);
4910	}
4911	touch_nmi_watchdog();
4912
4913	trace_printk_seq(&iter.seq);
4914	}
4915
4916	if (!cnt)
4917	printk(KERN_TRACE " (ftrace buffer empty)\n");
4918	else
4919	printk(KERN_TRACE "---------------------------------\n");
4920
4921	out_enable:
4922	/* Re-enable tracing if requested */
4923	if (!disable_tracing) {
4924	trace_flags \|= old_userobj;
4925
4926	for_each_tracing_cpu(cpu) {
4927	atomic_dec(&iter.tr->data[cpu]->disabled);
4928	}
4929	tracing_on();
4930	}
4931
4932	out:
4933	arch_spin_unlock(&ftrace_dump_lock);
4934	local_irq_restore(flags);
4935	}
4936
4937	/* By default: disable tracing after the dump */
4938	void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
4939	{
4940	__ftrace_dump(true, oops_dump_mode);
4941	}
4942	EXPORT_SYMBOL_GPL(ftrace_dump);
4943
4944	__init static int tracer_alloc_buffers(void)
4945	{
4946	int ring_buf_size;
4947	enum ring_buffer_flags rb_flags;
4948	int i;
4949	int ret = -ENOMEM;
4950
4951
4952	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4953	goto out;
4954
4955	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4956	goto out_free_buffer_mask;
4957
4958	/* To save memory, keep the ring buffer size to its minimum */
4959	if (ring_buffer_expanded)
4960	ring_buf_size = trace_buf_size;
4961	else
4962	ring_buf_size = 1;
4963
4964	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
4965
4966	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4967	cpumask_copy(tracing_cpumask, cpu_all_mask);
4968
4969	/* TODO: make the number of buffers hot pluggable with CPUS */
4970	global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
4971	if (!global_trace.buffer) {
4972	printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4973	WARN_ON(1);
4974	goto out_free_cpumask;
4975	}
4976	global_trace.entries = ring_buffer_size(global_trace.buffer);
4977	if (global_trace.buffer_disabled)
4978	tracing_off();
4979
4980
4981	#ifdef CONFIG_TRACER_MAX_TRACE
4982	max_tr.buffer = ring_buffer_alloc(1, rb_flags);
4983	if (!max_tr.buffer) {
4984	printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4985	WARN_ON(1);
4986	ring_buffer_free(global_trace.buffer);
4987	goto out_free_cpumask;
4988	}
4989	max_tr.entries = 1;
4990	#endif
4991
4992	/* Allocate the first page for all buffers */
4993	for_each_tracing_cpu(i) {
4994	global_trace.data[i] = &per_cpu(global_trace_cpu, i);
4995	max_tr.data[i] = &per_cpu(max_tr_data, i);
4996	}
4997
4998	trace_init_cmdlines();
4999
5000	register_tracer(&nop_trace);
5001	current_trace = &nop_trace;
5002	/* All seems OK, enable tracing */
5003	tracing_disabled = 0;
5004
5005	atomic_notifier_chain_register(&panic_notifier_list,
5006	&trace_panic_notifier);
5007
5008	register_die_notifier(&trace_die_notifier);
5009
5010	return 0;
5011
5012	out_free_cpumask:
5013	free_cpumask_var(tracing_cpumask);
5014	out_free_buffer_mask:
5015	free_cpumask_var(tracing_buffer_mask);
5016	out:
5017	return ret;
5018	}
5019
5020	__init static int clear_boot_tracer(void)
5021	{
5022	/*
5023	* The default tracer at boot buffer is an init section.
5024	* This function is called in lateinit. If we did not
5025	* find the boot tracer, then clear it out, to prevent
5026	* later registration from accessing the buffer that is
5027	* about to be freed.
5028	*/
5029	if (!default_bootup_tracer)
5030	return 0;
5031
5032	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5033	default_bootup_tracer);
5034	default_bootup_tracer = NULL;
5035
5036	return 0;
5037	}
5038
5039	early_initcall(tracer_alloc_buffers);
5040	fs_initcall(tracer_init_debugfs);
5041	late_initcall(clear_boot_tracer);