linux - kernel/trace/trace.c

6475 lines · 152.6 KiB

1	/*
2	* ring buffer based function tracer
3	*
4	* Copyright (C) 2007-2012 Steven Rostedt <[email protected]>
5	* Copyright (C) 2008 Ingo Molnar <[email protected]>
6	*
7	* Originally taken from the RT patch by:
8	* Arnaldo Carvalho de Melo <[email protected]>
9	*
10	* Based on code from the latency_tracer, that is:
11	* Copyright (C) 2004-2006 Ingo Molnar
12	* Copyright (C) 2004 Nadia Yvette Chambers
13	*/
14	#include <linux/ring_buffer.h>
15	#include <generated/utsrelease.h>
16	#include <linux/stacktrace.h>
17	#include <linux/writeback.h>
18	#include <linux/kallsyms.h>
19	#include <linux/seq_file.h>
20	#include <linux/notifier.h>
21	#include <linux/irqflags.h>
22	#include <linux/debugfs.h>
23	#include <linux/pagemap.h>
24	#include <linux/hardirq.h>
25	#include <linux/linkage.h>
26	#include <linux/uaccess.h>
27	#include <linux/kprobes.h>
28	#include <linux/ftrace.h>
29	#include <linux/module.h>
30	#include <linux/percpu.h>
31	#include <linux/splice.h>
32	#include <linux/kdebug.h>
33	#include <linux/string.h>
34	#include <linux/rwsem.h>
35	#include <linux/slab.h>
36	#include <linux/ctype.h>
37	#include <linux/init.h>
38	#include <linux/poll.h>
39	#include <linux/nmi.h>
40	#include <linux/fs.h>
41	#include <linux/sched/rt.h>
42
43	#include "trace.h"
44	#include "trace_output.h"
45
46	/*
47	* On boot up, the ring buffer is set to the minimum size, so that
48	* we do not waste memory on systems that are not using tracing.
49	*/
50	bool ring_buffer_expanded;
51
52	/*
53	* We need to change this state when a selftest is running.
54	* A selftest will lurk into the ring-buffer to count the
55	* entries inserted during the selftest although some concurrent
56	* insertions into the ring-buffer such as trace_printk could occurred
57	* at the same time, giving false positive or negative results.
58	*/
59	static bool __read_mostly tracing_selftest_running;
60
61	/*
62	* If a tracer is running, we do not want to run SELFTEST.
63	*/
64	bool __read_mostly tracing_selftest_disabled;
65
66	/* For tracers that don't implement custom flags */
67	static struct tracer_opt dummy_tracer_opt[] = {
68	{ }
69	};
70
71	static struct tracer_flags dummy_tracer_flags = {
72	.val = 0,
73	.opts = dummy_tracer_opt
74	};
75
76	static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77	{
78	return 0;
79	}
80
81	/*
82	* To prevent the comm cache from being overwritten when no
83	* tracing is active, only save the comm when a trace event
84	* occurred.
85	*/
86	static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88	/*
89	* Kill all tracing for good (never come back).
90	* It is initialized to 1 but will turn to zero if the initialization
91	* of the tracer is successful. But that is the only place that sets
92	* this back to zero.
93	*/
94	static int tracing_disabled = 1;
95
96	DEFINE_PER_CPU(int, ftrace_cpu_disabled);
97
98	cpumask_var_t __read_mostly tracing_buffer_mask;
99
100	/*
101	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102	*
103	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104	* is set, then ftrace_dump is called. This will output the contents
105	* of the ftrace buffers to the console. This is very useful for
106	* capturing traces that lead to crashes and outputing it to a
107	* serial console.
108	*
109	* It is default off, but you can enable it with either specifying
110	* "ftrace_dump_on_oops" in the kernel command line, or setting
111	* /proc/sys/kernel/ftrace_dump_on_oops
112	* Set 1 if you want to dump buffers of all CPUs
113	* Set 2 if you want to dump the buffer of the CPU that triggered oops
114	*/
115
116	enum ftrace_dump_mode ftrace_dump_on_oops;
117
118	/* When set, tracing will stop when a WARN() is hit /
119	int __disable_trace_on_warning;
120
121	static int tracing_set_tracer(const char *buf);
122
123	#define MAX_TRACER_SIZE 100
124	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
125	static char *default_bootup_tracer;
126
127	static bool allocate_snapshot;
128
129	static int __init set_cmdline_ftrace(char *str)
130	{
131	strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
132	default_bootup_tracer = bootup_tracer_buf;
133	/* We are using ftrace early, expand it */
134	ring_buffer_expanded = true;
135	return 1;
136	}
137	__setup("ftrace=", set_cmdline_ftrace);
138
139	static int __init set_ftrace_dump_on_oops(char *str)
140	{
141	if (str++ != '=' \|\| !str) {
142	ftrace_dump_on_oops = DUMP_ALL;
143	return 1;
144	}
145
146	if (!strcmp("orig_cpu", str)) {
147	ftrace_dump_on_oops = DUMP_ORIG;
148	return 1;
149	}
150
151	return 0;
152	}
153	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
154
155	static int __init stop_trace_on_warning(char *str)
156	{
157	__disable_trace_on_warning = 1;
158	return 1;
159	}
160	__setup("traceoff_on_warning=", stop_trace_on_warning);
161
162	static int __init boot_alloc_snapshot(char *str)
163	{
164	allocate_snapshot = true;
165	/* We also need the main ring buffer expanded */
166	ring_buffer_expanded = true;
167	return 1;
168	}
169	__setup("alloc_snapshot", boot_alloc_snapshot);
170
171
172	static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
173	static char *trace_boot_options __initdata;
174
175	static int __init set_trace_boot_options(char *str)
176	{
177	strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
178	trace_boot_options = trace_boot_options_buf;
179	return 0;
180	}
181	__setup("trace_options=", set_trace_boot_options);
182
183
184	unsigned long long ns2usecs(cycle_t nsec)
185	{
186	nsec += 500;
187	do_div(nsec, 1000);
188	return nsec;
189	}
190
191	/*
192	* The global_trace is the descriptor that holds the tracing
193	* buffers for the live tracing. For each CPU, it contains
194	* a link list of pages that will store trace entries. The
195	* page descriptor of the pages in the memory is used to hold
196	* the link list by linking the lru item in the page descriptor
197	* to each of the pages in the buffer per CPU.
198	*
199	* For each active CPU there is a data field that holds the
200	* pages for the buffer for that CPU. Each CPU has the same number
201	* of pages allocated for its buffer.
202	*/
203	static struct trace_array global_trace;
204
205	LIST_HEAD(ftrace_trace_arrays);
206
207	int trace_array_get(struct trace_array *this_tr)
208	{
209	struct trace_array *tr;
210	int ret = -ENODEV;
211
212	mutex_lock(&trace_types_lock);
213	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
214	if (tr == this_tr) {
215	tr->ref++;
216	ret = 0;
217	break;
218	}
219	}
220	mutex_unlock(&trace_types_lock);
221
222	return ret;
223	}
224
225	static void __trace_array_put(struct trace_array *this_tr)
226	{
227	WARN_ON(!this_tr->ref);
228	this_tr->ref--;
229	}
230
231	void trace_array_put(struct trace_array *this_tr)
232	{
233	mutex_lock(&trace_types_lock);
234	__trace_array_put(this_tr);
235	mutex_unlock(&trace_types_lock);
236	}
237
238	int filter_current_check_discard(struct ring_buffer *buffer,
239	struct ftrace_event_call call, void rec,
240	struct ring_buffer_event *event)
241	{
242	return filter_check_discard(call, rec, buffer, event);
243	}
244	EXPORT_SYMBOL_GPL(filter_current_check_discard);
245
246	cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
247	{
248	u64 ts;
249
250	/* Early boot up does not have a buffer yet */
251	if (!buf->buffer)
252	return trace_clock_local();
253
254	ts = ring_buffer_time_stamp(buf->buffer, cpu);
255	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
256
257	return ts;
258	}
259
260	cycle_t ftrace_now(int cpu)
261	{
262	return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
263	}
264
265	/**
266	* tracing_is_enabled - Show if global_trace has been disabled
267	*
268	* Shows if the global trace has been enabled or not. It uses the
269	* mirror flag "buffer_disabled" to be used in fast paths such as for
270	* the irqsoff tracer. But it may be inaccurate due to races. If you
271	* need to know the accurate state, use tracing_is_on() which is a little
272	* slower, but accurate.
273	*/
274	int tracing_is_enabled(void)
275	{
276	/*
277	* For quick access (irqsoff uses this in fast path), just
278	* return the mirror variable of the state of the ring buffer.
279	* It's a little racy, but we don't really care.
280	*/
281	smp_rmb();
282	return !global_trace.buffer_disabled;
283	}
284
285	/*
286	* trace_buf_size is the size in bytes that is allocated
287	* for a buffer. Note, the number of bytes is always rounded
288	* to page size.
289	*
290	* This number is purposely set to a low number of 16384.
291	* If the dump on oops happens, it will be much appreciated
292	* to not have to wait for all that output. Anyway this can be
293	* boot time and run time configurable.
294	*/
295	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
296
297	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
298
299	/* trace_types holds a link list of available tracers. */
300	static struct tracer *trace_types __read_mostly;
301
302	/*
303	* trace_types_lock is used to protect the trace_types list.
304	*/
305	DEFINE_MUTEX(trace_types_lock);
306
307	/*
308	* serialize the access of the ring buffer
309	*
310	* ring buffer serializes readers, but it is low level protection.
311	* The validity of the events (which returns by ring_buffer_peek() ..etc)
312	* are not protected by ring buffer.
313	*
314	* The content of events may become garbage if we allow other process consumes
315	* these events concurrently:
316	* A) the page of the consumed events may become a normal page
317	* (not reader page) in ring buffer, and this page will be rewrited
318	* by events producer.
319	* B) The page of the consumed events may become a page for splice_read,
320	* and this page will be returned to system.
321	*
322	* These primitives allow multi process access to different cpu ring buffer
323	* concurrently.
324	*
325	* These primitives don't distinguish read-only and read-consume access.
326	* Multi read-only access are also serialized.
327	*/
328
329	#ifdef CONFIG_SMP
330	static DECLARE_RWSEM(all_cpu_access_lock);
331	static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
332
333	static inline void trace_access_lock(int cpu)
334	{
335	if (cpu == RING_BUFFER_ALL_CPUS) {
336	/* gain it for accessing the whole ring buffer. */
337	down_write(&all_cpu_access_lock);
338	} else {
339	/* gain it for accessing a cpu ring buffer. */
340
341	/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
342	down_read(&all_cpu_access_lock);
343
344	/* Secondly block other access to this @cpu ring buffer. */
345	mutex_lock(&per_cpu(cpu_access_lock, cpu));
346	}
347	}
348
349	static inline void trace_access_unlock(int cpu)
350	{
351	if (cpu == RING_BUFFER_ALL_CPUS) {
352	up_write(&all_cpu_access_lock);
353	} else {
354	mutex_unlock(&per_cpu(cpu_access_lock, cpu));
355	up_read(&all_cpu_access_lock);
356	}
357	}
358
359	static inline void trace_access_lock_init(void)
360	{
361	int cpu;
362
363	for_each_possible_cpu(cpu)
364	mutex_init(&per_cpu(cpu_access_lock, cpu));
365	}
366
367	#else
368
369	static DEFINE_MUTEX(access_lock);
370
371	static inline void trace_access_lock(int cpu)
372	{
373	(void)cpu;
374	mutex_lock(&access_lock);
375	}
376
377	static inline void trace_access_unlock(int cpu)
378	{
379	(void)cpu;
380	mutex_unlock(&access_lock);
381	}
382
383	static inline void trace_access_lock_init(void)
384	{
385	}
386
387	#endif
388
389	/* trace_flags holds trace_options default values */
390	unsigned long trace_flags = TRACE_ITER_PRINT_PARENT \| TRACE_ITER_PRINTK \|
391	TRACE_ITER_ANNOTATE \| TRACE_ITER_CONTEXT_INFO \| TRACE_ITER_SLEEP_TIME \|
392	TRACE_ITER_GRAPH_TIME \| TRACE_ITER_RECORD_CMD \| TRACE_ITER_OVERWRITE \|
393	TRACE_ITER_IRQ_INFO \| TRACE_ITER_MARKERS \| TRACE_ITER_FUNCTION;
394
395	static void tracer_tracing_on(struct trace_array *tr)
396	{
397	if (tr->trace_buffer.buffer)
398	ring_buffer_record_on(tr->trace_buffer.buffer);
399	/*
400	* This flag is looked at when buffers haven't been allocated
401	* yet, or by some tracers (like irqsoff), that just want to
402	* know if the ring buffer has been disabled, but it can handle
403	* races of where it gets disabled but we still do a record.
404	* As the check is in the fast path of the tracers, it is more
405	* important to be fast than accurate.
406	*/
407	tr->buffer_disabled = 0;
408	/* Make the flag seen by readers */
409	smp_wmb();
410	}
411
412	/**
413	* tracing_on - enable tracing buffers
414	*
415	* This function enables tracing buffers that may have been
416	* disabled with tracing_off.
417	*/
418	void tracing_on(void)
419	{
420	tracer_tracing_on(&global_trace);
421	}
422	EXPORT_SYMBOL_GPL(tracing_on);
423
424	/**
425	* __trace_puts - write a constant string into the trace buffer.
426	* @ip: The address of the caller
427	* @str: The constant string to write
428	* @size: The size of the string.
429	*/
430	int __trace_puts(unsigned long ip, const char *str, int size)
431	{
432	struct ring_buffer_event *event;
433	struct ring_buffer *buffer;
434	struct print_entry *entry;
435	unsigned long irq_flags;
436	int alloc;
437
438	alloc = sizeof(entry) + size + 2; / possible \n added */
439
440	local_save_flags(irq_flags);
441	buffer = global_trace.trace_buffer.buffer;
442	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
443	irq_flags, preempt_count());
444	if (!event)
445	return 0;
446
447	entry = ring_buffer_event_data(event);
448	entry->ip = ip;
449
450	memcpy(&entry->buf, str, size);
451
452	/* Add a newline if necessary */
453	if (entry->buf[size - 1] != '\n') {
454	entry->buf[size] = '\n';
455	entry->buf[size + 1] = '\0';
456	} else
457	entry->buf[size] = '\0';
458
459	__buffer_unlock_commit(buffer, event);
460
461	return size;
462	}
463	EXPORT_SYMBOL_GPL(__trace_puts);
464
465	/**
466	* __trace_bputs - write the pointer to a constant string into trace buffer
467	* @ip: The address of the caller
468	* @str: The constant string to write to the buffer to
469	*/
470	int __trace_bputs(unsigned long ip, const char *str)
471	{
472	struct ring_buffer_event *event;
473	struct ring_buffer *buffer;
474	struct bputs_entry *entry;
475	unsigned long irq_flags;
476	int size = sizeof(struct bputs_entry);
477
478	local_save_flags(irq_flags);
479	buffer = global_trace.trace_buffer.buffer;
480	event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
481	irq_flags, preempt_count());
482	if (!event)
483	return 0;
484
485	entry = ring_buffer_event_data(event);
486	entry->ip = ip;
487	entry->str = str;
488
489	__buffer_unlock_commit(buffer, event);
490
491	return 1;
492	}
493	EXPORT_SYMBOL_GPL(__trace_bputs);
494
495	#ifdef CONFIG_TRACER_SNAPSHOT
496	/**
497	* trace_snapshot - take a snapshot of the current buffer.
498	*
499	* This causes a swap between the snapshot buffer and the current live
500	* tracing buffer. You can use this to take snapshots of the live
501	* trace when some condition is triggered, but continue to trace.
502	*
503	* Note, make sure to allocate the snapshot with either
504	* a tracing_snapshot_alloc(), or by doing it manually
505	* with: echo 1 > /sys/kernel/debug/tracing/snapshot
506	*
507	* If the snapshot buffer is not allocated, it will stop tracing.
508	* Basically making a permanent snapshot.
509	*/
510	void tracing_snapshot(void)
511	{
512	struct trace_array *tr = &global_trace;
513	struct tracer *tracer = tr->current_trace;
514	unsigned long flags;
515
516	if (in_nmi()) {
517	internal_trace_puts("* SNAPSHOT CALLED FROM NMI CONTEXT *\n");
518	internal_trace_puts("* snapshot is being ignored *\n");
519	return;
520	}
521
522	if (!tr->allocated_snapshot) {
523	internal_trace_puts("* SNAPSHOT NOT ALLOCATED *\n");
524	internal_trace_puts("* stopping trace here! *\n");
525	tracing_off();
526	return;
527	}
528
529	/* Note, snapshot can not be used when the tracer uses it */
530	if (tracer->use_max_tr) {
531	internal_trace_puts("* LATENCY TRACER ACTIVE *\n");
532	internal_trace_puts("* Can not use snapshot (sorry) *\n");
533	return;
534	}
535
536	local_irq_save(flags);
537	update_max_tr(tr, current, smp_processor_id());
538	local_irq_restore(flags);
539	}
540	EXPORT_SYMBOL_GPL(tracing_snapshot);
541
542	static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
543	struct trace_buffer *size_buf, int cpu_id);
544	static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
545
546	static int alloc_snapshot(struct trace_array *tr)
547	{
548	int ret;
549
550	if (!tr->allocated_snapshot) {
551
552	/* allocate spare buffer */
553	ret = resize_buffer_duplicate_size(&tr->max_buffer,
554	&tr->trace_buffer, RING_BUFFER_ALL_CPUS);
555	if (ret < 0)
556	return ret;
557
558	tr->allocated_snapshot = true;
559	}
560
561	return 0;
562	}
563
564	void free_snapshot(struct trace_array *tr)
565	{
566	/*
567	* We don't free the ring buffer. instead, resize it because
568	* The max_tr ring buffer has some state (e.g. ring->clock) and
569	* we want preserve it.
570	*/
571	ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
572	set_buffer_entries(&tr->max_buffer, 1);
573	tracing_reset_online_cpus(&tr->max_buffer);
574	tr->allocated_snapshot = false;
575	}
576
577	/**
578	* trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
579	*
580	* This is similar to trace_snapshot(), but it will allocate the
581	* snapshot buffer if it isn't already allocated. Use this only
582	* where it is safe to sleep, as the allocation may sleep.
583	*
584	* This causes a swap between the snapshot buffer and the current live
585	* tracing buffer. You can use this to take snapshots of the live
586	* trace when some condition is triggered, but continue to trace.
587	*/
588	void tracing_snapshot_alloc(void)
589	{
590	struct trace_array *tr = &global_trace;
591	int ret;
592
593	ret = alloc_snapshot(tr);
594	if (WARN_ON(ret < 0))
595	return;
596
597	tracing_snapshot();
598	}
599	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
600	#else
601	void tracing_snapshot(void)
602	{
603	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
604	}
605	EXPORT_SYMBOL_GPL(tracing_snapshot);
606	void tracing_snapshot_alloc(void)
607	{
608	/* Give warning */
609	tracing_snapshot();
610	}
611	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
612	#endif /* CONFIG_TRACER_SNAPSHOT */
613
614	static void tracer_tracing_off(struct trace_array *tr)
615	{
616	if (tr->trace_buffer.buffer)
617	ring_buffer_record_off(tr->trace_buffer.buffer);
618	/*
619	* This flag is looked at when buffers haven't been allocated
620	* yet, or by some tracers (like irqsoff), that just want to
621	* know if the ring buffer has been disabled, but it can handle
622	* races of where it gets disabled but we still do a record.
623	* As the check is in the fast path of the tracers, it is more
624	* important to be fast than accurate.
625	*/
626	tr->buffer_disabled = 1;
627	/* Make the flag seen by readers */
628	smp_wmb();
629	}
630
631	/**
632	* tracing_off - turn off tracing buffers
633	*
634	* This function stops the tracing buffers from recording data.
635	* It does not disable any overhead the tracers themselves may
636	* be causing. This function simply causes all recording to
637	* the ring buffers to fail.
638	*/
639	void tracing_off(void)
640	{
641	tracer_tracing_off(&global_trace);
642	}
643	EXPORT_SYMBOL_GPL(tracing_off);
644
645	void disable_trace_on_warning(void)
646	{
647	if (__disable_trace_on_warning)
648	tracing_off();
649	}
650
651	/**
652	* tracer_tracing_is_on - show real state of ring buffer enabled
653	* @tr : the trace array to know if ring buffer is enabled
654	*
655	* Shows real state of the ring buffer if it is enabled or not.
656	*/
657	static int tracer_tracing_is_on(struct trace_array *tr)
658	{
659	if (tr->trace_buffer.buffer)
660	return ring_buffer_record_is_on(tr->trace_buffer.buffer);
661	return !tr->buffer_disabled;
662	}
663
664	/**
665	* tracing_is_on - show state of ring buffers enabled
666	*/
667	int tracing_is_on(void)
668	{
669	return tracer_tracing_is_on(&global_trace);
670	}
671	EXPORT_SYMBOL_GPL(tracing_is_on);
672
673	static int __init set_buf_size(char *str)
674	{
675	unsigned long buf_size;
676
677	if (!str)
678	return 0;
679	buf_size = memparse(str, &str);
680	/* nr_entries can not be zero */
681	if (buf_size == 0)
682	return 0;
683	trace_buf_size = buf_size;
684	return 1;
685	}
686	__setup("trace_buf_size=", set_buf_size);
687
688	static int __init set_tracing_thresh(char *str)
689	{
690	unsigned long threshold;
691	int ret;
692
693	if (!str)
694	return 0;
695	ret = kstrtoul(str, 0, &threshold);
696	if (ret < 0)
697	return 0;
698	tracing_thresh = threshold * 1000;
699	return 1;
700	}
701	__setup("tracing_thresh=", set_tracing_thresh);
702
703	unsigned long nsecs_to_usecs(unsigned long nsecs)
704	{
705	return nsecs / 1000;
706	}
707
708	/* These must match the bit postions in trace_iterator_flags */
709	static const char *trace_options[] = {
710	"print-parent",
711	"sym-offset",
712	"sym-addr",
713	"verbose",
714	"raw",
715	"hex",
716	"bin",
717	"block",
718	"stacktrace",
719	"trace_printk",
720	"ftrace_preempt",
721	"branch",
722	"annotate",
723	"userstacktrace",
724	"sym-userobj",
725	"printk-msg-only",
726	"context-info",
727	"latency-format",
728	"sleep-time",
729	"graph-time",
730	"record-cmd",
731	"overwrite",
732	"disable_on_free",
733	"irq-info",
734	"markers",
735	"function-trace",
736	NULL
737	};
738
739	static struct {
740	u64 (*func)(void);
741	const char *name;
742	int in_ns; /* is this clock in nanoseconds? */
743	} trace_clocks[] = {
744	{ trace_clock_local, "local", 1 },
745	{ trace_clock_global, "global", 1 },
746	{ trace_clock_counter, "counter", 0 },
747	{ trace_clock_jiffies, "uptime", 1 },
748	{ trace_clock, "perf", 1 },
749	ARCH_TRACE_CLOCKS
750	};
751
752	/*
753	* trace_parser_get_init - gets the buffer for trace parser
754	*/
755	int trace_parser_get_init(struct trace_parser *parser, int size)
756	{
757	memset(parser, 0, sizeof(*parser));
758
759	parser->buffer = kmalloc(size, GFP_KERNEL);
760	if (!parser->buffer)
761	return 1;
762
763	parser->size = size;
764	return 0;
765	}
766
767	/*
768	* trace_parser_put - frees the buffer for trace parser
769	*/
770	void trace_parser_put(struct trace_parser *parser)
771	{
772	kfree(parser->buffer);
773	}
774
775	/*
776	* trace_get_user - reads the user input string separated by space
777	* (matched by isspace(ch))
778	*
779	* For each string found the 'struct trace_parser' is updated,
780	* and the function returns.
781	*
782	* Returns number of bytes read.
783	*
784	* See kernel/trace/trace.h for 'struct trace_parser' details.
785	*/
786	int trace_get_user(struct trace_parser parser, const char __user ubuf,
787	size_t cnt, loff_t *ppos)
788	{
789	char ch;
790	size_t read = 0;
791	ssize_t ret;
792
793	if (!*ppos)
794	trace_parser_clear(parser);
795
796	ret = get_user(ch, ubuf++);
797	if (ret)
798	goto out;
799
800	read++;
801	cnt--;
802
803	/*
804	* The parser is not finished with the last write,
805	* continue reading the user input without skipping spaces.
806	*/
807	if (!parser->cont) {
808	/* skip white space */
809	while (cnt && isspace(ch)) {
810	ret = get_user(ch, ubuf++);
811	if (ret)
812	goto out;
813	read++;
814	cnt--;
815	}
816
817	/* only spaces were written */
818	if (isspace(ch)) {
819	*ppos += read;
820	ret = read;
821	goto out;
822	}
823
824	parser->idx = 0;
825	}
826
827	/* read the non-space input */
828	while (cnt && !isspace(ch)) {
829	if (parser->idx < parser->size - 1)
830	parser->buffer[parser->idx++] = ch;
831	else {
832	ret = -EINVAL;
833	goto out;
834	}
835	ret = get_user(ch, ubuf++);
836	if (ret)
837	goto out;
838	read++;
839	cnt--;
840	}
841
842	/* We either got finished input or we have to wait for another call. */
843	if (isspace(ch)) {
844	parser->buffer[parser->idx] = 0;
845	parser->cont = false;
846	} else {
847	parser->cont = true;
848	parser->buffer[parser->idx++] = ch;
849	}
850
851	*ppos += read;
852	ret = read;
853
854	out:
855	return ret;
856	}
857
858	ssize_t trace_seq_to_user(struct trace_seq s, char __user ubuf, size_t cnt)
859	{
860	int len;
861	int ret;
862
863	if (!cnt)
864	return 0;
865
866	if (s->len <= s->readpos)
867	return -EBUSY;
868
869	len = s->len - s->readpos;
870	if (cnt > len)
871	cnt = len;
872	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
873	if (ret == cnt)
874	return -EFAULT;
875
876	cnt -= ret;
877
878	s->readpos += cnt;
879	return cnt;
880	}
881
882	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
883	{
884	int len;
885
886	if (s->len <= s->readpos)
887	return -EBUSY;
888
889	len = s->len - s->readpos;
890	if (cnt > len)
891	cnt = len;
892	memcpy(buf, s->buffer + s->readpos, cnt);
893
894	s->readpos += cnt;
895	return cnt;
896	}
897
898	/*
899	* ftrace_max_lock is used to protect the swapping of buffers
900	* when taking a max snapshot. The buffers themselves are
901	* protected by per_cpu spinlocks. But the action of the swap
902	* needs its own lock.
903	*
904	* This is defined as a arch_spinlock_t in order to help
905	* with performance when lockdep debugging is enabled.
906	*
907	* It is also used in other places outside the update_max_tr
908	* so it needs to be defined outside of the
909	* CONFIG_TRACER_MAX_TRACE.
910	*/
911	static arch_spinlock_t ftrace_max_lock =
912	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
913
914	unsigned long __read_mostly tracing_thresh;
915
916	#ifdef CONFIG_TRACER_MAX_TRACE
917	unsigned long __read_mostly tracing_max_latency;
918
919	/*
920	* Copy the new maximum trace into the separate maximum-trace
921	* structure. (this way the maximum trace is permanently saved,
922	* for later retrieval via /sys/kernel/debug/tracing/latency_trace)
923	*/
924	static void
925	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
926	{
927	struct trace_buffer *trace_buf = &tr->trace_buffer;
928	struct trace_buffer *max_buf = &tr->max_buffer;
929	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
930	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
931
932	max_buf->cpu = cpu;
933	max_buf->time_start = data->preempt_timestamp;
934
935	max_data->saved_latency = tracing_max_latency;
936	max_data->critical_start = data->critical_start;
937	max_data->critical_end = data->critical_end;
938
939	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
940	max_data->pid = tsk->pid;
941	/*
942	* If tsk == current, then use current_uid(), as that does not use
943	* RCU. The irq tracer can be called out of RCU scope.
944	*/
945	if (tsk == current)
946	max_data->uid = current_uid();
947	else
948	max_data->uid = task_uid(tsk);
949
950	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
951	max_data->policy = tsk->policy;
952	max_data->rt_priority = tsk->rt_priority;
953
954	/* record this tasks comm */
955	tracing_record_cmdline(tsk);
956	}
957
958	/**
959	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
960	* @tr: tracer
961	* @tsk: the task with the latency
962	* @cpu: The cpu that initiated the trace.
963	*
964	* Flip the buffers between the @tr and the max_tr and record information
965	* about which task was the cause of this latency.
966	*/
967	void
968	update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
969	{
970	struct ring_buffer *buf;
971
972	if (tr->stop_count)
973	return;
974
975	WARN_ON_ONCE(!irqs_disabled());
976
977	if (!tr->allocated_snapshot) {
978	/* Only the nop tracer should hit this when disabling */
979	WARN_ON_ONCE(tr->current_trace != &nop_trace);
980	return;
981	}
982
983	arch_spin_lock(&ftrace_max_lock);
984
985	buf = tr->trace_buffer.buffer;
986	tr->trace_buffer.buffer = tr->max_buffer.buffer;
987	tr->max_buffer.buffer = buf;
988
989	__update_max_tr(tr, tsk, cpu);
990	arch_spin_unlock(&ftrace_max_lock);
991	}
992
993	/**
994	* update_max_tr_single - only copy one trace over, and reset the rest
995	* @tr - tracer
996	* @tsk - task with the latency
997	* @cpu - the cpu of the buffer to copy.
998	*
999	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
1000	*/
1001	void
1002	update_max_tr_single(struct trace_array tr, struct task_struct tsk, int cpu)
1003	{
1004	int ret;
1005
1006	if (tr->stop_count)
1007	return;
1008
1009	WARN_ON_ONCE(!irqs_disabled());
1010	if (!tr->allocated_snapshot) {
1011	/* Only the nop tracer should hit this when disabling */
1012	WARN_ON_ONCE(tr->current_trace != &nop_trace);
1013	return;
1014	}
1015
1016	arch_spin_lock(&ftrace_max_lock);
1017
1018	ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1019
1020	if (ret == -EBUSY) {
1021	/*
1022	* We failed to swap the buffer due to a commit taking
1023	* place on this CPU. We fail to record, but we reset
1024	* the max trace buffer (no one writes directly to it)
1025	* and flag that it failed.
1026	*/
1027	trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1028	"Failed to swap buffers due to commit in progress\n");
1029	}
1030
1031	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1032
1033	__update_max_tr(tr, tsk, cpu);
1034	arch_spin_unlock(&ftrace_max_lock);
1035	}
1036	#endif /* CONFIG_TRACER_MAX_TRACE */
1037
1038	static void default_wait_pipe(struct trace_iterator *iter)
1039	{
1040	/* Iterators are static, they should be filled or empty */
1041	if (trace_buffer_iter(iter, iter->cpu_file))
1042	return;
1043
1044	ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1045	}
1046
1047	#ifdef CONFIG_FTRACE_STARTUP_TEST
1048	static int run_tracer_selftest(struct tracer *type)
1049	{
1050	struct trace_array *tr = &global_trace;
1051	struct tracer *saved_tracer = tr->current_trace;
1052	int ret;
1053
1054	if (!type->selftest \|\| tracing_selftest_disabled)
1055	return 0;
1056
1057	/*
1058	* Run a selftest on this tracer.
1059	* Here we reset the trace buffer, and set the current
1060	* tracer to be this tracer. The tracer can then run some
1061	* internal tracing to verify that everything is in order.
1062	* If we fail, we do not register this tracer.
1063	*/
1064	tracing_reset_online_cpus(&tr->trace_buffer);
1065
1066	tr->current_trace = type;
1067
1068	#ifdef CONFIG_TRACER_MAX_TRACE
1069	if (type->use_max_tr) {
1070	/* If we expanded the buffers, make sure the max is expanded too */
1071	if (ring_buffer_expanded)
1072	ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1073	RING_BUFFER_ALL_CPUS);
1074	tr->allocated_snapshot = true;
1075	}
1076	#endif
1077
1078	/* the test is responsible for initializing and enabling */
1079	pr_info("Testing tracer %s: ", type->name);
1080	ret = type->selftest(type, tr);
1081	/* the test is responsible for resetting too */
1082	tr->current_trace = saved_tracer;
1083	if (ret) {
1084	printk(KERN_CONT "FAILED!\n");
1085	/* Add the warning after printing 'FAILED' */
1086	WARN_ON(1);
1087	return -1;
1088	}
1089	/* Only reset on passing, to avoid touching corrupted buffers */
1090	tracing_reset_online_cpus(&tr->trace_buffer);
1091
1092	#ifdef CONFIG_TRACER_MAX_TRACE
1093	if (type->use_max_tr) {
1094	tr->allocated_snapshot = false;
1095
1096	/* Shrink the max buffer again */
1097	if (ring_buffer_expanded)
1098	ring_buffer_resize(tr->max_buffer.buffer, 1,
1099	RING_BUFFER_ALL_CPUS);
1100	}
1101	#endif
1102
1103	printk(KERN_CONT "PASSED\n");
1104	return 0;
1105	}
1106	#else
1107	static inline int run_tracer_selftest(struct tracer *type)
1108	{
1109	return 0;
1110	}
1111	#endif /* CONFIG_FTRACE_STARTUP_TEST */
1112
1113	/**
1114	* register_tracer - register a tracer with the ftrace system.
1115	* @type - the plugin for the tracer
1116	*
1117	* Register a new plugin tracer.
1118	*/
1119	int register_tracer(struct tracer *type)
1120	{
1121	struct tracer *t;
1122	int ret = 0;
1123
1124	if (!type->name) {
1125	pr_info("Tracer must have a name\n");
1126	return -1;
1127	}
1128
1129	if (strlen(type->name) >= MAX_TRACER_SIZE) {
1130	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1131	return -1;
1132	}
1133
1134	mutex_lock(&trace_types_lock);
1135
1136	tracing_selftest_running = true;
1137
1138	for (t = trace_types; t; t = t->next) {
1139	if (strcmp(type->name, t->name) == 0) {
1140	/* already found */
1141	pr_info("Tracer %s already registered\n",
1142	type->name);
1143	ret = -1;
1144	goto out;
1145	}
1146	}
1147
1148	if (!type->set_flag)
1149	type->set_flag = &dummy_set_flag;
1150	if (!type->flags)
1151	type->flags = &dummy_tracer_flags;
1152	else
1153	if (!type->flags->opts)
1154	type->flags->opts = dummy_tracer_opt;
1155	if (!type->wait_pipe)
1156	type->wait_pipe = default_wait_pipe;
1157
1158	ret = run_tracer_selftest(type);
1159	if (ret < 0)
1160	goto out;
1161
1162	type->next = trace_types;
1163	trace_types = type;
1164
1165	out:
1166	tracing_selftest_running = false;
1167	mutex_unlock(&trace_types_lock);
1168
1169	if (ret \|\| !default_bootup_tracer)
1170	goto out_unlock;
1171
1172	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1173	goto out_unlock;
1174
1175	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1176	/* Do we want this tracer to start on bootup? */
1177	tracing_set_tracer(type->name);
1178	default_bootup_tracer = NULL;
1179	/* disable other selftests, since this will break it. */
1180	tracing_selftest_disabled = true;
1181	#ifdef CONFIG_FTRACE_STARTUP_TEST
1182	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1183	type->name);
1184	#endif
1185
1186	out_unlock:
1187	return ret;
1188	}
1189
1190	void tracing_reset(struct trace_buffer *buf, int cpu)
1191	{
1192	struct ring_buffer *buffer = buf->buffer;
1193
1194	if (!buffer)
1195	return;
1196
1197	ring_buffer_record_disable(buffer);
1198
1199	/* Make sure all commits have finished */
1200	synchronize_sched();
1201	ring_buffer_reset_cpu(buffer, cpu);
1202
1203	ring_buffer_record_enable(buffer);
1204	}
1205
1206	void tracing_reset_online_cpus(struct trace_buffer *buf)
1207	{
1208	struct ring_buffer *buffer = buf->buffer;
1209	int cpu;
1210
1211	if (!buffer)
1212	return;
1213
1214	ring_buffer_record_disable(buffer);
1215
1216	/* Make sure all commits have finished */
1217	synchronize_sched();
1218
1219	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1220
1221	for_each_online_cpu(cpu)
1222	ring_buffer_reset_cpu(buffer, cpu);
1223
1224	ring_buffer_record_enable(buffer);
1225	}
1226
1227	/* Must have trace_types_lock held */
1228	void tracing_reset_all_online_cpus(void)
1229	{
1230	struct trace_array *tr;
1231
1232	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1233	tracing_reset_online_cpus(&tr->trace_buffer);
1234	#ifdef CONFIG_TRACER_MAX_TRACE
1235	tracing_reset_online_cpus(&tr->max_buffer);
1236	#endif
1237	}
1238	}
1239
1240	#define SAVED_CMDLINES 128
1241	#define NO_CMDLINE_MAP UINT_MAX
1242	static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1243	static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1244	static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1245	static int cmdline_idx;
1246	static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1247
1248	/* temporary disable recording */
1249	static atomic_t trace_record_cmdline_disabled __read_mostly;
1250
1251	static void trace_init_cmdlines(void)
1252	{
1253	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1254	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1255	cmdline_idx = 0;
1256	}
1257
1258	int is_tracing_stopped(void)
1259	{
1260	return global_trace.stop_count;
1261	}
1262
1263	/**
1264	* ftrace_off_permanent - disable all ftrace code permanently
1265	*
1266	* This should only be called when a serious anomally has
1267	* been detected. This will turn off the function tracing,
1268	* ring buffers, and other tracing utilites. It takes no
1269	* locks and can be called from any context.
1270	*/
1271	void ftrace_off_permanent(void)
1272	{
1273	tracing_disabled = 1;
1274	ftrace_stop();
1275	tracing_off_permanent();
1276	}
1277
1278	/**
1279	* tracing_start - quick start of the tracer
1280	*
1281	* If tracing is enabled but was stopped by tracing_stop,
1282	* this will start the tracer back up.
1283	*/
1284	void tracing_start(void)
1285	{
1286	struct ring_buffer *buffer;
1287	unsigned long flags;
1288
1289	if (tracing_disabled)
1290	return;
1291
1292	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1293	if (--global_trace.stop_count) {
1294	if (global_trace.stop_count < 0) {
1295	/* Someone screwed up their debugging */
1296	WARN_ON_ONCE(1);
1297	global_trace.stop_count = 0;
1298	}
1299	goto out;
1300	}
1301
1302	/* Prevent the buffers from switching */
1303	arch_spin_lock(&ftrace_max_lock);
1304
1305	buffer = global_trace.trace_buffer.buffer;
1306	if (buffer)
1307	ring_buffer_record_enable(buffer);
1308
1309	#ifdef CONFIG_TRACER_MAX_TRACE
1310	buffer = global_trace.max_buffer.buffer;
1311	if (buffer)
1312	ring_buffer_record_enable(buffer);
1313	#endif
1314
1315	arch_spin_unlock(&ftrace_max_lock);
1316
1317	ftrace_start();
1318	out:
1319	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1320	}
1321
1322	static void tracing_start_tr(struct trace_array *tr)
1323	{
1324	struct ring_buffer *buffer;
1325	unsigned long flags;
1326
1327	if (tracing_disabled)
1328	return;
1329
1330	/* If global, we need to also start the max tracer */
1331	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1332	return tracing_start();
1333
1334	raw_spin_lock_irqsave(&tr->start_lock, flags);
1335
1336	if (--tr->stop_count) {
1337	if (tr->stop_count < 0) {
1338	/* Someone screwed up their debugging */
1339	WARN_ON_ONCE(1);
1340	tr->stop_count = 0;
1341	}
1342	goto out;
1343	}
1344
1345	buffer = tr->trace_buffer.buffer;
1346	if (buffer)
1347	ring_buffer_record_enable(buffer);
1348
1349	out:
1350	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1351	}
1352
1353	/**
1354	* tracing_stop - quick stop of the tracer
1355	*
1356	* Light weight way to stop tracing. Use in conjunction with
1357	* tracing_start.
1358	*/
1359	void tracing_stop(void)
1360	{
1361	struct ring_buffer *buffer;
1362	unsigned long flags;
1363
1364	ftrace_stop();
1365	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1366	if (global_trace.stop_count++)
1367	goto out;
1368
1369	/* Prevent the buffers from switching */
1370	arch_spin_lock(&ftrace_max_lock);
1371
1372	buffer = global_trace.trace_buffer.buffer;
1373	if (buffer)
1374	ring_buffer_record_disable(buffer);
1375
1376	#ifdef CONFIG_TRACER_MAX_TRACE
1377	buffer = global_trace.max_buffer.buffer;
1378	if (buffer)
1379	ring_buffer_record_disable(buffer);
1380	#endif
1381
1382	arch_spin_unlock(&ftrace_max_lock);
1383
1384	out:
1385	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1386	}
1387
1388	static void tracing_stop_tr(struct trace_array *tr)
1389	{
1390	struct ring_buffer *buffer;
1391	unsigned long flags;
1392
1393	/* If global, we need to also stop the max tracer */
1394	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1395	return tracing_stop();
1396
1397	raw_spin_lock_irqsave(&tr->start_lock, flags);
1398	if (tr->stop_count++)
1399	goto out;
1400
1401	buffer = tr->trace_buffer.buffer;
1402	if (buffer)
1403	ring_buffer_record_disable(buffer);
1404
1405	out:
1406	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1407	}
1408
1409	void trace_stop_cmdline_recording(void);
1410
1411	static void trace_save_cmdline(struct task_struct *tsk)
1412	{
1413	unsigned pid, idx;
1414
1415	if (!tsk->pid \|\| unlikely(tsk->pid > PID_MAX_DEFAULT))
1416	return;
1417
1418	/*
1419	* It's not the end of the world if we don't get
1420	* the lock, but we also don't want to spin
1421	* nor do we want to disable interrupts,
1422	* so if we miss here, then better luck next time.
1423	*/
1424	if (!arch_spin_trylock(&trace_cmdline_lock))
1425	return;
1426
1427	idx = map_pid_to_cmdline[tsk->pid];
1428	if (idx == NO_CMDLINE_MAP) {
1429	idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1430
1431	/*
1432	* Check whether the cmdline buffer at idx has a pid
1433	* mapped. We are going to overwrite that entry so we
1434	* need to clear the map_pid_to_cmdline. Otherwise we
1435	* would read the new comm for the old pid.
1436	*/
1437	pid = map_cmdline_to_pid[idx];
1438	if (pid != NO_CMDLINE_MAP)
1439	map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1440
1441	map_cmdline_to_pid[idx] = tsk->pid;
1442	map_pid_to_cmdline[tsk->pid] = idx;
1443
1444	cmdline_idx = idx;
1445	}
1446
1447	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1448
1449	arch_spin_unlock(&trace_cmdline_lock);
1450	}
1451
1452	void trace_find_cmdline(int pid, char comm[])
1453	{
1454	unsigned map;
1455
1456	if (!pid) {
1457	strcpy(comm, "<idle>");
1458	return;
1459	}
1460
1461	if (WARN_ON_ONCE(pid < 0)) {
1462	strcpy(comm, "<XXX>");
1463	return;
1464	}
1465
1466	if (pid > PID_MAX_DEFAULT) {
1467	strcpy(comm, "<...>");
1468	return;
1469	}
1470
1471	preempt_disable();
1472	arch_spin_lock(&trace_cmdline_lock);
1473	map = map_pid_to_cmdline[pid];
1474	if (map != NO_CMDLINE_MAP)
1475	strcpy(comm, saved_cmdlines[map]);
1476	else
1477	strcpy(comm, "<...>");
1478
1479	arch_spin_unlock(&trace_cmdline_lock);
1480	preempt_enable();
1481	}
1482
1483	void tracing_record_cmdline(struct task_struct *tsk)
1484	{
1485	if (atomic_read(&trace_record_cmdline_disabled) \|\| !tracing_is_on())
1486	return;
1487
1488	if (!__this_cpu_read(trace_cmdline_save))
1489	return;
1490
1491	__this_cpu_write(trace_cmdline_save, false);
1492
1493	trace_save_cmdline(tsk);
1494	}
1495
1496	void
1497	tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1498	int pc)
1499	{
1500	struct task_struct *tsk = current;
1501
1502	entry->preempt_count = pc & 0xff;
1503	entry->pid = (tsk) ? tsk->pid : 0;
1504	entry->flags =
1505	#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1506	(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) \|
1507	#else
1508	TRACE_FLAG_IRQS_NOSUPPORT \|
1509	#endif
1510	((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) \|
1511	((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) \|
1512	(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1513	}
1514	EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1515
1516	struct ring_buffer_event *
1517	trace_buffer_lock_reserve(struct ring_buffer *buffer,
1518	int type,
1519	unsigned long len,
1520	unsigned long flags, int pc)
1521	{
1522	struct ring_buffer_event *event;
1523
1524	event = ring_buffer_lock_reserve(buffer, len);
1525	if (event != NULL) {
1526	struct trace_entry *ent = ring_buffer_event_data(event);
1527
1528	tracing_generic_entry_update(ent, flags, pc);
1529	ent->type = type;
1530	}
1531
1532	return event;
1533	}
1534
1535	void
1536	__buffer_unlock_commit(struct ring_buffer buffer, struct ring_buffer_event event)
1537	{
1538	__this_cpu_write(trace_cmdline_save, true);
1539	ring_buffer_unlock_commit(buffer, event);
1540	}
1541
1542	static inline void
1543	__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1544	struct ring_buffer_event *event,
1545	unsigned long flags, int pc)
1546	{
1547	__buffer_unlock_commit(buffer, event);
1548
1549	ftrace_trace_stack(buffer, flags, 6, pc);
1550	ftrace_trace_userstack(buffer, flags, pc);
1551	}
1552
1553	void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1554	struct ring_buffer_event *event,
1555	unsigned long flags, int pc)
1556	{
1557	__trace_buffer_unlock_commit(buffer, event, flags, pc);
1558	}
1559	EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1560
1561	struct ring_buffer_event *
1562	trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1563	struct ftrace_event_file *ftrace_file,
1564	int type, unsigned long len,
1565	unsigned long flags, int pc)
1566	{
1567	*current_rb = ftrace_file->tr->trace_buffer.buffer;
1568	return trace_buffer_lock_reserve(*current_rb,
1569	type, len, flags, pc);
1570	}
1571	EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1572
1573	struct ring_buffer_event *
1574	trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1575	int type, unsigned long len,
1576	unsigned long flags, int pc)
1577	{
1578	*current_rb = global_trace.trace_buffer.buffer;
1579	return trace_buffer_lock_reserve(*current_rb,
1580	type, len, flags, pc);
1581	}
1582	EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1583
1584	void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1585	struct ring_buffer_event *event,
1586	unsigned long flags, int pc)
1587	{
1588	__trace_buffer_unlock_commit(buffer, event, flags, pc);
1589	}
1590	EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1591
1592	void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1593	struct ring_buffer_event *event,
1594	unsigned long flags, int pc,
1595	struct pt_regs *regs)
1596	{
1597	__buffer_unlock_commit(buffer, event);
1598
1599	ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1600	ftrace_trace_userstack(buffer, flags, pc);
1601	}
1602	EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1603
1604	void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1605	struct ring_buffer_event *event)
1606	{
1607	ring_buffer_discard_commit(buffer, event);
1608	}
1609	EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1610
1611	void
1612	trace_function(struct trace_array *tr,
1613	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1614	int pc)
1615	{
1616	struct ftrace_event_call *call = &event_function;
1617	struct ring_buffer *buffer = tr->trace_buffer.buffer;
1618	struct ring_buffer_event *event;
1619	struct ftrace_entry *entry;
1620
1621	/* If we are reading the ring buffer, don't trace */
1622	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1623	return;
1624
1625	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1626	flags, pc);
1627	if (!event)
1628	return;
1629	entry = ring_buffer_event_data(event);
1630	entry->ip = ip;
1631	entry->parent_ip = parent_ip;
1632
1633	if (!filter_check_discard(call, entry, buffer, event))
1634	__buffer_unlock_commit(buffer, event);
1635	}
1636
1637	#ifdef CONFIG_STACKTRACE
1638
1639	#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1640	struct ftrace_stack {
1641	unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1642	};
1643
1644	static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1645	static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1646
1647	static void __ftrace_trace_stack(struct ring_buffer *buffer,
1648	unsigned long flags,
1649	int skip, int pc, struct pt_regs *regs)
1650	{
1651	struct ftrace_event_call *call = &event_kernel_stack;
1652	struct ring_buffer_event *event;
1653	struct stack_entry *entry;
1654	struct stack_trace trace;
1655	int use_stack;
1656	int size = FTRACE_STACK_ENTRIES;
1657
1658	trace.nr_entries = 0;
1659	trace.skip = skip;
1660
1661	/*
1662	* Since events can happen in NMIs there's no safe way to
1663	* use the per cpu ftrace_stacks. We reserve it and if an interrupt
1664	* or NMI comes in, it will just have to use the default
1665	* FTRACE_STACK_SIZE.
1666	*/
1667	preempt_disable_notrace();
1668
1669	use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1670	/*
1671	* We don't need any atomic variables, just a barrier.
1672	* If an interrupt comes in, we don't care, because it would
1673	* have exited and put the counter back to what we want.
1674	* We just need a barrier to keep gcc from moving things
1675	* around.
1676	*/
1677	barrier();
1678	if (use_stack == 1) {
1679	trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1680	trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1681
1682	if (regs)
1683	save_stack_trace_regs(regs, &trace);
1684	else
1685	save_stack_trace(&trace);
1686
1687	if (trace.nr_entries > size)
1688	size = trace.nr_entries;
1689	} else
1690	/* From now on, use_stack is a boolean */
1691	use_stack = 0;
1692
1693	size *= sizeof(unsigned long);
1694
1695	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1696	sizeof(*entry) + size, flags, pc);
1697	if (!event)
1698	goto out;
1699	entry = ring_buffer_event_data(event);
1700
1701	memset(&entry->caller, 0, size);
1702
1703	if (use_stack)
1704	memcpy(&entry->caller, trace.entries,
1705	trace.nr_entries * sizeof(unsigned long));
1706	else {
1707	trace.max_entries = FTRACE_STACK_ENTRIES;
1708	trace.entries = entry->caller;
1709	if (regs)
1710	save_stack_trace_regs(regs, &trace);
1711	else
1712	save_stack_trace(&trace);
1713	}
1714
1715	entry->size = trace.nr_entries;
1716
1717	if (!filter_check_discard(call, entry, buffer, event))
1718	__buffer_unlock_commit(buffer, event);
1719
1720	out:
1721	/* Again, don't let gcc optimize things here */
1722	barrier();
1723	__this_cpu_dec(ftrace_stack_reserve);
1724	preempt_enable_notrace();
1725
1726	}
1727
1728	void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1729	int skip, int pc, struct pt_regs *regs)
1730	{
1731	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1732	return;
1733
1734	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
1735	}
1736
1737	void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1738	int skip, int pc)
1739	{
1740	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1741	return;
1742
1743	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1744	}
1745
1746	void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1747	int pc)
1748	{
1749	__ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1750	}
1751
1752	/**
1753	* trace_dump_stack - record a stack back trace in the trace buffer
1754	* @skip: Number of functions to skip (helper handlers)
1755	*/
1756	void trace_dump_stack(int skip)
1757	{
1758	unsigned long flags;
1759
1760	if (tracing_disabled \|\| tracing_selftest_running)
1761	return;
1762
1763	local_save_flags(flags);
1764
1765	/*
1766	* Skip 3 more, seems to get us at the caller of
1767	* this function.
1768	*/
1769	skip += 3;
1770	__ftrace_trace_stack(global_trace.trace_buffer.buffer,
1771	flags, skip, preempt_count(), NULL);
1772	}
1773
1774	static DEFINE_PER_CPU(int, user_stack_count);
1775
1776	void
1777	ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1778	{
1779	struct ftrace_event_call *call = &event_user_stack;
1780	struct ring_buffer_event *event;
1781	struct userstack_entry *entry;
1782	struct stack_trace trace;
1783
1784	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1785	return;
1786
1787	/*
1788	* NMIs can not handle page faults, even with fix ups.
1789	* The save user stack can (and often does) fault.
1790	*/
1791	if (unlikely(in_nmi()))
1792	return;
1793
1794	/*
1795	* prevent recursion, since the user stack tracing may
1796	* trigger other kernel events.
1797	*/
1798	preempt_disable();
1799	if (__this_cpu_read(user_stack_count))
1800	goto out;
1801
1802	__this_cpu_inc(user_stack_count);
1803
1804	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1805	sizeof(*entry), flags, pc);
1806	if (!event)
1807	goto out_drop_count;
1808	entry = ring_buffer_event_data(event);
1809
1810	entry->tgid = current->tgid;
1811	memset(&entry->caller, 0, sizeof(entry->caller));
1812
1813	trace.nr_entries = 0;
1814	trace.max_entries = FTRACE_STACK_ENTRIES;
1815	trace.skip = 0;
1816	trace.entries = entry->caller;
1817
1818	save_stack_trace_user(&trace);
1819	if (!filter_check_discard(call, entry, buffer, event))
1820	__buffer_unlock_commit(buffer, event);
1821
1822	out_drop_count:
1823	__this_cpu_dec(user_stack_count);
1824	out:
1825	preempt_enable();
1826	}
1827
1828	#ifdef UNUSED
1829	static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1830	{
1831	ftrace_trace_userstack(tr, flags, preempt_count());
1832	}
1833	#endif /* UNUSED */
1834
1835	#endif /* CONFIG_STACKTRACE */
1836
1837	/* created for use with alloc_percpu */
1838	struct trace_buffer_struct {
1839	char buffer[TRACE_BUF_SIZE];
1840	};
1841
1842	static struct trace_buffer_struct *trace_percpu_buffer;
1843	static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1844	static struct trace_buffer_struct *trace_percpu_irq_buffer;
1845	static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1846
1847	/*
1848	* The buffer used is dependent on the context. There is a per cpu
1849	* buffer for normal context, softirq contex, hard irq context and
1850	* for NMI context. Thise allows for lockless recording.
1851	*
1852	* Note, if the buffers failed to be allocated, then this returns NULL
1853	*/
1854	static char *get_trace_buf(void)
1855	{
1856	struct trace_buffer_struct *percpu_buffer;
1857
1858	/*
1859	* If we have allocated per cpu buffers, then we do not
1860	* need to do any locking.
1861	*/
1862	if (in_nmi())
1863	percpu_buffer = trace_percpu_nmi_buffer;
1864	else if (in_irq())
1865	percpu_buffer = trace_percpu_irq_buffer;
1866	else if (in_softirq())
1867	percpu_buffer = trace_percpu_sirq_buffer;
1868	else
1869	percpu_buffer = trace_percpu_buffer;
1870
1871	if (!percpu_buffer)
1872	return NULL;
1873
1874	return this_cpu_ptr(&percpu_buffer->buffer[0]);
1875	}
1876
1877	static int alloc_percpu_trace_buffer(void)
1878	{
1879	struct trace_buffer_struct *buffers;
1880	struct trace_buffer_struct *sirq_buffers;
1881	struct trace_buffer_struct *irq_buffers;
1882	struct trace_buffer_struct *nmi_buffers;
1883
1884	buffers = alloc_percpu(struct trace_buffer_struct);
1885	if (!buffers)
1886	goto err_warn;
1887
1888	sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1889	if (!sirq_buffers)
1890	goto err_sirq;
1891
1892	irq_buffers = alloc_percpu(struct trace_buffer_struct);
1893	if (!irq_buffers)
1894	goto err_irq;
1895
1896	nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1897	if (!nmi_buffers)
1898	goto err_nmi;
1899
1900	trace_percpu_buffer = buffers;
1901	trace_percpu_sirq_buffer = sirq_buffers;
1902	trace_percpu_irq_buffer = irq_buffers;
1903	trace_percpu_nmi_buffer = nmi_buffers;
1904
1905	return 0;
1906
1907	err_nmi:
1908	free_percpu(irq_buffers);
1909	err_irq:
1910	free_percpu(sirq_buffers);
1911	err_sirq:
1912	free_percpu(buffers);
1913	err_warn:
1914	WARN(1, "Could not allocate percpu trace_printk buffer");
1915	return -ENOMEM;
1916	}
1917
1918	static int buffers_allocated;
1919
1920	void trace_printk_init_buffers(void)
1921	{
1922	if (buffers_allocated)
1923	return;
1924
1925	if (alloc_percpu_trace_buffer())
1926	return;
1927
1928	pr_info("ftrace: Allocated trace_printk buffers\n");
1929
1930	/* Expand the buffers to set size */
1931	tracing_update_buffers();
1932
1933	buffers_allocated = 1;
1934
1935	/*
1936	* trace_printk_init_buffers() can be called by modules.
1937	* If that happens, then we need to start cmdline recording
1938	* directly here. If the global_trace.buffer is already
1939	* allocated here, then this was called by module code.
1940	*/
1941	if (global_trace.trace_buffer.buffer)
1942	tracing_start_cmdline_record();
1943	}
1944
1945	void trace_printk_start_comm(void)
1946	{
1947	/* Start tracing comms if trace printk is set */
1948	if (!buffers_allocated)
1949	return;
1950	tracing_start_cmdline_record();
1951	}
1952
1953	static void trace_printk_start_stop_comm(int enabled)
1954	{
1955	if (!buffers_allocated)
1956	return;
1957
1958	if (enabled)
1959	tracing_start_cmdline_record();
1960	else
1961	tracing_stop_cmdline_record();
1962	}
1963
1964	/**
1965	* trace_vbprintk - write binary msg to tracing buffer
1966	*
1967	*/
1968	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1969	{
1970	struct ftrace_event_call *call = &event_bprint;
1971	struct ring_buffer_event *event;
1972	struct ring_buffer *buffer;
1973	struct trace_array *tr = &global_trace;
1974	struct bprint_entry *entry;
1975	unsigned long flags;
1976	char *tbuffer;
1977	int len = 0, size, pc;
1978
1979	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
1980	return 0;
1981
1982	/* Don't pollute graph traces with trace_vprintk internals */
1983	pause_graph_tracing();
1984
1985	pc = preempt_count();
1986	preempt_disable_notrace();
1987
1988	tbuffer = get_trace_buf();
1989	if (!tbuffer) {
1990	len = 0;
1991	goto out;
1992	}
1993
1994	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1995
1996	if (len > TRACE_BUF_SIZE/sizeof(int) \|\| len < 0)
1997	goto out;
1998
1999	local_save_flags(flags);
2000	size = sizeof(entry) + sizeof(u32) len;
2001	buffer = tr->trace_buffer.buffer;
2002	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2003	flags, pc);
2004	if (!event)
2005	goto out;
2006	entry = ring_buffer_event_data(event);
2007	entry->ip = ip;
2008	entry->fmt = fmt;
2009
2010	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2011	if (!filter_check_discard(call, entry, buffer, event)) {
2012	__buffer_unlock_commit(buffer, event);
2013	ftrace_trace_stack(buffer, flags, 6, pc);
2014	}
2015
2016	out:
2017	preempt_enable_notrace();
2018	unpause_graph_tracing();
2019
2020	return len;
2021	}
2022	EXPORT_SYMBOL_GPL(trace_vbprintk);
2023
2024	static int
2025	__trace_array_vprintk(struct ring_buffer *buffer,
2026	unsigned long ip, const char *fmt, va_list args)
2027	{
2028	struct ftrace_event_call *call = &event_print;
2029	struct ring_buffer_event *event;
2030	int len = 0, size, pc;
2031	struct print_entry *entry;
2032	unsigned long flags;
2033	char *tbuffer;
2034
2035	if (tracing_disabled \|\| tracing_selftest_running)
2036	return 0;
2037
2038	/* Don't pollute graph traces with trace_vprintk internals */
2039	pause_graph_tracing();
2040
2041	pc = preempt_count();
2042	preempt_disable_notrace();
2043
2044
2045	tbuffer = get_trace_buf();
2046	if (!tbuffer) {
2047	len = 0;
2048	goto out;
2049	}
2050
2051	len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2052	if (len > TRACE_BUF_SIZE)
2053	goto out;
2054
2055	local_save_flags(flags);
2056	size = sizeof(*entry) + len + 1;
2057	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2058	flags, pc);
2059	if (!event)
2060	goto out;
2061	entry = ring_buffer_event_data(event);
2062	entry->ip = ip;
2063
2064	memcpy(&entry->buf, tbuffer, len);
2065	entry->buf[len] = '\0';
2066	if (!filter_check_discard(call, entry, buffer, event)) {
2067	__buffer_unlock_commit(buffer, event);
2068	ftrace_trace_stack(buffer, flags, 6, pc);
2069	}
2070	out:
2071	preempt_enable_notrace();
2072	unpause_graph_tracing();
2073
2074	return len;
2075	}
2076
2077	int trace_array_vprintk(struct trace_array *tr,
2078	unsigned long ip, const char *fmt, va_list args)
2079	{
2080	return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2081	}
2082
2083	int trace_array_printk(struct trace_array *tr,
2084	unsigned long ip, const char *fmt, ...)
2085	{
2086	int ret;
2087	va_list ap;
2088
2089	if (!(trace_flags & TRACE_ITER_PRINTK))
2090	return 0;
2091
2092	va_start(ap, fmt);
2093	ret = trace_array_vprintk(tr, ip, fmt, ap);
2094	va_end(ap);
2095	return ret;
2096	}
2097
2098	int trace_array_printk_buf(struct ring_buffer *buffer,
2099	unsigned long ip, const char *fmt, ...)
2100	{
2101	int ret;
2102	va_list ap;
2103
2104	if (!(trace_flags & TRACE_ITER_PRINTK))
2105	return 0;
2106
2107	va_start(ap, fmt);
2108	ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2109	va_end(ap);
2110	return ret;
2111	}
2112
2113	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2114	{
2115	return trace_array_vprintk(&global_trace, ip, fmt, args);
2116	}
2117	EXPORT_SYMBOL_GPL(trace_vprintk);
2118
2119	static void trace_iterator_increment(struct trace_iterator *iter)
2120	{
2121	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2122
2123	iter->idx++;
2124	if (buf_iter)
2125	ring_buffer_read(buf_iter, NULL);
2126	}
2127
2128	static struct trace_entry *
2129	peek_next_entry(struct trace_iterator iter, int cpu, u64 ts,
2130	unsigned long *lost_events)
2131	{
2132	struct ring_buffer_event *event;
2133	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2134
2135	if (buf_iter)
2136	event = ring_buffer_iter_peek(buf_iter, ts);
2137	else
2138	event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2139	lost_events);
2140
2141	if (event) {
2142	iter->ent_size = ring_buffer_event_length(event);
2143	return ring_buffer_event_data(event);
2144	}
2145	iter->ent_size = 0;
2146	return NULL;
2147	}
2148
2149	static struct trace_entry *
2150	__find_next_entry(struct trace_iterator iter, int ent_cpu,
2151	unsigned long missing_events, u64 ent_ts)
2152	{
2153	struct ring_buffer *buffer = iter->trace_buffer->buffer;
2154	struct trace_entry ent, next = NULL;
2155	unsigned long lost_events = 0, next_lost = 0;
2156	int cpu_file = iter->cpu_file;
2157	u64 next_ts = 0, ts;
2158	int next_cpu = -1;
2159	int next_size = 0;
2160	int cpu;
2161
2162	/*
2163	* If we are in a per_cpu trace file, don't bother by iterating over
2164	* all cpu and peek directly.
2165	*/
2166	if (cpu_file > RING_BUFFER_ALL_CPUS) {
2167	if (ring_buffer_empty_cpu(buffer, cpu_file))
2168	return NULL;
2169	ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2170	if (ent_cpu)
2171	*ent_cpu = cpu_file;
2172
2173	return ent;
2174	}
2175
2176	for_each_tracing_cpu(cpu) {
2177
2178	if (ring_buffer_empty_cpu(buffer, cpu))
2179	continue;
2180
2181	ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2182
2183	/*
2184	* Pick the entry with the smallest timestamp:
2185	*/
2186	if (ent && (!next \|\| ts < next_ts)) {
2187	next = ent;
2188	next_cpu = cpu;
2189	next_ts = ts;
2190	next_lost = lost_events;
2191	next_size = iter->ent_size;
2192	}
2193	}
2194
2195	iter->ent_size = next_size;
2196
2197	if (ent_cpu)
2198	*ent_cpu = next_cpu;
2199
2200	if (ent_ts)
2201	*ent_ts = next_ts;
2202
2203	if (missing_events)
2204	*missing_events = next_lost;
2205
2206	return next;
2207	}
2208
2209	/* Find the next real entry, without updating the iterator itself */
2210	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
2211	int ent_cpu, u64 ent_ts)
2212	{
2213	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2214	}
2215
2216	/* Find the next real entry, and increment the iterator to the next entry */
2217	void trace_find_next_entry_inc(struct trace_iterator iter)
2218	{
2219	iter->ent = __find_next_entry(iter, &iter->cpu,
2220	&iter->lost_events, &iter->ts);
2221
2222	if (iter->ent)
2223	trace_iterator_increment(iter);
2224
2225	return iter->ent ? iter : NULL;
2226	}
2227
2228	static void trace_consume(struct trace_iterator *iter)
2229	{
2230	ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2231	&iter->lost_events);
2232	}
2233
2234	static void s_next(struct seq_file m, void v, loff_t pos)
2235	{
2236	struct trace_iterator *iter = m->private;
2237	int i = (int)*pos;
2238	void *ent;
2239
2240	WARN_ON_ONCE(iter->leftover);
2241
2242	(*pos)++;
2243
2244	/* can't go backwards */
2245	if (iter->idx > i)
2246	return NULL;
2247
2248	if (iter->idx < 0)
2249	ent = trace_find_next_entry_inc(iter);
2250	else
2251	ent = iter;
2252
2253	while (ent && iter->idx < i)
2254	ent = trace_find_next_entry_inc(iter);
2255
2256	iter->pos = *pos;
2257
2258	return ent;
2259	}
2260
2261	void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2262	{
2263	struct ring_buffer_event *event;
2264	struct ring_buffer_iter *buf_iter;
2265	unsigned long entries = 0;
2266	u64 ts;
2267
2268	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2269
2270	buf_iter = trace_buffer_iter(iter, cpu);
2271	if (!buf_iter)
2272	return;
2273
2274	ring_buffer_iter_reset(buf_iter);
2275
2276	/*
2277	* We could have the case with the max latency tracers
2278	* that a reset never took place on a cpu. This is evident
2279	* by the timestamp being before the start of the buffer.
2280	*/
2281	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2282	if (ts >= iter->trace_buffer->time_start)
2283	break;
2284	entries++;
2285	ring_buffer_read(buf_iter, NULL);
2286	}
2287
2288	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2289	}
2290
2291	/*
2292	* The current tracer is copied to avoid a global locking
2293	* all around.
2294	*/
2295	static void s_start(struct seq_file m, loff_t *pos)
2296	{
2297	struct trace_iterator *iter = m->private;
2298	struct trace_array *tr = iter->tr;
2299	int cpu_file = iter->cpu_file;
2300	void *p = NULL;
2301	loff_t l = 0;
2302	int cpu;
2303
2304	/*
2305	* copy the tracer to avoid using a global lock all around.
2306	* iter->trace is a copy of current_trace, the pointer to the
2307	* name may be used instead of a strcmp(), as iter->trace->name
2308	* will point to the same string as current_trace->name.
2309	*/
2310	mutex_lock(&trace_types_lock);
2311	if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2312	iter->trace = tr->current_trace;
2313	mutex_unlock(&trace_types_lock);
2314
2315	#ifdef CONFIG_TRACER_MAX_TRACE
2316	if (iter->snapshot && iter->trace->use_max_tr)
2317	return ERR_PTR(-EBUSY);
2318	#endif
2319
2320	if (!iter->snapshot)
2321	atomic_inc(&trace_record_cmdline_disabled);
2322
2323	if (*pos != iter->pos) {
2324	iter->ent = NULL;
2325	iter->cpu = 0;
2326	iter->idx = -1;
2327
2328	if (cpu_file == RING_BUFFER_ALL_CPUS) {
2329	for_each_tracing_cpu(cpu)
2330	tracing_iter_reset(iter, cpu);
2331	} else
2332	tracing_iter_reset(iter, cpu_file);
2333
2334	iter->leftover = 0;
2335	for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2336	;
2337
2338	} else {
2339	/*
2340	* If we overflowed the seq_file before, then we want
2341	* to just reuse the trace_seq buffer again.
2342	*/
2343	if (iter->leftover)
2344	p = iter;
2345	else {
2346	l = *pos - 1;
2347	p = s_next(m, p, &l);
2348	}
2349	}
2350
2351	trace_event_read_lock();
2352	trace_access_lock(cpu_file);
2353	return p;
2354	}
2355
2356	static void s_stop(struct seq_file m, void p)
2357	{
2358	struct trace_iterator *iter = m->private;
2359
2360	#ifdef CONFIG_TRACER_MAX_TRACE
2361	if (iter->snapshot && iter->trace->use_max_tr)
2362	return;
2363	#endif
2364
2365	if (!iter->snapshot)
2366	atomic_dec(&trace_record_cmdline_disabled);
2367
2368	trace_access_unlock(iter->cpu_file);
2369	trace_event_read_unlock();
2370	}
2371
2372	static void
2373	get_total_entries(struct trace_buffer *buf,
2374	unsigned long total, unsigned long entries)
2375	{
2376	unsigned long count;
2377	int cpu;
2378
2379	*total = 0;
2380	*entries = 0;
2381
2382	for_each_tracing_cpu(cpu) {
2383	count = ring_buffer_entries_cpu(buf->buffer, cpu);
2384	/*
2385	* If this buffer has skipped entries, then we hold all
2386	* entries for the trace and we need to ignore the
2387	* ones before the time stamp.
2388	*/
2389	if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2390	count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2391	/* total is the same as the entries */
2392	*total += count;
2393	} else
2394	*total += count +
2395	ring_buffer_overrun_cpu(buf->buffer, cpu);
2396	*entries += count;
2397	}
2398	}
2399
2400	static void print_lat_help_header(struct seq_file *m)
2401	{
2402	seq_puts(m, "# _------=> CPU# \n");
2403	seq_puts(m, "# / _-----=> irqs-off \n");
2404	seq_puts(m, "# \| / _----=> need-resched \n");
2405	seq_puts(m, "# \|\| / _---=> hardirq/softirq \n");
2406	seq_puts(m, "# \|\|\| / _--=> preempt-depth \n");
2407	seq_puts(m, "# \|\|\|\| / delay \n");
2408	seq_puts(m, "# cmd pid \|\|\|\|\| time \| caller \n");
2409	seq_puts(m, "# \\ / \|\|\|\|\| \\ \| / \n");
2410	}
2411
2412	static void print_event_info(struct trace_buffer buf, struct seq_file m)
2413	{
2414	unsigned long total;
2415	unsigned long entries;
2416
2417	get_total_entries(buf, &total, &entries);
2418	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2419	entries, total, num_online_cpus());
2420	seq_puts(m, "#\n");
2421	}
2422
2423	static void print_func_help_header(struct trace_buffer buf, struct seq_file m)
2424	{
2425	print_event_info(buf, m);
2426	seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2427	seq_puts(m, "# \| \| \| \| \|\n");
2428	}
2429
2430	static void print_func_help_header_irq(struct trace_buffer buf, struct seq_file m)
2431	{
2432	print_event_info(buf, m);
2433	seq_puts(m, "# _-----=> irqs-off\n");
2434	seq_puts(m, "# / _----=> need-resched\n");
2435	seq_puts(m, "# \| / _---=> hardirq/softirq\n");
2436	seq_puts(m, "# \|\| / _--=> preempt-depth\n");
2437	seq_puts(m, "# \|\|\| / delay\n");
2438	seq_puts(m, "# TASK-PID CPU# \|\|\|\| TIMESTAMP FUNCTION\n");
2439	seq_puts(m, "# \| \| \| \|\|\|\| \| \|\n");
2440	}
2441
2442	void
2443	print_trace_header(struct seq_file m, struct trace_iterator iter)
2444	{
2445	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2446	struct trace_buffer *buf = iter->trace_buffer;
2447	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2448	struct tracer *type = iter->trace;
2449	unsigned long entries;
2450	unsigned long total;
2451	const char *name = "preemption";
2452
2453	name = type->name;
2454
2455	get_total_entries(buf, &total, &entries);
2456
2457	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2458	name, UTS_RELEASE);
2459	seq_puts(m, "# -----------------------------------"
2460	"---------------------------------\n");
2461	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d \|"
2462	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2463	nsecs_to_usecs(data->saved_latency),
2464	entries,
2465	total,
2466	buf->cpu,
2467	#if defined(CONFIG_PREEMPT_NONE)
2468	"server",
2469	#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2470	"desktop",
2471	#elif defined(CONFIG_PREEMPT)
2472	"preempt",
2473	#else
2474	"unknown",
2475	#endif
2476	/* These are reserved for later use */
2477	0, 0, 0, 0);
2478	#ifdef CONFIG_SMP
2479	seq_printf(m, " #P:%d)\n", num_online_cpus());
2480	#else
2481	seq_puts(m, ")\n");
2482	#endif
2483	seq_puts(m, "# -----------------\n");
2484	seq_printf(m, "# \| task: %.16s-%d "
2485	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2486	data->comm, data->pid,
2487	from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2488	data->policy, data->rt_priority);
2489	seq_puts(m, "# -----------------\n");
2490
2491	if (data->critical_start) {
2492	seq_puts(m, "# => started at: ");
2493	seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2494	trace_print_seq(m, &iter->seq);
2495	seq_puts(m, "\n# => ended at: ");
2496	seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2497	trace_print_seq(m, &iter->seq);
2498	seq_puts(m, "\n#\n");
2499	}
2500
2501	seq_puts(m, "#\n");
2502	}
2503
2504	static void test_cpu_buff_start(struct trace_iterator *iter)
2505	{
2506	struct trace_seq *s = &iter->seq;
2507
2508	if (!(trace_flags & TRACE_ITER_ANNOTATE))
2509	return;
2510
2511	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2512	return;
2513
2514	if (cpumask_test_cpu(iter->cpu, iter->started))
2515	return;
2516
2517	if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2518	return;
2519
2520	cpumask_set_cpu(iter->cpu, iter->started);
2521
2522	/* Don't print started cpu buffer for the first entry of the trace */
2523	if (iter->idx > 1)
2524	trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2525	iter->cpu);
2526	}
2527
2528	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2529	{
2530	struct trace_seq *s = &iter->seq;
2531	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2532	struct trace_entry *entry;
2533	struct trace_event *event;
2534
2535	entry = iter->ent;
2536
2537	test_cpu_buff_start(iter);
2538
2539	event = ftrace_find_event(entry->type);
2540
2541	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2542	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2543	if (!trace_print_lat_context(iter))
2544	goto partial;
2545	} else {
2546	if (!trace_print_context(iter))
2547	goto partial;
2548	}
2549	}
2550
2551	if (event)
2552	return event->funcs->trace(iter, sym_flags, event);
2553
2554	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2555	goto partial;
2556
2557	return TRACE_TYPE_HANDLED;
2558	partial:
2559	return TRACE_TYPE_PARTIAL_LINE;
2560	}
2561
2562	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2563	{
2564	struct trace_seq *s = &iter->seq;
2565	struct trace_entry *entry;
2566	struct trace_event *event;
2567
2568	entry = iter->ent;
2569
2570	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2571	if (!trace_seq_printf(s, "%d %d %llu ",
2572	entry->pid, iter->cpu, iter->ts))
2573	goto partial;
2574	}
2575
2576	event = ftrace_find_event(entry->type);
2577	if (event)
2578	return event->funcs->raw(iter, 0, event);
2579
2580	if (!trace_seq_printf(s, "%d ?\n", entry->type))
2581	goto partial;
2582
2583	return TRACE_TYPE_HANDLED;
2584	partial:
2585	return TRACE_TYPE_PARTIAL_LINE;
2586	}
2587
2588	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2589	{
2590	struct trace_seq *s = &iter->seq;
2591	unsigned char newline = '\n';
2592	struct trace_entry *entry;
2593	struct trace_event *event;
2594
2595	entry = iter->ent;
2596
2597	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2598	SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2599	SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2600	SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2601	}
2602
2603	event = ftrace_find_event(entry->type);
2604	if (event) {
2605	enum print_line_t ret = event->funcs->hex(iter, 0, event);
2606	if (ret != TRACE_TYPE_HANDLED)
2607	return ret;
2608	}
2609
2610	SEQ_PUT_FIELD_RET(s, newline);
2611
2612	return TRACE_TYPE_HANDLED;
2613	}
2614
2615	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2616	{
2617	struct trace_seq *s = &iter->seq;
2618	struct trace_entry *entry;
2619	struct trace_event *event;
2620
2621	entry = iter->ent;
2622
2623	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2624	SEQ_PUT_FIELD_RET(s, entry->pid);
2625	SEQ_PUT_FIELD_RET(s, iter->cpu);
2626	SEQ_PUT_FIELD_RET(s, iter->ts);
2627	}
2628
2629	event = ftrace_find_event(entry->type);
2630	return event ? event->funcs->binary(iter, 0, event) :
2631	TRACE_TYPE_HANDLED;
2632	}
2633
2634	int trace_empty(struct trace_iterator *iter)
2635	{
2636	struct ring_buffer_iter *buf_iter;
2637	int cpu;
2638
2639	/* If we are looking at one CPU buffer, only check that one */
2640	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2641	cpu = iter->cpu_file;
2642	buf_iter = trace_buffer_iter(iter, cpu);
2643	if (buf_iter) {
2644	if (!ring_buffer_iter_empty(buf_iter))
2645	return 0;
2646	} else {
2647	if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2648	return 0;
2649	}
2650	return 1;
2651	}
2652
2653	for_each_tracing_cpu(cpu) {
2654	buf_iter = trace_buffer_iter(iter, cpu);
2655	if (buf_iter) {
2656	if (!ring_buffer_iter_empty(buf_iter))
2657	return 0;
2658	} else {
2659	if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2660	return 0;
2661	}
2662	}
2663
2664	return 1;
2665	}
2666
2667	/* Called with trace_event_read_lock() held. */
2668	enum print_line_t print_trace_line(struct trace_iterator *iter)
2669	{
2670	enum print_line_t ret;
2671
2672	if (iter->lost_events &&
2673	!trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2674	iter->cpu, iter->lost_events))
2675	return TRACE_TYPE_PARTIAL_LINE;
2676
2677	if (iter->trace && iter->trace->print_line) {
2678	ret = iter->trace->print_line(iter);
2679	if (ret != TRACE_TYPE_UNHANDLED)
2680	return ret;
2681	}
2682
2683	if (iter->ent->type == TRACE_BPUTS &&
2684	trace_flags & TRACE_ITER_PRINTK &&
2685	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2686	return trace_print_bputs_msg_only(iter);
2687
2688	if (iter->ent->type == TRACE_BPRINT &&
2689	trace_flags & TRACE_ITER_PRINTK &&
2690	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2691	return trace_print_bprintk_msg_only(iter);
2692
2693	if (iter->ent->type == TRACE_PRINT &&
2694	trace_flags & TRACE_ITER_PRINTK &&
2695	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2696	return trace_print_printk_msg_only(iter);
2697
2698	if (trace_flags & TRACE_ITER_BIN)
2699	return print_bin_fmt(iter);
2700
2701	if (trace_flags & TRACE_ITER_HEX)
2702	return print_hex_fmt(iter);
2703
2704	if (trace_flags & TRACE_ITER_RAW)
2705	return print_raw_fmt(iter);
2706
2707	return print_trace_fmt(iter);
2708	}
2709
2710	void trace_latency_header(struct seq_file *m)
2711	{
2712	struct trace_iterator *iter = m->private;
2713
2714	/* print nothing if the buffers are empty */
2715	if (trace_empty(iter))
2716	return;
2717
2718	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2719	print_trace_header(m, iter);
2720
2721	if (!(trace_flags & TRACE_ITER_VERBOSE))
2722	print_lat_help_header(m);
2723	}
2724
2725	void trace_default_header(struct seq_file *m)
2726	{
2727	struct trace_iterator *iter = m->private;
2728
2729	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2730	return;
2731
2732	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2733	/* print nothing if the buffers are empty */
2734	if (trace_empty(iter))
2735	return;
2736	print_trace_header(m, iter);
2737	if (!(trace_flags & TRACE_ITER_VERBOSE))
2738	print_lat_help_header(m);
2739	} else {
2740	if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2741	if (trace_flags & TRACE_ITER_IRQ_INFO)
2742	print_func_help_header_irq(iter->trace_buffer, m);
2743	else
2744	print_func_help_header(iter->trace_buffer, m);
2745	}
2746	}
2747	}
2748
2749	static void test_ftrace_alive(struct seq_file *m)
2750	{
2751	if (!ftrace_is_dead())
2752	return;
2753	seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2754	seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2755	}
2756
2757	#ifdef CONFIG_TRACER_MAX_TRACE
2758	static void show_snapshot_main_help(struct seq_file *m)
2759	{
2760	seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2761	seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2762	seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2763	seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2764	seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2765	seq_printf(m, "# is not a '0' or '1')\n");
2766	}
2767
2768	static void show_snapshot_percpu_help(struct seq_file *m)
2769	{
2770	seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2771	#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2772	seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2773	seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2774	#else
2775	seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2776	seq_printf(m, "# Must use main snapshot file to allocate.\n");
2777	#endif
2778	seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2779	seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2780	seq_printf(m, "# is not a '0' or '1')\n");
2781	}
2782
2783	static void print_snapshot_help(struct seq_file m, struct trace_iterator iter)
2784	{
2785	if (iter->tr->allocated_snapshot)
2786	seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2787	else
2788	seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2789
2790	seq_printf(m, "# Snapshot commands:\n");
2791	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2792	show_snapshot_main_help(m);
2793	else
2794	show_snapshot_percpu_help(m);
2795	}
2796	#else
2797	/* Should never be called */
2798	static inline void print_snapshot_help(struct seq_file m, struct trace_iterator iter) { }
2799	#endif
2800
2801	static int s_show(struct seq_file m, void v)
2802	{
2803	struct trace_iterator *iter = v;
2804	int ret;
2805
2806	if (iter->ent == NULL) {
2807	if (iter->tr) {
2808	seq_printf(m, "# tracer: %s\n", iter->trace->name);
2809	seq_puts(m, "#\n");
2810	test_ftrace_alive(m);
2811	}
2812	if (iter->snapshot && trace_empty(iter))
2813	print_snapshot_help(m, iter);
2814	else if (iter->trace && iter->trace->print_header)
2815	iter->trace->print_header(m);
2816	else
2817	trace_default_header(m);
2818
2819	} else if (iter->leftover) {
2820	/*
2821	* If we filled the seq_file buffer earlier, we
2822	* want to just show it now.
2823	*/
2824	ret = trace_print_seq(m, &iter->seq);
2825
2826	/* ret should this time be zero, but you never know */
2827	iter->leftover = ret;
2828
2829	} else {
2830	print_trace_line(iter);
2831	ret = trace_print_seq(m, &iter->seq);
2832	/*
2833	* If we overflow the seq_file buffer, then it will
2834	* ask us for this data again at start up.
2835	* Use that instead.
2836	* ret is 0 if seq_file write succeeded.
2837	* -1 otherwise.
2838	*/
2839	iter->leftover = ret;
2840	}
2841
2842	return 0;
2843	}
2844
2845	/*
2846	* Should be used after trace_array_get(), trace_types_lock
2847	* ensures that i_cdev was already initialized.
2848	*/
2849	static inline int tracing_get_cpu(struct inode *inode)
2850	{
2851	if (inode->i_cdev) /* See trace_create_cpu_file() */
2852	return (long)inode->i_cdev - 1;
2853	return RING_BUFFER_ALL_CPUS;
2854	}
2855
2856	static const struct seq_operations tracer_seq_ops = {
2857	.start = s_start,
2858	.next = s_next,
2859	.stop = s_stop,
2860	.show = s_show,
2861	};
2862
2863	static struct trace_iterator *
2864	__tracing_open(struct inode inode, struct file file, bool snapshot)
2865	{
2866	struct trace_array *tr = inode->i_private;
2867	struct trace_iterator *iter;
2868	int cpu;
2869
2870	if (tracing_disabled)
2871	return ERR_PTR(-ENODEV);
2872
2873	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2874	if (!iter)
2875	return ERR_PTR(-ENOMEM);
2876
2877	iter->buffer_iter = kzalloc(sizeof(iter->buffer_iter) num_possible_cpus(),
2878	GFP_KERNEL);
2879	if (!iter->buffer_iter)
2880	goto release;
2881
2882	/*
2883	* We make a copy of the current tracer to avoid concurrent
2884	* changes on it while we are reading.
2885	*/
2886	mutex_lock(&trace_types_lock);
2887	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2888	if (!iter->trace)
2889	goto fail;
2890
2891	iter->trace = tr->current_trace;
2892
2893	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2894	goto fail;
2895
2896	iter->tr = tr;
2897
2898	#ifdef CONFIG_TRACER_MAX_TRACE
2899	/* Currently only the top directory has a snapshot */
2900	if (tr->current_trace->print_max \|\| snapshot)
2901	iter->trace_buffer = &tr->max_buffer;
2902	else
2903	#endif
2904	iter->trace_buffer = &tr->trace_buffer;
2905	iter->snapshot = snapshot;
2906	iter->pos = -1;
2907	iter->cpu_file = tracing_get_cpu(inode);
2908	mutex_init(&iter->mutex);
2909
2910	/* Notify the tracer early; before we stop tracing. */
2911	if (iter->trace && iter->trace->open)
2912	iter->trace->open(iter);
2913
2914	/* Annotate start of buffers if we had overruns */
2915	if (ring_buffer_overruns(iter->trace_buffer->buffer))
2916	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
2917
2918	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
2919	if (trace_clocks[tr->clock_id].in_ns)
2920	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
2921
2922	/* stop the trace while dumping if we are not opening "snapshot" */
2923	if (!iter->snapshot)
2924	tracing_stop_tr(tr);
2925
2926	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2927	for_each_tracing_cpu(cpu) {
2928	iter->buffer_iter[cpu] =
2929	ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2930	}
2931	ring_buffer_read_prepare_sync();
2932	for_each_tracing_cpu(cpu) {
2933	ring_buffer_read_start(iter->buffer_iter[cpu]);
2934	tracing_iter_reset(iter, cpu);
2935	}
2936	} else {
2937	cpu = iter->cpu_file;
2938	iter->buffer_iter[cpu] =
2939	ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2940	ring_buffer_read_prepare_sync();
2941	ring_buffer_read_start(iter->buffer_iter[cpu]);
2942	tracing_iter_reset(iter, cpu);
2943	}
2944
2945	mutex_unlock(&trace_types_lock);
2946
2947	return iter;
2948
2949	fail:
2950	mutex_unlock(&trace_types_lock);
2951	kfree(iter->trace);
2952	kfree(iter->buffer_iter);
2953	release:
2954	seq_release_private(inode, file);
2955	return ERR_PTR(-ENOMEM);
2956	}
2957
2958	int tracing_open_generic(struct inode inode, struct file filp)
2959	{
2960	if (tracing_disabled)
2961	return -ENODEV;
2962
2963	filp->private_data = inode->i_private;
2964	return 0;
2965	}
2966
2967	/*
2968	* Open and update trace_array ref count.
2969	* Must have the current trace_array passed to it.
2970	*/
2971	static int tracing_open_generic_tr(struct inode inode, struct file filp)
2972	{
2973	struct trace_array *tr = inode->i_private;
2974
2975	if (tracing_disabled)
2976	return -ENODEV;
2977
2978	if (trace_array_get(tr) < 0)
2979	return -ENODEV;
2980
2981	filp->private_data = inode->i_private;
2982
2983	return 0;
2984	}
2985
2986	static int tracing_release(struct inode inode, struct file file)
2987	{
2988	struct trace_array *tr = inode->i_private;
2989	struct seq_file *m = file->private_data;
2990	struct trace_iterator *iter;
2991	int cpu;
2992
2993	if (!(file->f_mode & FMODE_READ)) {
2994	trace_array_put(tr);
2995	return 0;
2996	}
2997
2998	/* Writes do not use seq_file */
2999	iter = m->private;
3000	mutex_lock(&trace_types_lock);
3001
3002	for_each_tracing_cpu(cpu) {
3003	if (iter->buffer_iter[cpu])
3004	ring_buffer_read_finish(iter->buffer_iter[cpu]);
3005	}
3006
3007	if (iter->trace && iter->trace->close)
3008	iter->trace->close(iter);
3009
3010	if (!iter->snapshot)
3011	/* reenable tracing if it was previously enabled */
3012	tracing_start_tr(tr);
3013
3014	__trace_array_put(tr);
3015
3016	mutex_unlock(&trace_types_lock);
3017
3018	mutex_destroy(&iter->mutex);
3019	free_cpumask_var(iter->started);
3020	kfree(iter->trace);
3021	kfree(iter->buffer_iter);
3022	seq_release_private(inode, file);
3023
3024	return 0;
3025	}
3026
3027	static int tracing_release_generic_tr(struct inode inode, struct file file)
3028	{
3029	struct trace_array *tr = inode->i_private;
3030
3031	trace_array_put(tr);
3032	return 0;
3033	}
3034
3035	static int tracing_single_release_tr(struct inode inode, struct file file)
3036	{
3037	struct trace_array *tr = inode->i_private;
3038
3039	trace_array_put(tr);
3040
3041	return single_release(inode, file);
3042	}
3043
3044	static int tracing_open(struct inode inode, struct file file)
3045	{
3046	struct trace_array *tr = inode->i_private;
3047	struct trace_iterator *iter;
3048	int ret = 0;
3049
3050	if (trace_array_get(tr) < 0)
3051	return -ENODEV;
3052
3053	/* If this file was open for write, then erase contents */
3054	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3055	int cpu = tracing_get_cpu(inode);
3056
3057	if (cpu == RING_BUFFER_ALL_CPUS)
3058	tracing_reset_online_cpus(&tr->trace_buffer);
3059	else
3060	tracing_reset(&tr->trace_buffer, cpu);
3061	}
3062
3063	if (file->f_mode & FMODE_READ) {
3064	iter = __tracing_open(inode, file, false);
3065	if (IS_ERR(iter))
3066	ret = PTR_ERR(iter);
3067	else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3068	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
3069	}
3070
3071	if (ret < 0)
3072	trace_array_put(tr);
3073
3074	return ret;
3075	}
3076
3077	static void *
3078	t_next(struct seq_file m, void v, loff_t *pos)
3079	{
3080	struct tracer *t = v;
3081
3082	(*pos)++;
3083
3084	if (t)
3085	t = t->next;
3086
3087	return t;
3088	}
3089
3090	static void t_start(struct seq_file m, loff_t *pos)
3091	{
3092	struct tracer *t;
3093	loff_t l = 0;
3094
3095	mutex_lock(&trace_types_lock);
3096	for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
3097	;
3098
3099	return t;
3100	}
3101
3102	static void t_stop(struct seq_file m, void p)
3103	{
3104	mutex_unlock(&trace_types_lock);
3105	}
3106
3107	static int t_show(struct seq_file m, void v)
3108	{
3109	struct tracer *t = v;
3110
3111	if (!t)
3112	return 0;
3113
3114	seq_printf(m, "%s", t->name);
3115	if (t->next)
3116	seq_putc(m, ' ');
3117	else
3118	seq_putc(m, '\n');
3119
3120	return 0;
3121	}
3122
3123	static const struct seq_operations show_traces_seq_ops = {
3124	.start = t_start,
3125	.next = t_next,
3126	.stop = t_stop,
3127	.show = t_show,
3128	};
3129
3130	static int show_traces_open(struct inode inode, struct file file)
3131	{
3132	if (tracing_disabled)
3133	return -ENODEV;
3134
3135	return seq_open(file, &show_traces_seq_ops);
3136	}
3137
3138	static ssize_t
3139	tracing_write_stub(struct file filp, const char __user ubuf,
3140	size_t count, loff_t *ppos)
3141	{
3142	return count;
3143	}
3144
3145	static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3146	{
3147	if (file->f_mode & FMODE_READ)
3148	return seq_lseek(file, offset, origin);
3149	else
3150	return 0;
3151	}
3152
3153	static const struct file_operations tracing_fops = {
3154	.open = tracing_open,
3155	.read = seq_read,
3156	.write = tracing_write_stub,
3157	.llseek = tracing_seek,
3158	.release = tracing_release,
3159	};
3160
3161	static const struct file_operations show_traces_fops = {
3162	.open = show_traces_open,
3163	.read = seq_read,
3164	.release = seq_release,
3165	.llseek = seq_lseek,
3166	};
3167
3168	/*
3169	* The tracer itself will not take this lock, but still we want
3170	* to provide a consistent cpumask to user-space:
3171	*/
3172	static DEFINE_MUTEX(tracing_cpumask_update_lock);
3173
3174	/*
3175	* Temporary storage for the character representation of the
3176	* CPU bitmask (and one more byte for the newline):
3177	*/
3178	static char mask_str[NR_CPUS + 1];
3179
3180	static ssize_t
3181	tracing_cpumask_read(struct file filp, char __user ubuf,
3182	size_t count, loff_t *ppos)
3183	{
3184	struct trace_array *tr = file_inode(filp)->i_private;
3185	int len;
3186
3187	mutex_lock(&tracing_cpumask_update_lock);
3188
3189	len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3190	if (count - len < 2) {
3191	count = -EINVAL;
3192	goto out_err;
3193	}
3194	len += sprintf(mask_str + len, "\n");
3195	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3196
3197	out_err:
3198	mutex_unlock(&tracing_cpumask_update_lock);
3199
3200	return count;
3201	}
3202
3203	static ssize_t
3204	tracing_cpumask_write(struct file filp, const char __user ubuf,
3205	size_t count, loff_t *ppos)
3206	{
3207	struct trace_array *tr = file_inode(filp)->i_private;
3208	cpumask_var_t tracing_cpumask_new;
3209	int err, cpu;
3210
3211	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3212	return -ENOMEM;
3213
3214	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3215	if (err)
3216	goto err_unlock;
3217
3218	mutex_lock(&tracing_cpumask_update_lock);
3219
3220	local_irq_disable();
3221	arch_spin_lock(&ftrace_max_lock);
3222	for_each_tracing_cpu(cpu) {
3223	/*
3224	* Increase/decrease the disabled counter if we are
3225	* about to flip a bit in the cpumask:
3226	*/
3227	if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3228	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3229	atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3230	ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3231	}
3232	if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3233	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3234	atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3235	ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3236	}
3237	}
3238	arch_spin_unlock(&ftrace_max_lock);
3239	local_irq_enable();
3240
3241	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3242
3243	mutex_unlock(&tracing_cpumask_update_lock);
3244	free_cpumask_var(tracing_cpumask_new);
3245
3246	return count;
3247
3248	err_unlock:
3249	free_cpumask_var(tracing_cpumask_new);
3250
3251	return err;
3252	}
3253
3254	static const struct file_operations tracing_cpumask_fops = {
3255	.open = tracing_open_generic_tr,
3256	.read = tracing_cpumask_read,
3257	.write = tracing_cpumask_write,
3258	.release = tracing_release_generic_tr,
3259	.llseek = generic_file_llseek,
3260	};
3261
3262	static int tracing_trace_options_show(struct seq_file m, void v)
3263	{
3264	struct tracer_opt *trace_opts;
3265	struct trace_array *tr = m->private;
3266	u32 tracer_flags;
3267	int i;
3268
3269	mutex_lock(&trace_types_lock);
3270	tracer_flags = tr->current_trace->flags->val;
3271	trace_opts = tr->current_trace->flags->opts;
3272
3273	for (i = 0; trace_options[i]; i++) {
3274	if (trace_flags & (1 << i))
3275	seq_printf(m, "%s\n", trace_options[i]);
3276	else
3277	seq_printf(m, "no%s\n", trace_options[i]);
3278	}
3279
3280	for (i = 0; trace_opts[i].name; i++) {
3281	if (tracer_flags & trace_opts[i].bit)
3282	seq_printf(m, "%s\n", trace_opts[i].name);
3283	else
3284	seq_printf(m, "no%s\n", trace_opts[i].name);
3285	}
3286	mutex_unlock(&trace_types_lock);
3287
3288	return 0;
3289	}
3290
3291	static int __set_tracer_option(struct tracer *trace,
3292	struct tracer_flags *tracer_flags,
3293	struct tracer_opt *opts, int neg)
3294	{
3295	int ret;
3296
3297	ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3298	if (ret)
3299	return ret;
3300
3301	if (neg)
3302	tracer_flags->val &= ~opts->bit;
3303	else
3304	tracer_flags->val \|= opts->bit;
3305	return 0;
3306	}
3307
3308	/* Try to assign a tracer specific option */
3309	static int set_tracer_option(struct tracer trace, char cmp, int neg)
3310	{
3311	struct tracer_flags *tracer_flags = trace->flags;
3312	struct tracer_opt *opts = NULL;
3313	int i;
3314
3315	for (i = 0; tracer_flags->opts[i].name; i++) {
3316	opts = &tracer_flags->opts[i];
3317
3318	if (strcmp(cmp, opts->name) == 0)
3319	return __set_tracer_option(trace, trace->flags,
3320	opts, neg);
3321	}
3322
3323	return -EINVAL;
3324	}
3325
3326	/* Some tracers require overwrite to stay enabled */
3327	int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3328	{
3329	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3330	return -1;
3331
3332	return 0;
3333	}
3334
3335	int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3336	{
3337	/* do nothing if flag is already set */
3338	if (!!(trace_flags & mask) == !!enabled)
3339	return 0;
3340
3341	/* Give the tracer a chance to approve the change */
3342	if (tr->current_trace->flag_changed)
3343	if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
3344	return -EINVAL;
3345
3346	if (enabled)
3347	trace_flags \|= mask;
3348	else
3349	trace_flags &= ~mask;
3350
3351	if (mask == TRACE_ITER_RECORD_CMD)
3352	trace_event_enable_cmd_record(enabled);
3353
3354	if (mask == TRACE_ITER_OVERWRITE) {
3355	ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3356	#ifdef CONFIG_TRACER_MAX_TRACE
3357	ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3358	#endif
3359	}
3360
3361	if (mask == TRACE_ITER_PRINTK)
3362	trace_printk_start_stop_comm(enabled);
3363
3364	return 0;
3365	}
3366
3367	static int trace_set_options(struct trace_array tr, char option)
3368	{
3369	char *cmp;
3370	int neg = 0;
3371	int ret = -ENODEV;
3372	int i;
3373
3374	cmp = strstrip(option);
3375
3376	if (strncmp(cmp, "no", 2) == 0) {
3377	neg = 1;
3378	cmp += 2;
3379	}
3380
3381	mutex_lock(&trace_types_lock);
3382
3383	for (i = 0; trace_options[i]; i++) {
3384	if (strcmp(cmp, trace_options[i]) == 0) {
3385	ret = set_tracer_flag(tr, 1 << i, !neg);
3386	break;
3387	}
3388	}
3389
3390	/* If no option could be set, test the specific tracer options */
3391	if (!trace_options[i])
3392	ret = set_tracer_option(tr->current_trace, cmp, neg);
3393
3394	mutex_unlock(&trace_types_lock);
3395
3396	return ret;
3397	}
3398
3399	static ssize_t
3400	tracing_trace_options_write(struct file filp, const char __user ubuf,
3401	size_t cnt, loff_t *ppos)
3402	{
3403	struct seq_file *m = filp->private_data;
3404	struct trace_array *tr = m->private;
3405	char buf[64];
3406	int ret;
3407
3408	if (cnt >= sizeof(buf))
3409	return -EINVAL;
3410
3411	if (copy_from_user(&buf, ubuf, cnt))
3412	return -EFAULT;
3413
3414	buf[cnt] = 0;
3415
3416	ret = trace_set_options(tr, buf);
3417	if (ret < 0)
3418	return ret;
3419
3420	*ppos += cnt;
3421
3422	return cnt;
3423	}
3424
3425	static int tracing_trace_options_open(struct inode inode, struct file file)
3426	{
3427	struct trace_array *tr = inode->i_private;
3428	int ret;
3429
3430	if (tracing_disabled)
3431	return -ENODEV;
3432
3433	if (trace_array_get(tr) < 0)
3434	return -ENODEV;
3435
3436	ret = single_open(file, tracing_trace_options_show, inode->i_private);
3437	if (ret < 0)
3438	trace_array_put(tr);
3439
3440	return ret;
3441	}
3442
3443	static const struct file_operations tracing_iter_fops = {
3444	.open = tracing_trace_options_open,
3445	.read = seq_read,
3446	.llseek = seq_lseek,
3447	.release = tracing_single_release_tr,
3448	.write = tracing_trace_options_write,
3449	};
3450
3451	static const char readme_msg[] =
3452	"tracing mini-HOWTO:\n\n"
3453	"# echo 0 > tracing_on : quick way to disable tracing\n"
3454	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3455	" Important files:\n"
3456	" trace\t\t\t- The static contents of the buffer\n"
3457	"\t\t\t To clear the buffer write into this file: echo > trace\n"
3458	" trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3459	" current_tracer\t- function and latency tracers\n"
3460	" available_tracers\t- list of configured tracers for current_tracer\n"
3461	" buffer_size_kb\t- view and modify size of per cpu buffer\n"
3462	" buffer_total_size_kb - view total size of all cpu buffers\n\n"
3463	" trace_clock\t\t-change the clock used to order events\n"
3464	" local: Per cpu clock but may not be synced across CPUs\n"
3465	" global: Synced across CPUs but slows tracing down.\n"
3466	" counter: Not a clock, but just an increment\n"
3467	" uptime: Jiffy counter from time of boot\n"
3468	" perf: Same clock that perf events use\n"
3469	#ifdef CONFIG_X86_64
3470	" x86-tsc: TSC cycle counter\n"
3471	#endif
3472	"\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3473	" tracing_cpumask\t- Limit which CPUs to trace\n"
3474	" instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3475	"\t\t\t Remove sub-buffer with rmdir\n"
3476	" trace_options\t\t- Set format or modify how tracing happens\n"
3477	"\t\t\t Disable an option by adding a suffix 'no' to the option name\n"
3478	#ifdef CONFIG_DYNAMIC_FTRACE
3479	"\n available_filter_functions - list of functions that can be filtered on\n"
3480	" set_ftrace_filter\t- echo function name in here to only trace these functions\n"
3481	" accepts: func_full_name, func_end, func_begin, func_middle\n"
3482	" modules: Can select a group via module\n"
3483	" Format: :mod:<module-name>\n"
3484	" example: echo :mod:ext3 > set_ftrace_filter\n"
3485	" triggers: a command to perform when function is hit\n"
3486	" Format: <function>:<trigger>[:count]\n"
3487	" trigger: traceon, traceoff\n"
3488	" enable_event:<system>:<event>\n"
3489	" disable_event:<system>:<event>\n"
3490	#ifdef CONFIG_STACKTRACE
3491	" stacktrace\n"
3492	#endif
3493	#ifdef CONFIG_TRACER_SNAPSHOT
3494	" snapshot\n"
3495	#endif
3496	" example: echo do_fault:traceoff > set_ftrace_filter\n"
3497	" echo do_trap:traceoff:3 > set_ftrace_filter\n"
3498	" The first one will disable tracing every time do_fault is hit\n"
3499	" The second will disable tracing at most 3 times when do_trap is hit\n"
3500	" The first time do trap is hit and it disables tracing, the counter\n"
3501	" will decrement to 2. If tracing is already disabled, the counter\n"
3502	" will not decrement. It only decrements when the trigger did work\n"
3503	" To remove trigger without count:\n"
3504	" echo '!<function>:<trigger> > set_ftrace_filter\n"
3505	" To remove trigger with a count:\n"
3506	" echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3507	" set_ftrace_notrace\t- echo function name in here to never trace.\n"
3508	" accepts: func_full_name, func_end, func_begin, func_middle\n"
3509	" modules: Can select a group via module command :mod:\n"
3510	" Does not accept triggers\n"
3511	#endif /* CONFIG_DYNAMIC_FTRACE */
3512	#ifdef CONFIG_FUNCTION_TRACER
3513	" set_ftrace_pid\t- Write pid(s) to only function trace those pids (function)\n"
3514	#endif
3515	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3516	" set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3517	" max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3518	#endif
3519	#ifdef CONFIG_TRACER_SNAPSHOT
3520	"\n snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
3521	"\t\t\t Read the contents for more information\n"
3522	#endif
3523	#ifdef CONFIG_STACK_TRACER
3524	" stack_trace\t\t- Shows the max stack trace when active\n"
3525	" stack_max_size\t- Shows current max stack size that was traced\n"
3526	"\t\t\t Write into this file to reset the max size (trigger a new trace)\n"
3527	#ifdef CONFIG_DYNAMIC_FTRACE
3528	" stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
3529	#endif
3530	#endif /* CONFIG_STACK_TRACER */
3531	;
3532
3533	static ssize_t
3534	tracing_readme_read(struct file filp, char __user ubuf,
3535	size_t cnt, loff_t *ppos)
3536	{
3537	return simple_read_from_buffer(ubuf, cnt, ppos,
3538	readme_msg, strlen(readme_msg));
3539	}
3540
3541	static const struct file_operations tracing_readme_fops = {
3542	.open = tracing_open_generic,
3543	.read = tracing_readme_read,
3544	.llseek = generic_file_llseek,
3545	};
3546
3547	static ssize_t
3548	tracing_saved_cmdlines_read(struct file file, char __user ubuf,
3549	size_t cnt, loff_t *ppos)
3550	{
3551	char *buf_comm;
3552	char *file_buf;
3553	char *buf;
3554	int len = 0;
3555	int pid;
3556	int i;
3557
3558	file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3559	if (!file_buf)
3560	return -ENOMEM;
3561
3562	buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3563	if (!buf_comm) {
3564	kfree(file_buf);
3565	return -ENOMEM;
3566	}
3567
3568	buf = file_buf;
3569
3570	for (i = 0; i < SAVED_CMDLINES; i++) {
3571	int r;
3572
3573	pid = map_cmdline_to_pid[i];
3574	if (pid == -1 \|\| pid == NO_CMDLINE_MAP)
3575	continue;
3576
3577	trace_find_cmdline(pid, buf_comm);
3578	r = sprintf(buf, "%d %s\n", pid, buf_comm);
3579	buf += r;
3580	len += r;
3581	}
3582
3583	len = simple_read_from_buffer(ubuf, cnt, ppos,
3584	file_buf, len);
3585
3586	kfree(file_buf);
3587	kfree(buf_comm);
3588
3589	return len;
3590	}
3591
3592	static const struct file_operations tracing_saved_cmdlines_fops = {
3593	.open = tracing_open_generic,
3594	.read = tracing_saved_cmdlines_read,
3595	.llseek = generic_file_llseek,
3596	};
3597
3598	static ssize_t
3599	tracing_set_trace_read(struct file filp, char __user ubuf,
3600	size_t cnt, loff_t *ppos)
3601	{
3602	struct trace_array *tr = filp->private_data;
3603	char buf[MAX_TRACER_SIZE+2];
3604	int r;
3605
3606	mutex_lock(&trace_types_lock);
3607	r = sprintf(buf, "%s\n", tr->current_trace->name);
3608	mutex_unlock(&trace_types_lock);
3609
3610	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3611	}
3612
3613	int tracer_init(struct tracer t, struct trace_array tr)
3614	{
3615	tracing_reset_online_cpus(&tr->trace_buffer);
3616	return t->init(tr);
3617	}
3618
3619	static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3620	{
3621	int cpu;
3622
3623	for_each_tracing_cpu(cpu)
3624	per_cpu_ptr(buf->data, cpu)->entries = val;
3625	}
3626
3627	#ifdef CONFIG_TRACER_MAX_TRACE
3628	/* resize @tr's buffer to the size of @size_tr's entries */
3629	static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3630	struct trace_buffer *size_buf, int cpu_id)
3631	{
3632	int cpu, ret = 0;
3633
3634	if (cpu_id == RING_BUFFER_ALL_CPUS) {
3635	for_each_tracing_cpu(cpu) {
3636	ret = ring_buffer_resize(trace_buf->buffer,
3637	per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3638	if (ret < 0)
3639	break;
3640	per_cpu_ptr(trace_buf->data, cpu)->entries =
3641	per_cpu_ptr(size_buf->data, cpu)->entries;
3642	}
3643	} else {
3644	ret = ring_buffer_resize(trace_buf->buffer,
3645	per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3646	if (ret == 0)
3647	per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3648	per_cpu_ptr(size_buf->data, cpu_id)->entries;
3649	}
3650
3651	return ret;
3652	}
3653	#endif /* CONFIG_TRACER_MAX_TRACE */
3654
3655	static int __tracing_resize_ring_buffer(struct trace_array *tr,
3656	unsigned long size, int cpu)
3657	{
3658	int ret;
3659
3660	/*
3661	* If kernel or user changes the size of the ring buffer
3662	* we use the size that was given, and we can forget about
3663	* expanding it later.
3664	*/
3665	ring_buffer_expanded = true;
3666
3667	/* May be called before buffers are initialized */
3668	if (!tr->trace_buffer.buffer)
3669	return 0;
3670
3671	ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3672	if (ret < 0)
3673	return ret;
3674
3675	#ifdef CONFIG_TRACER_MAX_TRACE
3676	if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) \|\|
3677	!tr->current_trace->use_max_tr)
3678	goto out;
3679
3680	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3681	if (ret < 0) {
3682	int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3683	&tr->trace_buffer, cpu);
3684	if (r < 0) {
3685	/*
3686	* AARGH! We are left with different
3687	* size max buffer!!!!
3688	* The max buffer is our "snapshot" buffer.
3689	* When a tracer needs a snapshot (one of the
3690	* latency tracers), it swaps the max buffer
3691	* with the saved snap shot. We succeeded to
3692	* update the size of the main buffer, but failed to
3693	* update the size of the max buffer. But when we tried
3694	* to reset the main buffer to the original size, we
3695	* failed there too. This is very unlikely to
3696	* happen, but if it does, warn and kill all
3697	* tracing.
3698	*/
3699	WARN_ON(1);
3700	tracing_disabled = 1;
3701	}
3702	return ret;
3703	}
3704
3705	if (cpu == RING_BUFFER_ALL_CPUS)
3706	set_buffer_entries(&tr->max_buffer, size);
3707	else
3708	per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3709
3710	out:
3711	#endif /* CONFIG_TRACER_MAX_TRACE */
3712
3713	if (cpu == RING_BUFFER_ALL_CPUS)
3714	set_buffer_entries(&tr->trace_buffer, size);
3715	else
3716	per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3717
3718	return ret;
3719	}
3720
3721	static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3722	unsigned long size, int cpu_id)
3723	{
3724	int ret = size;
3725
3726	mutex_lock(&trace_types_lock);
3727
3728	if (cpu_id != RING_BUFFER_ALL_CPUS) {
3729	/* make sure, this cpu is enabled in the mask */
3730	if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3731	ret = -EINVAL;
3732	goto out;
3733	}
3734	}
3735
3736	ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3737	if (ret < 0)
3738	ret = -ENOMEM;
3739
3740	out:
3741	mutex_unlock(&trace_types_lock);
3742
3743	return ret;
3744	}
3745
3746
3747	/**
3748	* tracing_update_buffers - used by tracing facility to expand ring buffers
3749	*
3750	* To save on memory when the tracing is never used on a system with it
3751	* configured in. The ring buffers are set to a minimum size. But once
3752	* a user starts to use the tracing facility, then they need to grow
3753	* to their default size.
3754	*
3755	* This function is to be called when a tracer is about to be used.
3756	*/
3757	int tracing_update_buffers(void)
3758	{
3759	int ret = 0;
3760
3761	mutex_lock(&trace_types_lock);
3762	if (!ring_buffer_expanded)
3763	ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3764	RING_BUFFER_ALL_CPUS);
3765	mutex_unlock(&trace_types_lock);
3766
3767	return ret;
3768	}
3769
3770	struct trace_option_dentry;
3771
3772	static struct trace_option_dentry *
3773	create_trace_option_files(struct trace_array tr, struct tracer tracer);
3774
3775	static void
3776	destroy_trace_option_files(struct trace_option_dentry *topts);
3777
3778	static int tracing_set_tracer(const char *buf)
3779	{
3780	static struct trace_option_dentry *topts;
3781	struct trace_array *tr = &global_trace;
3782	struct tracer *t;
3783	#ifdef CONFIG_TRACER_MAX_TRACE
3784	bool had_max_tr;
3785	#endif
3786	int ret = 0;
3787
3788	mutex_lock(&trace_types_lock);
3789
3790	if (!ring_buffer_expanded) {
3791	ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3792	RING_BUFFER_ALL_CPUS);
3793	if (ret < 0)
3794	goto out;
3795	ret = 0;
3796	}
3797
3798	for (t = trace_types; t; t = t->next) {
3799	if (strcmp(t->name, buf) == 0)
3800	break;
3801	}
3802	if (!t) {
3803	ret = -EINVAL;
3804	goto out;
3805	}
3806	if (t == tr->current_trace)
3807	goto out;
3808
3809	trace_branch_disable();
3810
3811	tr->current_trace->enabled = false;
3812
3813	if (tr->current_trace->reset)
3814	tr->current_trace->reset(tr);
3815
3816	/* Current trace needs to be nop_trace before synchronize_sched */
3817	tr->current_trace = &nop_trace;
3818
3819	#ifdef CONFIG_TRACER_MAX_TRACE
3820	had_max_tr = tr->allocated_snapshot;
3821
3822	if (had_max_tr && !t->use_max_tr) {
3823	/*
3824	* We need to make sure that the update_max_tr sees that
3825	* current_trace changed to nop_trace to keep it from
3826	* swapping the buffers after we resize it.
3827	* The update_max_tr is called from interrupts disabled
3828	* so a synchronized_sched() is sufficient.
3829	*/
3830	synchronize_sched();
3831	free_snapshot(tr);
3832	}
3833	#endif
3834	destroy_trace_option_files(topts);
3835
3836	topts = create_trace_option_files(tr, t);
3837
3838	#ifdef CONFIG_TRACER_MAX_TRACE
3839	if (t->use_max_tr && !had_max_tr) {
3840	ret = alloc_snapshot(tr);
3841	if (ret < 0)
3842	goto out;
3843	}
3844	#endif
3845
3846	if (t->init) {
3847	ret = tracer_init(t, tr);
3848	if (ret)
3849	goto out;
3850	}
3851
3852	tr->current_trace = t;
3853	tr->current_trace->enabled = true;
3854	trace_branch_enable(tr);
3855	out:
3856	mutex_unlock(&trace_types_lock);
3857
3858	return ret;
3859	}
3860
3861	static ssize_t
3862	tracing_set_trace_write(struct file filp, const char __user ubuf,
3863	size_t cnt, loff_t *ppos)
3864	{
3865	char buf[MAX_TRACER_SIZE+1];
3866	int i;
3867	size_t ret;
3868	int err;
3869
3870	ret = cnt;
3871
3872	if (cnt > MAX_TRACER_SIZE)
3873	cnt = MAX_TRACER_SIZE;
3874
3875	if (copy_from_user(&buf, ubuf, cnt))
3876	return -EFAULT;
3877
3878	buf[cnt] = 0;
3879
3880	/* strip ending whitespace. */
3881	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3882	buf[i] = 0;
3883
3884	err = tracing_set_tracer(buf);
3885	if (err)
3886	return err;
3887
3888	*ppos += ret;
3889
3890	return ret;
3891	}
3892
3893	static ssize_t
3894	tracing_max_lat_read(struct file filp, char __user ubuf,
3895	size_t cnt, loff_t *ppos)
3896	{
3897	unsigned long *ptr = filp->private_data;
3898	char buf[64];
3899	int r;
3900
3901	r = snprintf(buf, sizeof(buf), "%ld\n",
3902	ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(ptr));
3903	if (r > sizeof(buf))
3904	r = sizeof(buf);
3905	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3906	}
3907
3908	static ssize_t
3909	tracing_max_lat_write(struct file filp, const char __user ubuf,
3910	size_t cnt, loff_t *ppos)
3911	{
3912	unsigned long *ptr = filp->private_data;
3913	unsigned long val;
3914	int ret;
3915
3916	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3917	if (ret)
3918	return ret;
3919
3920	ptr = val 1000;
3921
3922	return cnt;
3923	}
3924
3925	static int tracing_open_pipe(struct inode inode, struct file filp)
3926	{
3927	struct trace_array *tr = inode->i_private;
3928	struct trace_iterator *iter;
3929	int ret = 0;
3930
3931	if (tracing_disabled)
3932	return -ENODEV;
3933
3934	if (trace_array_get(tr) < 0)
3935	return -ENODEV;
3936
3937	mutex_lock(&trace_types_lock);
3938
3939	/* create a buffer to store the information to pass to userspace */
3940	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3941	if (!iter) {
3942	ret = -ENOMEM;
3943	__trace_array_put(tr);
3944	goto out;
3945	}
3946
3947	/*
3948	* We make a copy of the current tracer to avoid concurrent
3949	* changes on it while we are reading.
3950	*/
3951	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3952	if (!iter->trace) {
3953	ret = -ENOMEM;
3954	goto fail;
3955	}
3956	iter->trace = tr->current_trace;
3957
3958	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3959	ret = -ENOMEM;
3960	goto fail;
3961	}
3962
3963	/* trace pipe does not show start of buffer */
3964	cpumask_setall(iter->started);
3965
3966	if (trace_flags & TRACE_ITER_LATENCY_FMT)
3967	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
3968
3969	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
3970	if (trace_clocks[tr->clock_id].in_ns)
3971	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
3972
3973	iter->tr = tr;
3974	iter->trace_buffer = &tr->trace_buffer;
3975	iter->cpu_file = tracing_get_cpu(inode);
3976	mutex_init(&iter->mutex);
3977	filp->private_data = iter;
3978
3979	if (iter->trace->pipe_open)
3980	iter->trace->pipe_open(iter);
3981
3982	nonseekable_open(inode, filp);
3983	out:
3984	mutex_unlock(&trace_types_lock);
3985	return ret;
3986
3987	fail:
3988	kfree(iter->trace);
3989	kfree(iter);
3990	__trace_array_put(tr);
3991	mutex_unlock(&trace_types_lock);
3992	return ret;
3993	}
3994
3995	static int tracing_release_pipe(struct inode inode, struct file file)
3996	{
3997	struct trace_iterator *iter = file->private_data;
3998	struct trace_array *tr = inode->i_private;
3999
4000	mutex_lock(&trace_types_lock);
4001
4002	if (iter->trace->pipe_close)
4003	iter->trace->pipe_close(iter);
4004
4005	mutex_unlock(&trace_types_lock);
4006
4007	free_cpumask_var(iter->started);
4008	mutex_destroy(&iter->mutex);
4009	kfree(iter->trace);
4010	kfree(iter);
4011
4012	trace_array_put(tr);
4013
4014	return 0;
4015	}
4016
4017	static unsigned int
4018	trace_poll(struct trace_iterator iter, struct file filp, poll_table *poll_table)
4019	{
4020	/* Iterators are static, they should be filled or empty */
4021	if (trace_buffer_iter(iter, iter->cpu_file))
4022	return POLLIN \| POLLRDNORM;
4023
4024	if (trace_flags & TRACE_ITER_BLOCK)
4025	/*
4026	* Always select as readable when in blocking mode
4027	*/
4028	return POLLIN \| POLLRDNORM;
4029	else
4030	return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4031	filp, poll_table);
4032	}
4033
4034	static unsigned int
4035	tracing_poll_pipe(struct file filp, poll_table poll_table)
4036	{
4037	struct trace_iterator *iter = filp->private_data;
4038
4039	return trace_poll(iter, filp, poll_table);
4040	}
4041
4042	/*
4043	* This is a make-shift waitqueue.
4044	* A tracer might use this callback on some rare cases:
4045	*
4046	* 1) the current tracer might hold the runqueue lock when it wakes up
4047	* a reader, hence a deadlock (sched, function, and function graph tracers)
4048	* 2) the function tracers, trace all functions, we don't want
4049	* the overhead of calling wake_up and friends
4050	* (and tracing them too)
4051	*
4052	* Anyway, this is really very primitive wakeup.
4053	*/
4054	void poll_wait_pipe(struct trace_iterator *iter)
4055	{
4056	set_current_state(TASK_INTERRUPTIBLE);
4057	/* sleep for 100 msecs, and try again. */
4058	schedule_timeout(HZ / 10);
4059	}
4060
4061	/* Must be called with trace_types_lock mutex held. */
4062	static int tracing_wait_pipe(struct file *filp)
4063	{
4064	struct trace_iterator *iter = filp->private_data;
4065
4066	while (trace_empty(iter)) {
4067
4068	if ((filp->f_flags & O_NONBLOCK)) {
4069	return -EAGAIN;
4070	}
4071
4072	mutex_unlock(&iter->mutex);
4073
4074	iter->trace->wait_pipe(iter);
4075
4076	mutex_lock(&iter->mutex);
4077
4078	if (signal_pending(current))
4079	return -EINTR;
4080
4081	/*
4082	* We block until we read something and tracing is disabled.
4083	* We still block if tracing is disabled, but we have never
4084	* read anything. This allows a user to cat this file, and
4085	* then enable tracing. But after we have read something,
4086	* we give an EOF when tracing is again disabled.
4087	*
4088	* iter->pos will be 0 if we haven't read anything.
4089	*/
4090	if (!tracing_is_on() && iter->pos)
4091	break;
4092	}
4093
4094	return 1;
4095	}
4096
4097	/*
4098	* Consumer reader.
4099	*/
4100	static ssize_t
4101	tracing_read_pipe(struct file filp, char __user ubuf,
4102	size_t cnt, loff_t *ppos)
4103	{
4104	struct trace_iterator *iter = filp->private_data;
4105	struct trace_array *tr = iter->tr;
4106	ssize_t sret;
4107
4108	/* return any leftover data */
4109	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4110	if (sret != -EBUSY)
4111	return sret;
4112
4113	trace_seq_init(&iter->seq);
4114
4115	/* copy the tracer to avoid using a global lock all around */
4116	mutex_lock(&trace_types_lock);
4117	if (unlikely(iter->trace->name != tr->current_trace->name))
4118	iter->trace = tr->current_trace;
4119	mutex_unlock(&trace_types_lock);
4120
4121	/*
4122	* Avoid more than one consumer on a single file descriptor
4123	* This is just a matter of traces coherency, the ring buffer itself
4124	* is protected.
4125	*/
4126	mutex_lock(&iter->mutex);
4127	if (iter->trace->read) {
4128	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4129	if (sret)
4130	goto out;
4131	}
4132
4133	waitagain:
4134	sret = tracing_wait_pipe(filp);
4135	if (sret <= 0)
4136	goto out;
4137
4138	/* stop when tracing is finished */
4139	if (trace_empty(iter)) {
4140	sret = 0;
4141	goto out;
4142	}
4143
4144	if (cnt >= PAGE_SIZE)
4145	cnt = PAGE_SIZE - 1;
4146
4147	/* reset all but tr, trace, and overruns */
4148	memset(&iter->seq, 0,
4149	sizeof(struct trace_iterator) -
4150	offsetof(struct trace_iterator, seq));
4151	cpumask_clear(iter->started);
4152	iter->pos = -1;
4153
4154	trace_event_read_lock();
4155	trace_access_lock(iter->cpu_file);
4156	while (trace_find_next_entry_inc(iter) != NULL) {
4157	enum print_line_t ret;
4158	int len = iter->seq.len;
4159
4160	ret = print_trace_line(iter);
4161	if (ret == TRACE_TYPE_PARTIAL_LINE) {
4162	/* don't print partial lines */
4163	iter->seq.len = len;
4164	break;
4165	}
4166	if (ret != TRACE_TYPE_NO_CONSUME)
4167	trace_consume(iter);
4168
4169	if (iter->seq.len >= cnt)
4170	break;
4171
4172	/*
4173	* Setting the full flag means we reached the trace_seq buffer
4174	* size and we should leave by partial output condition above.
4175	* One of the trace_seq_* functions is not used properly.
4176	*/
4177	WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4178	iter->ent->type);
4179	}
4180	trace_access_unlock(iter->cpu_file);
4181	trace_event_read_unlock();
4182
4183	/* Now copy what we have to the user */
4184	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4185	if (iter->seq.readpos >= iter->seq.len)
4186	trace_seq_init(&iter->seq);
4187
4188	/*
4189	* If there was nothing to send to user, in spite of consuming trace
4190	* entries, go back to wait for more entries.
4191	*/
4192	if (sret == -EBUSY)
4193	goto waitagain;
4194
4195	out:
4196	mutex_unlock(&iter->mutex);
4197
4198	return sret;
4199	}
4200
4201	static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
4202	struct pipe_buffer *buf)
4203	{
4204	__free_page(buf->page);
4205	}
4206
4207	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4208	unsigned int idx)
4209	{
4210	__free_page(spd->pages[idx]);
4211	}
4212
4213	static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4214	.can_merge = 0,
4215	.map = generic_pipe_buf_map,
4216	.unmap = generic_pipe_buf_unmap,
4217	.confirm = generic_pipe_buf_confirm,
4218	.release = tracing_pipe_buf_release,
4219	.steal = generic_pipe_buf_steal,
4220	.get = generic_pipe_buf_get,
4221	};
4222
4223	static size_t
4224	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4225	{
4226	size_t count;
4227	int ret;
4228
4229	/* Seq buffer is page-sized, exactly what we need. */
4230	for (;;) {
4231	count = iter->seq.len;
4232	ret = print_trace_line(iter);
4233	count = iter->seq.len - count;
4234	if (rem < count) {
4235	rem = 0;
4236	iter->seq.len -= count;
4237	break;
4238	}
4239	if (ret == TRACE_TYPE_PARTIAL_LINE) {
4240	iter->seq.len -= count;
4241	break;
4242	}
4243
4244	if (ret != TRACE_TYPE_NO_CONSUME)
4245	trace_consume(iter);
4246	rem -= count;
4247	if (!trace_find_next_entry_inc(iter)) {
4248	rem = 0;
4249	iter->ent = NULL;
4250	break;
4251	}
4252	}
4253
4254	return rem;
4255	}
4256
4257	static ssize_t tracing_splice_read_pipe(struct file *filp,
4258	loff_t *ppos,
4259	struct pipe_inode_info *pipe,
4260	size_t len,
4261	unsigned int flags)
4262	{
4263	struct page *pages_def[PIPE_DEF_BUFFERS];
4264	struct partial_page partial_def[PIPE_DEF_BUFFERS];
4265	struct trace_iterator *iter = filp->private_data;
4266	struct splice_pipe_desc spd = {
4267	.pages = pages_def,
4268	.partial = partial_def,
4269	.nr_pages = 0, /* This gets updated below. */
4270	.nr_pages_max = PIPE_DEF_BUFFERS,
4271	.flags = flags,
4272	.ops = &tracing_pipe_buf_ops,
4273	.spd_release = tracing_spd_release_pipe,
4274	};
4275	struct trace_array *tr = iter->tr;
4276	ssize_t ret;
4277	size_t rem;
4278	unsigned int i;
4279
4280	if (splice_grow_spd(pipe, &spd))
4281	return -ENOMEM;
4282
4283	/* copy the tracer to avoid using a global lock all around */
4284	mutex_lock(&trace_types_lock);
4285	if (unlikely(iter->trace->name != tr->current_trace->name))
4286	iter->trace = tr->current_trace;
4287	mutex_unlock(&trace_types_lock);
4288
4289	mutex_lock(&iter->mutex);
4290
4291	if (iter->trace->splice_read) {
4292	ret = iter->trace->splice_read(iter, filp,
4293	ppos, pipe, len, flags);
4294	if (ret)
4295	goto out_err;
4296	}
4297
4298	ret = tracing_wait_pipe(filp);
4299	if (ret <= 0)
4300	goto out_err;
4301
4302	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4303	ret = -EFAULT;
4304	goto out_err;
4305	}
4306
4307	trace_event_read_lock();
4308	trace_access_lock(iter->cpu_file);
4309
4310	/* Fill as many pages as possible. */
4311	for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4312	spd.pages[i] = alloc_page(GFP_KERNEL);
4313	if (!spd.pages[i])
4314	break;
4315
4316	rem = tracing_fill_pipe_page(rem, iter);
4317
4318	/* Copy the data into the page, so we can start over. */
4319	ret = trace_seq_to_buffer(&iter->seq,
4320	page_address(spd.pages[i]),
4321	iter->seq.len);
4322	if (ret < 0) {
4323	__free_page(spd.pages[i]);
4324	break;
4325	}
4326	spd.partial[i].offset = 0;
4327	spd.partial[i].len = iter->seq.len;
4328
4329	trace_seq_init(&iter->seq);
4330	}
4331
4332	trace_access_unlock(iter->cpu_file);
4333	trace_event_read_unlock();
4334	mutex_unlock(&iter->mutex);
4335
4336	spd.nr_pages = i;
4337
4338	ret = splice_to_pipe(pipe, &spd);
4339	out:
4340	splice_shrink_spd(&spd);
4341	return ret;
4342
4343	out_err:
4344	mutex_unlock(&iter->mutex);
4345	goto out;
4346	}
4347
4348	static ssize_t
4349	tracing_entries_read(struct file filp, char __user ubuf,
4350	size_t cnt, loff_t *ppos)
4351	{
4352	struct inode *inode = file_inode(filp);
4353	struct trace_array *tr = inode->i_private;
4354	int cpu = tracing_get_cpu(inode);
4355	char buf[64];
4356	int r = 0;
4357	ssize_t ret;
4358
4359	mutex_lock(&trace_types_lock);
4360
4361	if (cpu == RING_BUFFER_ALL_CPUS) {
4362	int cpu, buf_size_same;
4363	unsigned long size;
4364
4365	size = 0;
4366	buf_size_same = 1;
4367	/* check if all cpu sizes are same */
4368	for_each_tracing_cpu(cpu) {
4369	/* fill in the size from first enabled cpu */
4370	if (size == 0)
4371	size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4372	if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4373	buf_size_same = 0;
4374	break;
4375	}
4376	}
4377
4378	if (buf_size_same) {
4379	if (!ring_buffer_expanded)
4380	r = sprintf(buf, "%lu (expanded: %lu)\n",
4381	size >> 10,
4382	trace_buf_size >> 10);
4383	else
4384	r = sprintf(buf, "%lu\n", size >> 10);
4385	} else
4386	r = sprintf(buf, "X\n");
4387	} else
4388	r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4389
4390	mutex_unlock(&trace_types_lock);
4391
4392	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4393	return ret;
4394	}
4395
4396	static ssize_t
4397	tracing_entries_write(struct file filp, const char __user ubuf,
4398	size_t cnt, loff_t *ppos)
4399	{
4400	struct inode *inode = file_inode(filp);
4401	struct trace_array *tr = inode->i_private;
4402	unsigned long val;
4403	int ret;
4404
4405	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4406	if (ret)
4407	return ret;
4408
4409	/* must have at least 1 entry */
4410	if (!val)
4411	return -EINVAL;
4412
4413	/* value is in KB */
4414	val <<= 10;
4415	ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4416	if (ret < 0)
4417	return ret;
4418
4419	*ppos += cnt;
4420
4421	return cnt;
4422	}
4423
4424	static ssize_t
4425	tracing_total_entries_read(struct file filp, char __user ubuf,
4426	size_t cnt, loff_t *ppos)
4427	{
4428	struct trace_array *tr = filp->private_data;
4429	char buf[64];
4430	int r, cpu;
4431	unsigned long size = 0, expanded_size = 0;
4432
4433	mutex_lock(&trace_types_lock);
4434	for_each_tracing_cpu(cpu) {
4435	size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4436	if (!ring_buffer_expanded)
4437	expanded_size += trace_buf_size >> 10;
4438	}
4439	if (ring_buffer_expanded)
4440	r = sprintf(buf, "%lu\n", size);
4441	else
4442	r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4443	mutex_unlock(&trace_types_lock);
4444
4445	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4446	}
4447
4448	static ssize_t
4449	tracing_free_buffer_write(struct file filp, const char __user ubuf,
4450	size_t cnt, loff_t *ppos)
4451	{
4452	/*
4453	* There is no need to read what the user has written, this function
4454	* is just to make sure that there is no error when "echo" is used
4455	*/
4456
4457	*ppos += cnt;
4458
4459	return cnt;
4460	}
4461
4462	static int
4463	tracing_free_buffer_release(struct inode inode, struct file filp)
4464	{
4465	struct trace_array *tr = inode->i_private;
4466
4467	/* disable tracing ? */
4468	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4469	tracer_tracing_off(tr);
4470	/* resize the ring buffer to 0 */
4471	tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4472
4473	trace_array_put(tr);
4474
4475	return 0;
4476	}
4477
4478	static ssize_t
4479	tracing_mark_write(struct file filp, const char __user ubuf,
4480	size_t cnt, loff_t *fpos)
4481	{
4482	unsigned long addr = (unsigned long)ubuf;
4483	struct trace_array *tr = filp->private_data;
4484	struct ring_buffer_event *event;
4485	struct ring_buffer *buffer;
4486	struct print_entry *entry;
4487	unsigned long irq_flags;
4488	struct page *pages[2];
4489	void *map_page[2];
4490	int nr_pages = 1;
4491	ssize_t written;
4492	int offset;
4493	int size;
4494	int len;
4495	int ret;
4496	int i;
4497
4498	if (tracing_disabled)
4499	return -EINVAL;
4500
4501	if (!(trace_flags & TRACE_ITER_MARKERS))
4502	return -EINVAL;
4503
4504	if (cnt > TRACE_BUF_SIZE)
4505	cnt = TRACE_BUF_SIZE;
4506
4507	/*
4508	* Userspace is injecting traces into the kernel trace buffer.
4509	* We want to be as non intrusive as possible.
4510	* To do so, we do not want to allocate any special buffers
4511	* or take any locks, but instead write the userspace data
4512	* straight into the ring buffer.
4513	*
4514	* First we need to pin the userspace buffer into memory,
4515	* which, most likely it is, because it just referenced it.
4516	* But there's no guarantee that it is. By using get_user_pages_fast()
4517	* and kmap_atomic/kunmap_atomic() we can get access to the
4518	* pages directly. We then write the data directly into the
4519	* ring buffer.
4520	*/
4521	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4522
4523	/* check if we cross pages */
4524	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4525	nr_pages = 2;
4526
4527	offset = addr & (PAGE_SIZE - 1);
4528	addr &= PAGE_MASK;
4529
4530	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4531	if (ret < nr_pages) {
4532	while (--ret >= 0)
4533	put_page(pages[ret]);
4534	written = -EFAULT;
4535	goto out;
4536	}
4537
4538	for (i = 0; i < nr_pages; i++)
4539	map_page[i] = kmap_atomic(pages[i]);
4540
4541	local_save_flags(irq_flags);
4542	size = sizeof(entry) + cnt + 2; / possible \n added */
4543	buffer = tr->trace_buffer.buffer;
4544	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4545	irq_flags, preempt_count());
4546	if (!event) {
4547	/* Ring buffer disabled, return as if not open for write */
4548	written = -EBADF;
4549	goto out_unlock;
4550	}
4551
4552	entry = ring_buffer_event_data(event);
4553	entry->ip = _THIS_IP_;
4554
4555	if (nr_pages == 2) {
4556	len = PAGE_SIZE - offset;
4557	memcpy(&entry->buf, map_page[0] + offset, len);
4558	memcpy(&entry->buf[len], map_page[1], cnt - len);
4559	} else
4560	memcpy(&entry->buf, map_page[0] + offset, cnt);
4561
4562	if (entry->buf[cnt - 1] != '\n') {
4563	entry->buf[cnt] = '\n';
4564	entry->buf[cnt + 1] = '\0';
4565	} else
4566	entry->buf[cnt] = '\0';
4567
4568	__buffer_unlock_commit(buffer, event);
4569
4570	written = cnt;
4571
4572	*fpos += written;
4573
4574	out_unlock:
4575	for (i = 0; i < nr_pages; i++){
4576	kunmap_atomic(map_page[i]);
4577	put_page(pages[i]);
4578	}
4579	out:
4580	return written;
4581	}
4582
4583	static int tracing_clock_show(struct seq_file m, void v)
4584	{
4585	struct trace_array *tr = m->private;
4586	int i;
4587
4588	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4589	seq_printf(m,
4590	"%s%s%s%s", i ? " " : "",
4591	i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4592	i == tr->clock_id ? "]" : "");
4593	seq_putc(m, '\n');
4594
4595	return 0;
4596	}
4597
4598	static ssize_t tracing_clock_write(struct file filp, const char __user ubuf,
4599	size_t cnt, loff_t *fpos)
4600	{
4601	struct seq_file *m = filp->private_data;
4602	struct trace_array *tr = m->private;
4603	char buf[64];
4604	const char *clockstr;
4605	int i;
4606
4607	if (cnt >= sizeof(buf))
4608	return -EINVAL;
4609
4610	if (copy_from_user(&buf, ubuf, cnt))
4611	return -EFAULT;
4612
4613	buf[cnt] = 0;
4614
4615	clockstr = strstrip(buf);
4616
4617	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4618	if (strcmp(trace_clocks[i].name, clockstr) == 0)
4619	break;
4620	}
4621	if (i == ARRAY_SIZE(trace_clocks))
4622	return -EINVAL;
4623
4624	mutex_lock(&trace_types_lock);
4625
4626	tr->clock_id = i;
4627
4628	ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4629
4630	/*
4631	* New clock may not be consistent with the previous clock.
4632	* Reset the buffer so that it doesn't have incomparable timestamps.
4633	*/
4634	tracing_reset_online_cpus(&tr->trace_buffer);
4635
4636	#ifdef CONFIG_TRACER_MAX_TRACE
4637	if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4638	ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4639	tracing_reset_online_cpus(&tr->max_buffer);
4640	#endif
4641
4642	mutex_unlock(&trace_types_lock);
4643
4644	*fpos += cnt;
4645
4646	return cnt;
4647	}
4648
4649	static int tracing_clock_open(struct inode inode, struct file file)
4650	{
4651	struct trace_array *tr = inode->i_private;
4652	int ret;
4653
4654	if (tracing_disabled)
4655	return -ENODEV;
4656
4657	if (trace_array_get(tr))
4658	return -ENODEV;
4659
4660	ret = single_open(file, tracing_clock_show, inode->i_private);
4661	if (ret < 0)
4662	trace_array_put(tr);
4663
4664	return ret;
4665	}
4666
4667	struct ftrace_buffer_info {
4668	struct trace_iterator iter;
4669	void *spare;
4670	unsigned int read;
4671	};
4672
4673	#ifdef CONFIG_TRACER_SNAPSHOT
4674	static int tracing_snapshot_open(struct inode inode, struct file file)
4675	{
4676	struct trace_array *tr = inode->i_private;
4677	struct trace_iterator *iter;
4678	struct seq_file *m;
4679	int ret = 0;
4680
4681	if (trace_array_get(tr) < 0)
4682	return -ENODEV;
4683
4684	if (file->f_mode & FMODE_READ) {
4685	iter = __tracing_open(inode, file, true);
4686	if (IS_ERR(iter))
4687	ret = PTR_ERR(iter);
4688	} else {
4689	/* Writes still need the seq_file to hold the private data */
4690	ret = -ENOMEM;
4691	m = kzalloc(sizeof(*m), GFP_KERNEL);
4692	if (!m)
4693	goto out;
4694	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4695	if (!iter) {
4696	kfree(m);
4697	goto out;
4698	}
4699	ret = 0;
4700
4701	iter->tr = tr;
4702	iter->trace_buffer = &tr->max_buffer;
4703	iter->cpu_file = tracing_get_cpu(inode);
4704	m->private = iter;
4705	file->private_data = m;
4706	}
4707	out:
4708	if (ret < 0)
4709	trace_array_put(tr);
4710
4711	return ret;
4712	}
4713
4714	static ssize_t
4715	tracing_snapshot_write(struct file filp, const char __user ubuf, size_t cnt,
4716	loff_t *ppos)
4717	{
4718	struct seq_file *m = filp->private_data;
4719	struct trace_iterator *iter = m->private;
4720	struct trace_array *tr = iter->tr;
4721	unsigned long val;
4722	int ret;
4723
4724	ret = tracing_update_buffers();
4725	if (ret < 0)
4726	return ret;
4727
4728	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4729	if (ret)
4730	return ret;
4731
4732	mutex_lock(&trace_types_lock);
4733
4734	if (tr->current_trace->use_max_tr) {
4735	ret = -EBUSY;
4736	goto out;
4737	}
4738
4739	switch (val) {
4740	case 0:
4741	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4742	ret = -EINVAL;
4743	break;
4744	}
4745	if (tr->allocated_snapshot)
4746	free_snapshot(tr);
4747	break;
4748	case 1:
4749	/* Only allow per-cpu swap if the ring buffer supports it */
4750	#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4751	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4752	ret = -EINVAL;
4753	break;
4754	}
4755	#endif
4756	if (!tr->allocated_snapshot) {
4757	ret = alloc_snapshot(tr);
4758	if (ret < 0)
4759	break;
4760	}
4761	local_irq_disable();
4762	/* Now, we're going to swap */
4763	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4764	update_max_tr(tr, current, smp_processor_id());
4765	else
4766	update_max_tr_single(tr, current, iter->cpu_file);
4767	local_irq_enable();
4768	break;
4769	default:
4770	if (tr->allocated_snapshot) {
4771	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4772	tracing_reset_online_cpus(&tr->max_buffer);
4773	else
4774	tracing_reset(&tr->max_buffer, iter->cpu_file);
4775	}
4776	break;
4777	}
4778
4779	if (ret >= 0) {
4780	*ppos += cnt;
4781	ret = cnt;
4782	}
4783	out:
4784	mutex_unlock(&trace_types_lock);
4785	return ret;
4786	}
4787
4788	static int tracing_snapshot_release(struct inode inode, struct file file)
4789	{
4790	struct seq_file *m = file->private_data;
4791	int ret;
4792
4793	ret = tracing_release(inode, file);
4794
4795	if (file->f_mode & FMODE_READ)
4796	return ret;
4797
4798	/* If write only, the seq_file is just a stub */
4799	if (m)
4800	kfree(m->private);
4801	kfree(m);
4802
4803	return 0;
4804	}
4805
4806	static int tracing_buffers_open(struct inode inode, struct file filp);
4807	static ssize_t tracing_buffers_read(struct file filp, char __user ubuf,
4808	size_t count, loff_t *ppos);
4809	static int tracing_buffers_release(struct inode inode, struct file file);
4810	static ssize_t tracing_buffers_splice_read(struct file file, loff_t ppos,
4811	struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4812
4813	static int snapshot_raw_open(struct inode inode, struct file filp)
4814	{
4815	struct ftrace_buffer_info *info;
4816	int ret;
4817
4818	ret = tracing_buffers_open(inode, filp);
4819	if (ret < 0)
4820	return ret;
4821
4822	info = filp->private_data;
4823
4824	if (info->iter.trace->use_max_tr) {
4825	tracing_buffers_release(inode, filp);
4826	return -EBUSY;
4827	}
4828
4829	info->iter.snapshot = true;
4830	info->iter.trace_buffer = &info->iter.tr->max_buffer;
4831
4832	return ret;
4833	}
4834
4835	#endif /* CONFIG_TRACER_SNAPSHOT */
4836
4837
4838	static const struct file_operations tracing_max_lat_fops = {
4839	.open = tracing_open_generic,
4840	.read = tracing_max_lat_read,
4841	.write = tracing_max_lat_write,
4842	.llseek = generic_file_llseek,
4843	};
4844
4845	static const struct file_operations set_tracer_fops = {
4846	.open = tracing_open_generic,
4847	.read = tracing_set_trace_read,
4848	.write = tracing_set_trace_write,
4849	.llseek = generic_file_llseek,
4850	};
4851
4852	static const struct file_operations tracing_pipe_fops = {
4853	.open = tracing_open_pipe,
4854	.poll = tracing_poll_pipe,
4855	.read = tracing_read_pipe,
4856	.splice_read = tracing_splice_read_pipe,
4857	.release = tracing_release_pipe,
4858	.llseek = no_llseek,
4859	};
4860
4861	static const struct file_operations tracing_entries_fops = {
4862	.open = tracing_open_generic_tr,
4863	.read = tracing_entries_read,
4864	.write = tracing_entries_write,
4865	.llseek = generic_file_llseek,
4866	.release = tracing_release_generic_tr,
4867	};
4868
4869	static const struct file_operations tracing_total_entries_fops = {
4870	.open = tracing_open_generic_tr,
4871	.read = tracing_total_entries_read,
4872	.llseek = generic_file_llseek,
4873	.release = tracing_release_generic_tr,
4874	};
4875
4876	static const struct file_operations tracing_free_buffer_fops = {
4877	.open = tracing_open_generic_tr,
4878	.write = tracing_free_buffer_write,
4879	.release = tracing_free_buffer_release,
4880	};
4881
4882	static const struct file_operations tracing_mark_fops = {
4883	.open = tracing_open_generic_tr,
4884	.write = tracing_mark_write,
4885	.llseek = generic_file_llseek,
4886	.release = tracing_release_generic_tr,
4887	};
4888
4889	static const struct file_operations trace_clock_fops = {
4890	.open = tracing_clock_open,
4891	.read = seq_read,
4892	.llseek = seq_lseek,
4893	.release = tracing_single_release_tr,
4894	.write = tracing_clock_write,
4895	};
4896
4897	#ifdef CONFIG_TRACER_SNAPSHOT
4898	static const struct file_operations snapshot_fops = {
4899	.open = tracing_snapshot_open,
4900	.read = seq_read,
4901	.write = tracing_snapshot_write,
4902	.llseek = tracing_seek,
4903	.release = tracing_snapshot_release,
4904	};
4905
4906	static const struct file_operations snapshot_raw_fops = {
4907	.open = snapshot_raw_open,
4908	.read = tracing_buffers_read,
4909	.release = tracing_buffers_release,
4910	.splice_read = tracing_buffers_splice_read,
4911	.llseek = no_llseek,
4912	};
4913
4914	#endif /* CONFIG_TRACER_SNAPSHOT */
4915
4916	static int tracing_buffers_open(struct inode inode, struct file filp)
4917	{
4918	struct trace_array *tr = inode->i_private;
4919	struct ftrace_buffer_info *info;
4920	int ret;
4921
4922	if (tracing_disabled)
4923	return -ENODEV;
4924
4925	if (trace_array_get(tr) < 0)
4926	return -ENODEV;
4927
4928	info = kzalloc(sizeof(*info), GFP_KERNEL);
4929	if (!info) {
4930	trace_array_put(tr);
4931	return -ENOMEM;
4932	}
4933
4934	mutex_lock(&trace_types_lock);
4935
4936	info->iter.tr = tr;
4937	info->iter.cpu_file = tracing_get_cpu(inode);
4938	info->iter.trace = tr->current_trace;
4939	info->iter.trace_buffer = &tr->trace_buffer;
4940	info->spare = NULL;
4941	/* Force reading ring buffer for first read */
4942	info->read = (unsigned int)-1;
4943
4944	filp->private_data = info;
4945
4946	mutex_unlock(&trace_types_lock);
4947
4948	ret = nonseekable_open(inode, filp);
4949	if (ret < 0)
4950	trace_array_put(tr);
4951
4952	return ret;
4953	}
4954
4955	static unsigned int
4956	tracing_buffers_poll(struct file filp, poll_table poll_table)
4957	{
4958	struct ftrace_buffer_info *info = filp->private_data;
4959	struct trace_iterator *iter = &info->iter;
4960
4961	return trace_poll(iter, filp, poll_table);
4962	}
4963
4964	static ssize_t
4965	tracing_buffers_read(struct file filp, char __user ubuf,
4966	size_t count, loff_t *ppos)
4967	{
4968	struct ftrace_buffer_info *info = filp->private_data;
4969	struct trace_iterator *iter = &info->iter;
4970	ssize_t ret;
4971	ssize_t size;
4972
4973	if (!count)
4974	return 0;
4975
4976	mutex_lock(&trace_types_lock);
4977
4978	#ifdef CONFIG_TRACER_MAX_TRACE
4979	if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4980	size = -EBUSY;
4981	goto out_unlock;
4982	}
4983	#endif
4984
4985	if (!info->spare)
4986	info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4987	iter->cpu_file);
4988	size = -ENOMEM;
4989	if (!info->spare)
4990	goto out_unlock;
4991
4992	/* Do we have previous read data to read? */
4993	if (info->read < PAGE_SIZE)
4994	goto read;
4995
4996	again:
4997	trace_access_lock(iter->cpu_file);
4998	ret = ring_buffer_read_page(iter->trace_buffer->buffer,
4999	&info->spare,
5000	count,
5001	iter->cpu_file, 0);
5002	trace_access_unlock(iter->cpu_file);
5003
5004	if (ret < 0) {
5005	if (trace_empty(iter)) {
5006	if ((filp->f_flags & O_NONBLOCK)) {
5007	size = -EAGAIN;
5008	goto out_unlock;
5009	}
5010	mutex_unlock(&trace_types_lock);
5011	iter->trace->wait_pipe(iter);
5012	mutex_lock(&trace_types_lock);
5013	if (signal_pending(current)) {
5014	size = -EINTR;
5015	goto out_unlock;
5016	}
5017	goto again;
5018	}
5019	size = 0;
5020	goto out_unlock;
5021	}
5022
5023	info->read = 0;
5024	read:
5025	size = PAGE_SIZE - info->read;
5026	if (size > count)
5027	size = count;
5028
5029	ret = copy_to_user(ubuf, info->spare + info->read, size);
5030	if (ret == size) {
5031	size = -EFAULT;
5032	goto out_unlock;
5033	}
5034	size -= ret;
5035
5036	*ppos += size;
5037	info->read += size;
5038
5039	out_unlock:
5040	mutex_unlock(&trace_types_lock);
5041
5042	return size;
5043	}
5044
5045	static int tracing_buffers_release(struct inode inode, struct file file)
5046	{
5047	struct ftrace_buffer_info *info = file->private_data;
5048	struct trace_iterator *iter = &info->iter;
5049
5050	mutex_lock(&trace_types_lock);
5051
5052	__trace_array_put(iter->tr);
5053
5054	if (info->spare)
5055	ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5056	kfree(info);
5057
5058	mutex_unlock(&trace_types_lock);
5059
5060	return 0;
5061	}
5062
5063	struct buffer_ref {
5064	struct ring_buffer *buffer;
5065	void *page;
5066	int ref;
5067	};
5068
5069	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5070	struct pipe_buffer *buf)
5071	{
5072	struct buffer_ref ref = (struct buffer_ref )buf->private;
5073
5074	if (--ref->ref)
5075	return;
5076
5077	ring_buffer_free_read_page(ref->buffer, ref->page);
5078	kfree(ref);
5079	buf->private = 0;
5080	}
5081
5082	static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5083	struct pipe_buffer *buf)
5084	{
5085	struct buffer_ref ref = (struct buffer_ref )buf->private;
5086
5087	ref->ref++;
5088	}
5089
5090	/* Pipe buffer operations for a buffer. */
5091	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5092	.can_merge = 0,
5093	.map = generic_pipe_buf_map,
5094	.unmap = generic_pipe_buf_unmap,
5095	.confirm = generic_pipe_buf_confirm,
5096	.release = buffer_pipe_buf_release,
5097	.steal = generic_pipe_buf_steal,
5098	.get = buffer_pipe_buf_get,
5099	};
5100
5101	/*
5102	* Callback from splice_to_pipe(), if we need to release some pages
5103	* at the end of the spd in case we error'ed out in filling the pipe.
5104	*/
5105	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5106	{
5107	struct buffer_ref *ref =
5108	(struct buffer_ref *)spd->partial[i].private;
5109
5110	if (--ref->ref)
5111	return;
5112
5113	ring_buffer_free_read_page(ref->buffer, ref->page);
5114	kfree(ref);
5115	spd->partial[i].private = 0;
5116	}
5117
5118	static ssize_t
5119	tracing_buffers_splice_read(struct file file, loff_t ppos,
5120	struct pipe_inode_info *pipe, size_t len,
5121	unsigned int flags)
5122	{
5123	struct ftrace_buffer_info *info = file->private_data;
5124	struct trace_iterator *iter = &info->iter;
5125	struct partial_page partial_def[PIPE_DEF_BUFFERS];
5126	struct page *pages_def[PIPE_DEF_BUFFERS];
5127	struct splice_pipe_desc spd = {
5128	.pages = pages_def,
5129	.partial = partial_def,
5130	.nr_pages_max = PIPE_DEF_BUFFERS,
5131	.flags = flags,
5132	.ops = &buffer_pipe_buf_ops,
5133	.spd_release = buffer_spd_release,
5134	};
5135	struct buffer_ref *ref;
5136	int entries, size, i;
5137	ssize_t ret;
5138
5139	mutex_lock(&trace_types_lock);
5140
5141	#ifdef CONFIG_TRACER_MAX_TRACE
5142	if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5143	ret = -EBUSY;
5144	goto out;
5145	}
5146	#endif
5147
5148	if (splice_grow_spd(pipe, &spd)) {
5149	ret = -ENOMEM;
5150	goto out;
5151	}
5152
5153	if (*ppos & (PAGE_SIZE - 1)) {
5154	ret = -EINVAL;
5155	goto out;
5156	}
5157
5158	if (len & (PAGE_SIZE - 1)) {
5159	if (len < PAGE_SIZE) {
5160	ret = -EINVAL;
5161	goto out;
5162	}
5163	len &= PAGE_MASK;
5164	}
5165
5166	again:
5167	trace_access_lock(iter->cpu_file);
5168	entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5169
5170	for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
5171	struct page *page;
5172	int r;
5173
5174	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5175	if (!ref)
5176	break;
5177
5178	ref->ref = 1;
5179	ref->buffer = iter->trace_buffer->buffer;
5180	ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5181	if (!ref->page) {
5182	kfree(ref);
5183	break;
5184	}
5185
5186	r = ring_buffer_read_page(ref->buffer, &ref->page,
5187	len, iter->cpu_file, 1);
5188	if (r < 0) {
5189	ring_buffer_free_read_page(ref->buffer, ref->page);
5190	kfree(ref);
5191	break;
5192	}
5193
5194	/*
5195	* zero out any left over data, this is going to
5196	* user land.
5197	*/
5198	size = ring_buffer_page_len(ref->page);
5199	if (size < PAGE_SIZE)
5200	memset(ref->page + size, 0, PAGE_SIZE - size);
5201
5202	page = virt_to_page(ref->page);
5203
5204	spd.pages[i] = page;
5205	spd.partial[i].len = PAGE_SIZE;
5206	spd.partial[i].offset = 0;
5207	spd.partial[i].private = (unsigned long)ref;
5208	spd.nr_pages++;
5209	*ppos += PAGE_SIZE;
5210
5211	entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5212	}
5213
5214	trace_access_unlock(iter->cpu_file);
5215	spd.nr_pages = i;
5216
5217	/* did we read anything? */
5218	if (!spd.nr_pages) {
5219	if ((file->f_flags & O_NONBLOCK) \|\| (flags & SPLICE_F_NONBLOCK)) {
5220	ret = -EAGAIN;
5221	goto out;
5222	}
5223	mutex_unlock(&trace_types_lock);
5224	iter->trace->wait_pipe(iter);
5225	mutex_lock(&trace_types_lock);
5226	if (signal_pending(current)) {
5227	ret = -EINTR;
5228	goto out;
5229	}
5230	goto again;
5231	}
5232
5233	ret = splice_to_pipe(pipe, &spd);
5234	splice_shrink_spd(&spd);
5235	out:
5236	mutex_unlock(&trace_types_lock);
5237
5238	return ret;
5239	}
5240
5241	static const struct file_operations tracing_buffers_fops = {
5242	.open = tracing_buffers_open,
5243	.read = tracing_buffers_read,
5244	.poll = tracing_buffers_poll,
5245	.release = tracing_buffers_release,
5246	.splice_read = tracing_buffers_splice_read,
5247	.llseek = no_llseek,
5248	};
5249
5250	static ssize_t
5251	tracing_stats_read(struct file filp, char __user ubuf,
5252	size_t count, loff_t *ppos)
5253	{
5254	struct inode *inode = file_inode(filp);
5255	struct trace_array *tr = inode->i_private;
5256	struct trace_buffer *trace_buf = &tr->trace_buffer;
5257	int cpu = tracing_get_cpu(inode);
5258	struct trace_seq *s;
5259	unsigned long cnt;
5260	unsigned long long t;
5261	unsigned long usec_rem;
5262
5263	s = kmalloc(sizeof(*s), GFP_KERNEL);
5264	if (!s)
5265	return -ENOMEM;
5266
5267	trace_seq_init(s);
5268
5269	cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5270	trace_seq_printf(s, "entries: %ld\n", cnt);
5271
5272	cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5273	trace_seq_printf(s, "overrun: %ld\n", cnt);
5274
5275	cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5276	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5277
5278	cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5279	trace_seq_printf(s, "bytes: %ld\n", cnt);
5280
5281	if (trace_clocks[tr->clock_id].in_ns) {
5282	/* local or global for trace_clock */
5283	t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5284	usec_rem = do_div(t, USEC_PER_SEC);
5285	trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5286	t, usec_rem);
5287
5288	t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5289	usec_rem = do_div(t, USEC_PER_SEC);
5290	trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5291	} else {
5292	/* counter or tsc mode for trace_clock */
5293	trace_seq_printf(s, "oldest event ts: %llu\n",
5294	ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5295
5296	trace_seq_printf(s, "now ts: %llu\n",
5297	ring_buffer_time_stamp(trace_buf->buffer, cpu));
5298	}
5299
5300	cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5301	trace_seq_printf(s, "dropped events: %ld\n", cnt);
5302
5303	cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5304	trace_seq_printf(s, "read events: %ld\n", cnt);
5305
5306	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5307
5308	kfree(s);
5309
5310	return count;
5311	}
5312
5313	static const struct file_operations tracing_stats_fops = {
5314	.open = tracing_open_generic_tr,
5315	.read = tracing_stats_read,
5316	.llseek = generic_file_llseek,
5317	.release = tracing_release_generic_tr,
5318	};
5319
5320	#ifdef CONFIG_DYNAMIC_FTRACE
5321
5322	int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5323	{
5324	return 0;
5325	}
5326
5327	static ssize_t
5328	tracing_read_dyn_info(struct file filp, char __user ubuf,
5329	size_t cnt, loff_t *ppos)
5330	{
5331	static char ftrace_dyn_info_buffer[1024];
5332	static DEFINE_MUTEX(dyn_info_mutex);
5333	unsigned long *p = filp->private_data;
5334	char *buf = ftrace_dyn_info_buffer;
5335	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5336	int r;
5337
5338	mutex_lock(&dyn_info_mutex);
5339	r = sprintf(buf, "%ld ", *p);
5340
5341	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5342	buf[r++] = '\n';
5343
5344	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5345
5346	mutex_unlock(&dyn_info_mutex);
5347
5348	return r;
5349	}
5350
5351	static const struct file_operations tracing_dyn_info_fops = {
5352	.open = tracing_open_generic,
5353	.read = tracing_read_dyn_info,
5354	.llseek = generic_file_llseek,
5355	};
5356	#endif /* CONFIG_DYNAMIC_FTRACE */
5357
5358	#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5359	static void
5360	ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5361	{
5362	tracing_snapshot();
5363	}
5364
5365	static void
5366	ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5367	{
5368	unsigned long count = (long )data;
5369
5370	if (!*count)
5371	return;
5372
5373	if (*count != -1)
5374	(*count)--;
5375
5376	tracing_snapshot();
5377	}
5378
5379	static int
5380	ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5381	struct ftrace_probe_ops ops, void data)
5382	{
5383	long count = (long)data;
5384
5385	seq_printf(m, "%ps:", (void *)ip);
5386
5387	seq_printf(m, "snapshot");
5388
5389	if (count == -1)
5390	seq_printf(m, ":unlimited\n");
5391	else
5392	seq_printf(m, ":count=%ld\n", count);
5393
5394	return 0;
5395	}
5396
5397	static struct ftrace_probe_ops snapshot_probe_ops = {
5398	.func = ftrace_snapshot,
5399	.print = ftrace_snapshot_print,
5400	};
5401
5402	static struct ftrace_probe_ops snapshot_count_probe_ops = {
5403	.func = ftrace_count_snapshot,
5404	.print = ftrace_snapshot_print,
5405	};
5406
5407	static int
5408	ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5409	char glob, char cmd, char *param, int enable)
5410	{
5411	struct ftrace_probe_ops *ops;
5412	void count = (void )-1;
5413	char *number;
5414	int ret;
5415
5416	/* hash funcs only work with set_ftrace_filter */
5417	if (!enable)
5418	return -EINVAL;
5419
5420	ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5421
5422	if (glob[0] == '!') {
5423	unregister_ftrace_function_probe_func(glob+1, ops);
5424	return 0;
5425	}
5426
5427	if (!param)
5428	goto out_reg;
5429
5430	number = strsep(&param, ":");
5431
5432	if (!strlen(number))
5433	goto out_reg;
5434
5435	/*
5436	* We use the callback data field (which is a pointer)
5437	* as our counter.
5438	*/
5439	ret = kstrtoul(number, 0, (unsigned long *)&count);
5440	if (ret)
5441	return ret;
5442
5443	out_reg:
5444	ret = register_ftrace_function_probe(glob, ops, count);
5445
5446	if (ret >= 0)
5447	alloc_snapshot(&global_trace);
5448
5449	return ret < 0 ? ret : 0;
5450	}
5451
5452	static struct ftrace_func_command ftrace_snapshot_cmd = {
5453	.name = "snapshot",
5454	.func = ftrace_trace_snapshot_callback,
5455	};
5456
5457	static int register_snapshot_cmd(void)
5458	{
5459	return register_ftrace_command(&ftrace_snapshot_cmd);
5460	}
5461	#else
5462	static inline int register_snapshot_cmd(void) { return 0; }
5463	#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5464
5465	struct dentry tracing_init_dentry_tr(struct trace_array tr)
5466	{
5467	if (tr->dir)
5468	return tr->dir;
5469
5470	if (!debugfs_initialized())
5471	return NULL;
5472
5473	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5474	tr->dir = debugfs_create_dir("tracing", NULL);
5475
5476	if (!tr->dir)
5477	pr_warn_once("Could not create debugfs directory 'tracing'\n");
5478
5479	return tr->dir;
5480	}
5481
5482	struct dentry *tracing_init_dentry(void)
5483	{
5484	return tracing_init_dentry_tr(&global_trace);
5485	}
5486
5487	static struct dentry tracing_dentry_percpu(struct trace_array tr, int cpu)
5488	{
5489	struct dentry *d_tracer;
5490
5491	if (tr->percpu_dir)
5492	return tr->percpu_dir;
5493
5494	d_tracer = tracing_init_dentry_tr(tr);
5495	if (!d_tracer)
5496	return NULL;
5497
5498	tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5499
5500	WARN_ONCE(!tr->percpu_dir,
5501	"Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5502
5503	return tr->percpu_dir;
5504	}
5505
5506	static struct dentry *
5507	trace_create_cpu_file(const char name, umode_t mode, struct dentry parent,
5508	void data, long cpu, const struct file_operations fops)
5509	{
5510	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5511
5512	if (ret) /* See tracing_get_cpu() */
5513	ret->d_inode->i_cdev = (void *)(cpu + 1);
5514	return ret;
5515	}
5516
5517	static void
5518	tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5519	{
5520	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5521	struct dentry *d_cpu;
5522	char cpu_dir[30]; /* 30 characters should be more than enough */
5523
5524	if (!d_percpu)
5525	return;
5526
5527	snprintf(cpu_dir, 30, "cpu%ld", cpu);
5528	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5529	if (!d_cpu) {
5530	pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5531	return;
5532	}
5533
5534	/* per cpu trace_pipe */
5535	trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5536	tr, cpu, &tracing_pipe_fops);
5537
5538	/* per cpu trace */
5539	trace_create_cpu_file("trace", 0644, d_cpu,
5540	tr, cpu, &tracing_fops);
5541
5542	trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5543	tr, cpu, &tracing_buffers_fops);
5544
5545	trace_create_cpu_file("stats", 0444, d_cpu,
5546	tr, cpu, &tracing_stats_fops);
5547
5548	trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5549	tr, cpu, &tracing_entries_fops);
5550
5551	#ifdef CONFIG_TRACER_SNAPSHOT
5552	trace_create_cpu_file("snapshot", 0644, d_cpu,
5553	tr, cpu, &snapshot_fops);
5554
5555	trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5556	tr, cpu, &snapshot_raw_fops);
5557	#endif
5558	}
5559
5560	#ifdef CONFIG_FTRACE_SELFTEST
5561	/* Let selftest have access to static functions in this file */
5562	#include "trace_selftest.c"
5563	#endif
5564
5565	struct trace_option_dentry {
5566	struct tracer_opt *opt;
5567	struct tracer_flags *flags;
5568	struct trace_array *tr;
5569	struct dentry *entry;
5570	};
5571
5572	static ssize_t
5573	trace_options_read(struct file filp, char __user ubuf, size_t cnt,
5574	loff_t *ppos)
5575	{
5576	struct trace_option_dentry *topt = filp->private_data;
5577	char *buf;
5578
5579	if (topt->flags->val & topt->opt->bit)
5580	buf = "1\n";
5581	else
5582	buf = "0\n";
5583
5584	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5585	}
5586
5587	static ssize_t
5588	trace_options_write(struct file filp, const char __user ubuf, size_t cnt,
5589	loff_t *ppos)
5590	{
5591	struct trace_option_dentry *topt = filp->private_data;
5592	unsigned long val;
5593	int ret;
5594
5595	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5596	if (ret)
5597	return ret;
5598
5599	if (val != 0 && val != 1)
5600	return -EINVAL;
5601
5602	if (!!(topt->flags->val & topt->opt->bit) != val) {
5603	mutex_lock(&trace_types_lock);
5604	ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
5605	topt->opt, !val);
5606	mutex_unlock(&trace_types_lock);
5607	if (ret)
5608	return ret;
5609	}
5610
5611	*ppos += cnt;
5612
5613	return cnt;
5614	}
5615
5616
5617	static const struct file_operations trace_options_fops = {
5618	.open = tracing_open_generic,
5619	.read = trace_options_read,
5620	.write = trace_options_write,
5621	.llseek = generic_file_llseek,
5622	};
5623
5624	static ssize_t
5625	trace_options_core_read(struct file filp, char __user ubuf, size_t cnt,
5626	loff_t *ppos)
5627	{
5628	long index = (long)filp->private_data;
5629	char *buf;
5630
5631	if (trace_flags & (1 << index))
5632	buf = "1\n";
5633	else
5634	buf = "0\n";
5635
5636	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5637	}
5638
5639	static ssize_t
5640	trace_options_core_write(struct file filp, const char __user ubuf, size_t cnt,
5641	loff_t *ppos)
5642	{
5643	struct trace_array *tr = &global_trace;
5644	long index = (long)filp->private_data;
5645	unsigned long val;
5646	int ret;
5647
5648	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5649	if (ret)
5650	return ret;
5651
5652	if (val != 0 && val != 1)
5653	return -EINVAL;
5654
5655	mutex_lock(&trace_types_lock);
5656	ret = set_tracer_flag(tr, 1 << index, val);
5657	mutex_unlock(&trace_types_lock);
5658
5659	if (ret < 0)
5660	return ret;
5661
5662	*ppos += cnt;
5663
5664	return cnt;
5665	}
5666
5667	static const struct file_operations trace_options_core_fops = {
5668	.open = tracing_open_generic,
5669	.read = trace_options_core_read,
5670	.write = trace_options_core_write,
5671	.llseek = generic_file_llseek,
5672	};
5673
5674	struct dentry trace_create_file(const char name,
5675	umode_t mode,
5676	struct dentry *parent,
5677	void *data,
5678	const struct file_operations *fops)
5679	{
5680	struct dentry *ret;
5681
5682	ret = debugfs_create_file(name, mode, parent, data, fops);
5683	if (!ret)
5684	pr_warning("Could not create debugfs '%s' entry\n", name);
5685
5686	return ret;
5687	}
5688
5689
5690	static struct dentry trace_options_init_dentry(struct trace_array tr)
5691	{
5692	struct dentry *d_tracer;
5693
5694	if (tr->options)
5695	return tr->options;
5696
5697	d_tracer = tracing_init_dentry_tr(tr);
5698	if (!d_tracer)
5699	return NULL;
5700
5701	tr->options = debugfs_create_dir("options", d_tracer);
5702	if (!tr->options) {
5703	pr_warning("Could not create debugfs directory 'options'\n");
5704	return NULL;
5705	}
5706
5707	return tr->options;
5708	}
5709
5710	static void
5711	create_trace_option_file(struct trace_array *tr,
5712	struct trace_option_dentry *topt,
5713	struct tracer_flags *flags,
5714	struct tracer_opt *opt)
5715	{
5716	struct dentry *t_options;
5717
5718	t_options = trace_options_init_dentry(tr);
5719	if (!t_options)
5720	return;
5721
5722	topt->flags = flags;
5723	topt->opt = opt;
5724	topt->tr = tr;
5725
5726	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5727	&trace_options_fops);
5728
5729	}
5730
5731	static struct trace_option_dentry *
5732	create_trace_option_files(struct trace_array tr, struct tracer tracer)
5733	{
5734	struct trace_option_dentry *topts;
5735	struct tracer_flags *flags;
5736	struct tracer_opt *opts;
5737	int cnt;
5738
5739	if (!tracer)
5740	return NULL;
5741
5742	flags = tracer->flags;
5743
5744	if (!flags \|\| !flags->opts)
5745	return NULL;
5746
5747	opts = flags->opts;
5748
5749	for (cnt = 0; opts[cnt].name; cnt++)
5750	;
5751
5752	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5753	if (!topts)
5754	return NULL;
5755
5756	for (cnt = 0; opts[cnt].name; cnt++)
5757	create_trace_option_file(tr, &topts[cnt], flags,
5758	&opts[cnt]);
5759
5760	return topts;
5761	}
5762
5763	static void
5764	destroy_trace_option_files(struct trace_option_dentry *topts)
5765	{
5766	int cnt;
5767
5768	if (!topts)
5769	return;
5770
5771	for (cnt = 0; topts[cnt].opt; cnt++) {
5772	if (topts[cnt].entry)
5773	debugfs_remove(topts[cnt].entry);
5774	}
5775
5776	kfree(topts);
5777	}
5778
5779	static struct dentry *
5780	create_trace_option_core_file(struct trace_array *tr,
5781	const char *option, long index)
5782	{
5783	struct dentry *t_options;
5784
5785	t_options = trace_options_init_dentry(tr);
5786	if (!t_options)
5787	return NULL;
5788
5789	return trace_create_file(option, 0644, t_options, (void *)index,
5790	&trace_options_core_fops);
5791	}
5792
5793	static __init void create_trace_options_dir(struct trace_array *tr)
5794	{
5795	struct dentry *t_options;
5796	int i;
5797
5798	t_options = trace_options_init_dentry(tr);
5799	if (!t_options)
5800	return;
5801
5802	for (i = 0; trace_options[i]; i++)
5803	create_trace_option_core_file(tr, trace_options[i], i);
5804	}
5805
5806	static ssize_t
5807	rb_simple_read(struct file filp, char __user ubuf,
5808	size_t cnt, loff_t *ppos)
5809	{
5810	struct trace_array *tr = filp->private_data;
5811	char buf[64];
5812	int r;
5813
5814	r = tracer_tracing_is_on(tr);
5815	r = sprintf(buf, "%d\n", r);
5816
5817	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5818	}
5819
5820	static ssize_t
5821	rb_simple_write(struct file filp, const char __user ubuf,
5822	size_t cnt, loff_t *ppos)
5823	{
5824	struct trace_array *tr = filp->private_data;
5825	struct ring_buffer *buffer = tr->trace_buffer.buffer;
5826	unsigned long val;
5827	int ret;
5828
5829	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5830	if (ret)
5831	return ret;
5832
5833	if (buffer) {
5834	mutex_lock(&trace_types_lock);
5835	if (val) {
5836	tracer_tracing_on(tr);
5837	if (tr->current_trace->start)
5838	tr->current_trace->start(tr);
5839	} else {
5840	tracer_tracing_off(tr);
5841	if (tr->current_trace->stop)
5842	tr->current_trace->stop(tr);
5843	}
5844	mutex_unlock(&trace_types_lock);
5845	}
5846
5847	(*ppos)++;
5848
5849	return cnt;
5850	}
5851
5852	static const struct file_operations rb_simple_fops = {
5853	.open = tracing_open_generic_tr,
5854	.read = rb_simple_read,
5855	.write = rb_simple_write,
5856	.release = tracing_release_generic_tr,
5857	.llseek = default_llseek,
5858	};
5859
5860	struct dentry *trace_instance_dir;
5861
5862	static void
5863	init_tracer_debugfs(struct trace_array tr, struct dentry d_tracer);
5864
5865	static int
5866	allocate_trace_buffer(struct trace_array tr, struct trace_buffer buf, int size)
5867	{
5868	enum ring_buffer_flags rb_flags;
5869
5870	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5871
5872	buf->buffer = ring_buffer_alloc(size, rb_flags);
5873	if (!buf->buffer)
5874	return -ENOMEM;
5875
5876	buf->data = alloc_percpu(struct trace_array_cpu);
5877	if (!buf->data) {
5878	ring_buffer_free(buf->buffer);
5879	return -ENOMEM;
5880	}
5881
5882	/* Allocate the first page for all buffers */
5883	set_buffer_entries(&tr->trace_buffer,
5884	ring_buffer_size(tr->trace_buffer.buffer, 0));
5885
5886	return 0;
5887	}
5888
5889	static int allocate_trace_buffers(struct trace_array *tr, int size)
5890	{
5891	int ret;
5892
5893	ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5894	if (ret)
5895	return ret;
5896
5897	#ifdef CONFIG_TRACER_MAX_TRACE
5898	ret = allocate_trace_buffer(tr, &tr->max_buffer,
5899	allocate_snapshot ? size : 1);
5900	if (WARN_ON(ret)) {
5901	ring_buffer_free(tr->trace_buffer.buffer);
5902	free_percpu(tr->trace_buffer.data);
5903	return -ENOMEM;
5904	}
5905	tr->allocated_snapshot = allocate_snapshot;
5906
5907	/*
5908	* Only the top level trace array gets its snapshot allocated
5909	* from the kernel command line.
5910	*/
5911	allocate_snapshot = false;
5912	#endif
5913	return 0;
5914	}
5915
5916	static int new_instance_create(const char *name)
5917	{
5918	struct trace_array *tr;
5919	int ret;
5920
5921	mutex_lock(&trace_types_lock);
5922
5923	ret = -EEXIST;
5924	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5925	if (tr->name && strcmp(tr->name, name) == 0)
5926	goto out_unlock;
5927	}
5928
5929	ret = -ENOMEM;
5930	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5931	if (!tr)
5932	goto out_unlock;
5933
5934	tr->name = kstrdup(name, GFP_KERNEL);
5935	if (!tr->name)
5936	goto out_free_tr;
5937
5938	if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
5939	goto out_free_tr;
5940
5941	cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
5942
5943	raw_spin_lock_init(&tr->start_lock);
5944
5945	tr->current_trace = &nop_trace;
5946
5947	INIT_LIST_HEAD(&tr->systems);
5948	INIT_LIST_HEAD(&tr->events);
5949
5950	if (allocate_trace_buffers(tr, trace_buf_size) < 0)
5951	goto out_free_tr;
5952
5953	tr->dir = debugfs_create_dir(name, trace_instance_dir);
5954	if (!tr->dir)
5955	goto out_free_tr;
5956
5957	ret = event_trace_add_tracer(tr->dir, tr);
5958	if (ret) {
5959	debugfs_remove_recursive(tr->dir);
5960	goto out_free_tr;
5961	}
5962
5963	init_tracer_debugfs(tr, tr->dir);
5964
5965	list_add(&tr->list, &ftrace_trace_arrays);
5966
5967	mutex_unlock(&trace_types_lock);
5968
5969	return 0;
5970
5971	out_free_tr:
5972	if (tr->trace_buffer.buffer)
5973	ring_buffer_free(tr->trace_buffer.buffer);
5974	free_cpumask_var(tr->tracing_cpumask);
5975	kfree(tr->name);
5976	kfree(tr);
5977
5978	out_unlock:
5979	mutex_unlock(&trace_types_lock);
5980
5981	return ret;
5982
5983	}
5984
5985	static int instance_delete(const char *name)
5986	{
5987	struct trace_array *tr;
5988	int found = 0;
5989	int ret;
5990
5991	mutex_lock(&trace_types_lock);
5992
5993	ret = -ENODEV;
5994	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5995	if (tr->name && strcmp(tr->name, name) == 0) {
5996	found = 1;
5997	break;
5998	}
5999	}
6000	if (!found)
6001	goto out_unlock;
6002
6003	ret = -EBUSY;
6004	if (tr->ref)
6005	goto out_unlock;
6006
6007	list_del(&tr->list);
6008
6009	event_trace_del_tracer(tr);
6010	debugfs_remove_recursive(tr->dir);
6011	free_percpu(tr->trace_buffer.data);
6012	ring_buffer_free(tr->trace_buffer.buffer);
6013
6014	kfree(tr->name);
6015	kfree(tr);
6016
6017	ret = 0;
6018
6019	out_unlock:
6020	mutex_unlock(&trace_types_lock);
6021
6022	return ret;
6023	}
6024
6025	static int instance_mkdir (struct inode inode, struct dentry dentry, umode_t mode)
6026	{
6027	struct dentry *parent;
6028	int ret;
6029
6030	/* Paranoid: Make sure the parent is the "instances" directory */
6031	parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6032	if (WARN_ON_ONCE(parent != trace_instance_dir))
6033	return -ENOENT;
6034
6035	/*
6036	* The inode mutex is locked, but debugfs_create_dir() will also
6037	* take the mutex. As the instances directory can not be destroyed
6038	* or changed in any other way, it is safe to unlock it, and
6039	* let the dentry try. If two users try to make the same dir at
6040	* the same time, then the new_instance_create() will determine the
6041	* winner.
6042	*/
6043	mutex_unlock(&inode->i_mutex);
6044
6045	ret = new_instance_create(dentry->d_iname);
6046
6047	mutex_lock(&inode->i_mutex);
6048
6049	return ret;
6050	}
6051
6052	static int instance_rmdir(struct inode inode, struct dentry dentry)
6053	{
6054	struct dentry *parent;
6055	int ret;
6056
6057	/* Paranoid: Make sure the parent is the "instances" directory */
6058	parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6059	if (WARN_ON_ONCE(parent != trace_instance_dir))
6060	return -ENOENT;
6061
6062	/* The caller did a dget() on dentry */
6063	mutex_unlock(&dentry->d_inode->i_mutex);
6064
6065	/*
6066	* The inode mutex is locked, but debugfs_create_dir() will also
6067	* take the mutex. As the instances directory can not be destroyed
6068	* or changed in any other way, it is safe to unlock it, and
6069	* let the dentry try. If two users try to make the same dir at
6070	* the same time, then the instance_delete() will determine the
6071	* winner.
6072	*/
6073	mutex_unlock(&inode->i_mutex);
6074
6075	ret = instance_delete(dentry->d_iname);
6076
6077	mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6078	mutex_lock(&dentry->d_inode->i_mutex);
6079
6080	return ret;
6081	}
6082
6083	static const struct inode_operations instance_dir_inode_operations = {
6084	.lookup = simple_lookup,
6085	.mkdir = instance_mkdir,
6086	.rmdir = instance_rmdir,
6087	};
6088
6089	static __init void create_trace_instances(struct dentry *d_tracer)
6090	{
6091	trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6092	if (WARN_ON(!trace_instance_dir))
6093	return;
6094
6095	/* Hijack the dir inode operations, to allow mkdir */
6096	trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6097	}
6098
6099	static void
6100	init_tracer_debugfs(struct trace_array tr, struct dentry d_tracer)
6101	{
6102	int cpu;
6103
6104	trace_create_file("tracing_cpumask", 0644, d_tracer,
6105	tr, &tracing_cpumask_fops);
6106
6107	trace_create_file("trace_options", 0644, d_tracer,
6108	tr, &tracing_iter_fops);
6109
6110	trace_create_file("trace", 0644, d_tracer,
6111	tr, &tracing_fops);
6112
6113	trace_create_file("trace_pipe", 0444, d_tracer,
6114	tr, &tracing_pipe_fops);
6115
6116	trace_create_file("buffer_size_kb", 0644, d_tracer,
6117	tr, &tracing_entries_fops);
6118
6119	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6120	tr, &tracing_total_entries_fops);
6121
6122	trace_create_file("free_buffer", 0200, d_tracer,
6123	tr, &tracing_free_buffer_fops);
6124
6125	trace_create_file("trace_marker", 0220, d_tracer,
6126	tr, &tracing_mark_fops);
6127
6128	trace_create_file("trace_clock", 0644, d_tracer, tr,
6129	&trace_clock_fops);
6130
6131	trace_create_file("tracing_on", 0644, d_tracer,
6132	tr, &rb_simple_fops);
6133
6134	#ifdef CONFIG_TRACER_SNAPSHOT
6135	trace_create_file("snapshot", 0644, d_tracer,
6136	tr, &snapshot_fops);
6137	#endif
6138
6139	for_each_tracing_cpu(cpu)
6140	tracing_init_debugfs_percpu(tr, cpu);
6141
6142	}
6143
6144	static __init int tracer_init_debugfs(void)
6145	{
6146	struct dentry *d_tracer;
6147
6148	trace_access_lock_init();
6149
6150	d_tracer = tracing_init_dentry();
6151	if (!d_tracer)
6152	return 0;
6153
6154	init_tracer_debugfs(&global_trace, d_tracer);
6155
6156	trace_create_file("available_tracers", 0444, d_tracer,
6157	&global_trace, &show_traces_fops);
6158
6159	trace_create_file("current_tracer", 0644, d_tracer,
6160	&global_trace, &set_tracer_fops);
6161
6162	#ifdef CONFIG_TRACER_MAX_TRACE
6163	trace_create_file("tracing_max_latency", 0644, d_tracer,
6164	&tracing_max_latency, &tracing_max_lat_fops);
6165	#endif
6166
6167	trace_create_file("tracing_thresh", 0644, d_tracer,
6168	&tracing_thresh, &tracing_max_lat_fops);
6169
6170	trace_create_file("README", 0444, d_tracer,
6171	NULL, &tracing_readme_fops);
6172
6173	trace_create_file("saved_cmdlines", 0444, d_tracer,
6174	NULL, &tracing_saved_cmdlines_fops);
6175
6176	#ifdef CONFIG_DYNAMIC_FTRACE
6177	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6178	&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6179	#endif
6180
6181	create_trace_instances(d_tracer);
6182
6183	create_trace_options_dir(&global_trace);
6184
6185	return 0;
6186	}
6187
6188	static int trace_panic_handler(struct notifier_block *this,
6189	unsigned long event, void *unused)
6190	{
6191	if (ftrace_dump_on_oops)
6192	ftrace_dump(ftrace_dump_on_oops);
6193	return NOTIFY_OK;
6194	}
6195
6196	static struct notifier_block trace_panic_notifier = {
6197	.notifier_call = trace_panic_handler,
6198	.next = NULL,
6199	.priority = 150 /* priority: INT_MAX >= x >= 0 */
6200	};
6201
6202	static int trace_die_handler(struct notifier_block *self,
6203	unsigned long val,
6204	void *data)
6205	{
6206	switch (val) {
6207	case DIE_OOPS:
6208	if (ftrace_dump_on_oops)
6209	ftrace_dump(ftrace_dump_on_oops);
6210	break;
6211	default:
6212	break;
6213	}
6214	return NOTIFY_OK;
6215	}
6216
6217	static struct notifier_block trace_die_notifier = {
6218	.notifier_call = trace_die_handler,
6219	.priority = 200
6220	};
6221
6222	/*
6223	* printk is set to max of 1024, we really don't need it that big.
6224	* Nothing should be printing 1000 characters anyway.
6225	*/
6226	#define TRACE_MAX_PRINT 1000
6227
6228	/*
6229	* Define here KERN_TRACE so that we have one place to modify
6230	* it if we decide to change what log level the ftrace dump
6231	* should be at.
6232	*/
6233	#define KERN_TRACE KERN_EMERG
6234
6235	void
6236	trace_printk_seq(struct trace_seq *s)
6237	{
6238	/* Probably should print a warning here. */
6239	if (s->len >= TRACE_MAX_PRINT)
6240	s->len = TRACE_MAX_PRINT;
6241
6242	/* should be zero ended, but we are paranoid. */
6243	s->buffer[s->len] = 0;
6244
6245	printk(KERN_TRACE "%s", s->buffer);
6246
6247	trace_seq_init(s);
6248	}
6249
6250	void trace_init_global_iter(struct trace_iterator *iter)
6251	{
6252	iter->tr = &global_trace;
6253	iter->trace = iter->tr->current_trace;
6254	iter->cpu_file = RING_BUFFER_ALL_CPUS;
6255	iter->trace_buffer = &global_trace.trace_buffer;
6256	}
6257
6258	void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6259	{
6260	/* use static because iter can be a bit big for the stack */
6261	static struct trace_iterator iter;
6262	static atomic_t dump_running;
6263	unsigned int old_userobj;
6264	unsigned long flags;
6265	int cnt = 0, cpu;
6266
6267	/* Only allow one dump user at a time. */
6268	if (atomic_inc_return(&dump_running) != 1) {
6269	atomic_dec(&dump_running);
6270	return;
6271	}
6272
6273	/*
6274	* Always turn off tracing when we dump.
6275	* We don't need to show trace output of what happens
6276	* between multiple crashes.
6277	*
6278	* If the user does a sysrq-z, then they can re-enable
6279	* tracing with echo 1 > tracing_on.
6280	*/
6281	tracing_off();
6282
6283	local_irq_save(flags);
6284
6285	/* Simulate the iterator */
6286	trace_init_global_iter(&iter);
6287
6288	for_each_tracing_cpu(cpu) {
6289	atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6290	}
6291
6292	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6293
6294	/* don't look at user memory in panic mode */
6295	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6296
6297	switch (oops_dump_mode) {
6298	case DUMP_ALL:
6299	iter.cpu_file = RING_BUFFER_ALL_CPUS;
6300	break;
6301	case DUMP_ORIG:
6302	iter.cpu_file = raw_smp_processor_id();
6303	break;
6304	case DUMP_NONE:
6305	goto out_enable;
6306	default:
6307	printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6308	iter.cpu_file = RING_BUFFER_ALL_CPUS;
6309	}
6310
6311	printk(KERN_TRACE "Dumping ftrace buffer:\n");
6312
6313	/* Did function tracer already get disabled? */
6314	if (ftrace_is_dead()) {
6315	printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6316	printk("# MAY BE MISSING FUNCTION EVENTS\n");
6317	}
6318
6319	/*
6320	* We need to stop all tracing on all CPUS to read the
6321	* the next buffer. This is a bit expensive, but is
6322	* not done often. We fill all what we can read,
6323	* and then release the locks again.
6324	*/
6325
6326	while (!trace_empty(&iter)) {
6327
6328	if (!cnt)
6329	printk(KERN_TRACE "---------------------------------\n");
6330
6331	cnt++;
6332
6333	/* reset all but tr, trace, and overruns */
6334	memset(&iter.seq, 0,
6335	sizeof(struct trace_iterator) -
6336	offsetof(struct trace_iterator, seq));
6337	iter.iter_flags \|= TRACE_FILE_LAT_FMT;
6338	iter.pos = -1;
6339
6340	if (trace_find_next_entry_inc(&iter) != NULL) {
6341	int ret;
6342
6343	ret = print_trace_line(&iter);
6344	if (ret != TRACE_TYPE_NO_CONSUME)
6345	trace_consume(&iter);
6346	}
6347	touch_nmi_watchdog();
6348
6349	trace_printk_seq(&iter.seq);
6350	}
6351
6352	if (!cnt)
6353	printk(KERN_TRACE " (ftrace buffer empty)\n");
6354	else
6355	printk(KERN_TRACE "---------------------------------\n");
6356
6357	out_enable:
6358	trace_flags \|= old_userobj;
6359
6360	for_each_tracing_cpu(cpu) {
6361	atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6362	}
6363	atomic_dec(&dump_running);
6364	local_irq_restore(flags);
6365	}
6366	EXPORT_SYMBOL_GPL(ftrace_dump);
6367
6368	__init static int tracer_alloc_buffers(void)
6369	{
6370	int ring_buf_size;
6371	int ret = -ENOMEM;
6372
6373
6374	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6375	goto out;
6376
6377	if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6378	goto out_free_buffer_mask;
6379
6380	/* Only allocate trace_printk buffers if a trace_printk exists */
6381	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6382	/* Must be called before global_trace.buffer is allocated */
6383	trace_printk_init_buffers();
6384
6385	/* To save memory, keep the ring buffer size to its minimum */
6386	if (ring_buffer_expanded)
6387	ring_buf_size = trace_buf_size;
6388	else
6389	ring_buf_size = 1;
6390
6391	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6392	cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6393
6394	raw_spin_lock_init(&global_trace.start_lock);
6395
6396	/* TODO: make the number of buffers hot pluggable with CPUS */
6397	if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6398	printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6399	WARN_ON(1);
6400	goto out_free_cpumask;
6401	}
6402
6403	if (global_trace.buffer_disabled)
6404	tracing_off();
6405
6406	trace_init_cmdlines();
6407
6408	/*
6409	* register_tracer() might reference current_trace, so it
6410	* needs to be set before we register anything. This is
6411	* just a bootstrap of current_trace anyway.
6412	*/
6413	global_trace.current_trace = &nop_trace;
6414
6415	register_tracer(&nop_trace);
6416
6417	/* All seems OK, enable tracing */
6418	tracing_disabled = 0;
6419
6420	atomic_notifier_chain_register(&panic_notifier_list,
6421	&trace_panic_notifier);
6422
6423	register_die_notifier(&trace_die_notifier);
6424
6425	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6426
6427	INIT_LIST_HEAD(&global_trace.systems);
6428	INIT_LIST_HEAD(&global_trace.events);
6429	list_add(&global_trace.list, &ftrace_trace_arrays);
6430
6431	while (trace_boot_options) {
6432	char *option;
6433
6434	option = strsep(&trace_boot_options, ",");
6435	trace_set_options(&global_trace, option);
6436	}
6437
6438	register_snapshot_cmd();
6439
6440	return 0;
6441
6442	out_free_cpumask:
6443	free_percpu(global_trace.trace_buffer.data);
6444	#ifdef CONFIG_TRACER_MAX_TRACE
6445	free_percpu(global_trace.max_buffer.data);
6446	#endif
6447	free_cpumask_var(global_trace.tracing_cpumask);
6448	out_free_buffer_mask:
6449	free_cpumask_var(tracing_buffer_mask);
6450	out:
6451	return ret;
6452	}
6453
6454	__init static int clear_boot_tracer(void)
6455	{
6456	/*
6457	* The default tracer at boot buffer is an init section.
6458	* This function is called in lateinit. If we did not
6459	* find the boot tracer, then clear it out, to prevent
6460	* later registration from accessing the buffer that is
6461	* about to be freed.
6462	*/
6463	if (!default_bootup_tracer)
6464	return 0;
6465
6466	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6467	default_bootup_tracer);
6468	default_bootup_tracer = NULL;
6469
6470	return 0;
6471	}
6472
6473	early_initcall(tracer_alloc_buffers);
6474	fs_initcall(tracer_init_debugfs);
6475	late_initcall(clear_boot_tracer);