linux - kernel/trace/trace.c

7340 lines · 173.4 KiB

1	/*
2	* ring buffer based function tracer
3	*
4	* Copyright (C) 2007-2012 Steven Rostedt <[email protected]>
5	* Copyright (C) 2008 Ingo Molnar <[email protected]>
6	*
7	* Originally taken from the RT patch by:
8	* Arnaldo Carvalho de Melo <[email protected]>
9	*
10	* Based on code from the latency_tracer, that is:
11	* Copyright (C) 2004-2006 Ingo Molnar
12	* Copyright (C) 2004 Nadia Yvette Chambers
13	*/
14	#include <linux/ring_buffer.h>
15	#include <generated/utsrelease.h>
16	#include <linux/stacktrace.h>
17	#include <linux/writeback.h>
18	#include <linux/kallsyms.h>
19	#include <linux/seq_file.h>
20	#include <linux/notifier.h>
21	#include <linux/irqflags.h>
22	#include <linux/debugfs.h>
23	#include <linux/tracefs.h>
24	#include <linux/pagemap.h>
25	#include <linux/hardirq.h>
26	#include <linux/linkage.h>
27	#include <linux/uaccess.h>
28	#include <linux/kprobes.h>
29	#include <linux/ftrace.h>
30	#include <linux/module.h>
31	#include <linux/percpu.h>
32	#include <linux/splice.h>
33	#include <linux/kdebug.h>
34	#include <linux/string.h>
35	#include <linux/mount.h>
36	#include <linux/rwsem.h>
37	#include <linux/slab.h>
38	#include <linux/ctype.h>
39	#include <linux/init.h>
40	#include <linux/poll.h>
41	#include <linux/nmi.h>
42	#include <linux/fs.h>
43	#include <linux/sched/rt.h>
44
45	#include "trace.h"
46	#include "trace_output.h"
47
48	/*
49	* On boot up, the ring buffer is set to the minimum size, so that
50	* we do not waste memory on systems that are not using tracing.
51	*/
52	bool ring_buffer_expanded;
53
54	/*
55	* We need to change this state when a selftest is running.
56	* A selftest will lurk into the ring-buffer to count the
57	* entries inserted during the selftest although some concurrent
58	* insertions into the ring-buffer such as trace_printk could occurred
59	* at the same time, giving false positive or negative results.
60	*/
61	static bool __read_mostly tracing_selftest_running;
62
63	/*
64	* If a tracer is running, we do not want to run SELFTEST.
65	*/
66	bool __read_mostly tracing_selftest_disabled;
67
68	/* Pipe tracepoints to printk */
69	struct trace_iterator *tracepoint_print_iter;
70	int tracepoint_printk;
71
72	/* For tracers that don't implement custom flags */
73	static struct tracer_opt dummy_tracer_opt[] = {
74	{ }
75	};
76
77	static int
78	dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
79	{
80	return 0;
81	}
82
83	/*
84	* To prevent the comm cache from being overwritten when no
85	* tracing is active, only save the comm when a trace event
86	* occurred.
87	*/
88	static DEFINE_PER_CPU(bool, trace_cmdline_save);
89
90	/*
91	* Kill all tracing for good (never come back).
92	* It is initialized to 1 but will turn to zero if the initialization
93	* of the tracer is successful. But that is the only place that sets
94	* this back to zero.
95	*/
96	static int tracing_disabled = 1;
97
98	cpumask_var_t __read_mostly tracing_buffer_mask;
99
100	/*
101	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102	*
103	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104	* is set, then ftrace_dump is called. This will output the contents
105	* of the ftrace buffers to the console. This is very useful for
106	* capturing traces that lead to crashes and outputing it to a
107	* serial console.
108	*
109	* It is default off, but you can enable it with either specifying
110	* "ftrace_dump_on_oops" in the kernel command line, or setting
111	* /proc/sys/kernel/ftrace_dump_on_oops
112	* Set 1 if you want to dump buffers of all CPUs
113	* Set 2 if you want to dump the buffer of the CPU that triggered oops
114	*/
115
116	enum ftrace_dump_mode ftrace_dump_on_oops;
117
118	/* When set, tracing will stop when a WARN() is hit /
119	int __disable_trace_on_warning;
120
121	#ifdef CONFIG_TRACE_ENUM_MAP_FILE
122	/* Map of enums to their values, for "enum_map" file */
123	struct trace_enum_map_head {
124	struct module *mod;
125	unsigned long length;
126	};
127
128	union trace_enum_map_item;
129
130	struct trace_enum_map_tail {
131	/*
132	* "end" is first and points to NULL as it must be different
133	* than "mod" or "enum_string"
134	*/
135	union trace_enum_map_item *next;
136	const char end; / points to NULL */
137	};
138
139	static DEFINE_MUTEX(trace_enum_mutex);
140
141	/*
142	* The trace_enum_maps are saved in an array with two extra elements,
143	* one at the beginning, and one at the end. The beginning item contains
144	* the count of the saved maps (head.length), and the module they
145	* belong to if not built in (head.mod). The ending item contains a
146	* pointer to the next array of saved enum_map items.
147	*/
148	union trace_enum_map_item {
149	struct trace_enum_map map;
150	struct trace_enum_map_head head;
151	struct trace_enum_map_tail tail;
152	};
153
154	static union trace_enum_map_item *trace_enum_maps;
155	#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
156
157	static int tracing_set_tracer(struct trace_array tr, const char buf);
158
159	#define MAX_TRACER_SIZE 100
160	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
161	static char *default_bootup_tracer;
162
163	static bool allocate_snapshot;
164
165	static int __init set_cmdline_ftrace(char *str)
166	{
167	strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
168	default_bootup_tracer = bootup_tracer_buf;
169	/* We are using ftrace early, expand it */
170	ring_buffer_expanded = true;
171	return 1;
172	}
173	__setup("ftrace=", set_cmdline_ftrace);
174
175	static int __init set_ftrace_dump_on_oops(char *str)
176	{
177	if (str++ != '=' \|\| !str) {
178	ftrace_dump_on_oops = DUMP_ALL;
179	return 1;
180	}
181
182	if (!strcmp("orig_cpu", str)) {
183	ftrace_dump_on_oops = DUMP_ORIG;
184	return 1;
185	}
186
187	return 0;
188	}
189	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
190
191	static int __init stop_trace_on_warning(char *str)
192	{
193	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
194	__disable_trace_on_warning = 1;
195	return 1;
196	}
197	__setup("traceoff_on_warning", stop_trace_on_warning);
198
199	static int __init boot_alloc_snapshot(char *str)
200	{
201	allocate_snapshot = true;
202	/* We also need the main ring buffer expanded */
203	ring_buffer_expanded = true;
204	return 1;
205	}
206	__setup("alloc_snapshot", boot_alloc_snapshot);
207
208
209	static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
210
211	static int __init set_trace_boot_options(char *str)
212	{
213	strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
214	return 0;
215	}
216	__setup("trace_options=", set_trace_boot_options);
217
218	static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
219	static char *trace_boot_clock __initdata;
220
221	static int __init set_trace_boot_clock(char *str)
222	{
223	strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
224	trace_boot_clock = trace_boot_clock_buf;
225	return 0;
226	}
227	__setup("trace_clock=", set_trace_boot_clock);
228
229	static int __init set_tracepoint_printk(char *str)
230	{
231	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
232	tracepoint_printk = 1;
233	return 1;
234	}
235	__setup("tp_printk", set_tracepoint_printk);
236
237	unsigned long long ns2usecs(cycle_t nsec)
238	{
239	nsec += 500;
240	do_div(nsec, 1000);
241	return nsec;
242	}
243
244	/* trace_flags holds trace_options default values */
245	#define TRACE_DEFAULT_FLAGS \
246	(FUNCTION_DEFAULT_FLAGS \| \
247	TRACE_ITER_PRINT_PARENT \| TRACE_ITER_PRINTK \| \
248	TRACE_ITER_ANNOTATE \| TRACE_ITER_CONTEXT_INFO \| \
249	TRACE_ITER_RECORD_CMD \| TRACE_ITER_OVERWRITE \| \
250	TRACE_ITER_IRQ_INFO \| TRACE_ITER_MARKERS)
251
252	/* trace_options that are only supported by global_trace */
253	#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK \| \
254	TRACE_ITER_PRINTK_MSGONLY \| TRACE_ITER_RECORD_CMD)
255
256
257	/*
258	* The global_trace is the descriptor that holds the tracing
259	* buffers for the live tracing. For each CPU, it contains
260	* a link list of pages that will store trace entries. The
261	* page descriptor of the pages in the memory is used to hold
262	* the link list by linking the lru item in the page descriptor
263	* to each of the pages in the buffer per CPU.
264	*
265	* For each active CPU there is a data field that holds the
266	* pages for the buffer for that CPU. Each CPU has the same number
267	* of pages allocated for its buffer.
268	*/
269	static struct trace_array global_trace = {
270	.trace_flags = TRACE_DEFAULT_FLAGS,
271	};
272
273	LIST_HEAD(ftrace_trace_arrays);
274
275	int trace_array_get(struct trace_array *this_tr)
276	{
277	struct trace_array *tr;
278	int ret = -ENODEV;
279
280	mutex_lock(&trace_types_lock);
281	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
282	if (tr == this_tr) {
283	tr->ref++;
284	ret = 0;
285	break;
286	}
287	}
288	mutex_unlock(&trace_types_lock);
289
290	return ret;
291	}
292
293	static void __trace_array_put(struct trace_array *this_tr)
294	{
295	WARN_ON(!this_tr->ref);
296	this_tr->ref--;
297	}
298
299	void trace_array_put(struct trace_array *this_tr)
300	{
301	mutex_lock(&trace_types_lock);
302	__trace_array_put(this_tr);
303	mutex_unlock(&trace_types_lock);
304	}
305
306	int filter_check_discard(struct trace_event_file file, void rec,
307	struct ring_buffer *buffer,
308	struct ring_buffer_event *event)
309	{
310	if (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
311	!filter_match_preds(file->filter, rec)) {
312	ring_buffer_discard_commit(buffer, event);
313	return 1;
314	}
315
316	return 0;
317	}
318	EXPORT_SYMBOL_GPL(filter_check_discard);
319
320	int call_filter_check_discard(struct trace_event_call call, void rec,
321	struct ring_buffer *buffer,
322	struct ring_buffer_event *event)
323	{
324	if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
325	!filter_match_preds(call->filter, rec)) {
326	ring_buffer_discard_commit(buffer, event);
327	return 1;
328	}
329
330	return 0;
331	}
332	EXPORT_SYMBOL_GPL(call_filter_check_discard);
333
334	static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
335	{
336	u64 ts;
337
338	/* Early boot up does not have a buffer yet */
339	if (!buf->buffer)
340	return trace_clock_local();
341
342	ts = ring_buffer_time_stamp(buf->buffer, cpu);
343	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
344
345	return ts;
346	}
347
348	cycle_t ftrace_now(int cpu)
349	{
350	return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
351	}
352
353	/**
354	* tracing_is_enabled - Show if global_trace has been disabled
355	*
356	* Shows if the global trace has been enabled or not. It uses the
357	* mirror flag "buffer_disabled" to be used in fast paths such as for
358	* the irqsoff tracer. But it may be inaccurate due to races. If you
359	* need to know the accurate state, use tracing_is_on() which is a little
360	* slower, but accurate.
361	*/
362	int tracing_is_enabled(void)
363	{
364	/*
365	* For quick access (irqsoff uses this in fast path), just
366	* return the mirror variable of the state of the ring buffer.
367	* It's a little racy, but we don't really care.
368	*/
369	smp_rmb();
370	return !global_trace.buffer_disabled;
371	}
372
373	/*
374	* trace_buf_size is the size in bytes that is allocated
375	* for a buffer. Note, the number of bytes is always rounded
376	* to page size.
377	*
378	* This number is purposely set to a low number of 16384.
379	* If the dump on oops happens, it will be much appreciated
380	* to not have to wait for all that output. Anyway this can be
381	* boot time and run time configurable.
382	*/
383	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
384
385	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
386
387	/* trace_types holds a link list of available tracers. */
388	static struct tracer *trace_types __read_mostly;
389
390	/*
391	* trace_types_lock is used to protect the trace_types list.
392	*/
393	DEFINE_MUTEX(trace_types_lock);
394
395	/*
396	* serialize the access of the ring buffer
397	*
398	* ring buffer serializes readers, but it is low level protection.
399	* The validity of the events (which returns by ring_buffer_peek() ..etc)
400	* are not protected by ring buffer.
401	*
402	* The content of events may become garbage if we allow other process consumes
403	* these events concurrently:
404	* A) the page of the consumed events may become a normal page
405	* (not reader page) in ring buffer, and this page will be rewrited
406	* by events producer.
407	* B) The page of the consumed events may become a page for splice_read,
408	* and this page will be returned to system.
409	*
410	* These primitives allow multi process access to different cpu ring buffer
411	* concurrently.
412	*
413	* These primitives don't distinguish read-only and read-consume access.
414	* Multi read-only access are also serialized.
415	*/
416
417	#ifdef CONFIG_SMP
418	static DECLARE_RWSEM(all_cpu_access_lock);
419	static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
420
421	static inline void trace_access_lock(int cpu)
422	{
423	if (cpu == RING_BUFFER_ALL_CPUS) {
424	/* gain it for accessing the whole ring buffer. */
425	down_write(&all_cpu_access_lock);
426	} else {
427	/* gain it for accessing a cpu ring buffer. */
428
429	/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
430	down_read(&all_cpu_access_lock);
431
432	/* Secondly block other access to this @cpu ring buffer. */
433	mutex_lock(&per_cpu(cpu_access_lock, cpu));
434	}
435	}
436
437	static inline void trace_access_unlock(int cpu)
438	{
439	if (cpu == RING_BUFFER_ALL_CPUS) {
440	up_write(&all_cpu_access_lock);
441	} else {
442	mutex_unlock(&per_cpu(cpu_access_lock, cpu));
443	up_read(&all_cpu_access_lock);
444	}
445	}
446
447	static inline void trace_access_lock_init(void)
448	{
449	int cpu;
450
451	for_each_possible_cpu(cpu)
452	mutex_init(&per_cpu(cpu_access_lock, cpu));
453	}
454
455	#else
456
457	static DEFINE_MUTEX(access_lock);
458
459	static inline void trace_access_lock(int cpu)
460	{
461	(void)cpu;
462	mutex_lock(&access_lock);
463	}
464
465	static inline void trace_access_unlock(int cpu)
466	{
467	(void)cpu;
468	mutex_unlock(&access_lock);
469	}
470
471	static inline void trace_access_lock_init(void)
472	{
473	}
474
475	#endif
476
477	#ifdef CONFIG_STACKTRACE
478	static void __ftrace_trace_stack(struct ring_buffer *buffer,
479	unsigned long flags,
480	int skip, int pc, struct pt_regs *regs);
481	static inline void ftrace_trace_stack(struct trace_array *tr,
482	struct ring_buffer *buffer,
483	unsigned long flags,
484	int skip, int pc, struct pt_regs *regs);
485
486	#else
487	static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
488	unsigned long flags,
489	int skip, int pc, struct pt_regs *regs)
490	{
491	}
492	static inline void ftrace_trace_stack(struct trace_array *tr,
493	struct ring_buffer *buffer,
494	unsigned long flags,
495	int skip, int pc, struct pt_regs *regs)
496	{
497	}
498
499	#endif
500
501	static void tracer_tracing_on(struct trace_array *tr)
502	{
503	if (tr->trace_buffer.buffer)
504	ring_buffer_record_on(tr->trace_buffer.buffer);
505	/*
506	* This flag is looked at when buffers haven't been allocated
507	* yet, or by some tracers (like irqsoff), that just want to
508	* know if the ring buffer has been disabled, but it can handle
509	* races of where it gets disabled but we still do a record.
510	* As the check is in the fast path of the tracers, it is more
511	* important to be fast than accurate.
512	*/
513	tr->buffer_disabled = 0;
514	/* Make the flag seen by readers */
515	smp_wmb();
516	}
517
518	/**
519	* tracing_on - enable tracing buffers
520	*
521	* This function enables tracing buffers that may have been
522	* disabled with tracing_off.
523	*/
524	void tracing_on(void)
525	{
526	tracer_tracing_on(&global_trace);
527	}
528	EXPORT_SYMBOL_GPL(tracing_on);
529
530	/**
531	* __trace_puts - write a constant string into the trace buffer.
532	* @ip: The address of the caller
533	* @str: The constant string to write
534	* @size: The size of the string.
535	*/
536	int __trace_puts(unsigned long ip, const char *str, int size)
537	{
538	struct ring_buffer_event *event;
539	struct ring_buffer *buffer;
540	struct print_entry *entry;
541	unsigned long irq_flags;
542	int alloc;
543	int pc;
544
545	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
546	return 0;
547
548	pc = preempt_count();
549
550	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
551	return 0;
552
553	alloc = sizeof(entry) + size + 2; / possible \n added */
554
555	local_save_flags(irq_flags);
556	buffer = global_trace.trace_buffer.buffer;
557	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
558	irq_flags, pc);
559	if (!event)
560	return 0;
561
562	entry = ring_buffer_event_data(event);
563	entry->ip = ip;
564
565	memcpy(&entry->buf, str, size);
566
567	/* Add a newline if necessary */
568	if (entry->buf[size - 1] != '\n') {
569	entry->buf[size] = '\n';
570	entry->buf[size + 1] = '\0';
571	} else
572	entry->buf[size] = '\0';
573
574	__buffer_unlock_commit(buffer, event);
575	ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
576
577	return size;
578	}
579	EXPORT_SYMBOL_GPL(__trace_puts);
580
581	/**
582	* __trace_bputs - write the pointer to a constant string into trace buffer
583	* @ip: The address of the caller
584	* @str: The constant string to write to the buffer to
585	*/
586	int __trace_bputs(unsigned long ip, const char *str)
587	{
588	struct ring_buffer_event *event;
589	struct ring_buffer *buffer;
590	struct bputs_entry *entry;
591	unsigned long irq_flags;
592	int size = sizeof(struct bputs_entry);
593	int pc;
594
595	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
596	return 0;
597
598	pc = preempt_count();
599
600	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
601	return 0;
602
603	local_save_flags(irq_flags);
604	buffer = global_trace.trace_buffer.buffer;
605	event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
606	irq_flags, pc);
607	if (!event)
608	return 0;
609
610	entry = ring_buffer_event_data(event);
611	entry->ip = ip;
612	entry->str = str;
613
614	__buffer_unlock_commit(buffer, event);
615	ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
616
617	return 1;
618	}
619	EXPORT_SYMBOL_GPL(__trace_bputs);
620
621	#ifdef CONFIG_TRACER_SNAPSHOT
622	/**
623	* trace_snapshot - take a snapshot of the current buffer.
624	*
625	* This causes a swap between the snapshot buffer and the current live
626	* tracing buffer. You can use this to take snapshots of the live
627	* trace when some condition is triggered, but continue to trace.
628	*
629	* Note, make sure to allocate the snapshot with either
630	* a tracing_snapshot_alloc(), or by doing it manually
631	* with: echo 1 > /sys/kernel/debug/tracing/snapshot
632	*
633	* If the snapshot buffer is not allocated, it will stop tracing.
634	* Basically making a permanent snapshot.
635	*/
636	void tracing_snapshot(void)
637	{
638	struct trace_array *tr = &global_trace;
639	struct tracer *tracer = tr->current_trace;
640	unsigned long flags;
641
642	if (in_nmi()) {
643	internal_trace_puts("* SNAPSHOT CALLED FROM NMI CONTEXT *\n");
644	internal_trace_puts("* snapshot is being ignored *\n");
645	return;
646	}
647
648	if (!tr->allocated_snapshot) {
649	internal_trace_puts("* SNAPSHOT NOT ALLOCATED *\n");
650	internal_trace_puts("* stopping trace here! *\n");
651	tracing_off();
652	return;
653	}
654
655	/* Note, snapshot can not be used when the tracer uses it */
656	if (tracer->use_max_tr) {
657	internal_trace_puts("* LATENCY TRACER ACTIVE *\n");
658	internal_trace_puts("* Can not use snapshot (sorry) *\n");
659	return;
660	}
661
662	local_irq_save(flags);
663	update_max_tr(tr, current, smp_processor_id());
664	local_irq_restore(flags);
665	}
666	EXPORT_SYMBOL_GPL(tracing_snapshot);
667
668	static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
669	struct trace_buffer *size_buf, int cpu_id);
670	static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
671
672	static int alloc_snapshot(struct trace_array *tr)
673	{
674	int ret;
675
676	if (!tr->allocated_snapshot) {
677
678	/* allocate spare buffer */
679	ret = resize_buffer_duplicate_size(&tr->max_buffer,
680	&tr->trace_buffer, RING_BUFFER_ALL_CPUS);
681	if (ret < 0)
682	return ret;
683
684	tr->allocated_snapshot = true;
685	}
686
687	return 0;
688	}
689
690	static void free_snapshot(struct trace_array *tr)
691	{
692	/*
693	* We don't free the ring buffer. instead, resize it because
694	* The max_tr ring buffer has some state (e.g. ring->clock) and
695	* we want preserve it.
696	*/
697	ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
698	set_buffer_entries(&tr->max_buffer, 1);
699	tracing_reset_online_cpus(&tr->max_buffer);
700	tr->allocated_snapshot = false;
701	}
702
703	/**
704	* tracing_alloc_snapshot - allocate snapshot buffer.
705	*
706	* This only allocates the snapshot buffer if it isn't already
707	* allocated - it doesn't also take a snapshot.
708	*
709	* This is meant to be used in cases where the snapshot buffer needs
710	* to be set up for events that can't sleep but need to be able to
711	* trigger a snapshot.
712	*/
713	int tracing_alloc_snapshot(void)
714	{
715	struct trace_array *tr = &global_trace;
716	int ret;
717
718	ret = alloc_snapshot(tr);
719	WARN_ON(ret < 0);
720
721	return ret;
722	}
723	EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
724
725	/**
726	* trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
727	*
728	* This is similar to trace_snapshot(), but it will allocate the
729	* snapshot buffer if it isn't already allocated. Use this only
730	* where it is safe to sleep, as the allocation may sleep.
731	*
732	* This causes a swap between the snapshot buffer and the current live
733	* tracing buffer. You can use this to take snapshots of the live
734	* trace when some condition is triggered, but continue to trace.
735	*/
736	void tracing_snapshot_alloc(void)
737	{
738	int ret;
739
740	ret = tracing_alloc_snapshot();
741	if (ret < 0)
742	return;
743
744	tracing_snapshot();
745	}
746	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
747	#else
748	void tracing_snapshot(void)
749	{
750	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
751	}
752	EXPORT_SYMBOL_GPL(tracing_snapshot);
753	int tracing_alloc_snapshot(void)
754	{
755	WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
756	return -ENODEV;
757	}
758	EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
759	void tracing_snapshot_alloc(void)
760	{
761	/* Give warning */
762	tracing_snapshot();
763	}
764	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
765	#endif /* CONFIG_TRACER_SNAPSHOT */
766
767	static void tracer_tracing_off(struct trace_array *tr)
768	{
769	if (tr->trace_buffer.buffer)
770	ring_buffer_record_off(tr->trace_buffer.buffer);
771	/*
772	* This flag is looked at when buffers haven't been allocated
773	* yet, or by some tracers (like irqsoff), that just want to
774	* know if the ring buffer has been disabled, but it can handle
775	* races of where it gets disabled but we still do a record.
776	* As the check is in the fast path of the tracers, it is more
777	* important to be fast than accurate.
778	*/
779	tr->buffer_disabled = 1;
780	/* Make the flag seen by readers */
781	smp_wmb();
782	}
783
784	/**
785	* tracing_off - turn off tracing buffers
786	*
787	* This function stops the tracing buffers from recording data.
788	* It does not disable any overhead the tracers themselves may
789	* be causing. This function simply causes all recording to
790	* the ring buffers to fail.
791	*/
792	void tracing_off(void)
793	{
794	tracer_tracing_off(&global_trace);
795	}
796	EXPORT_SYMBOL_GPL(tracing_off);
797
798	void disable_trace_on_warning(void)
799	{
800	if (__disable_trace_on_warning)
801	tracing_off();
802	}
803
804	/**
805	* tracer_tracing_is_on - show real state of ring buffer enabled
806	* @tr : the trace array to know if ring buffer is enabled
807	*
808	* Shows real state of the ring buffer if it is enabled or not.
809	*/
810	static int tracer_tracing_is_on(struct trace_array *tr)
811	{
812	if (tr->trace_buffer.buffer)
813	return ring_buffer_record_is_on(tr->trace_buffer.buffer);
814	return !tr->buffer_disabled;
815	}
816
817	/**
818	* tracing_is_on - show state of ring buffers enabled
819	*/
820	int tracing_is_on(void)
821	{
822	return tracer_tracing_is_on(&global_trace);
823	}
824	EXPORT_SYMBOL_GPL(tracing_is_on);
825
826	static int __init set_buf_size(char *str)
827	{
828	unsigned long buf_size;
829
830	if (!str)
831	return 0;
832	buf_size = memparse(str, &str);
833	/* nr_entries can not be zero */
834	if (buf_size == 0)
835	return 0;
836	trace_buf_size = buf_size;
837	return 1;
838	}
839	__setup("trace_buf_size=", set_buf_size);
840
841	static int __init set_tracing_thresh(char *str)
842	{
843	unsigned long threshold;
844	int ret;
845
846	if (!str)
847	return 0;
848	ret = kstrtoul(str, 0, &threshold);
849	if (ret < 0)
850	return 0;
851	tracing_thresh = threshold * 1000;
852	return 1;
853	}
854	__setup("tracing_thresh=", set_tracing_thresh);
855
856	unsigned long nsecs_to_usecs(unsigned long nsecs)
857	{
858	return nsecs / 1000;
859	}
860
861	/*
862	* TRACE_FLAGS is defined as a tuple matching bit masks with strings.
863	* It uses C(a, b) where 'a' is the enum name and 'b' is the string that
864	* matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
865	* of strings in the order that the enums were defined.
866	*/
867	#undef C
868	#define C(a, b) b
869
870	/* These must match the bit postions in trace_iterator_flags */
871	static const char *trace_options[] = {
872	TRACE_FLAGS
873	NULL
874	};
875
876	static struct {
877	u64 (*func)(void);
878	const char *name;
879	int in_ns; /* is this clock in nanoseconds? */
880	} trace_clocks[] = {
881	{ trace_clock_local, "local", 1 },
882	{ trace_clock_global, "global", 1 },
883	{ trace_clock_counter, "counter", 0 },
884	{ trace_clock_jiffies, "uptime", 0 },
885	{ trace_clock, "perf", 1 },
886	{ ktime_get_mono_fast_ns, "mono", 1 },
887	{ ktime_get_raw_fast_ns, "mono_raw", 1 },
888	ARCH_TRACE_CLOCKS
889	};
890
891	/*
892	* trace_parser_get_init - gets the buffer for trace parser
893	*/
894	int trace_parser_get_init(struct trace_parser *parser, int size)
895	{
896	memset(parser, 0, sizeof(*parser));
897
898	parser->buffer = kmalloc(size, GFP_KERNEL);
899	if (!parser->buffer)
900	return 1;
901
902	parser->size = size;
903	return 0;
904	}
905
906	/*
907	* trace_parser_put - frees the buffer for trace parser
908	*/
909	void trace_parser_put(struct trace_parser *parser)
910	{
911	kfree(parser->buffer);
912	}
913
914	/*
915	* trace_get_user - reads the user input string separated by space
916	* (matched by isspace(ch))
917	*
918	* For each string found the 'struct trace_parser' is updated,
919	* and the function returns.
920	*
921	* Returns number of bytes read.
922	*
923	* See kernel/trace/trace.h for 'struct trace_parser' details.
924	*/
925	int trace_get_user(struct trace_parser parser, const char __user ubuf,
926	size_t cnt, loff_t *ppos)
927	{
928	char ch;
929	size_t read = 0;
930	ssize_t ret;
931
932	if (!*ppos)
933	trace_parser_clear(parser);
934
935	ret = get_user(ch, ubuf++);
936	if (ret)
937	goto out;
938
939	read++;
940	cnt--;
941
942	/*
943	* The parser is not finished with the last write,
944	* continue reading the user input without skipping spaces.
945	*/
946	if (!parser->cont) {
947	/* skip white space */
948	while (cnt && isspace(ch)) {
949	ret = get_user(ch, ubuf++);
950	if (ret)
951	goto out;
952	read++;
953	cnt--;
954	}
955
956	/* only spaces were written */
957	if (isspace(ch)) {
958	*ppos += read;
959	ret = read;
960	goto out;
961	}
962
963	parser->idx = 0;
964	}
965
966	/* read the non-space input */
967	while (cnt && !isspace(ch)) {
968	if (parser->idx < parser->size - 1)
969	parser->buffer[parser->idx++] = ch;
970	else {
971	ret = -EINVAL;
972	goto out;
973	}
974	ret = get_user(ch, ubuf++);
975	if (ret)
976	goto out;
977	read++;
978	cnt--;
979	}
980
981	/* We either got finished input or we have to wait for another call. */
982	if (isspace(ch)) {
983	parser->buffer[parser->idx] = 0;
984	parser->cont = false;
985	} else if (parser->idx < parser->size - 1) {
986	parser->cont = true;
987	parser->buffer[parser->idx++] = ch;
988	} else {
989	ret = -EINVAL;
990	goto out;
991	}
992
993	*ppos += read;
994	ret = read;
995
996	out:
997	return ret;
998	}
999
1000	/* TODO add a seq_buf_to_buffer() */
1001	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
1002	{
1003	int len;
1004
1005	if (trace_seq_used(s) <= s->seq.readpos)
1006	return -EBUSY;
1007
1008	len = trace_seq_used(s) - s->seq.readpos;
1009	if (cnt > len)
1010	cnt = len;
1011	memcpy(buf, s->buffer + s->seq.readpos, cnt);
1012
1013	s->seq.readpos += cnt;
1014	return cnt;
1015	}
1016
1017	unsigned long __read_mostly tracing_thresh;
1018
1019	#ifdef CONFIG_TRACER_MAX_TRACE
1020	/*
1021	* Copy the new maximum trace into the separate maximum-trace
1022	* structure. (this way the maximum trace is permanently saved,
1023	* for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1024	*/
1025	static void
1026	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
1027	{
1028	struct trace_buffer *trace_buf = &tr->trace_buffer;
1029	struct trace_buffer *max_buf = &tr->max_buffer;
1030	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1031	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1032
1033	max_buf->cpu = cpu;
1034	max_buf->time_start = data->preempt_timestamp;
1035
1036	max_data->saved_latency = tr->max_latency;
1037	max_data->critical_start = data->critical_start;
1038	max_data->critical_end = data->critical_end;
1039
1040	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1041	max_data->pid = tsk->pid;
1042	/*
1043	* If tsk == current, then use current_uid(), as that does not use
1044	* RCU. The irq tracer can be called out of RCU scope.
1045	*/
1046	if (tsk == current)
1047	max_data->uid = current_uid();
1048	else
1049	max_data->uid = task_uid(tsk);
1050
1051	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1052	max_data->policy = tsk->policy;
1053	max_data->rt_priority = tsk->rt_priority;
1054
1055	/* record this tasks comm */
1056	tracing_record_cmdline(tsk);
1057	}
1058
1059	/**
1060	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
1061	* @tr: tracer
1062	* @tsk: the task with the latency
1063	* @cpu: The cpu that initiated the trace.
1064	*
1065	* Flip the buffers between the @tr and the max_tr and record information
1066	* about which task was the cause of this latency.
1067	*/
1068	void
1069	update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
1070	{
1071	struct ring_buffer *buf;
1072
1073	if (tr->stop_count)
1074	return;
1075
1076	WARN_ON_ONCE(!irqs_disabled());
1077
1078	if (!tr->allocated_snapshot) {
1079	/* Only the nop tracer should hit this when disabling */
1080	WARN_ON_ONCE(tr->current_trace != &nop_trace);
1081	return;
1082	}
1083
1084	arch_spin_lock(&tr->max_lock);
1085
1086	buf = tr->trace_buffer.buffer;
1087	tr->trace_buffer.buffer = tr->max_buffer.buffer;
1088	tr->max_buffer.buffer = buf;
1089
1090	__update_max_tr(tr, tsk, cpu);
1091	arch_spin_unlock(&tr->max_lock);
1092	}
1093
1094	/**
1095	* update_max_tr_single - only copy one trace over, and reset the rest
1096	* @tr - tracer
1097	* @tsk - task with the latency
1098	* @cpu - the cpu of the buffer to copy.
1099	*
1100	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
1101	*/
1102	void
1103	update_max_tr_single(struct trace_array tr, struct task_struct tsk, int cpu)
1104	{
1105	int ret;
1106
1107	if (tr->stop_count)
1108	return;
1109
1110	WARN_ON_ONCE(!irqs_disabled());
1111	if (!tr->allocated_snapshot) {
1112	/* Only the nop tracer should hit this when disabling */
1113	WARN_ON_ONCE(tr->current_trace != &nop_trace);
1114	return;
1115	}
1116
1117	arch_spin_lock(&tr->max_lock);
1118
1119	ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1120
1121	if (ret == -EBUSY) {
1122	/*
1123	* We failed to swap the buffer due to a commit taking
1124	* place on this CPU. We fail to record, but we reset
1125	* the max trace buffer (no one writes directly to it)
1126	* and flag that it failed.
1127	*/
1128	trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1129	"Failed to swap buffers due to commit in progress\n");
1130	}
1131
1132	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1133
1134	__update_max_tr(tr, tsk, cpu);
1135	arch_spin_unlock(&tr->max_lock);
1136	}
1137	#endif /* CONFIG_TRACER_MAX_TRACE */
1138
1139	static int wait_on_pipe(struct trace_iterator *iter, bool full)
1140	{
1141	/* Iterators are static, they should be filled or empty */
1142	if (trace_buffer_iter(iter, iter->cpu_file))
1143	return 0;
1144
1145	return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1146	full);
1147	}
1148
1149	#ifdef CONFIG_FTRACE_STARTUP_TEST
1150	static int run_tracer_selftest(struct tracer *type)
1151	{
1152	struct trace_array *tr = &global_trace;
1153	struct tracer *saved_tracer = tr->current_trace;
1154	int ret;
1155
1156	if (!type->selftest \|\| tracing_selftest_disabled)
1157	return 0;
1158
1159	/*
1160	* Run a selftest on this tracer.
1161	* Here we reset the trace buffer, and set the current
1162	* tracer to be this tracer. The tracer can then run some
1163	* internal tracing to verify that everything is in order.
1164	* If we fail, we do not register this tracer.
1165	*/
1166	tracing_reset_online_cpus(&tr->trace_buffer);
1167
1168	tr->current_trace = type;
1169
1170	#ifdef CONFIG_TRACER_MAX_TRACE
1171	if (type->use_max_tr) {
1172	/* If we expanded the buffers, make sure the max is expanded too */
1173	if (ring_buffer_expanded)
1174	ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1175	RING_BUFFER_ALL_CPUS);
1176	tr->allocated_snapshot = true;
1177	}
1178	#endif
1179
1180	/* the test is responsible for initializing and enabling */
1181	pr_info("Testing tracer %s: ", type->name);
1182	ret = type->selftest(type, tr);
1183	/* the test is responsible for resetting too */
1184	tr->current_trace = saved_tracer;
1185	if (ret) {
1186	printk(KERN_CONT "FAILED!\n");
1187	/* Add the warning after printing 'FAILED' */
1188	WARN_ON(1);
1189	return -1;
1190	}
1191	/* Only reset on passing, to avoid touching corrupted buffers */
1192	tracing_reset_online_cpus(&tr->trace_buffer);
1193
1194	#ifdef CONFIG_TRACER_MAX_TRACE
1195	if (type->use_max_tr) {
1196	tr->allocated_snapshot = false;
1197
1198	/* Shrink the max buffer again */
1199	if (ring_buffer_expanded)
1200	ring_buffer_resize(tr->max_buffer.buffer, 1,
1201	RING_BUFFER_ALL_CPUS);
1202	}
1203	#endif
1204
1205	printk(KERN_CONT "PASSED\n");
1206	return 0;
1207	}
1208	#else
1209	static inline int run_tracer_selftest(struct tracer *type)
1210	{
1211	return 0;
1212	}
1213	#endif /* CONFIG_FTRACE_STARTUP_TEST */
1214
1215	static void add_tracer_options(struct trace_array tr, struct tracer t);
1216
1217	static void __init apply_trace_boot_options(void);
1218
1219	/**
1220	* register_tracer - register a tracer with the ftrace system.
1221	* @type - the plugin for the tracer
1222	*
1223	* Register a new plugin tracer.
1224	*/
1225	int __init register_tracer(struct tracer *type)
1226	{
1227	struct tracer *t;
1228	int ret = 0;
1229
1230	if (!type->name) {
1231	pr_info("Tracer must have a name\n");
1232	return -1;
1233	}
1234
1235	if (strlen(type->name) >= MAX_TRACER_SIZE) {
1236	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1237	return -1;
1238	}
1239
1240	mutex_lock(&trace_types_lock);
1241
1242	tracing_selftest_running = true;
1243
1244	for (t = trace_types; t; t = t->next) {
1245	if (strcmp(type->name, t->name) == 0) {
1246	/* already found */
1247	pr_info("Tracer %s already registered\n",
1248	type->name);
1249	ret = -1;
1250	goto out;
1251	}
1252	}
1253
1254	if (!type->set_flag)
1255	type->set_flag = &dummy_set_flag;
1256	if (!type->flags) {
1257	/allocate a dummy tracer_flags/
1258	type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1259	if (!type->flags) {
1260	ret = -ENOMEM;
1261	goto out;
1262	}
1263	type->flags->val = 0;
1264	type->flags->opts = dummy_tracer_opt;
1265	} else
1266	if (!type->flags->opts)
1267	type->flags->opts = dummy_tracer_opt;
1268
1269	/* store the tracer for __set_tracer_option */
1270	type->flags->trace = type;
1271
1272	ret = run_tracer_selftest(type);
1273	if (ret < 0)
1274	goto out;
1275
1276	type->next = trace_types;
1277	trace_types = type;
1278	add_tracer_options(&global_trace, type);
1279
1280	out:
1281	tracing_selftest_running = false;
1282	mutex_unlock(&trace_types_lock);
1283
1284	if (ret \|\| !default_bootup_tracer)
1285	goto out_unlock;
1286
1287	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1288	goto out_unlock;
1289
1290	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1291	/* Do we want this tracer to start on bootup? */
1292	tracing_set_tracer(&global_trace, type->name);
1293	default_bootup_tracer = NULL;
1294
1295	apply_trace_boot_options();
1296
1297	/* disable other selftests, since this will break it. */
1298	tracing_selftest_disabled = true;
1299	#ifdef CONFIG_FTRACE_STARTUP_TEST
1300	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1301	type->name);
1302	#endif
1303
1304	out_unlock:
1305	return ret;
1306	}
1307
1308	void tracing_reset(struct trace_buffer *buf, int cpu)
1309	{
1310	struct ring_buffer *buffer = buf->buffer;
1311
1312	if (!buffer)
1313	return;
1314
1315	ring_buffer_record_disable(buffer);
1316
1317	/* Make sure all commits have finished */
1318	synchronize_sched();
1319	ring_buffer_reset_cpu(buffer, cpu);
1320
1321	ring_buffer_record_enable(buffer);
1322	}
1323
1324	void tracing_reset_online_cpus(struct trace_buffer *buf)
1325	{
1326	struct ring_buffer *buffer = buf->buffer;
1327	int cpu;
1328
1329	if (!buffer)
1330	return;
1331
1332	ring_buffer_record_disable(buffer);
1333
1334	/* Make sure all commits have finished */
1335	synchronize_sched();
1336
1337	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1338
1339	for_each_online_cpu(cpu)
1340	ring_buffer_reset_cpu(buffer, cpu);
1341
1342	ring_buffer_record_enable(buffer);
1343	}
1344
1345	/* Must have trace_types_lock held */
1346	void tracing_reset_all_online_cpus(void)
1347	{
1348	struct trace_array *tr;
1349
1350	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1351	tracing_reset_online_cpus(&tr->trace_buffer);
1352	#ifdef CONFIG_TRACER_MAX_TRACE
1353	tracing_reset_online_cpus(&tr->max_buffer);
1354	#endif
1355	}
1356	}
1357
1358	#define SAVED_CMDLINES_DEFAULT 128
1359	#define NO_CMDLINE_MAP UINT_MAX
1360	static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1361	struct saved_cmdlines_buffer {
1362	unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1363	unsigned *map_cmdline_to_pid;
1364	unsigned cmdline_num;
1365	int cmdline_idx;
1366	char *saved_cmdlines;
1367	};
1368	static struct saved_cmdlines_buffer *savedcmd;
1369
1370	/* temporary disable recording */
1371	static atomic_t trace_record_cmdline_disabled __read_mostly;
1372
1373	static inline char *get_saved_cmdlines(int idx)
1374	{
1375	return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1376	}
1377
1378	static inline void set_cmdline(int idx, const char *cmdline)
1379	{
1380	memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1381	}
1382
1383	static int allocate_cmdlines_buffer(unsigned int val,
1384	struct saved_cmdlines_buffer *s)
1385	{
1386	s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1387	GFP_KERNEL);
1388	if (!s->map_cmdline_to_pid)
1389	return -ENOMEM;
1390
1391	s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1392	if (!s->saved_cmdlines) {
1393	kfree(s->map_cmdline_to_pid);
1394	return -ENOMEM;
1395	}
1396
1397	s->cmdline_idx = 0;
1398	s->cmdline_num = val;
1399	memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1400	sizeof(s->map_pid_to_cmdline));
1401	memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1402	val * sizeof(*s->map_cmdline_to_pid));
1403
1404	return 0;
1405	}
1406
1407	static int trace_create_savedcmd(void)
1408	{
1409	int ret;
1410
1411	savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1412	if (!savedcmd)
1413	return -ENOMEM;
1414
1415	ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1416	if (ret < 0) {
1417	kfree(savedcmd);
1418	savedcmd = NULL;
1419	return -ENOMEM;
1420	}
1421
1422	return 0;
1423	}
1424
1425	int is_tracing_stopped(void)
1426	{
1427	return global_trace.stop_count;
1428	}
1429
1430	/**
1431	* tracing_start - quick start of the tracer
1432	*
1433	* If tracing is enabled but was stopped by tracing_stop,
1434	* this will start the tracer back up.
1435	*/
1436	void tracing_start(void)
1437	{
1438	struct ring_buffer *buffer;
1439	unsigned long flags;
1440
1441	if (tracing_disabled)
1442	return;
1443
1444	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1445	if (--global_trace.stop_count) {
1446	if (global_trace.stop_count < 0) {
1447	/* Someone screwed up their debugging */
1448	WARN_ON_ONCE(1);
1449	global_trace.stop_count = 0;
1450	}
1451	goto out;
1452	}
1453
1454	/* Prevent the buffers from switching */
1455	arch_spin_lock(&global_trace.max_lock);
1456
1457	buffer = global_trace.trace_buffer.buffer;
1458	if (buffer)
1459	ring_buffer_record_enable(buffer);
1460
1461	#ifdef CONFIG_TRACER_MAX_TRACE
1462	buffer = global_trace.max_buffer.buffer;
1463	if (buffer)
1464	ring_buffer_record_enable(buffer);
1465	#endif
1466
1467	arch_spin_unlock(&global_trace.max_lock);
1468
1469	out:
1470	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1471	}
1472
1473	static void tracing_start_tr(struct trace_array *tr)
1474	{
1475	struct ring_buffer *buffer;
1476	unsigned long flags;
1477
1478	if (tracing_disabled)
1479	return;
1480
1481	/* If global, we need to also start the max tracer */
1482	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1483	return tracing_start();
1484
1485	raw_spin_lock_irqsave(&tr->start_lock, flags);
1486
1487	if (--tr->stop_count) {
1488	if (tr->stop_count < 0) {
1489	/* Someone screwed up their debugging */
1490	WARN_ON_ONCE(1);
1491	tr->stop_count = 0;
1492	}
1493	goto out;
1494	}
1495
1496	buffer = tr->trace_buffer.buffer;
1497	if (buffer)
1498	ring_buffer_record_enable(buffer);
1499
1500	out:
1501	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1502	}
1503
1504	/**
1505	* tracing_stop - quick stop of the tracer
1506	*
1507	* Light weight way to stop tracing. Use in conjunction with
1508	* tracing_start.
1509	*/
1510	void tracing_stop(void)
1511	{
1512	struct ring_buffer *buffer;
1513	unsigned long flags;
1514
1515	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1516	if (global_trace.stop_count++)
1517	goto out;
1518
1519	/* Prevent the buffers from switching */
1520	arch_spin_lock(&global_trace.max_lock);
1521
1522	buffer = global_trace.trace_buffer.buffer;
1523	if (buffer)
1524	ring_buffer_record_disable(buffer);
1525
1526	#ifdef CONFIG_TRACER_MAX_TRACE
1527	buffer = global_trace.max_buffer.buffer;
1528	if (buffer)
1529	ring_buffer_record_disable(buffer);
1530	#endif
1531
1532	arch_spin_unlock(&global_trace.max_lock);
1533
1534	out:
1535	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1536	}
1537
1538	static void tracing_stop_tr(struct trace_array *tr)
1539	{
1540	struct ring_buffer *buffer;
1541	unsigned long flags;
1542
1543	/* If global, we need to also stop the max tracer */
1544	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1545	return tracing_stop();
1546
1547	raw_spin_lock_irqsave(&tr->start_lock, flags);
1548	if (tr->stop_count++)
1549	goto out;
1550
1551	buffer = tr->trace_buffer.buffer;
1552	if (buffer)
1553	ring_buffer_record_disable(buffer);
1554
1555	out:
1556	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1557	}
1558
1559	void trace_stop_cmdline_recording(void);
1560
1561	static int trace_save_cmdline(struct task_struct *tsk)
1562	{
1563	unsigned pid, idx;
1564
1565	if (!tsk->pid \|\| unlikely(tsk->pid > PID_MAX_DEFAULT))
1566	return 0;
1567
1568	/*
1569	* It's not the end of the world if we don't get
1570	* the lock, but we also don't want to spin
1571	* nor do we want to disable interrupts,
1572	* so if we miss here, then better luck next time.
1573	*/
1574	if (!arch_spin_trylock(&trace_cmdline_lock))
1575	return 0;
1576
1577	idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1578	if (idx == NO_CMDLINE_MAP) {
1579	idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1580
1581	/*
1582	* Check whether the cmdline buffer at idx has a pid
1583	* mapped. We are going to overwrite that entry so we
1584	* need to clear the map_pid_to_cmdline. Otherwise we
1585	* would read the new comm for the old pid.
1586	*/
1587	pid = savedcmd->map_cmdline_to_pid[idx];
1588	if (pid != NO_CMDLINE_MAP)
1589	savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1590
1591	savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1592	savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1593
1594	savedcmd->cmdline_idx = idx;
1595	}
1596
1597	set_cmdline(idx, tsk->comm);
1598
1599	arch_spin_unlock(&trace_cmdline_lock);
1600
1601	return 1;
1602	}
1603
1604	static void __trace_find_cmdline(int pid, char comm[])
1605	{
1606	unsigned map;
1607
1608	if (!pid) {
1609	strcpy(comm, "<idle>");
1610	return;
1611	}
1612
1613	if (WARN_ON_ONCE(pid < 0)) {
1614	strcpy(comm, "<XXX>");
1615	return;
1616	}
1617
1618	if (pid > PID_MAX_DEFAULT) {
1619	strcpy(comm, "<...>");
1620	return;
1621	}
1622
1623	map = savedcmd->map_pid_to_cmdline[pid];
1624	if (map != NO_CMDLINE_MAP)
1625	strcpy(comm, get_saved_cmdlines(map));
1626	else
1627	strcpy(comm, "<...>");
1628	}
1629
1630	void trace_find_cmdline(int pid, char comm[])
1631	{
1632	preempt_disable();
1633	arch_spin_lock(&trace_cmdline_lock);
1634
1635	__trace_find_cmdline(pid, comm);
1636
1637	arch_spin_unlock(&trace_cmdline_lock);
1638	preempt_enable();
1639	}
1640
1641	void tracing_record_cmdline(struct task_struct *tsk)
1642	{
1643	if (atomic_read(&trace_record_cmdline_disabled) \|\| !tracing_is_on())
1644	return;
1645
1646	if (!__this_cpu_read(trace_cmdline_save))
1647	return;
1648
1649	if (trace_save_cmdline(tsk))
1650	__this_cpu_write(trace_cmdline_save, false);
1651	}
1652
1653	void
1654	tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1655	int pc)
1656	{
1657	struct task_struct *tsk = current;
1658
1659	entry->preempt_count = pc & 0xff;
1660	entry->pid = (tsk) ? tsk->pid : 0;
1661	entry->flags =
1662	#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1663	(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) \|
1664	#else
1665	TRACE_FLAG_IRQS_NOSUPPORT \|
1666	#endif
1667	((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) \|
1668	((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) \|
1669	((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) \|
1670	(tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) \|
1671	(test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1672	}
1673	EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1674
1675	struct ring_buffer_event *
1676	trace_buffer_lock_reserve(struct ring_buffer *buffer,
1677	int type,
1678	unsigned long len,
1679	unsigned long flags, int pc)
1680	{
1681	struct ring_buffer_event *event;
1682
1683	event = ring_buffer_lock_reserve(buffer, len);
1684	if (event != NULL) {
1685	struct trace_entry *ent = ring_buffer_event_data(event);
1686
1687	tracing_generic_entry_update(ent, flags, pc);
1688	ent->type = type;
1689	}
1690
1691	return event;
1692	}
1693
1694	void
1695	__buffer_unlock_commit(struct ring_buffer buffer, struct ring_buffer_event event)
1696	{
1697	__this_cpu_write(trace_cmdline_save, true);
1698	ring_buffer_unlock_commit(buffer, event);
1699	}
1700
1701	void trace_buffer_unlock_commit(struct trace_array *tr,
1702	struct ring_buffer *buffer,
1703	struct ring_buffer_event *event,
1704	unsigned long flags, int pc)
1705	{
1706	__buffer_unlock_commit(buffer, event);
1707
1708	ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
1709	ftrace_trace_userstack(buffer, flags, pc);
1710	}
1711	EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1712
1713	static struct ring_buffer *temp_buffer;
1714
1715	struct ring_buffer_event *
1716	trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1717	struct trace_event_file *trace_file,
1718	int type, unsigned long len,
1719	unsigned long flags, int pc)
1720	{
1721	struct ring_buffer_event *entry;
1722
1723	*current_rb = trace_file->tr->trace_buffer.buffer;
1724	entry = trace_buffer_lock_reserve(*current_rb,
1725	type, len, flags, pc);
1726	/*
1727	* If tracing is off, but we have triggers enabled
1728	* we still need to look at the event data. Use the temp_buffer
1729	* to store the trace event for the tigger to use. It's recusive
1730	* safe and will not be recorded anywhere.
1731	*/
1732	if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
1733	*current_rb = temp_buffer;
1734	entry = trace_buffer_lock_reserve(*current_rb,
1735	type, len, flags, pc);
1736	}
1737	return entry;
1738	}
1739	EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1740
1741	struct ring_buffer_event *
1742	trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1743	int type, unsigned long len,
1744	unsigned long flags, int pc)
1745	{
1746	*current_rb = global_trace.trace_buffer.buffer;
1747	return trace_buffer_lock_reserve(*current_rb,
1748	type, len, flags, pc);
1749	}
1750	EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1751
1752	void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1753	struct ring_buffer *buffer,
1754	struct ring_buffer_event *event,
1755	unsigned long flags, int pc,
1756	struct pt_regs *regs)
1757	{
1758	__buffer_unlock_commit(buffer, event);
1759
1760	ftrace_trace_stack(tr, buffer, flags, 0, pc, regs);
1761	ftrace_trace_userstack(buffer, flags, pc);
1762	}
1763	EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1764
1765	void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1766	struct ring_buffer_event *event)
1767	{
1768	ring_buffer_discard_commit(buffer, event);
1769	}
1770	EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1771
1772	void
1773	trace_function(struct trace_array *tr,
1774	unsigned long ip, unsigned long parent_ip, unsigned long flags,
1775	int pc)
1776	{
1777	struct trace_event_call *call = &event_function;
1778	struct ring_buffer *buffer = tr->trace_buffer.buffer;
1779	struct ring_buffer_event *event;
1780	struct ftrace_entry *entry;
1781
1782	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1783	flags, pc);
1784	if (!event)
1785	return;
1786	entry = ring_buffer_event_data(event);
1787	entry->ip = ip;
1788	entry->parent_ip = parent_ip;
1789
1790	if (!call_filter_check_discard(call, entry, buffer, event))
1791	__buffer_unlock_commit(buffer, event);
1792	}
1793
1794	#ifdef CONFIG_STACKTRACE
1795
1796	#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1797	struct ftrace_stack {
1798	unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1799	};
1800
1801	static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1802	static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1803
1804	static void __ftrace_trace_stack(struct ring_buffer *buffer,
1805	unsigned long flags,
1806	int skip, int pc, struct pt_regs *regs)
1807	{
1808	struct trace_event_call *call = &event_kernel_stack;
1809	struct ring_buffer_event *event;
1810	struct stack_entry *entry;
1811	struct stack_trace trace;
1812	int use_stack;
1813	int size = FTRACE_STACK_ENTRIES;
1814
1815	trace.nr_entries = 0;
1816	trace.skip = skip;
1817
1818	/*
1819	* Since events can happen in NMIs there's no safe way to
1820	* use the per cpu ftrace_stacks. We reserve it and if an interrupt
1821	* or NMI comes in, it will just have to use the default
1822	* FTRACE_STACK_SIZE.
1823	*/
1824	preempt_disable_notrace();
1825
1826	use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1827	/*
1828	* We don't need any atomic variables, just a barrier.
1829	* If an interrupt comes in, we don't care, because it would
1830	* have exited and put the counter back to what we want.
1831	* We just need a barrier to keep gcc from moving things
1832	* around.
1833	*/
1834	barrier();
1835	if (use_stack == 1) {
1836	trace.entries = this_cpu_ptr(ftrace_stack.calls);
1837	trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1838
1839	if (regs)
1840	save_stack_trace_regs(regs, &trace);
1841	else
1842	save_stack_trace(&trace);
1843
1844	if (trace.nr_entries > size)
1845	size = trace.nr_entries;
1846	} else
1847	/* From now on, use_stack is a boolean */
1848	use_stack = 0;
1849
1850	size *= sizeof(unsigned long);
1851
1852	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1853	sizeof(*entry) + size, flags, pc);
1854	if (!event)
1855	goto out;
1856	entry = ring_buffer_event_data(event);
1857
1858	memset(&entry->caller, 0, size);
1859
1860	if (use_stack)
1861	memcpy(&entry->caller, trace.entries,
1862	trace.nr_entries * sizeof(unsigned long));
1863	else {
1864	trace.max_entries = FTRACE_STACK_ENTRIES;
1865	trace.entries = entry->caller;
1866	if (regs)
1867	save_stack_trace_regs(regs, &trace);
1868	else
1869	save_stack_trace(&trace);
1870	}
1871
1872	entry->size = trace.nr_entries;
1873
1874	if (!call_filter_check_discard(call, entry, buffer, event))
1875	__buffer_unlock_commit(buffer, event);
1876
1877	out:
1878	/* Again, don't let gcc optimize things here */
1879	barrier();
1880	__this_cpu_dec(ftrace_stack_reserve);
1881	preempt_enable_notrace();
1882
1883	}
1884
1885	static inline void ftrace_trace_stack(struct trace_array *tr,
1886	struct ring_buffer *buffer,
1887	unsigned long flags,
1888	int skip, int pc, struct pt_regs *regs)
1889	{
1890	if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
1891	return;
1892
1893	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
1894	}
1895
1896	void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1897	int pc)
1898	{
1899	__ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1900	}
1901
1902	/**
1903	* trace_dump_stack - record a stack back trace in the trace buffer
1904	* @skip: Number of functions to skip (helper handlers)
1905	*/
1906	void trace_dump_stack(int skip)
1907	{
1908	unsigned long flags;
1909
1910	if (tracing_disabled \|\| tracing_selftest_running)
1911	return;
1912
1913	local_save_flags(flags);
1914
1915	/*
1916	* Skip 3 more, seems to get us at the caller of
1917	* this function.
1918	*/
1919	skip += 3;
1920	__ftrace_trace_stack(global_trace.trace_buffer.buffer,
1921	flags, skip, preempt_count(), NULL);
1922	}
1923
1924	static DEFINE_PER_CPU(int, user_stack_count);
1925
1926	void
1927	ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1928	{
1929	struct trace_event_call *call = &event_user_stack;
1930	struct ring_buffer_event *event;
1931	struct userstack_entry *entry;
1932	struct stack_trace trace;
1933
1934	if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
1935	return;
1936
1937	/*
1938	* NMIs can not handle page faults, even with fix ups.
1939	* The save user stack can (and often does) fault.
1940	*/
1941	if (unlikely(in_nmi()))
1942	return;
1943
1944	/*
1945	* prevent recursion, since the user stack tracing may
1946	* trigger other kernel events.
1947	*/
1948	preempt_disable();
1949	if (__this_cpu_read(user_stack_count))
1950	goto out;
1951
1952	__this_cpu_inc(user_stack_count);
1953
1954	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1955	sizeof(*entry), flags, pc);
1956	if (!event)
1957	goto out_drop_count;
1958	entry = ring_buffer_event_data(event);
1959
1960	entry->tgid = current->tgid;
1961	memset(&entry->caller, 0, sizeof(entry->caller));
1962
1963	trace.nr_entries = 0;
1964	trace.max_entries = FTRACE_STACK_ENTRIES;
1965	trace.skip = 0;
1966	trace.entries = entry->caller;
1967
1968	save_stack_trace_user(&trace);
1969	if (!call_filter_check_discard(call, entry, buffer, event))
1970	__buffer_unlock_commit(buffer, event);
1971
1972	out_drop_count:
1973	__this_cpu_dec(user_stack_count);
1974	out:
1975	preempt_enable();
1976	}
1977
1978	#ifdef UNUSED
1979	static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1980	{
1981	ftrace_trace_userstack(tr, flags, preempt_count());
1982	}
1983	#endif /* UNUSED */
1984
1985	#endif /* CONFIG_STACKTRACE */
1986
1987	/* created for use with alloc_percpu */
1988	struct trace_buffer_struct {
1989	char buffer[TRACE_BUF_SIZE];
1990	};
1991
1992	static struct trace_buffer_struct *trace_percpu_buffer;
1993	static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1994	static struct trace_buffer_struct *trace_percpu_irq_buffer;
1995	static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1996
1997	/*
1998	* The buffer used is dependent on the context. There is a per cpu
1999	* buffer for normal context, softirq contex, hard irq context and
2000	* for NMI context. Thise allows for lockless recording.
2001	*
2002	* Note, if the buffers failed to be allocated, then this returns NULL
2003	*/
2004	static char *get_trace_buf(void)
2005	{
2006	struct trace_buffer_struct *percpu_buffer;
2007
2008	/*
2009	* If we have allocated per cpu buffers, then we do not
2010	* need to do any locking.
2011	*/
2012	if (in_nmi())
2013	percpu_buffer = trace_percpu_nmi_buffer;
2014	else if (in_irq())
2015	percpu_buffer = trace_percpu_irq_buffer;
2016	else if (in_softirq())
2017	percpu_buffer = trace_percpu_sirq_buffer;
2018	else
2019	percpu_buffer = trace_percpu_buffer;
2020
2021	if (!percpu_buffer)
2022	return NULL;
2023
2024	return this_cpu_ptr(&percpu_buffer->buffer[0]);
2025	}
2026
2027	static int alloc_percpu_trace_buffer(void)
2028	{
2029	struct trace_buffer_struct *buffers;
2030	struct trace_buffer_struct *sirq_buffers;
2031	struct trace_buffer_struct *irq_buffers;
2032	struct trace_buffer_struct *nmi_buffers;
2033
2034	buffers = alloc_percpu(struct trace_buffer_struct);
2035	if (!buffers)
2036	goto err_warn;
2037
2038	sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2039	if (!sirq_buffers)
2040	goto err_sirq;
2041
2042	irq_buffers = alloc_percpu(struct trace_buffer_struct);
2043	if (!irq_buffers)
2044	goto err_irq;
2045
2046	nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2047	if (!nmi_buffers)
2048	goto err_nmi;
2049
2050	trace_percpu_buffer = buffers;
2051	trace_percpu_sirq_buffer = sirq_buffers;
2052	trace_percpu_irq_buffer = irq_buffers;
2053	trace_percpu_nmi_buffer = nmi_buffers;
2054
2055	return 0;
2056
2057	err_nmi:
2058	free_percpu(irq_buffers);
2059	err_irq:
2060	free_percpu(sirq_buffers);
2061	err_sirq:
2062	free_percpu(buffers);
2063	err_warn:
2064	WARN(1, "Could not allocate percpu trace_printk buffer");
2065	return -ENOMEM;
2066	}
2067
2068	static int buffers_allocated;
2069
2070	void trace_printk_init_buffers(void)
2071	{
2072	if (buffers_allocated)
2073	return;
2074
2075	if (alloc_percpu_trace_buffer())
2076	return;
2077
2078	/* trace_printk() is for debug use only. Don't use it in production. */
2079
2080	pr_warn("\n");
2081	pr_warn("**********************************************************\n");
2082	pr_warn(" NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE \n");
2083	pr_warn(" \n");
2084	pr_warn(" trace_printk() being used. Allocating extra memory. \n");
2085	pr_warn(" \n");
2086	pr_warn(" This means that this is a DEBUG kernel and it is \n");
2087	pr_warn(" unsafe for production use. \n");
2088	pr_warn(" \n");
2089	pr_warn(" If you see this message and you are not debugging \n");
2090	pr_warn(" the kernel, report this immediately to your vendor! \n");
2091	pr_warn(" \n");
2092	pr_warn(" NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE \n");
2093	pr_warn("**********************************************************\n");
2094
2095	/* Expand the buffers to set size */
2096	tracing_update_buffers();
2097
2098	buffers_allocated = 1;
2099
2100	/*
2101	* trace_printk_init_buffers() can be called by modules.
2102	* If that happens, then we need to start cmdline recording
2103	* directly here. If the global_trace.buffer is already
2104	* allocated here, then this was called by module code.
2105	*/
2106	if (global_trace.trace_buffer.buffer)
2107	tracing_start_cmdline_record();
2108	}
2109
2110	void trace_printk_start_comm(void)
2111	{
2112	/* Start tracing comms if trace printk is set */
2113	if (!buffers_allocated)
2114	return;
2115	tracing_start_cmdline_record();
2116	}
2117
2118	static void trace_printk_start_stop_comm(int enabled)
2119	{
2120	if (!buffers_allocated)
2121	return;
2122
2123	if (enabled)
2124	tracing_start_cmdline_record();
2125	else
2126	tracing_stop_cmdline_record();
2127	}
2128
2129	/**
2130	* trace_vbprintk - write binary msg to tracing buffer
2131	*
2132	*/
2133	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2134	{
2135	struct trace_event_call *call = &event_bprint;
2136	struct ring_buffer_event *event;
2137	struct ring_buffer *buffer;
2138	struct trace_array *tr = &global_trace;
2139	struct bprint_entry *entry;
2140	unsigned long flags;
2141	char *tbuffer;
2142	int len = 0, size, pc;
2143
2144	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
2145	return 0;
2146
2147	/* Don't pollute graph traces with trace_vprintk internals */
2148	pause_graph_tracing();
2149
2150	pc = preempt_count();
2151	preempt_disable_notrace();
2152
2153	tbuffer = get_trace_buf();
2154	if (!tbuffer) {
2155	len = 0;
2156	goto out;
2157	}
2158
2159	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2160
2161	if (len > TRACE_BUF_SIZE/sizeof(int) \|\| len < 0)
2162	goto out;
2163
2164	local_save_flags(flags);
2165	size = sizeof(entry) + sizeof(u32) len;
2166	buffer = tr->trace_buffer.buffer;
2167	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2168	flags, pc);
2169	if (!event)
2170	goto out;
2171	entry = ring_buffer_event_data(event);
2172	entry->ip = ip;
2173	entry->fmt = fmt;
2174
2175	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2176	if (!call_filter_check_discard(call, entry, buffer, event)) {
2177	__buffer_unlock_commit(buffer, event);
2178	ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2179	}
2180
2181	out:
2182	preempt_enable_notrace();
2183	unpause_graph_tracing();
2184
2185	return len;
2186	}
2187	EXPORT_SYMBOL_GPL(trace_vbprintk);
2188
2189	static int
2190	__trace_array_vprintk(struct ring_buffer *buffer,
2191	unsigned long ip, const char *fmt, va_list args)
2192	{
2193	struct trace_event_call *call = &event_print;
2194	struct ring_buffer_event *event;
2195	int len = 0, size, pc;
2196	struct print_entry *entry;
2197	unsigned long flags;
2198	char *tbuffer;
2199
2200	if (tracing_disabled \|\| tracing_selftest_running)
2201	return 0;
2202
2203	/* Don't pollute graph traces with trace_vprintk internals */
2204	pause_graph_tracing();
2205
2206	pc = preempt_count();
2207	preempt_disable_notrace();
2208
2209
2210	tbuffer = get_trace_buf();
2211	if (!tbuffer) {
2212	len = 0;
2213	goto out;
2214	}
2215
2216	len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2217
2218	local_save_flags(flags);
2219	size = sizeof(*entry) + len + 1;
2220	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2221	flags, pc);
2222	if (!event)
2223	goto out;
2224	entry = ring_buffer_event_data(event);
2225	entry->ip = ip;
2226
2227	memcpy(&entry->buf, tbuffer, len + 1);
2228	if (!call_filter_check_discard(call, entry, buffer, event)) {
2229	__buffer_unlock_commit(buffer, event);
2230	ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
2231	}
2232	out:
2233	preempt_enable_notrace();
2234	unpause_graph_tracing();
2235
2236	return len;
2237	}
2238
2239	int trace_array_vprintk(struct trace_array *tr,
2240	unsigned long ip, const char *fmt, va_list args)
2241	{
2242	return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2243	}
2244
2245	int trace_array_printk(struct trace_array *tr,
2246	unsigned long ip, const char *fmt, ...)
2247	{
2248	int ret;
2249	va_list ap;
2250
2251	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2252	return 0;
2253
2254	va_start(ap, fmt);
2255	ret = trace_array_vprintk(tr, ip, fmt, ap);
2256	va_end(ap);
2257	return ret;
2258	}
2259
2260	int trace_array_printk_buf(struct ring_buffer *buffer,
2261	unsigned long ip, const char *fmt, ...)
2262	{
2263	int ret;
2264	va_list ap;
2265
2266	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2267	return 0;
2268
2269	va_start(ap, fmt);
2270	ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2271	va_end(ap);
2272	return ret;
2273	}
2274
2275	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2276	{
2277	return trace_array_vprintk(&global_trace, ip, fmt, args);
2278	}
2279	EXPORT_SYMBOL_GPL(trace_vprintk);
2280
2281	static void trace_iterator_increment(struct trace_iterator *iter)
2282	{
2283	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2284
2285	iter->idx++;
2286	if (buf_iter)
2287	ring_buffer_read(buf_iter, NULL);
2288	}
2289
2290	static struct trace_entry *
2291	peek_next_entry(struct trace_iterator iter, int cpu, u64 ts,
2292	unsigned long *lost_events)
2293	{
2294	struct ring_buffer_event *event;
2295	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2296
2297	if (buf_iter)
2298	event = ring_buffer_iter_peek(buf_iter, ts);
2299	else
2300	event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2301	lost_events);
2302
2303	if (event) {
2304	iter->ent_size = ring_buffer_event_length(event);
2305	return ring_buffer_event_data(event);
2306	}
2307	iter->ent_size = 0;
2308	return NULL;
2309	}
2310
2311	static struct trace_entry *
2312	__find_next_entry(struct trace_iterator iter, int ent_cpu,
2313	unsigned long missing_events, u64 ent_ts)
2314	{
2315	struct ring_buffer *buffer = iter->trace_buffer->buffer;
2316	struct trace_entry ent, next = NULL;
2317	unsigned long lost_events = 0, next_lost = 0;
2318	int cpu_file = iter->cpu_file;
2319	u64 next_ts = 0, ts;
2320	int next_cpu = -1;
2321	int next_size = 0;
2322	int cpu;
2323
2324	/*
2325	* If we are in a per_cpu trace file, don't bother by iterating over
2326	* all cpu and peek directly.
2327	*/
2328	if (cpu_file > RING_BUFFER_ALL_CPUS) {
2329	if (ring_buffer_empty_cpu(buffer, cpu_file))
2330	return NULL;
2331	ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2332	if (ent_cpu)
2333	*ent_cpu = cpu_file;
2334
2335	return ent;
2336	}
2337
2338	for_each_tracing_cpu(cpu) {
2339
2340	if (ring_buffer_empty_cpu(buffer, cpu))
2341	continue;
2342
2343	ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2344
2345	/*
2346	* Pick the entry with the smallest timestamp:
2347	*/
2348	if (ent && (!next \|\| ts < next_ts)) {
2349	next = ent;
2350	next_cpu = cpu;
2351	next_ts = ts;
2352	next_lost = lost_events;
2353	next_size = iter->ent_size;
2354	}
2355	}
2356
2357	iter->ent_size = next_size;
2358
2359	if (ent_cpu)
2360	*ent_cpu = next_cpu;
2361
2362	if (ent_ts)
2363	*ent_ts = next_ts;
2364
2365	if (missing_events)
2366	*missing_events = next_lost;
2367
2368	return next;
2369	}
2370
2371	/* Find the next real entry, without updating the iterator itself */
2372	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
2373	int ent_cpu, u64 ent_ts)
2374	{
2375	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2376	}
2377
2378	/* Find the next real entry, and increment the iterator to the next entry */
2379	void trace_find_next_entry_inc(struct trace_iterator iter)
2380	{
2381	iter->ent = __find_next_entry(iter, &iter->cpu,
2382	&iter->lost_events, &iter->ts);
2383
2384	if (iter->ent)
2385	trace_iterator_increment(iter);
2386
2387	return iter->ent ? iter : NULL;
2388	}
2389
2390	static void trace_consume(struct trace_iterator *iter)
2391	{
2392	ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2393	&iter->lost_events);
2394	}
2395
2396	static void s_next(struct seq_file m, void v, loff_t pos)
2397	{
2398	struct trace_iterator *iter = m->private;
2399	int i = (int)*pos;
2400	void *ent;
2401
2402	WARN_ON_ONCE(iter->leftover);
2403
2404	(*pos)++;
2405
2406	/* can't go backwards */
2407	if (iter->idx > i)
2408	return NULL;
2409
2410	if (iter->idx < 0)
2411	ent = trace_find_next_entry_inc(iter);
2412	else
2413	ent = iter;
2414
2415	while (ent && iter->idx < i)
2416	ent = trace_find_next_entry_inc(iter);
2417
2418	iter->pos = *pos;
2419
2420	return ent;
2421	}
2422
2423	void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2424	{
2425	struct ring_buffer_event *event;
2426	struct ring_buffer_iter *buf_iter;
2427	unsigned long entries = 0;
2428	u64 ts;
2429
2430	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2431
2432	buf_iter = trace_buffer_iter(iter, cpu);
2433	if (!buf_iter)
2434	return;
2435
2436	ring_buffer_iter_reset(buf_iter);
2437
2438	/*
2439	* We could have the case with the max latency tracers
2440	* that a reset never took place on a cpu. This is evident
2441	* by the timestamp being before the start of the buffer.
2442	*/
2443	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2444	if (ts >= iter->trace_buffer->time_start)
2445	break;
2446	entries++;
2447	ring_buffer_read(buf_iter, NULL);
2448	}
2449
2450	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2451	}
2452
2453	/*
2454	* The current tracer is copied to avoid a global locking
2455	* all around.
2456	*/
2457	static void s_start(struct seq_file m, loff_t *pos)
2458	{
2459	struct trace_iterator *iter = m->private;
2460	struct trace_array *tr = iter->tr;
2461	int cpu_file = iter->cpu_file;
2462	void *p = NULL;
2463	loff_t l = 0;
2464	int cpu;
2465
2466	/*
2467	* copy the tracer to avoid using a global lock all around.
2468	* iter->trace is a copy of current_trace, the pointer to the
2469	* name may be used instead of a strcmp(), as iter->trace->name
2470	* will point to the same string as current_trace->name.
2471	*/
2472	mutex_lock(&trace_types_lock);
2473	if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2474	iter->trace = tr->current_trace;
2475	mutex_unlock(&trace_types_lock);
2476
2477	#ifdef CONFIG_TRACER_MAX_TRACE
2478	if (iter->snapshot && iter->trace->use_max_tr)
2479	return ERR_PTR(-EBUSY);
2480	#endif
2481
2482	if (!iter->snapshot)
2483	atomic_inc(&trace_record_cmdline_disabled);
2484
2485	if (*pos != iter->pos) {
2486	iter->ent = NULL;
2487	iter->cpu = 0;
2488	iter->idx = -1;
2489
2490	if (cpu_file == RING_BUFFER_ALL_CPUS) {
2491	for_each_tracing_cpu(cpu)
2492	tracing_iter_reset(iter, cpu);
2493	} else
2494	tracing_iter_reset(iter, cpu_file);
2495
2496	iter->leftover = 0;
2497	for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2498	;
2499
2500	} else {
2501	/*
2502	* If we overflowed the seq_file before, then we want
2503	* to just reuse the trace_seq buffer again.
2504	*/
2505	if (iter->leftover)
2506	p = iter;
2507	else {
2508	l = *pos - 1;
2509	p = s_next(m, p, &l);
2510	}
2511	}
2512
2513	trace_event_read_lock();
2514	trace_access_lock(cpu_file);
2515	return p;
2516	}
2517
2518	static void s_stop(struct seq_file m, void p)
2519	{
2520	struct trace_iterator *iter = m->private;
2521
2522	#ifdef CONFIG_TRACER_MAX_TRACE
2523	if (iter->snapshot && iter->trace->use_max_tr)
2524	return;
2525	#endif
2526
2527	if (!iter->snapshot)
2528	atomic_dec(&trace_record_cmdline_disabled);
2529
2530	trace_access_unlock(iter->cpu_file);
2531	trace_event_read_unlock();
2532	}
2533
2534	static void
2535	get_total_entries(struct trace_buffer *buf,
2536	unsigned long total, unsigned long entries)
2537	{
2538	unsigned long count;
2539	int cpu;
2540
2541	*total = 0;
2542	*entries = 0;
2543
2544	for_each_tracing_cpu(cpu) {
2545	count = ring_buffer_entries_cpu(buf->buffer, cpu);
2546	/*
2547	* If this buffer has skipped entries, then we hold all
2548	* entries for the trace and we need to ignore the
2549	* ones before the time stamp.
2550	*/
2551	if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2552	count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2553	/* total is the same as the entries */
2554	*total += count;
2555	} else
2556	*total += count +
2557	ring_buffer_overrun_cpu(buf->buffer, cpu);
2558	*entries += count;
2559	}
2560	}
2561
2562	static void print_lat_help_header(struct seq_file *m)
2563	{
2564	seq_puts(m, "# _------=> CPU# \n"
2565	"# / _-----=> irqs-off \n"
2566	"# \| / _----=> need-resched \n"
2567	"# \|\| / _---=> hardirq/softirq \n"
2568	"# \|\|\| / _--=> preempt-depth \n"
2569	"# \|\|\|\| / delay \n"
2570	"# cmd pid \|\|\|\|\| time \| caller \n"
2571	"# \\ / \|\|\|\|\| \\ \| / \n");
2572	}
2573
2574	static void print_event_info(struct trace_buffer buf, struct seq_file m)
2575	{
2576	unsigned long total;
2577	unsigned long entries;
2578
2579	get_total_entries(buf, &total, &entries);
2580	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2581	entries, total, num_online_cpus());
2582	seq_puts(m, "#\n");
2583	}
2584
2585	static void print_func_help_header(struct trace_buffer buf, struct seq_file m)
2586	{
2587	print_event_info(buf, m);
2588	seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2589	"# \| \| \| \| \|\n");
2590	}
2591
2592	static void print_func_help_header_irq(struct trace_buffer buf, struct seq_file m)
2593	{
2594	print_event_info(buf, m);
2595	seq_puts(m, "# _-----=> irqs-off\n"
2596	"# / _----=> need-resched\n"
2597	"# \| / _---=> hardirq/softirq\n"
2598	"# \|\| / _--=> preempt-depth\n"
2599	"# \|\|\| / delay\n"
2600	"# TASK-PID CPU# \|\|\|\| TIMESTAMP FUNCTION\n"
2601	"# \| \| \| \|\|\|\| \| \|\n");
2602	}
2603
2604	void
2605	print_trace_header(struct seq_file m, struct trace_iterator iter)
2606	{
2607	unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2608	struct trace_buffer *buf = iter->trace_buffer;
2609	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2610	struct tracer *type = iter->trace;
2611	unsigned long entries;
2612	unsigned long total;
2613	const char *name = "preemption";
2614
2615	name = type->name;
2616
2617	get_total_entries(buf, &total, &entries);
2618
2619	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2620	name, UTS_RELEASE);
2621	seq_puts(m, "# -----------------------------------"
2622	"---------------------------------\n");
2623	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d \|"
2624	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2625	nsecs_to_usecs(data->saved_latency),
2626	entries,
2627	total,
2628	buf->cpu,
2629	#if defined(CONFIG_PREEMPT_NONE)
2630	"server",
2631	#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2632	"desktop",
2633	#elif defined(CONFIG_PREEMPT)
2634	"preempt",
2635	#else
2636	"unknown",
2637	#endif
2638	/* These are reserved for later use */
2639	0, 0, 0, 0);
2640	#ifdef CONFIG_SMP
2641	seq_printf(m, " #P:%d)\n", num_online_cpus());
2642	#else
2643	seq_puts(m, ")\n");
2644	#endif
2645	seq_puts(m, "# -----------------\n");
2646	seq_printf(m, "# \| task: %.16s-%d "
2647	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2648	data->comm, data->pid,
2649	from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2650	data->policy, data->rt_priority);
2651	seq_puts(m, "# -----------------\n");
2652
2653	if (data->critical_start) {
2654	seq_puts(m, "# => started at: ");
2655	seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2656	trace_print_seq(m, &iter->seq);
2657	seq_puts(m, "\n# => ended at: ");
2658	seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2659	trace_print_seq(m, &iter->seq);
2660	seq_puts(m, "\n#\n");
2661	}
2662
2663	seq_puts(m, "#\n");
2664	}
2665
2666	static void test_cpu_buff_start(struct trace_iterator *iter)
2667	{
2668	struct trace_seq *s = &iter->seq;
2669	struct trace_array *tr = iter->tr;
2670
2671	if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
2672	return;
2673
2674	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2675	return;
2676
2677	if (iter->started && cpumask_test_cpu(iter->cpu, iter->started))
2678	return;
2679
2680	if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2681	return;
2682
2683	if (iter->started)
2684	cpumask_set_cpu(iter->cpu, iter->started);
2685
2686	/* Don't print started cpu buffer for the first entry of the trace */
2687	if (iter->idx > 1)
2688	trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2689	iter->cpu);
2690	}
2691
2692	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2693	{
2694	struct trace_array *tr = iter->tr;
2695	struct trace_seq *s = &iter->seq;
2696	unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
2697	struct trace_entry *entry;
2698	struct trace_event *event;
2699
2700	entry = iter->ent;
2701
2702	test_cpu_buff_start(iter);
2703
2704	event = ftrace_find_event(entry->type);
2705
2706	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2707	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2708	trace_print_lat_context(iter);
2709	else
2710	trace_print_context(iter);
2711	}
2712
2713	if (trace_seq_has_overflowed(s))
2714	return TRACE_TYPE_PARTIAL_LINE;
2715
2716	if (event)
2717	return event->funcs->trace(iter, sym_flags, event);
2718
2719	trace_seq_printf(s, "Unknown type %d\n", entry->type);
2720
2721	return trace_handle_return(s);
2722	}
2723
2724	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2725	{
2726	struct trace_array *tr = iter->tr;
2727	struct trace_seq *s = &iter->seq;
2728	struct trace_entry *entry;
2729	struct trace_event *event;
2730
2731	entry = iter->ent;
2732
2733	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
2734	trace_seq_printf(s, "%d %d %llu ",
2735	entry->pid, iter->cpu, iter->ts);
2736
2737	if (trace_seq_has_overflowed(s))
2738	return TRACE_TYPE_PARTIAL_LINE;
2739
2740	event = ftrace_find_event(entry->type);
2741	if (event)
2742	return event->funcs->raw(iter, 0, event);
2743
2744	trace_seq_printf(s, "%d ?\n", entry->type);
2745
2746	return trace_handle_return(s);
2747	}
2748
2749	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2750	{
2751	struct trace_array *tr = iter->tr;
2752	struct trace_seq *s = &iter->seq;
2753	unsigned char newline = '\n';
2754	struct trace_entry *entry;
2755	struct trace_event *event;
2756
2757	entry = iter->ent;
2758
2759	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2760	SEQ_PUT_HEX_FIELD(s, entry->pid);
2761	SEQ_PUT_HEX_FIELD(s, iter->cpu);
2762	SEQ_PUT_HEX_FIELD(s, iter->ts);
2763	if (trace_seq_has_overflowed(s))
2764	return TRACE_TYPE_PARTIAL_LINE;
2765	}
2766
2767	event = ftrace_find_event(entry->type);
2768	if (event) {
2769	enum print_line_t ret = event->funcs->hex(iter, 0, event);
2770	if (ret != TRACE_TYPE_HANDLED)
2771	return ret;
2772	}
2773
2774	SEQ_PUT_FIELD(s, newline);
2775
2776	return trace_handle_return(s);
2777	}
2778
2779	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2780	{
2781	struct trace_array *tr = iter->tr;
2782	struct trace_seq *s = &iter->seq;
2783	struct trace_entry *entry;
2784	struct trace_event *event;
2785
2786	entry = iter->ent;
2787
2788	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2789	SEQ_PUT_FIELD(s, entry->pid);
2790	SEQ_PUT_FIELD(s, iter->cpu);
2791	SEQ_PUT_FIELD(s, iter->ts);
2792	if (trace_seq_has_overflowed(s))
2793	return TRACE_TYPE_PARTIAL_LINE;
2794	}
2795
2796	event = ftrace_find_event(entry->type);
2797	return event ? event->funcs->binary(iter, 0, event) :
2798	TRACE_TYPE_HANDLED;
2799	}
2800
2801	int trace_empty(struct trace_iterator *iter)
2802	{
2803	struct ring_buffer_iter *buf_iter;
2804	int cpu;
2805
2806	/* If we are looking at one CPU buffer, only check that one */
2807	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2808	cpu = iter->cpu_file;
2809	buf_iter = trace_buffer_iter(iter, cpu);
2810	if (buf_iter) {
2811	if (!ring_buffer_iter_empty(buf_iter))
2812	return 0;
2813	} else {
2814	if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2815	return 0;
2816	}
2817	return 1;
2818	}
2819
2820	for_each_tracing_cpu(cpu) {
2821	buf_iter = trace_buffer_iter(iter, cpu);
2822	if (buf_iter) {
2823	if (!ring_buffer_iter_empty(buf_iter))
2824	return 0;
2825	} else {
2826	if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2827	return 0;
2828	}
2829	}
2830
2831	return 1;
2832	}
2833
2834	/* Called with trace_event_read_lock() held. */
2835	enum print_line_t print_trace_line(struct trace_iterator *iter)
2836	{
2837	struct trace_array *tr = iter->tr;
2838	unsigned long trace_flags = tr->trace_flags;
2839	enum print_line_t ret;
2840
2841	if (iter->lost_events) {
2842	trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2843	iter->cpu, iter->lost_events);
2844	if (trace_seq_has_overflowed(&iter->seq))
2845	return TRACE_TYPE_PARTIAL_LINE;
2846	}
2847
2848	if (iter->trace && iter->trace->print_line) {
2849	ret = iter->trace->print_line(iter);
2850	if (ret != TRACE_TYPE_UNHANDLED)
2851	return ret;
2852	}
2853
2854	if (iter->ent->type == TRACE_BPUTS &&
2855	trace_flags & TRACE_ITER_PRINTK &&
2856	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2857	return trace_print_bputs_msg_only(iter);
2858
2859	if (iter->ent->type == TRACE_BPRINT &&
2860	trace_flags & TRACE_ITER_PRINTK &&
2861	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2862	return trace_print_bprintk_msg_only(iter);
2863
2864	if (iter->ent->type == TRACE_PRINT &&
2865	trace_flags & TRACE_ITER_PRINTK &&
2866	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2867	return trace_print_printk_msg_only(iter);
2868
2869	if (trace_flags & TRACE_ITER_BIN)
2870	return print_bin_fmt(iter);
2871
2872	if (trace_flags & TRACE_ITER_HEX)
2873	return print_hex_fmt(iter);
2874
2875	if (trace_flags & TRACE_ITER_RAW)
2876	return print_raw_fmt(iter);
2877
2878	return print_trace_fmt(iter);
2879	}
2880
2881	void trace_latency_header(struct seq_file *m)
2882	{
2883	struct trace_iterator *iter = m->private;
2884	struct trace_array *tr = iter->tr;
2885
2886	/* print nothing if the buffers are empty */
2887	if (trace_empty(iter))
2888	return;
2889
2890	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2891	print_trace_header(m, iter);
2892
2893	if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
2894	print_lat_help_header(m);
2895	}
2896
2897	void trace_default_header(struct seq_file *m)
2898	{
2899	struct trace_iterator *iter = m->private;
2900	struct trace_array *tr = iter->tr;
2901	unsigned long trace_flags = tr->trace_flags;
2902
2903	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2904	return;
2905
2906	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2907	/* print nothing if the buffers are empty */
2908	if (trace_empty(iter))
2909	return;
2910	print_trace_header(m, iter);
2911	if (!(trace_flags & TRACE_ITER_VERBOSE))
2912	print_lat_help_header(m);
2913	} else {
2914	if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2915	if (trace_flags & TRACE_ITER_IRQ_INFO)
2916	print_func_help_header_irq(iter->trace_buffer, m);
2917	else
2918	print_func_help_header(iter->trace_buffer, m);
2919	}
2920	}
2921	}
2922
2923	static void test_ftrace_alive(struct seq_file *m)
2924	{
2925	if (!ftrace_is_dead())
2926	return;
2927	seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2928	"# MAY BE MISSING FUNCTION EVENTS\n");
2929	}
2930
2931	#ifdef CONFIG_TRACER_MAX_TRACE
2932	static void show_snapshot_main_help(struct seq_file *m)
2933	{
2934	seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2935	"# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2936	"# Takes a snapshot of the main buffer.\n"
2937	"# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2938	"# (Doesn't have to be '2' works with any number that\n"
2939	"# is not a '0' or '1')\n");
2940	}
2941
2942	static void show_snapshot_percpu_help(struct seq_file *m)
2943	{
2944	seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2945	#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2946	seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2947	"# Takes a snapshot of the main buffer for this cpu.\n");
2948	#else
2949	seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2950	"# Must use main snapshot file to allocate.\n");
2951	#endif
2952	seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2953	"# (Doesn't have to be '2' works with any number that\n"
2954	"# is not a '0' or '1')\n");
2955	}
2956
2957	static void print_snapshot_help(struct seq_file m, struct trace_iterator iter)
2958	{
2959	if (iter->tr->allocated_snapshot)
2960	seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
2961	else
2962	seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
2963
2964	seq_puts(m, "# Snapshot commands:\n");
2965	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2966	show_snapshot_main_help(m);
2967	else
2968	show_snapshot_percpu_help(m);
2969	}
2970	#else
2971	/* Should never be called */
2972	static inline void print_snapshot_help(struct seq_file m, struct trace_iterator iter) { }
2973	#endif
2974
2975	static int s_show(struct seq_file m, void v)
2976	{
2977	struct trace_iterator *iter = v;
2978	int ret;
2979
2980	if (iter->ent == NULL) {
2981	if (iter->tr) {
2982	seq_printf(m, "# tracer: %s\n", iter->trace->name);
2983	seq_puts(m, "#\n");
2984	test_ftrace_alive(m);
2985	}
2986	if (iter->snapshot && trace_empty(iter))
2987	print_snapshot_help(m, iter);
2988	else if (iter->trace && iter->trace->print_header)
2989	iter->trace->print_header(m);
2990	else
2991	trace_default_header(m);
2992
2993	} else if (iter->leftover) {
2994	/*
2995	* If we filled the seq_file buffer earlier, we
2996	* want to just show it now.
2997	*/
2998	ret = trace_print_seq(m, &iter->seq);
2999
3000	/* ret should this time be zero, but you never know */
3001	iter->leftover = ret;
3002
3003	} else {
3004	print_trace_line(iter);
3005	ret = trace_print_seq(m, &iter->seq);
3006	/*
3007	* If we overflow the seq_file buffer, then it will
3008	* ask us for this data again at start up.
3009	* Use that instead.
3010	* ret is 0 if seq_file write succeeded.
3011	* -1 otherwise.
3012	*/
3013	iter->leftover = ret;
3014	}
3015
3016	return 0;
3017	}
3018
3019	/*
3020	* Should be used after trace_array_get(), trace_types_lock
3021	* ensures that i_cdev was already initialized.
3022	*/
3023	static inline int tracing_get_cpu(struct inode *inode)
3024	{
3025	if (inode->i_cdev) /* See trace_create_cpu_file() */
3026	return (long)inode->i_cdev - 1;
3027	return RING_BUFFER_ALL_CPUS;
3028	}
3029
3030	static const struct seq_operations tracer_seq_ops = {
3031	.start = s_start,
3032	.next = s_next,
3033	.stop = s_stop,
3034	.show = s_show,
3035	};
3036
3037	static struct trace_iterator *
3038	__tracing_open(struct inode inode, struct file file, bool snapshot)
3039	{
3040	struct trace_array *tr = inode->i_private;
3041	struct trace_iterator *iter;
3042	int cpu;
3043
3044	if (tracing_disabled)
3045	return ERR_PTR(-ENODEV);
3046
3047	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3048	if (!iter)
3049	return ERR_PTR(-ENOMEM);
3050
3051	iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3052	GFP_KERNEL);
3053	if (!iter->buffer_iter)
3054	goto release;
3055
3056	/*
3057	* We make a copy of the current tracer to avoid concurrent
3058	* changes on it while we are reading.
3059	*/
3060	mutex_lock(&trace_types_lock);
3061	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3062	if (!iter->trace)
3063	goto fail;
3064
3065	iter->trace = tr->current_trace;
3066
3067	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3068	goto fail;
3069
3070	iter->tr = tr;
3071
3072	#ifdef CONFIG_TRACER_MAX_TRACE
3073	/* Currently only the top directory has a snapshot */
3074	if (tr->current_trace->print_max \|\| snapshot)
3075	iter->trace_buffer = &tr->max_buffer;
3076	else
3077	#endif
3078	iter->trace_buffer = &tr->trace_buffer;
3079	iter->snapshot = snapshot;
3080	iter->pos = -1;
3081	iter->cpu_file = tracing_get_cpu(inode);
3082	mutex_init(&iter->mutex);
3083
3084	/* Notify the tracer early; before we stop tracing. */
3085	if (iter->trace && iter->trace->open)
3086	iter->trace->open(iter);
3087
3088	/* Annotate start of buffers if we had overruns */
3089	if (ring_buffer_overruns(iter->trace_buffer->buffer))
3090	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
3091
3092	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
3093	if (trace_clocks[tr->clock_id].in_ns)
3094	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
3095
3096	/* stop the trace while dumping if we are not opening "snapshot" */
3097	if (!iter->snapshot)
3098	tracing_stop_tr(tr);
3099
3100	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3101	for_each_tracing_cpu(cpu) {
3102	iter->buffer_iter[cpu] =
3103	ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3104	}
3105	ring_buffer_read_prepare_sync();
3106	for_each_tracing_cpu(cpu) {
3107	ring_buffer_read_start(iter->buffer_iter[cpu]);
3108	tracing_iter_reset(iter, cpu);
3109	}
3110	} else {
3111	cpu = iter->cpu_file;
3112	iter->buffer_iter[cpu] =
3113	ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3114	ring_buffer_read_prepare_sync();
3115	ring_buffer_read_start(iter->buffer_iter[cpu]);
3116	tracing_iter_reset(iter, cpu);
3117	}
3118
3119	mutex_unlock(&trace_types_lock);
3120
3121	return iter;
3122
3123	fail:
3124	mutex_unlock(&trace_types_lock);
3125	kfree(iter->trace);
3126	kfree(iter->buffer_iter);
3127	release:
3128	seq_release_private(inode, file);
3129	return ERR_PTR(-ENOMEM);
3130	}
3131
3132	int tracing_open_generic(struct inode inode, struct file filp)
3133	{
3134	if (tracing_disabled)
3135	return -ENODEV;
3136
3137	filp->private_data = inode->i_private;
3138	return 0;
3139	}
3140
3141	bool tracing_is_disabled(void)
3142	{
3143	return (tracing_disabled) ? true: false;
3144	}
3145
3146	/*
3147	* Open and update trace_array ref count.
3148	* Must have the current trace_array passed to it.
3149	*/
3150	static int tracing_open_generic_tr(struct inode inode, struct file filp)
3151	{
3152	struct trace_array *tr = inode->i_private;
3153
3154	if (tracing_disabled)
3155	return -ENODEV;
3156
3157	if (trace_array_get(tr) < 0)
3158	return -ENODEV;
3159
3160	filp->private_data = inode->i_private;
3161
3162	return 0;
3163	}
3164
3165	static int tracing_release(struct inode inode, struct file file)
3166	{
3167	struct trace_array *tr = inode->i_private;
3168	struct seq_file *m = file->private_data;
3169	struct trace_iterator *iter;
3170	int cpu;
3171
3172	if (!(file->f_mode & FMODE_READ)) {
3173	trace_array_put(tr);
3174	return 0;
3175	}
3176
3177	/* Writes do not use seq_file */
3178	iter = m->private;
3179	mutex_lock(&trace_types_lock);
3180
3181	for_each_tracing_cpu(cpu) {
3182	if (iter->buffer_iter[cpu])
3183	ring_buffer_read_finish(iter->buffer_iter[cpu]);
3184	}
3185
3186	if (iter->trace && iter->trace->close)
3187	iter->trace->close(iter);
3188
3189	if (!iter->snapshot)
3190	/* reenable tracing if it was previously enabled */
3191	tracing_start_tr(tr);
3192
3193	__trace_array_put(tr);
3194
3195	mutex_unlock(&trace_types_lock);
3196
3197	mutex_destroy(&iter->mutex);
3198	free_cpumask_var(iter->started);
3199	kfree(iter->trace);
3200	kfree(iter->buffer_iter);
3201	seq_release_private(inode, file);
3202
3203	return 0;
3204	}
3205
3206	static int tracing_release_generic_tr(struct inode inode, struct file file)
3207	{
3208	struct trace_array *tr = inode->i_private;
3209
3210	trace_array_put(tr);
3211	return 0;
3212	}
3213
3214	static int tracing_single_release_tr(struct inode inode, struct file file)
3215	{
3216	struct trace_array *tr = inode->i_private;
3217
3218	trace_array_put(tr);
3219
3220	return single_release(inode, file);
3221	}
3222
3223	static int tracing_open(struct inode inode, struct file file)
3224	{
3225	struct trace_array *tr = inode->i_private;
3226	struct trace_iterator *iter;
3227	int ret = 0;
3228
3229	if (trace_array_get(tr) < 0)
3230	return -ENODEV;
3231
3232	/* If this file was open for write, then erase contents */
3233	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3234	int cpu = tracing_get_cpu(inode);
3235
3236	if (cpu == RING_BUFFER_ALL_CPUS)
3237	tracing_reset_online_cpus(&tr->trace_buffer);
3238	else
3239	tracing_reset(&tr->trace_buffer, cpu);
3240	}
3241
3242	if (file->f_mode & FMODE_READ) {
3243	iter = __tracing_open(inode, file, false);
3244	if (IS_ERR(iter))
3245	ret = PTR_ERR(iter);
3246	else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
3247	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
3248	}
3249
3250	if (ret < 0)
3251	trace_array_put(tr);
3252
3253	return ret;
3254	}
3255
3256	/*
3257	* Some tracers are not suitable for instance buffers.
3258	* A tracer is always available for the global array (toplevel)
3259	* or if it explicitly states that it is.
3260	*/
3261	static bool
3262	trace_ok_for_array(struct tracer t, struct trace_array tr)
3263	{
3264	return (tr->flags & TRACE_ARRAY_FL_GLOBAL) \|\| t->allow_instances;
3265	}
3266
3267	/* Find the next tracer that this trace array may use */
3268	static struct tracer *
3269	get_tracer_for_array(struct trace_array tr, struct tracer t)
3270	{
3271	while (t && !trace_ok_for_array(t, tr))
3272	t = t->next;
3273
3274	return t;
3275	}
3276
3277	static void *
3278	t_next(struct seq_file m, void v, loff_t *pos)
3279	{
3280	struct trace_array *tr = m->private;
3281	struct tracer *t = v;
3282
3283	(*pos)++;
3284
3285	if (t)
3286	t = get_tracer_for_array(tr, t->next);
3287
3288	return t;
3289	}
3290
3291	static void t_start(struct seq_file m, loff_t *pos)
3292	{
3293	struct trace_array *tr = m->private;
3294	struct tracer *t;
3295	loff_t l = 0;
3296
3297	mutex_lock(&trace_types_lock);
3298
3299	t = get_tracer_for_array(tr, trace_types);
3300	for (; t && l < *pos; t = t_next(m, t, &l))
3301	;
3302
3303	return t;
3304	}
3305
3306	static void t_stop(struct seq_file m, void p)
3307	{
3308	mutex_unlock(&trace_types_lock);
3309	}
3310
3311	static int t_show(struct seq_file m, void v)
3312	{
3313	struct tracer *t = v;
3314
3315	if (!t)
3316	return 0;
3317
3318	seq_puts(m, t->name);
3319	if (t->next)
3320	seq_putc(m, ' ');
3321	else
3322	seq_putc(m, '\n');
3323
3324	return 0;
3325	}
3326
3327	static const struct seq_operations show_traces_seq_ops = {
3328	.start = t_start,
3329	.next = t_next,
3330	.stop = t_stop,
3331	.show = t_show,
3332	};
3333
3334	static int show_traces_open(struct inode inode, struct file file)
3335	{
3336	struct trace_array *tr = inode->i_private;
3337	struct seq_file *m;
3338	int ret;
3339
3340	if (tracing_disabled)
3341	return -ENODEV;
3342
3343	ret = seq_open(file, &show_traces_seq_ops);
3344	if (ret)
3345	return ret;
3346
3347	m = file->private_data;
3348	m->private = tr;
3349
3350	return 0;
3351	}
3352
3353	static ssize_t
3354	tracing_write_stub(struct file filp, const char __user ubuf,
3355	size_t count, loff_t *ppos)
3356	{
3357	return count;
3358	}
3359
3360	loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3361	{
3362	int ret;
3363
3364	if (file->f_mode & FMODE_READ)
3365	ret = seq_lseek(file, offset, whence);
3366	else
3367	file->f_pos = ret = 0;
3368
3369	return ret;
3370	}
3371
3372	static const struct file_operations tracing_fops = {
3373	.open = tracing_open,
3374	.read = seq_read,
3375	.write = tracing_write_stub,
3376	.llseek = tracing_lseek,
3377	.release = tracing_release,
3378	};
3379
3380	static const struct file_operations show_traces_fops = {
3381	.open = show_traces_open,
3382	.read = seq_read,
3383	.release = seq_release,
3384	.llseek = seq_lseek,
3385	};
3386
3387	/*
3388	* The tracer itself will not take this lock, but still we want
3389	* to provide a consistent cpumask to user-space:
3390	*/
3391	static DEFINE_MUTEX(tracing_cpumask_update_lock);
3392
3393	/*
3394	* Temporary storage for the character representation of the
3395	* CPU bitmask (and one more byte for the newline):
3396	*/
3397	static char mask_str[NR_CPUS + 1];
3398
3399	static ssize_t
3400	tracing_cpumask_read(struct file filp, char __user ubuf,
3401	size_t count, loff_t *ppos)
3402	{
3403	struct trace_array *tr = file_inode(filp)->i_private;
3404	int len;
3405
3406	mutex_lock(&tracing_cpumask_update_lock);
3407
3408	len = snprintf(mask_str, count, "%*pb\n",
3409	cpumask_pr_args(tr->tracing_cpumask));
3410	if (len >= count) {
3411	count = -EINVAL;
3412	goto out_err;
3413	}
3414	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3415
3416	out_err:
3417	mutex_unlock(&tracing_cpumask_update_lock);
3418
3419	return count;
3420	}
3421
3422	static ssize_t
3423	tracing_cpumask_write(struct file filp, const char __user ubuf,
3424	size_t count, loff_t *ppos)
3425	{
3426	struct trace_array *tr = file_inode(filp)->i_private;
3427	cpumask_var_t tracing_cpumask_new;
3428	int err, cpu;
3429
3430	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3431	return -ENOMEM;
3432
3433	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3434	if (err)
3435	goto err_unlock;
3436
3437	mutex_lock(&tracing_cpumask_update_lock);
3438
3439	local_irq_disable();
3440	arch_spin_lock(&tr->max_lock);
3441	for_each_tracing_cpu(cpu) {
3442	/*
3443	* Increase/decrease the disabled counter if we are
3444	* about to flip a bit in the cpumask:
3445	*/
3446	if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3447	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3448	atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3449	ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3450	}
3451	if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3452	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3453	atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3454	ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3455	}
3456	}
3457	arch_spin_unlock(&tr->max_lock);
3458	local_irq_enable();
3459
3460	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3461
3462	mutex_unlock(&tracing_cpumask_update_lock);
3463	free_cpumask_var(tracing_cpumask_new);
3464
3465	return count;
3466
3467	err_unlock:
3468	free_cpumask_var(tracing_cpumask_new);
3469
3470	return err;
3471	}
3472
3473	static const struct file_operations tracing_cpumask_fops = {
3474	.open = tracing_open_generic_tr,
3475	.read = tracing_cpumask_read,
3476	.write = tracing_cpumask_write,
3477	.release = tracing_release_generic_tr,
3478	.llseek = generic_file_llseek,
3479	};
3480
3481	static int tracing_trace_options_show(struct seq_file m, void v)
3482	{
3483	struct tracer_opt *trace_opts;
3484	struct trace_array *tr = m->private;
3485	u32 tracer_flags;
3486	int i;
3487
3488	mutex_lock(&trace_types_lock);
3489	tracer_flags = tr->current_trace->flags->val;
3490	trace_opts = tr->current_trace->flags->opts;
3491
3492	for (i = 0; trace_options[i]; i++) {
3493	if (tr->trace_flags & (1 << i))
3494	seq_printf(m, "%s\n", trace_options[i]);
3495	else
3496	seq_printf(m, "no%s\n", trace_options[i]);
3497	}
3498
3499	for (i = 0; trace_opts[i].name; i++) {
3500	if (tracer_flags & trace_opts[i].bit)
3501	seq_printf(m, "%s\n", trace_opts[i].name);
3502	else
3503	seq_printf(m, "no%s\n", trace_opts[i].name);
3504	}
3505	mutex_unlock(&trace_types_lock);
3506
3507	return 0;
3508	}
3509
3510	static int __set_tracer_option(struct trace_array *tr,
3511	struct tracer_flags *tracer_flags,
3512	struct tracer_opt *opts, int neg)
3513	{
3514	struct tracer *trace = tracer_flags->trace;
3515	int ret;
3516
3517	ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3518	if (ret)
3519	return ret;
3520
3521	if (neg)
3522	tracer_flags->val &= ~opts->bit;
3523	else
3524	tracer_flags->val \|= opts->bit;
3525	return 0;
3526	}
3527
3528	/* Try to assign a tracer specific option */
3529	static int set_tracer_option(struct trace_array tr, char cmp, int neg)
3530	{
3531	struct tracer *trace = tr->current_trace;
3532	struct tracer_flags *tracer_flags = trace->flags;
3533	struct tracer_opt *opts = NULL;
3534	int i;
3535
3536	for (i = 0; tracer_flags->opts[i].name; i++) {
3537	opts = &tracer_flags->opts[i];
3538
3539	if (strcmp(cmp, opts->name) == 0)
3540	return __set_tracer_option(tr, trace->flags, opts, neg);
3541	}
3542
3543	return -EINVAL;
3544	}
3545
3546	/* Some tracers require overwrite to stay enabled */
3547	int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3548	{
3549	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3550	return -1;
3551
3552	return 0;
3553	}
3554
3555	int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3556	{
3557	/* do nothing if flag is already set */
3558	if (!!(tr->trace_flags & mask) == !!enabled)
3559	return 0;
3560
3561	/* Give the tracer a chance to approve the change */
3562	if (tr->current_trace->flag_changed)
3563	if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3564	return -EINVAL;
3565
3566	if (enabled)
3567	tr->trace_flags \|= mask;
3568	else
3569	tr->trace_flags &= ~mask;
3570
3571	if (mask == TRACE_ITER_RECORD_CMD)
3572	trace_event_enable_cmd_record(enabled);
3573
3574	if (mask == TRACE_ITER_OVERWRITE) {
3575	ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3576	#ifdef CONFIG_TRACER_MAX_TRACE
3577	ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3578	#endif
3579	}
3580
3581	if (mask == TRACE_ITER_PRINTK) {
3582	trace_printk_start_stop_comm(enabled);
3583	trace_printk_control(enabled);
3584	}
3585
3586	return 0;
3587	}
3588
3589	static int trace_set_options(struct trace_array tr, char option)
3590	{
3591	char *cmp;
3592	int neg = 0;
3593	int ret = -ENODEV;
3594	int i;
3595	size_t orig_len = strlen(option);
3596
3597	cmp = strstrip(option);
3598
3599	if (strncmp(cmp, "no", 2) == 0) {
3600	neg = 1;
3601	cmp += 2;
3602	}
3603
3604	mutex_lock(&trace_types_lock);
3605
3606	for (i = 0; trace_options[i]; i++) {
3607	if (strcmp(cmp, trace_options[i]) == 0) {
3608	ret = set_tracer_flag(tr, 1 << i, !neg);
3609	break;
3610	}
3611	}
3612
3613	/* If no option could be set, test the specific tracer options */
3614	if (!trace_options[i])
3615	ret = set_tracer_option(tr, cmp, neg);
3616
3617	mutex_unlock(&trace_types_lock);
3618
3619	/*
3620	* If the first trailing whitespace is replaced with '\0' by strstrip,
3621	* turn it back into a space.
3622	*/
3623	if (orig_len > strlen(option))
3624	option[strlen(option)] = ' ';
3625
3626	return ret;
3627	}
3628
3629	static void __init apply_trace_boot_options(void)
3630	{
3631	char *buf = trace_boot_options_buf;
3632	char *option;
3633
3634	while (true) {
3635	option = strsep(&buf, ",");
3636
3637	if (!option)
3638	break;
3639
3640	if (*option)
3641	trace_set_options(&global_trace, option);
3642
3643	/* Put back the comma to allow this to be called again */
3644	if (buf)
3645	*(buf - 1) = ',';
3646	}
3647	}
3648
3649	static ssize_t
3650	tracing_trace_options_write(struct file filp, const char __user ubuf,
3651	size_t cnt, loff_t *ppos)
3652	{
3653	struct seq_file *m = filp->private_data;
3654	struct trace_array *tr = m->private;
3655	char buf[64];
3656	int ret;
3657
3658	if (cnt >= sizeof(buf))
3659	return -EINVAL;
3660
3661	if (copy_from_user(&buf, ubuf, cnt))
3662	return -EFAULT;
3663
3664	buf[cnt] = 0;
3665
3666	ret = trace_set_options(tr, buf);
3667	if (ret < 0)
3668	return ret;
3669
3670	*ppos += cnt;
3671
3672	return cnt;
3673	}
3674
3675	static int tracing_trace_options_open(struct inode inode, struct file file)
3676	{
3677	struct trace_array *tr = inode->i_private;
3678	int ret;
3679
3680	if (tracing_disabled)
3681	return -ENODEV;
3682
3683	if (trace_array_get(tr) < 0)
3684	return -ENODEV;
3685
3686	ret = single_open(file, tracing_trace_options_show, inode->i_private);
3687	if (ret < 0)
3688	trace_array_put(tr);
3689
3690	return ret;
3691	}
3692
3693	static const struct file_operations tracing_iter_fops = {
3694	.open = tracing_trace_options_open,
3695	.read = seq_read,
3696	.llseek = seq_lseek,
3697	.release = tracing_single_release_tr,
3698	.write = tracing_trace_options_write,
3699	};
3700
3701	static const char readme_msg[] =
3702	"tracing mini-HOWTO:\n\n"
3703	"# echo 0 > tracing_on : quick way to disable tracing\n"
3704	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3705	" Important files:\n"
3706	" trace\t\t\t- The static contents of the buffer\n"
3707	"\t\t\t To clear the buffer write into this file: echo > trace\n"
3708	" trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3709	" current_tracer\t- function and latency tracers\n"
3710	" available_tracers\t- list of configured tracers for current_tracer\n"
3711	" buffer_size_kb\t- view and modify size of per cpu buffer\n"
3712	" buffer_total_size_kb - view total size of all cpu buffers\n\n"
3713	" trace_clock\t\t-change the clock used to order events\n"
3714	" local: Per cpu clock but may not be synced across CPUs\n"
3715	" global: Synced across CPUs but slows tracing down.\n"
3716	" counter: Not a clock, but just an increment\n"
3717	" uptime: Jiffy counter from time of boot\n"
3718	" perf: Same clock that perf events use\n"
3719	#ifdef CONFIG_X86_64
3720	" x86-tsc: TSC cycle counter\n"
3721	#endif
3722	"\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3723	" tracing_cpumask\t- Limit which CPUs to trace\n"
3724	" instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3725	"\t\t\t Remove sub-buffer with rmdir\n"
3726	" trace_options\t\t- Set format or modify how tracing happens\n"
3727	"\t\t\t Disable an option by adding a suffix 'no' to the\n"
3728	"\t\t\t option name\n"
3729	" saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3730	#ifdef CONFIG_DYNAMIC_FTRACE
3731	"\n available_filter_functions - list of functions that can be filtered on\n"
3732	" set_ftrace_filter\t- echo function name in here to only trace these\n"
3733	"\t\t\t functions\n"
3734	"\t accepts: func_full_name, func_end, func_begin, func_middle\n"
3735	"\t modules: Can select a group via module\n"
3736	"\t Format: :mod:<module-name>\n"
3737	"\t example: echo :mod:ext3 > set_ftrace_filter\n"
3738	"\t triggers: a command to perform when function is hit\n"
3739	"\t Format: <function>:<trigger>[:count]\n"
3740	"\t trigger: traceon, traceoff\n"
3741	"\t\t enable_event:<system>:<event>\n"
3742	"\t\t disable_event:<system>:<event>\n"
3743	#ifdef CONFIG_STACKTRACE
3744	"\t\t stacktrace\n"
3745	#endif
3746	#ifdef CONFIG_TRACER_SNAPSHOT
3747	"\t\t snapshot\n"
3748	#endif
3749	"\t\t dump\n"
3750	"\t\t cpudump\n"
3751	"\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3752	"\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3753	"\t The first one will disable tracing every time do_fault is hit\n"
3754	"\t The second will disable tracing at most 3 times when do_trap is hit\n"
3755	"\t The first time do trap is hit and it disables tracing, the\n"
3756	"\t counter will decrement to 2. If tracing is already disabled,\n"
3757	"\t the counter will not decrement. It only decrements when the\n"
3758	"\t trigger did work\n"
3759	"\t To remove trigger without count:\n"
3760	"\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3761	"\t To remove trigger with a count:\n"
3762	"\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3763	" set_ftrace_notrace\t- echo function name in here to never trace.\n"
3764	"\t accepts: func_full_name, func_end, func_begin, func_middle\n"
3765	"\t modules: Can select a group via module command :mod:\n"
3766	"\t Does not accept triggers\n"
3767	#endif /* CONFIG_DYNAMIC_FTRACE */
3768	#ifdef CONFIG_FUNCTION_TRACER
3769	" set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3770	"\t\t (function)\n"
3771	#endif
3772	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3773	" set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3774	" set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3775	" max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3776	#endif
3777	#ifdef CONFIG_TRACER_SNAPSHOT
3778	"\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3779	"\t\t\t snapshot buffer. Read the contents for more\n"
3780	"\t\t\t information\n"
3781	#endif
3782	#ifdef CONFIG_STACK_TRACER
3783	" stack_trace\t\t- Shows the max stack trace when active\n"
3784	" stack_max_size\t- Shows current max stack size that was traced\n"
3785	"\t\t\t Write into this file to reset the max size (trigger a\n"
3786	"\t\t\t new trace)\n"
3787	#ifdef CONFIG_DYNAMIC_FTRACE
3788	" stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3789	"\t\t\t traces\n"
3790	#endif
3791	#endif /* CONFIG_STACK_TRACER */
3792	" events/\t\t- Directory containing all trace event subsystems:\n"
3793	" enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3794	" events/<system>/\t- Directory containing all trace events for <system>:\n"
3795	" enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3796	"\t\t\t events\n"
3797	" filter\t\t- If set, only events passing filter are traced\n"
3798	" events/<system>/<event>/\t- Directory containing control files for\n"
3799	"\t\t\t <event>:\n"
3800	" enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3801	" filter\t\t- If set, only events passing filter are traced\n"
3802	" trigger\t\t- If set, a command to perform when event is hit\n"
3803	"\t Format: <trigger>[:count][if <filter>]\n"
3804	"\t trigger: traceon, traceoff\n"
3805	"\t enable_event:<system>:<event>\n"
3806	"\t disable_event:<system>:<event>\n"
3807	#ifdef CONFIG_STACKTRACE
3808	"\t\t stacktrace\n"
3809	#endif
3810	#ifdef CONFIG_TRACER_SNAPSHOT
3811	"\t\t snapshot\n"
3812	#endif
3813	"\t example: echo traceoff > events/block/block_unplug/trigger\n"
3814	"\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3815	"\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3816	"\t events/block/block_unplug/trigger\n"
3817	"\t The first disables tracing every time block_unplug is hit.\n"
3818	"\t The second disables tracing the first 3 times block_unplug is hit.\n"
3819	"\t The third enables the kmalloc event the first 3 times block_unplug\n"
3820	"\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3821	"\t Like function triggers, the counter is only decremented if it\n"
3822	"\t enabled or disabled tracing.\n"
3823	"\t To remove a trigger without a count:\n"
3824	"\t echo '!<trigger> > <system>/<event>/trigger\n"
3825	"\t To remove a trigger with a count:\n"
3826	"\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3827	"\t Filters can be ignored when removing a trigger.\n"
3828	;
3829
3830	static ssize_t
3831	tracing_readme_read(struct file filp, char __user ubuf,
3832	size_t cnt, loff_t *ppos)
3833	{
3834	return simple_read_from_buffer(ubuf, cnt, ppos,
3835	readme_msg, strlen(readme_msg));
3836	}
3837
3838	static const struct file_operations tracing_readme_fops = {
3839	.open = tracing_open_generic,
3840	.read = tracing_readme_read,
3841	.llseek = generic_file_llseek,
3842	};
3843
3844	static void saved_cmdlines_next(struct seq_file m, void v, loff_t pos)
3845	{
3846	unsigned int *ptr = v;
3847
3848	if (*pos \|\| m->count)
3849	ptr++;
3850
3851	(*pos)++;
3852
3853	for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3854	ptr++) {
3855	if (ptr == -1 \|\| ptr == NO_CMDLINE_MAP)
3856	continue;
3857
3858	return ptr;
3859	}
3860
3861	return NULL;
3862	}
3863
3864	static void saved_cmdlines_start(struct seq_file m, loff_t *pos)
3865	{
3866	void *v;
3867	loff_t l = 0;
3868
3869	preempt_disable();
3870	arch_spin_lock(&trace_cmdline_lock);
3871
3872	v = &savedcmd->map_cmdline_to_pid[0];
3873	while (l <= *pos) {
3874	v = saved_cmdlines_next(m, v, &l);
3875	if (!v)
3876	return NULL;
3877	}
3878
3879	return v;
3880	}
3881
3882	static void saved_cmdlines_stop(struct seq_file m, void v)
3883	{
3884	arch_spin_unlock(&trace_cmdline_lock);
3885	preempt_enable();
3886	}
3887
3888	static int saved_cmdlines_show(struct seq_file m, void v)
3889	{
3890	char buf[TASK_COMM_LEN];
3891	unsigned int *pid = v;
3892
3893	__trace_find_cmdline(*pid, buf);
3894	seq_printf(m, "%d %s\n", *pid, buf);
3895	return 0;
3896	}
3897
3898	static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3899	.start = saved_cmdlines_start,
3900	.next = saved_cmdlines_next,
3901	.stop = saved_cmdlines_stop,
3902	.show = saved_cmdlines_show,
3903	};
3904
3905	static int tracing_saved_cmdlines_open(struct inode inode, struct file filp)
3906	{
3907	if (tracing_disabled)
3908	return -ENODEV;
3909
3910	return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3911	}
3912
3913	static const struct file_operations tracing_saved_cmdlines_fops = {
3914	.open = tracing_saved_cmdlines_open,
3915	.read = seq_read,
3916	.llseek = seq_lseek,
3917	.release = seq_release,
3918	};
3919
3920	static ssize_t
3921	tracing_saved_cmdlines_size_read(struct file filp, char __user ubuf,
3922	size_t cnt, loff_t *ppos)
3923	{
3924	char buf[64];
3925	int r;
3926
3927	arch_spin_lock(&trace_cmdline_lock);
3928	r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3929	arch_spin_unlock(&trace_cmdline_lock);
3930
3931	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3932	}
3933
3934	static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3935	{
3936	kfree(s->saved_cmdlines);
3937	kfree(s->map_cmdline_to_pid);
3938	kfree(s);
3939	}
3940
3941	static int tracing_resize_saved_cmdlines(unsigned int val)
3942	{
3943	struct saved_cmdlines_buffer s, savedcmd_temp;
3944
3945	s = kmalloc(sizeof(*s), GFP_KERNEL);
3946	if (!s)
3947	return -ENOMEM;
3948
3949	if (allocate_cmdlines_buffer(val, s) < 0) {
3950	kfree(s);
3951	return -ENOMEM;
3952	}
3953
3954	arch_spin_lock(&trace_cmdline_lock);
3955	savedcmd_temp = savedcmd;
3956	savedcmd = s;
3957	arch_spin_unlock(&trace_cmdline_lock);
3958	free_saved_cmdlines_buffer(savedcmd_temp);
3959
3960	return 0;
3961	}
3962
3963	static ssize_t
3964	tracing_saved_cmdlines_size_write(struct file filp, const char __user ubuf,
3965	size_t cnt, loff_t *ppos)
3966	{
3967	unsigned long val;
3968	int ret;
3969
3970	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3971	if (ret)
3972	return ret;
3973
3974	/* must have at least 1 entry or less than PID_MAX_DEFAULT */
3975	if (!val \|\| val > PID_MAX_DEFAULT)
3976	return -EINVAL;
3977
3978	ret = tracing_resize_saved_cmdlines((unsigned int)val);
3979	if (ret < 0)
3980	return ret;
3981
3982	*ppos += cnt;
3983
3984	return cnt;
3985	}
3986
3987	static const struct file_operations tracing_saved_cmdlines_size_fops = {
3988	.open = tracing_open_generic,
3989	.read = tracing_saved_cmdlines_size_read,
3990	.write = tracing_saved_cmdlines_size_write,
3991	};
3992
3993	#ifdef CONFIG_TRACE_ENUM_MAP_FILE
3994	static union trace_enum_map_item *
3995	update_enum_map(union trace_enum_map_item *ptr)
3996	{
3997	if (!ptr->map.enum_string) {
3998	if (ptr->tail.next) {
3999	ptr = ptr->tail.next;
4000	/* Set ptr to the next real item (skip head) */
4001	ptr++;
4002	} else
4003	return NULL;
4004	}
4005	return ptr;
4006	}
4007
4008	static void enum_map_next(struct seq_file m, void v, loff_t pos)
4009	{
4010	union trace_enum_map_item *ptr = v;
4011
4012	/*
4013	* Paranoid! If ptr points to end, we don't want to increment past it.
4014	* This really should never happen.
4015	*/
4016	ptr = update_enum_map(ptr);
4017	if (WARN_ON_ONCE(!ptr))
4018	return NULL;
4019
4020	ptr++;
4021
4022	(*pos)++;
4023
4024	ptr = update_enum_map(ptr);
4025
4026	return ptr;
4027	}
4028
4029	static void enum_map_start(struct seq_file m, loff_t *pos)
4030	{
4031	union trace_enum_map_item *v;
4032	loff_t l = 0;
4033
4034	mutex_lock(&trace_enum_mutex);
4035
4036	v = trace_enum_maps;
4037	if (v)
4038	v++;
4039
4040	while (v && l < *pos) {
4041	v = enum_map_next(m, v, &l);
4042	}
4043
4044	return v;
4045	}
4046
4047	static void enum_map_stop(struct seq_file m, void v)
4048	{
4049	mutex_unlock(&trace_enum_mutex);
4050	}
4051
4052	static int enum_map_show(struct seq_file m, void v)
4053	{
4054	union trace_enum_map_item *ptr = v;
4055
4056	seq_printf(m, "%s %ld (%s)\n",
4057	ptr->map.enum_string, ptr->map.enum_value,
4058	ptr->map.system);
4059
4060	return 0;
4061	}
4062
4063	static const struct seq_operations tracing_enum_map_seq_ops = {
4064	.start = enum_map_start,
4065	.next = enum_map_next,
4066	.stop = enum_map_stop,
4067	.show = enum_map_show,
4068	};
4069
4070	static int tracing_enum_map_open(struct inode inode, struct file filp)
4071	{
4072	if (tracing_disabled)
4073	return -ENODEV;
4074
4075	return seq_open(filp, &tracing_enum_map_seq_ops);
4076	}
4077
4078	static const struct file_operations tracing_enum_map_fops = {
4079	.open = tracing_enum_map_open,
4080	.read = seq_read,
4081	.llseek = seq_lseek,
4082	.release = seq_release,
4083	};
4084
4085	static inline union trace_enum_map_item *
4086	trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4087	{
4088	/* Return tail of array given the head */
4089	return ptr + ptr->head.length + 1;
4090	}
4091
4092	static void
4093	trace_insert_enum_map_file(struct module mod, struct trace_enum_map *start,
4094	int len)
4095	{
4096	struct trace_enum_map **stop;
4097	struct trace_enum_map **map;
4098	union trace_enum_map_item *map_array;
4099	union trace_enum_map_item *ptr;
4100
4101	stop = start + len;
4102
4103	/*
4104	* The trace_enum_maps contains the map plus a head and tail item,
4105	* where the head holds the module and length of array, and the
4106	* tail holds a pointer to the next list.
4107	*/
4108	map_array = kmalloc(sizeof(map_array) (len + 2), GFP_KERNEL);
4109	if (!map_array) {
4110	pr_warn("Unable to allocate trace enum mapping\n");
4111	return;
4112	}
4113
4114	mutex_lock(&trace_enum_mutex);
4115
4116	if (!trace_enum_maps)
4117	trace_enum_maps = map_array;
4118	else {
4119	ptr = trace_enum_maps;
4120	for (;;) {
4121	ptr = trace_enum_jmp_to_tail(ptr);
4122	if (!ptr->tail.next)
4123	break;
4124	ptr = ptr->tail.next;
4125
4126	}
4127	ptr->tail.next = map_array;
4128	}
4129	map_array->head.mod = mod;
4130	map_array->head.length = len;
4131	map_array++;
4132
4133	for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4134	map_array->map = **map;
4135	map_array++;
4136	}
4137	memset(map_array, 0, sizeof(*map_array));
4138
4139	mutex_unlock(&trace_enum_mutex);
4140	}
4141
4142	static void trace_create_enum_file(struct dentry *d_tracer)
4143	{
4144	trace_create_file("enum_map", 0444, d_tracer,
4145	NULL, &tracing_enum_map_fops);
4146	}
4147
4148	#else /* CONFIG_TRACE_ENUM_MAP_FILE */
4149	static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4150	static inline void trace_insert_enum_map_file(struct module *mod,
4151	struct trace_enum_map **start, int len) { }
4152	#endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4153
4154	static void trace_insert_enum_map(struct module *mod,
4155	struct trace_enum_map **start, int len)
4156	{
4157	struct trace_enum_map **map;
4158
4159	if (len <= 0)
4160	return;
4161
4162	map = start;
4163
4164	trace_event_enum_update(map, len);
4165
4166	trace_insert_enum_map_file(mod, start, len);
4167	}
4168
4169	static ssize_t
4170	tracing_set_trace_read(struct file filp, char __user ubuf,
4171	size_t cnt, loff_t *ppos)
4172	{
4173	struct trace_array *tr = filp->private_data;
4174	char buf[MAX_TRACER_SIZE+2];
4175	int r;
4176
4177	mutex_lock(&trace_types_lock);
4178	r = sprintf(buf, "%s\n", tr->current_trace->name);
4179	mutex_unlock(&trace_types_lock);
4180
4181	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4182	}
4183
4184	int tracer_init(struct tracer t, struct trace_array tr)
4185	{
4186	tracing_reset_online_cpus(&tr->trace_buffer);
4187	return t->init(tr);
4188	}
4189
4190	static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4191	{
4192	int cpu;
4193
4194	for_each_tracing_cpu(cpu)
4195	per_cpu_ptr(buf->data, cpu)->entries = val;
4196	}
4197
4198	#ifdef CONFIG_TRACER_MAX_TRACE
4199	/* resize @tr's buffer to the size of @size_tr's entries */
4200	static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4201	struct trace_buffer *size_buf, int cpu_id)
4202	{
4203	int cpu, ret = 0;
4204
4205	if (cpu_id == RING_BUFFER_ALL_CPUS) {
4206	for_each_tracing_cpu(cpu) {
4207	ret = ring_buffer_resize(trace_buf->buffer,
4208	per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4209	if (ret < 0)
4210	break;
4211	per_cpu_ptr(trace_buf->data, cpu)->entries =
4212	per_cpu_ptr(size_buf->data, cpu)->entries;
4213	}
4214	} else {
4215	ret = ring_buffer_resize(trace_buf->buffer,
4216	per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4217	if (ret == 0)
4218	per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4219	per_cpu_ptr(size_buf->data, cpu_id)->entries;
4220	}
4221
4222	return ret;
4223	}
4224	#endif /* CONFIG_TRACER_MAX_TRACE */
4225
4226	static int __tracing_resize_ring_buffer(struct trace_array *tr,
4227	unsigned long size, int cpu)
4228	{
4229	int ret;
4230
4231	/*
4232	* If kernel or user changes the size of the ring buffer
4233	* we use the size that was given, and we can forget about
4234	* expanding it later.
4235	*/
4236	ring_buffer_expanded = true;
4237
4238	/* May be called before buffers are initialized */
4239	if (!tr->trace_buffer.buffer)
4240	return 0;
4241
4242	ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4243	if (ret < 0)
4244	return ret;
4245
4246	#ifdef CONFIG_TRACER_MAX_TRACE
4247	if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) \|\|
4248	!tr->current_trace->use_max_tr)
4249	goto out;
4250
4251	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4252	if (ret < 0) {
4253	int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4254	&tr->trace_buffer, cpu);
4255	if (r < 0) {
4256	/*
4257	* AARGH! We are left with different
4258	* size max buffer!!!!
4259	* The max buffer is our "snapshot" buffer.
4260	* When a tracer needs a snapshot (one of the
4261	* latency tracers), it swaps the max buffer
4262	* with the saved snap shot. We succeeded to
4263	* update the size of the main buffer, but failed to
4264	* update the size of the max buffer. But when we tried
4265	* to reset the main buffer to the original size, we
4266	* failed there too. This is very unlikely to
4267	* happen, but if it does, warn and kill all
4268	* tracing.
4269	*/
4270	WARN_ON(1);
4271	tracing_disabled = 1;
4272	}
4273	return ret;
4274	}
4275
4276	if (cpu == RING_BUFFER_ALL_CPUS)
4277	set_buffer_entries(&tr->max_buffer, size);
4278	else
4279	per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4280
4281	out:
4282	#endif /* CONFIG_TRACER_MAX_TRACE */
4283
4284	if (cpu == RING_BUFFER_ALL_CPUS)
4285	set_buffer_entries(&tr->trace_buffer, size);
4286	else
4287	per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4288
4289	return ret;
4290	}
4291
4292	static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4293	unsigned long size, int cpu_id)
4294	{
4295	int ret = size;
4296
4297	mutex_lock(&trace_types_lock);
4298
4299	if (cpu_id != RING_BUFFER_ALL_CPUS) {
4300	/* make sure, this cpu is enabled in the mask */
4301	if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4302	ret = -EINVAL;
4303	goto out;
4304	}
4305	}
4306
4307	ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4308	if (ret < 0)
4309	ret = -ENOMEM;
4310
4311	out:
4312	mutex_unlock(&trace_types_lock);
4313
4314	return ret;
4315	}
4316
4317
4318	/**
4319	* tracing_update_buffers - used by tracing facility to expand ring buffers
4320	*
4321	* To save on memory when the tracing is never used on a system with it
4322	* configured in. The ring buffers are set to a minimum size. But once
4323	* a user starts to use the tracing facility, then they need to grow
4324	* to their default size.
4325	*
4326	* This function is to be called when a tracer is about to be used.
4327	*/
4328	int tracing_update_buffers(void)
4329	{
4330	int ret = 0;
4331
4332	mutex_lock(&trace_types_lock);
4333	if (!ring_buffer_expanded)
4334	ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4335	RING_BUFFER_ALL_CPUS);
4336	mutex_unlock(&trace_types_lock);
4337
4338	return ret;
4339	}
4340
4341	struct trace_option_dentry;
4342
4343	static void
4344	create_trace_option_files(struct trace_array tr, struct tracer tracer);
4345
4346	/*
4347	* Used to clear out the tracer before deletion of an instance.
4348	* Must have trace_types_lock held.
4349	*/
4350	static void tracing_set_nop(struct trace_array *tr)
4351	{
4352	if (tr->current_trace == &nop_trace)
4353	return;
4354
4355	tr->current_trace->enabled--;
4356
4357	if (tr->current_trace->reset)
4358	tr->current_trace->reset(tr);
4359
4360	tr->current_trace = &nop_trace;
4361	}
4362
4363	static void add_tracer_options(struct trace_array tr, struct tracer t)
4364	{
4365	/* Only enable if the directory has been created already. */
4366	if (!tr->dir)
4367	return;
4368
4369	create_trace_option_files(tr, t);
4370	}
4371
4372	static int tracing_set_tracer(struct trace_array tr, const char buf)
4373	{
4374	struct tracer *t;
4375	#ifdef CONFIG_TRACER_MAX_TRACE
4376	bool had_max_tr;
4377	#endif
4378	int ret = 0;
4379
4380	mutex_lock(&trace_types_lock);
4381
4382	if (!ring_buffer_expanded) {
4383	ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4384	RING_BUFFER_ALL_CPUS);
4385	if (ret < 0)
4386	goto out;
4387	ret = 0;
4388	}
4389
4390	for (t = trace_types; t; t = t->next) {
4391	if (strcmp(t->name, buf) == 0)
4392	break;
4393	}
4394	if (!t) {
4395	ret = -EINVAL;
4396	goto out;
4397	}
4398	if (t == tr->current_trace)
4399	goto out;
4400
4401	/* Some tracers are only allowed for the top level buffer */
4402	if (!trace_ok_for_array(t, tr)) {
4403	ret = -EINVAL;
4404	goto out;
4405	}
4406
4407	/* If trace pipe files are being read, we can't change the tracer */
4408	if (tr->current_trace->ref) {
4409	ret = -EBUSY;
4410	goto out;
4411	}
4412
4413	trace_branch_disable();
4414
4415	tr->current_trace->enabled--;
4416
4417	if (tr->current_trace->reset)
4418	tr->current_trace->reset(tr);
4419
4420	/* Current trace needs to be nop_trace before synchronize_sched */
4421	tr->current_trace = &nop_trace;
4422
4423	#ifdef CONFIG_TRACER_MAX_TRACE
4424	had_max_tr = tr->allocated_snapshot;
4425
4426	if (had_max_tr && !t->use_max_tr) {
4427	/*
4428	* We need to make sure that the update_max_tr sees that
4429	* current_trace changed to nop_trace to keep it from
4430	* swapping the buffers after we resize it.
4431	* The update_max_tr is called from interrupts disabled
4432	* so a synchronized_sched() is sufficient.
4433	*/
4434	synchronize_sched();
4435	free_snapshot(tr);
4436	}
4437	#endif
4438
4439	#ifdef CONFIG_TRACER_MAX_TRACE
4440	if (t->use_max_tr && !had_max_tr) {
4441	ret = alloc_snapshot(tr);
4442	if (ret < 0)
4443	goto out;
4444	}
4445	#endif
4446
4447	if (t->init) {
4448	ret = tracer_init(t, tr);
4449	if (ret)
4450	goto out;
4451	}
4452
4453	tr->current_trace = t;
4454	tr->current_trace->enabled++;
4455	trace_branch_enable(tr);
4456	out:
4457	mutex_unlock(&trace_types_lock);
4458
4459	return ret;
4460	}
4461
4462	static ssize_t
4463	tracing_set_trace_write(struct file filp, const char __user ubuf,
4464	size_t cnt, loff_t *ppos)
4465	{
4466	struct trace_array *tr = filp->private_data;
4467	char buf[MAX_TRACER_SIZE+1];
4468	int i;
4469	size_t ret;
4470	int err;
4471
4472	ret = cnt;
4473
4474	if (cnt > MAX_TRACER_SIZE)
4475	cnt = MAX_TRACER_SIZE;
4476
4477	if (copy_from_user(&buf, ubuf, cnt))
4478	return -EFAULT;
4479
4480	buf[cnt] = 0;
4481
4482	/* strip ending whitespace. */
4483	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4484	buf[i] = 0;
4485
4486	err = tracing_set_tracer(tr, buf);
4487	if (err)
4488	return err;
4489
4490	*ppos += ret;
4491
4492	return ret;
4493	}
4494
4495	static ssize_t
4496	tracing_nsecs_read(unsigned long ptr, char __user ubuf,
4497	size_t cnt, loff_t *ppos)
4498	{
4499	char buf[64];
4500	int r;
4501
4502	r = snprintf(buf, sizeof(buf), "%ld\n",
4503	ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(ptr));
4504	if (r > sizeof(buf))
4505	r = sizeof(buf);
4506	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4507	}
4508
4509	static ssize_t
4510	tracing_nsecs_write(unsigned long ptr, const char __user ubuf,
4511	size_t cnt, loff_t *ppos)
4512	{
4513	unsigned long val;
4514	int ret;
4515
4516	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4517	if (ret)
4518	return ret;
4519
4520	ptr = val 1000;
4521
4522	return cnt;
4523	}
4524
4525	static ssize_t
4526	tracing_thresh_read(struct file filp, char __user ubuf,
4527	size_t cnt, loff_t *ppos)
4528	{
4529	return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4530	}
4531
4532	static ssize_t
4533	tracing_thresh_write(struct file filp, const char __user ubuf,
4534	size_t cnt, loff_t *ppos)
4535	{
4536	struct trace_array *tr = filp->private_data;
4537	int ret;
4538
4539	mutex_lock(&trace_types_lock);
4540	ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4541	if (ret < 0)
4542	goto out;
4543
4544	if (tr->current_trace->update_thresh) {
4545	ret = tr->current_trace->update_thresh(tr);
4546	if (ret < 0)
4547	goto out;
4548	}
4549
4550	ret = cnt;
4551	out:
4552	mutex_unlock(&trace_types_lock);
4553
4554	return ret;
4555	}
4556
4557	#ifdef CONFIG_TRACER_MAX_TRACE
4558
4559	static ssize_t
4560	tracing_max_lat_read(struct file filp, char __user ubuf,
4561	size_t cnt, loff_t *ppos)
4562	{
4563	return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4564	}
4565
4566	static ssize_t
4567	tracing_max_lat_write(struct file filp, const char __user ubuf,
4568	size_t cnt, loff_t *ppos)
4569	{
4570	return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4571	}
4572
4573	#endif
4574
4575	static int tracing_open_pipe(struct inode inode, struct file filp)
4576	{
4577	struct trace_array *tr = inode->i_private;
4578	struct trace_iterator *iter;
4579	int ret = 0;
4580
4581	if (tracing_disabled)
4582	return -ENODEV;
4583
4584	if (trace_array_get(tr) < 0)
4585	return -ENODEV;
4586
4587	mutex_lock(&trace_types_lock);
4588
4589	/* create a buffer to store the information to pass to userspace */
4590	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4591	if (!iter) {
4592	ret = -ENOMEM;
4593	__trace_array_put(tr);
4594	goto out;
4595	}
4596
4597	trace_seq_init(&iter->seq);
4598	iter->trace = tr->current_trace;
4599
4600	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4601	ret = -ENOMEM;
4602	goto fail;
4603	}
4604
4605	/* trace pipe does not show start of buffer */
4606	cpumask_setall(iter->started);
4607
4608	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4609	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
4610
4611	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
4612	if (trace_clocks[tr->clock_id].in_ns)
4613	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
4614
4615	iter->tr = tr;
4616	iter->trace_buffer = &tr->trace_buffer;
4617	iter->cpu_file = tracing_get_cpu(inode);
4618	mutex_init(&iter->mutex);
4619	filp->private_data = iter;
4620
4621	if (iter->trace->pipe_open)
4622	iter->trace->pipe_open(iter);
4623
4624	nonseekable_open(inode, filp);
4625
4626	tr->current_trace->ref++;
4627	out:
4628	mutex_unlock(&trace_types_lock);
4629	return ret;
4630
4631	fail:
4632	kfree(iter->trace);
4633	kfree(iter);
4634	__trace_array_put(tr);
4635	mutex_unlock(&trace_types_lock);
4636	return ret;
4637	}
4638
4639	static int tracing_release_pipe(struct inode inode, struct file file)
4640	{
4641	struct trace_iterator *iter = file->private_data;
4642	struct trace_array *tr = inode->i_private;
4643
4644	mutex_lock(&trace_types_lock);
4645
4646	tr->current_trace->ref--;
4647
4648	if (iter->trace->pipe_close)
4649	iter->trace->pipe_close(iter);
4650
4651	mutex_unlock(&trace_types_lock);
4652
4653	free_cpumask_var(iter->started);
4654	mutex_destroy(&iter->mutex);
4655	kfree(iter);
4656
4657	trace_array_put(tr);
4658
4659	return 0;
4660	}
4661
4662	static unsigned int
4663	trace_poll(struct trace_iterator iter, struct file filp, poll_table *poll_table)
4664	{
4665	struct trace_array *tr = iter->tr;
4666
4667	/* Iterators are static, they should be filled or empty */
4668	if (trace_buffer_iter(iter, iter->cpu_file))
4669	return POLLIN \| POLLRDNORM;
4670
4671	if (tr->trace_flags & TRACE_ITER_BLOCK)
4672	/*
4673	* Always select as readable when in blocking mode
4674	*/
4675	return POLLIN \| POLLRDNORM;
4676	else
4677	return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4678	filp, poll_table);
4679	}
4680
4681	static unsigned int
4682	tracing_poll_pipe(struct file filp, poll_table poll_table)
4683	{
4684	struct trace_iterator *iter = filp->private_data;
4685
4686	return trace_poll(iter, filp, poll_table);
4687	}
4688
4689	/* Must be called with iter->mutex held. */
4690	static int tracing_wait_pipe(struct file *filp)
4691	{
4692	struct trace_iterator *iter = filp->private_data;
4693	int ret;
4694
4695	while (trace_empty(iter)) {
4696
4697	if ((filp->f_flags & O_NONBLOCK)) {
4698	return -EAGAIN;
4699	}
4700
4701	/*
4702	* We block until we read something and tracing is disabled.
4703	* We still block if tracing is disabled, but we have never
4704	* read anything. This allows a user to cat this file, and
4705	* then enable tracing. But after we have read something,
4706	* we give an EOF when tracing is again disabled.
4707	*
4708	* iter->pos will be 0 if we haven't read anything.
4709	*/
4710	if (!tracing_is_on() && iter->pos)
4711	break;
4712
4713	mutex_unlock(&iter->mutex);
4714
4715	ret = wait_on_pipe(iter, false);
4716
4717	mutex_lock(&iter->mutex);
4718
4719	if (ret)
4720	return ret;
4721	}
4722
4723	return 1;
4724	}
4725
4726	/*
4727	* Consumer reader.
4728	*/
4729	static ssize_t
4730	tracing_read_pipe(struct file filp, char __user ubuf,
4731	size_t cnt, loff_t *ppos)
4732	{
4733	struct trace_iterator *iter = filp->private_data;
4734	ssize_t sret;
4735
4736	/* return any leftover data */
4737	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4738	if (sret != -EBUSY)
4739	return sret;
4740
4741	trace_seq_init(&iter->seq);
4742
4743	/*
4744	* Avoid more than one consumer on a single file descriptor
4745	* This is just a matter of traces coherency, the ring buffer itself
4746	* is protected.
4747	*/
4748	mutex_lock(&iter->mutex);
4749	if (iter->trace->read) {
4750	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4751	if (sret)
4752	goto out;
4753	}
4754
4755	waitagain:
4756	sret = tracing_wait_pipe(filp);
4757	if (sret <= 0)
4758	goto out;
4759
4760	/* stop when tracing is finished */
4761	if (trace_empty(iter)) {
4762	sret = 0;
4763	goto out;
4764	}
4765
4766	if (cnt >= PAGE_SIZE)
4767	cnt = PAGE_SIZE - 1;
4768
4769	/* reset all but tr, trace, and overruns */
4770	memset(&iter->seq, 0,
4771	sizeof(struct trace_iterator) -
4772	offsetof(struct trace_iterator, seq));
4773	cpumask_clear(iter->started);
4774	iter->pos = -1;
4775
4776	trace_event_read_lock();
4777	trace_access_lock(iter->cpu_file);
4778	while (trace_find_next_entry_inc(iter) != NULL) {
4779	enum print_line_t ret;
4780	int save_len = iter->seq.seq.len;
4781
4782	ret = print_trace_line(iter);
4783	if (ret == TRACE_TYPE_PARTIAL_LINE) {
4784	/* don't print partial lines */
4785	iter->seq.seq.len = save_len;
4786	break;
4787	}
4788	if (ret != TRACE_TYPE_NO_CONSUME)
4789	trace_consume(iter);
4790
4791	if (trace_seq_used(&iter->seq) >= cnt)
4792	break;
4793
4794	/*
4795	* Setting the full flag means we reached the trace_seq buffer
4796	* size and we should leave by partial output condition above.
4797	* One of the trace_seq_* functions is not used properly.
4798	*/
4799	WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4800	iter->ent->type);
4801	}
4802	trace_access_unlock(iter->cpu_file);
4803	trace_event_read_unlock();
4804
4805	/* Now copy what we have to the user */
4806	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4807	if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
4808	trace_seq_init(&iter->seq);
4809
4810	/*
4811	* If there was nothing to send to user, in spite of consuming trace
4812	* entries, go back to wait for more entries.
4813	*/
4814	if (sret == -EBUSY)
4815	goto waitagain;
4816
4817	out:
4818	mutex_unlock(&iter->mutex);
4819
4820	return sret;
4821	}
4822
4823	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4824	unsigned int idx)
4825	{
4826	__free_page(spd->pages[idx]);
4827	}
4828
4829	static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4830	.can_merge = 0,
4831	.confirm = generic_pipe_buf_confirm,
4832	.release = generic_pipe_buf_release,
4833	.steal = generic_pipe_buf_steal,
4834	.get = generic_pipe_buf_get,
4835	};
4836
4837	static size_t
4838	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4839	{
4840	size_t count;
4841	int save_len;
4842	int ret;
4843
4844	/* Seq buffer is page-sized, exactly what we need. */
4845	for (;;) {
4846	save_len = iter->seq.seq.len;
4847	ret = print_trace_line(iter);
4848
4849	if (trace_seq_has_overflowed(&iter->seq)) {
4850	iter->seq.seq.len = save_len;
4851	break;
4852	}
4853
4854	/*
4855	* This should not be hit, because it should only
4856	* be set if the iter->seq overflowed. But check it
4857	* anyway to be safe.
4858	*/
4859	if (ret == TRACE_TYPE_PARTIAL_LINE) {
4860	iter->seq.seq.len = save_len;
4861	break;
4862	}
4863
4864	count = trace_seq_used(&iter->seq) - save_len;
4865	if (rem < count) {
4866	rem = 0;
4867	iter->seq.seq.len = save_len;
4868	break;
4869	}
4870
4871	if (ret != TRACE_TYPE_NO_CONSUME)
4872	trace_consume(iter);
4873	rem -= count;
4874	if (!trace_find_next_entry_inc(iter)) {
4875	rem = 0;
4876	iter->ent = NULL;
4877	break;
4878	}
4879	}
4880
4881	return rem;
4882	}
4883
4884	static ssize_t tracing_splice_read_pipe(struct file *filp,
4885	loff_t *ppos,
4886	struct pipe_inode_info *pipe,
4887	size_t len,
4888	unsigned int flags)
4889	{
4890	struct page *pages_def[PIPE_DEF_BUFFERS];
4891	struct partial_page partial_def[PIPE_DEF_BUFFERS];
4892	struct trace_iterator *iter = filp->private_data;
4893	struct splice_pipe_desc spd = {
4894	.pages = pages_def,
4895	.partial = partial_def,
4896	.nr_pages = 0, /* This gets updated below. */
4897	.nr_pages_max = PIPE_DEF_BUFFERS,
4898	.flags = flags,
4899	.ops = &tracing_pipe_buf_ops,
4900	.spd_release = tracing_spd_release_pipe,
4901	};
4902	ssize_t ret;
4903	size_t rem;
4904	unsigned int i;
4905
4906	if (splice_grow_spd(pipe, &spd))
4907	return -ENOMEM;
4908
4909	mutex_lock(&iter->mutex);
4910
4911	if (iter->trace->splice_read) {
4912	ret = iter->trace->splice_read(iter, filp,
4913	ppos, pipe, len, flags);
4914	if (ret)
4915	goto out_err;
4916	}
4917
4918	ret = tracing_wait_pipe(filp);
4919	if (ret <= 0)
4920	goto out_err;
4921
4922	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4923	ret = -EFAULT;
4924	goto out_err;
4925	}
4926
4927	trace_event_read_lock();
4928	trace_access_lock(iter->cpu_file);
4929
4930	/* Fill as many pages as possible. */
4931	for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4932	spd.pages[i] = alloc_page(GFP_KERNEL);
4933	if (!spd.pages[i])
4934	break;
4935
4936	rem = tracing_fill_pipe_page(rem, iter);
4937
4938	/* Copy the data into the page, so we can start over. */
4939	ret = trace_seq_to_buffer(&iter->seq,
4940	page_address(spd.pages[i]),
4941	trace_seq_used(&iter->seq));
4942	if (ret < 0) {
4943	__free_page(spd.pages[i]);
4944	break;
4945	}
4946	spd.partial[i].offset = 0;
4947	spd.partial[i].len = trace_seq_used(&iter->seq);
4948
4949	trace_seq_init(&iter->seq);
4950	}
4951
4952	trace_access_unlock(iter->cpu_file);
4953	trace_event_read_unlock();
4954	mutex_unlock(&iter->mutex);
4955
4956	spd.nr_pages = i;
4957
4958	if (i)
4959	ret = splice_to_pipe(pipe, &spd);
4960	else
4961	ret = 0;
4962	out:
4963	splice_shrink_spd(&spd);
4964	return ret;
4965
4966	out_err:
4967	mutex_unlock(&iter->mutex);
4968	goto out;
4969	}
4970
4971	static ssize_t
4972	tracing_entries_read(struct file filp, char __user ubuf,
4973	size_t cnt, loff_t *ppos)
4974	{
4975	struct inode *inode = file_inode(filp);
4976	struct trace_array *tr = inode->i_private;
4977	int cpu = tracing_get_cpu(inode);
4978	char buf[64];
4979	int r = 0;
4980	ssize_t ret;
4981
4982	mutex_lock(&trace_types_lock);
4983
4984	if (cpu == RING_BUFFER_ALL_CPUS) {
4985	int cpu, buf_size_same;
4986	unsigned long size;
4987
4988	size = 0;
4989	buf_size_same = 1;
4990	/* check if all cpu sizes are same */
4991	for_each_tracing_cpu(cpu) {
4992	/* fill in the size from first enabled cpu */
4993	if (size == 0)
4994	size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4995	if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4996	buf_size_same = 0;
4997	break;
4998	}
4999	}
5000
5001	if (buf_size_same) {
5002	if (!ring_buffer_expanded)
5003	r = sprintf(buf, "%lu (expanded: %lu)\n",
5004	size >> 10,
5005	trace_buf_size >> 10);
5006	else
5007	r = sprintf(buf, "%lu\n", size >> 10);
5008	} else
5009	r = sprintf(buf, "X\n");
5010	} else
5011	r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
5012
5013	mutex_unlock(&trace_types_lock);
5014
5015	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5016	return ret;
5017	}
5018
5019	static ssize_t
5020	tracing_entries_write(struct file filp, const char __user ubuf,
5021	size_t cnt, loff_t *ppos)
5022	{
5023	struct inode *inode = file_inode(filp);
5024	struct trace_array *tr = inode->i_private;
5025	unsigned long val;
5026	int ret;
5027
5028	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5029	if (ret)
5030	return ret;
5031
5032	/* must have at least 1 entry */
5033	if (!val)
5034	return -EINVAL;
5035
5036	/* value is in KB */
5037	val <<= 10;
5038	ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5039	if (ret < 0)
5040	return ret;
5041
5042	*ppos += cnt;
5043
5044	return cnt;
5045	}
5046
5047	static ssize_t
5048	tracing_total_entries_read(struct file filp, char __user ubuf,
5049	size_t cnt, loff_t *ppos)
5050	{
5051	struct trace_array *tr = filp->private_data;
5052	char buf[64];
5053	int r, cpu;
5054	unsigned long size = 0, expanded_size = 0;
5055
5056	mutex_lock(&trace_types_lock);
5057	for_each_tracing_cpu(cpu) {
5058	size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5059	if (!ring_buffer_expanded)
5060	expanded_size += trace_buf_size >> 10;
5061	}
5062	if (ring_buffer_expanded)
5063	r = sprintf(buf, "%lu\n", size);
5064	else
5065	r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5066	mutex_unlock(&trace_types_lock);
5067
5068	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5069	}
5070
5071	static ssize_t
5072	tracing_free_buffer_write(struct file filp, const char __user ubuf,
5073	size_t cnt, loff_t *ppos)
5074	{
5075	/*
5076	* There is no need to read what the user has written, this function
5077	* is just to make sure that there is no error when "echo" is used
5078	*/
5079
5080	*ppos += cnt;
5081
5082	return cnt;
5083	}
5084
5085	static int
5086	tracing_free_buffer_release(struct inode inode, struct file filp)
5087	{
5088	struct trace_array *tr = inode->i_private;
5089
5090	/* disable tracing ? */
5091	if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
5092	tracer_tracing_off(tr);
5093	/* resize the ring buffer to 0 */
5094	tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5095
5096	trace_array_put(tr);
5097
5098	return 0;
5099	}
5100
5101	static ssize_t
5102	tracing_mark_write(struct file filp, const char __user ubuf,
5103	size_t cnt, loff_t *fpos)
5104	{
5105	unsigned long addr = (unsigned long)ubuf;
5106	struct trace_array *tr = filp->private_data;
5107	struct ring_buffer_event *event;
5108	struct ring_buffer *buffer;
5109	struct print_entry *entry;
5110	unsigned long irq_flags;
5111	struct page *pages[2];
5112	void *map_page[2];
5113	int nr_pages = 1;
5114	ssize_t written;
5115	int offset;
5116	int size;
5117	int len;
5118	int ret;
5119	int i;
5120
5121	if (tracing_disabled)
5122	return -EINVAL;
5123
5124	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5125	return -EINVAL;
5126
5127	if (cnt > TRACE_BUF_SIZE)
5128	cnt = TRACE_BUF_SIZE;
5129
5130	/*
5131	* Userspace is injecting traces into the kernel trace buffer.
5132	* We want to be as non intrusive as possible.
5133	* To do so, we do not want to allocate any special buffers
5134	* or take any locks, but instead write the userspace data
5135	* straight into the ring buffer.
5136	*
5137	* First we need to pin the userspace buffer into memory,
5138	* which, most likely it is, because it just referenced it.
5139	* But there's no guarantee that it is. By using get_user_pages_fast()
5140	* and kmap_atomic/kunmap_atomic() we can get access to the
5141	* pages directly. We then write the data directly into the
5142	* ring buffer.
5143	*/
5144	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5145
5146	/* check if we cross pages */
5147	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5148	nr_pages = 2;
5149
5150	offset = addr & (PAGE_SIZE - 1);
5151	addr &= PAGE_MASK;
5152
5153	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5154	if (ret < nr_pages) {
5155	while (--ret >= 0)
5156	put_page(pages[ret]);
5157	written = -EFAULT;
5158	goto out;
5159	}
5160
5161	for (i = 0; i < nr_pages; i++)
5162	map_page[i] = kmap_atomic(pages[i]);
5163
5164	local_save_flags(irq_flags);
5165	size = sizeof(entry) + cnt + 2; / possible \n added */
5166	buffer = tr->trace_buffer.buffer;
5167	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5168	irq_flags, preempt_count());
5169	if (!event) {
5170	/* Ring buffer disabled, return as if not open for write */
5171	written = -EBADF;
5172	goto out_unlock;
5173	}
5174
5175	entry = ring_buffer_event_data(event);
5176	entry->ip = _THIS_IP_;
5177
5178	if (nr_pages == 2) {
5179	len = PAGE_SIZE - offset;
5180	memcpy(&entry->buf, map_page[0] + offset, len);
5181	memcpy(&entry->buf[len], map_page[1], cnt - len);
5182	} else
5183	memcpy(&entry->buf, map_page[0] + offset, cnt);
5184
5185	if (entry->buf[cnt - 1] != '\n') {
5186	entry->buf[cnt] = '\n';
5187	entry->buf[cnt + 1] = '\0';
5188	} else
5189	entry->buf[cnt] = '\0';
5190
5191	__buffer_unlock_commit(buffer, event);
5192
5193	written = cnt;
5194
5195	*fpos += written;
5196
5197	out_unlock:
5198	for (i = nr_pages - 1; i >= 0; i--) {
5199	kunmap_atomic(map_page[i]);
5200	put_page(pages[i]);
5201	}
5202	out:
5203	return written;
5204	}
5205
5206	static int tracing_clock_show(struct seq_file m, void v)
5207	{
5208	struct trace_array *tr = m->private;
5209	int i;
5210
5211	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5212	seq_printf(m,
5213	"%s%s%s%s", i ? " " : "",
5214	i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5215	i == tr->clock_id ? "]" : "");
5216	seq_putc(m, '\n');
5217
5218	return 0;
5219	}
5220
5221	static int tracing_set_clock(struct trace_array tr, const char clockstr)
5222	{
5223	int i;
5224
5225	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5226	if (strcmp(trace_clocks[i].name, clockstr) == 0)
5227	break;
5228	}
5229	if (i == ARRAY_SIZE(trace_clocks))
5230	return -EINVAL;
5231
5232	mutex_lock(&trace_types_lock);
5233
5234	tr->clock_id = i;
5235
5236	ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5237
5238	/*
5239	* New clock may not be consistent with the previous clock.
5240	* Reset the buffer so that it doesn't have incomparable timestamps.
5241	*/
5242	tracing_reset_online_cpus(&tr->trace_buffer);
5243
5244	#ifdef CONFIG_TRACER_MAX_TRACE
5245	if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5246	ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5247	tracing_reset_online_cpus(&tr->max_buffer);
5248	#endif
5249
5250	mutex_unlock(&trace_types_lock);
5251
5252	return 0;
5253	}
5254
5255	static ssize_t tracing_clock_write(struct file filp, const char __user ubuf,
5256	size_t cnt, loff_t *fpos)
5257	{
5258	struct seq_file *m = filp->private_data;
5259	struct trace_array *tr = m->private;
5260	char buf[64];
5261	const char *clockstr;
5262	int ret;
5263
5264	if (cnt >= sizeof(buf))
5265	return -EINVAL;
5266
5267	if (copy_from_user(&buf, ubuf, cnt))
5268	return -EFAULT;
5269
5270	buf[cnt] = 0;
5271
5272	clockstr = strstrip(buf);
5273
5274	ret = tracing_set_clock(tr, clockstr);
5275	if (ret)
5276	return ret;
5277
5278	*fpos += cnt;
5279
5280	return cnt;
5281	}
5282
5283	static int tracing_clock_open(struct inode inode, struct file file)
5284	{
5285	struct trace_array *tr = inode->i_private;
5286	int ret;
5287
5288	if (tracing_disabled)
5289	return -ENODEV;
5290
5291	if (trace_array_get(tr))
5292	return -ENODEV;
5293
5294	ret = single_open(file, tracing_clock_show, inode->i_private);
5295	if (ret < 0)
5296	trace_array_put(tr);
5297
5298	return ret;
5299	}
5300
5301	struct ftrace_buffer_info {
5302	struct trace_iterator iter;
5303	void *spare;
5304	unsigned int read;
5305	};
5306
5307	#ifdef CONFIG_TRACER_SNAPSHOT
5308	static int tracing_snapshot_open(struct inode inode, struct file file)
5309	{
5310	struct trace_array *tr = inode->i_private;
5311	struct trace_iterator *iter;
5312	struct seq_file *m;
5313	int ret = 0;
5314
5315	if (trace_array_get(tr) < 0)
5316	return -ENODEV;
5317
5318	if (file->f_mode & FMODE_READ) {
5319	iter = __tracing_open(inode, file, true);
5320	if (IS_ERR(iter))
5321	ret = PTR_ERR(iter);
5322	} else {
5323	/* Writes still need the seq_file to hold the private data */
5324	ret = -ENOMEM;
5325	m = kzalloc(sizeof(*m), GFP_KERNEL);
5326	if (!m)
5327	goto out;
5328	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5329	if (!iter) {
5330	kfree(m);
5331	goto out;
5332	}
5333	ret = 0;
5334
5335	iter->tr = tr;
5336	iter->trace_buffer = &tr->max_buffer;
5337	iter->cpu_file = tracing_get_cpu(inode);
5338	m->private = iter;
5339	file->private_data = m;
5340	}
5341	out:
5342	if (ret < 0)
5343	trace_array_put(tr);
5344
5345	return ret;
5346	}
5347
5348	static ssize_t
5349	tracing_snapshot_write(struct file filp, const char __user ubuf, size_t cnt,
5350	loff_t *ppos)
5351	{
5352	struct seq_file *m = filp->private_data;
5353	struct trace_iterator *iter = m->private;
5354	struct trace_array *tr = iter->tr;
5355	unsigned long val;
5356	int ret;
5357
5358	ret = tracing_update_buffers();
5359	if (ret < 0)
5360	return ret;
5361
5362	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5363	if (ret)
5364	return ret;
5365
5366	mutex_lock(&trace_types_lock);
5367
5368	if (tr->current_trace->use_max_tr) {
5369	ret = -EBUSY;
5370	goto out;
5371	}
5372
5373	switch (val) {
5374	case 0:
5375	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5376	ret = -EINVAL;
5377	break;
5378	}
5379	if (tr->allocated_snapshot)
5380	free_snapshot(tr);
5381	break;
5382	case 1:
5383	/* Only allow per-cpu swap if the ring buffer supports it */
5384	#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5385	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5386	ret = -EINVAL;
5387	break;
5388	}
5389	#endif
5390	if (!tr->allocated_snapshot) {
5391	ret = alloc_snapshot(tr);
5392	if (ret < 0)
5393	break;
5394	}
5395	local_irq_disable();
5396	/* Now, we're going to swap */
5397	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5398	update_max_tr(tr, current, smp_processor_id());
5399	else
5400	update_max_tr_single(tr, current, iter->cpu_file);
5401	local_irq_enable();
5402	break;
5403	default:
5404	if (tr->allocated_snapshot) {
5405	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5406	tracing_reset_online_cpus(&tr->max_buffer);
5407	else
5408	tracing_reset(&tr->max_buffer, iter->cpu_file);
5409	}
5410	break;
5411	}
5412
5413	if (ret >= 0) {
5414	*ppos += cnt;
5415	ret = cnt;
5416	}
5417	out:
5418	mutex_unlock(&trace_types_lock);
5419	return ret;
5420	}
5421
5422	static int tracing_snapshot_release(struct inode inode, struct file file)
5423	{
5424	struct seq_file *m = file->private_data;
5425	int ret;
5426
5427	ret = tracing_release(inode, file);
5428
5429	if (file->f_mode & FMODE_READ)
5430	return ret;
5431
5432	/* If write only, the seq_file is just a stub */
5433	if (m)
5434	kfree(m->private);
5435	kfree(m);
5436
5437	return 0;
5438	}
5439
5440	static int tracing_buffers_open(struct inode inode, struct file filp);
5441	static ssize_t tracing_buffers_read(struct file filp, char __user ubuf,
5442	size_t count, loff_t *ppos);
5443	static int tracing_buffers_release(struct inode inode, struct file file);
5444	static ssize_t tracing_buffers_splice_read(struct file file, loff_t ppos,
5445	struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5446
5447	static int snapshot_raw_open(struct inode inode, struct file filp)
5448	{
5449	struct ftrace_buffer_info *info;
5450	int ret;
5451
5452	ret = tracing_buffers_open(inode, filp);
5453	if (ret < 0)
5454	return ret;
5455
5456	info = filp->private_data;
5457
5458	if (info->iter.trace->use_max_tr) {
5459	tracing_buffers_release(inode, filp);
5460	return -EBUSY;
5461	}
5462
5463	info->iter.snapshot = true;
5464	info->iter.trace_buffer = &info->iter.tr->max_buffer;
5465
5466	return ret;
5467	}
5468
5469	#endif /* CONFIG_TRACER_SNAPSHOT */
5470
5471
5472	static const struct file_operations tracing_thresh_fops = {
5473	.open = tracing_open_generic,
5474	.read = tracing_thresh_read,
5475	.write = tracing_thresh_write,
5476	.llseek = generic_file_llseek,
5477	};
5478
5479	#ifdef CONFIG_TRACER_MAX_TRACE
5480	static const struct file_operations tracing_max_lat_fops = {
5481	.open = tracing_open_generic,
5482	.read = tracing_max_lat_read,
5483	.write = tracing_max_lat_write,
5484	.llseek = generic_file_llseek,
5485	};
5486	#endif
5487
5488	static const struct file_operations set_tracer_fops = {
5489	.open = tracing_open_generic,
5490	.read = tracing_set_trace_read,
5491	.write = tracing_set_trace_write,
5492	.llseek = generic_file_llseek,
5493	};
5494
5495	static const struct file_operations tracing_pipe_fops = {
5496	.open = tracing_open_pipe,
5497	.poll = tracing_poll_pipe,
5498	.read = tracing_read_pipe,
5499	.splice_read = tracing_splice_read_pipe,
5500	.release = tracing_release_pipe,
5501	.llseek = no_llseek,
5502	};
5503
5504	static const struct file_operations tracing_entries_fops = {
5505	.open = tracing_open_generic_tr,
5506	.read = tracing_entries_read,
5507	.write = tracing_entries_write,
5508	.llseek = generic_file_llseek,
5509	.release = tracing_release_generic_tr,
5510	};
5511
5512	static const struct file_operations tracing_total_entries_fops = {
5513	.open = tracing_open_generic_tr,
5514	.read = tracing_total_entries_read,
5515	.llseek = generic_file_llseek,
5516	.release = tracing_release_generic_tr,
5517	};
5518
5519	static const struct file_operations tracing_free_buffer_fops = {
5520	.open = tracing_open_generic_tr,
5521	.write = tracing_free_buffer_write,
5522	.release = tracing_free_buffer_release,
5523	};
5524
5525	static const struct file_operations tracing_mark_fops = {
5526	.open = tracing_open_generic_tr,
5527	.write = tracing_mark_write,
5528	.llseek = generic_file_llseek,
5529	.release = tracing_release_generic_tr,
5530	};
5531
5532	static const struct file_operations trace_clock_fops = {
5533	.open = tracing_clock_open,
5534	.read = seq_read,
5535	.llseek = seq_lseek,
5536	.release = tracing_single_release_tr,
5537	.write = tracing_clock_write,
5538	};
5539
5540	#ifdef CONFIG_TRACER_SNAPSHOT
5541	static const struct file_operations snapshot_fops = {
5542	.open = tracing_snapshot_open,
5543	.read = seq_read,
5544	.write = tracing_snapshot_write,
5545	.llseek = tracing_lseek,
5546	.release = tracing_snapshot_release,
5547	};
5548
5549	static const struct file_operations snapshot_raw_fops = {
5550	.open = snapshot_raw_open,
5551	.read = tracing_buffers_read,
5552	.release = tracing_buffers_release,
5553	.splice_read = tracing_buffers_splice_read,
5554	.llseek = no_llseek,
5555	};
5556
5557	#endif /* CONFIG_TRACER_SNAPSHOT */
5558
5559	static int tracing_buffers_open(struct inode inode, struct file filp)
5560	{
5561	struct trace_array *tr = inode->i_private;
5562	struct ftrace_buffer_info *info;
5563	int ret;
5564
5565	if (tracing_disabled)
5566	return -ENODEV;
5567
5568	if (trace_array_get(tr) < 0)
5569	return -ENODEV;
5570
5571	info = kzalloc(sizeof(*info), GFP_KERNEL);
5572	if (!info) {
5573	trace_array_put(tr);
5574	return -ENOMEM;
5575	}
5576
5577	mutex_lock(&trace_types_lock);
5578
5579	info->iter.tr = tr;
5580	info->iter.cpu_file = tracing_get_cpu(inode);
5581	info->iter.trace = tr->current_trace;
5582	info->iter.trace_buffer = &tr->trace_buffer;
5583	info->spare = NULL;
5584	/* Force reading ring buffer for first read */
5585	info->read = (unsigned int)-1;
5586
5587	filp->private_data = info;
5588
5589	tr->current_trace->ref++;
5590
5591	mutex_unlock(&trace_types_lock);
5592
5593	ret = nonseekable_open(inode, filp);
5594	if (ret < 0)
5595	trace_array_put(tr);
5596
5597	return ret;
5598	}
5599
5600	static unsigned int
5601	tracing_buffers_poll(struct file filp, poll_table poll_table)
5602	{
5603	struct ftrace_buffer_info *info = filp->private_data;
5604	struct trace_iterator *iter = &info->iter;
5605
5606	return trace_poll(iter, filp, poll_table);
5607	}
5608
5609	static ssize_t
5610	tracing_buffers_read(struct file filp, char __user ubuf,
5611	size_t count, loff_t *ppos)
5612	{
5613	struct ftrace_buffer_info *info = filp->private_data;
5614	struct trace_iterator *iter = &info->iter;
5615	ssize_t ret;
5616	ssize_t size;
5617
5618	if (!count)
5619	return 0;
5620
5621	#ifdef CONFIG_TRACER_MAX_TRACE
5622	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5623	return -EBUSY;
5624	#endif
5625
5626	if (!info->spare)
5627	info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5628	iter->cpu_file);
5629	if (!info->spare)
5630	return -ENOMEM;
5631
5632	/* Do we have previous read data to read? */
5633	if (info->read < PAGE_SIZE)
5634	goto read;
5635
5636	again:
5637	trace_access_lock(iter->cpu_file);
5638	ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5639	&info->spare,
5640	count,
5641	iter->cpu_file, 0);
5642	trace_access_unlock(iter->cpu_file);
5643
5644	if (ret < 0) {
5645	if (trace_empty(iter)) {
5646	if ((filp->f_flags & O_NONBLOCK))
5647	return -EAGAIN;
5648
5649	ret = wait_on_pipe(iter, false);
5650	if (ret)
5651	return ret;
5652
5653	goto again;
5654	}
5655	return 0;
5656	}
5657
5658	info->read = 0;
5659	read:
5660	size = PAGE_SIZE - info->read;
5661	if (size > count)
5662	size = count;
5663
5664	ret = copy_to_user(ubuf, info->spare + info->read, size);
5665	if (ret == size)
5666	return -EFAULT;
5667
5668	size -= ret;
5669
5670	*ppos += size;
5671	info->read += size;
5672
5673	return size;
5674	}
5675
5676	static int tracing_buffers_release(struct inode inode, struct file file)
5677	{
5678	struct ftrace_buffer_info *info = file->private_data;
5679	struct trace_iterator *iter = &info->iter;
5680
5681	mutex_lock(&trace_types_lock);
5682
5683	iter->tr->current_trace->ref--;
5684
5685	__trace_array_put(iter->tr);
5686
5687	if (info->spare)
5688	ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5689	kfree(info);
5690
5691	mutex_unlock(&trace_types_lock);
5692
5693	return 0;
5694	}
5695
5696	struct buffer_ref {
5697	struct ring_buffer *buffer;
5698	void *page;
5699	int ref;
5700	};
5701
5702	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5703	struct pipe_buffer *buf)
5704	{
5705	struct buffer_ref ref = (struct buffer_ref )buf->private;
5706
5707	if (--ref->ref)
5708	return;
5709
5710	ring_buffer_free_read_page(ref->buffer, ref->page);
5711	kfree(ref);
5712	buf->private = 0;
5713	}
5714
5715	static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5716	struct pipe_buffer *buf)
5717	{
5718	struct buffer_ref ref = (struct buffer_ref )buf->private;
5719
5720	ref->ref++;
5721	}
5722
5723	/* Pipe buffer operations for a buffer. */
5724	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5725	.can_merge = 0,
5726	.confirm = generic_pipe_buf_confirm,
5727	.release = buffer_pipe_buf_release,
5728	.steal = generic_pipe_buf_steal,
5729	.get = buffer_pipe_buf_get,
5730	};
5731
5732	/*
5733	* Callback from splice_to_pipe(), if we need to release some pages
5734	* at the end of the spd in case we error'ed out in filling the pipe.
5735	*/
5736	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5737	{
5738	struct buffer_ref *ref =
5739	(struct buffer_ref *)spd->partial[i].private;
5740
5741	if (--ref->ref)
5742	return;
5743
5744	ring_buffer_free_read_page(ref->buffer, ref->page);
5745	kfree(ref);
5746	spd->partial[i].private = 0;
5747	}
5748
5749	static ssize_t
5750	tracing_buffers_splice_read(struct file file, loff_t ppos,
5751	struct pipe_inode_info *pipe, size_t len,
5752	unsigned int flags)
5753	{
5754	struct ftrace_buffer_info *info = file->private_data;
5755	struct trace_iterator *iter = &info->iter;
5756	struct partial_page partial_def[PIPE_DEF_BUFFERS];
5757	struct page *pages_def[PIPE_DEF_BUFFERS];
5758	struct splice_pipe_desc spd = {
5759	.pages = pages_def,
5760	.partial = partial_def,
5761	.nr_pages_max = PIPE_DEF_BUFFERS,
5762	.flags = flags,
5763	.ops = &buffer_pipe_buf_ops,
5764	.spd_release = buffer_spd_release,
5765	};
5766	struct buffer_ref *ref;
5767	int entries, size, i;
5768	ssize_t ret = 0;
5769
5770	#ifdef CONFIG_TRACER_MAX_TRACE
5771	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5772	return -EBUSY;
5773	#endif
5774
5775	if (splice_grow_spd(pipe, &spd))
5776	return -ENOMEM;
5777
5778	if (*ppos & (PAGE_SIZE - 1))
5779	return -EINVAL;
5780
5781	if (len & (PAGE_SIZE - 1)) {
5782	if (len < PAGE_SIZE)
5783	return -EINVAL;
5784	len &= PAGE_MASK;
5785	}
5786
5787	again:
5788	trace_access_lock(iter->cpu_file);
5789	entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5790
5791	for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5792	struct page *page;
5793	int r;
5794
5795	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5796	if (!ref) {
5797	ret = -ENOMEM;
5798	break;
5799	}
5800
5801	ref->ref = 1;
5802	ref->buffer = iter->trace_buffer->buffer;
5803	ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5804	if (!ref->page) {
5805	ret = -ENOMEM;
5806	kfree(ref);
5807	break;
5808	}
5809
5810	r = ring_buffer_read_page(ref->buffer, &ref->page,
5811	len, iter->cpu_file, 1);
5812	if (r < 0) {
5813	ring_buffer_free_read_page(ref->buffer, ref->page);
5814	kfree(ref);
5815	break;
5816	}
5817
5818	/*
5819	* zero out any left over data, this is going to
5820	* user land.
5821	*/
5822	size = ring_buffer_page_len(ref->page);
5823	if (size < PAGE_SIZE)
5824	memset(ref->page + size, 0, PAGE_SIZE - size);
5825
5826	page = virt_to_page(ref->page);
5827
5828	spd.pages[i] = page;
5829	spd.partial[i].len = PAGE_SIZE;
5830	spd.partial[i].offset = 0;
5831	spd.partial[i].private = (unsigned long)ref;
5832	spd.nr_pages++;
5833	*ppos += PAGE_SIZE;
5834
5835	entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5836	}
5837
5838	trace_access_unlock(iter->cpu_file);
5839	spd.nr_pages = i;
5840
5841	/* did we read anything? */
5842	if (!spd.nr_pages) {
5843	if (ret)
5844	return ret;
5845
5846	if ((file->f_flags & O_NONBLOCK) \|\| (flags & SPLICE_F_NONBLOCK))
5847	return -EAGAIN;
5848
5849	ret = wait_on_pipe(iter, true);
5850	if (ret)
5851	return ret;
5852
5853	goto again;
5854	}
5855
5856	ret = splice_to_pipe(pipe, &spd);
5857	splice_shrink_spd(&spd);
5858
5859	return ret;
5860	}
5861
5862	static const struct file_operations tracing_buffers_fops = {
5863	.open = tracing_buffers_open,
5864	.read = tracing_buffers_read,
5865	.poll = tracing_buffers_poll,
5866	.release = tracing_buffers_release,
5867	.splice_read = tracing_buffers_splice_read,
5868	.llseek = no_llseek,
5869	};
5870
5871	static ssize_t
5872	tracing_stats_read(struct file filp, char __user ubuf,
5873	size_t count, loff_t *ppos)
5874	{
5875	struct inode *inode = file_inode(filp);
5876	struct trace_array *tr = inode->i_private;
5877	struct trace_buffer *trace_buf = &tr->trace_buffer;
5878	int cpu = tracing_get_cpu(inode);
5879	struct trace_seq *s;
5880	unsigned long cnt;
5881	unsigned long long t;
5882	unsigned long usec_rem;
5883
5884	s = kmalloc(sizeof(*s), GFP_KERNEL);
5885	if (!s)
5886	return -ENOMEM;
5887
5888	trace_seq_init(s);
5889
5890	cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5891	trace_seq_printf(s, "entries: %ld\n", cnt);
5892
5893	cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5894	trace_seq_printf(s, "overrun: %ld\n", cnt);
5895
5896	cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5897	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5898
5899	cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5900	trace_seq_printf(s, "bytes: %ld\n", cnt);
5901
5902	if (trace_clocks[tr->clock_id].in_ns) {
5903	/* local or global for trace_clock */
5904	t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5905	usec_rem = do_div(t, USEC_PER_SEC);
5906	trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5907	t, usec_rem);
5908
5909	t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5910	usec_rem = do_div(t, USEC_PER_SEC);
5911	trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5912	} else {
5913	/* counter or tsc mode for trace_clock */
5914	trace_seq_printf(s, "oldest event ts: %llu\n",
5915	ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5916
5917	trace_seq_printf(s, "now ts: %llu\n",
5918	ring_buffer_time_stamp(trace_buf->buffer, cpu));
5919	}
5920
5921	cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5922	trace_seq_printf(s, "dropped events: %ld\n", cnt);
5923
5924	cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5925	trace_seq_printf(s, "read events: %ld\n", cnt);
5926
5927	count = simple_read_from_buffer(ubuf, count, ppos,
5928	s->buffer, trace_seq_used(s));
5929
5930	kfree(s);
5931
5932	return count;
5933	}
5934
5935	static const struct file_operations tracing_stats_fops = {
5936	.open = tracing_open_generic_tr,
5937	.read = tracing_stats_read,
5938	.llseek = generic_file_llseek,
5939	.release = tracing_release_generic_tr,
5940	};
5941
5942	#ifdef CONFIG_DYNAMIC_FTRACE
5943
5944	int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5945	{
5946	return 0;
5947	}
5948
5949	static ssize_t
5950	tracing_read_dyn_info(struct file filp, char __user ubuf,
5951	size_t cnt, loff_t *ppos)
5952	{
5953	static char ftrace_dyn_info_buffer[1024];
5954	static DEFINE_MUTEX(dyn_info_mutex);
5955	unsigned long *p = filp->private_data;
5956	char *buf = ftrace_dyn_info_buffer;
5957	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5958	int r;
5959
5960	mutex_lock(&dyn_info_mutex);
5961	r = sprintf(buf, "%ld ", *p);
5962
5963	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5964	buf[r++] = '\n';
5965
5966	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5967
5968	mutex_unlock(&dyn_info_mutex);
5969
5970	return r;
5971	}
5972
5973	static const struct file_operations tracing_dyn_info_fops = {
5974	.open = tracing_open_generic,
5975	.read = tracing_read_dyn_info,
5976	.llseek = generic_file_llseek,
5977	};
5978	#endif /* CONFIG_DYNAMIC_FTRACE */
5979
5980	#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5981	static void
5982	ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5983	{
5984	tracing_snapshot();
5985	}
5986
5987	static void
5988	ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5989	{
5990	unsigned long count = (long )data;
5991
5992	if (!*count)
5993	return;
5994
5995	if (*count != -1)
5996	(*count)--;
5997
5998	tracing_snapshot();
5999	}
6000
6001	static int
6002	ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
6003	struct ftrace_probe_ops ops, void data)
6004	{
6005	long count = (long)data;
6006
6007	seq_printf(m, "%ps:", (void *)ip);
6008
6009	seq_puts(m, "snapshot");
6010
6011	if (count == -1)
6012	seq_puts(m, ":unlimited\n");
6013	else
6014	seq_printf(m, ":count=%ld\n", count);
6015
6016	return 0;
6017	}
6018
6019	static struct ftrace_probe_ops snapshot_probe_ops = {
6020	.func = ftrace_snapshot,
6021	.print = ftrace_snapshot_print,
6022	};
6023
6024	static struct ftrace_probe_ops snapshot_count_probe_ops = {
6025	.func = ftrace_count_snapshot,
6026	.print = ftrace_snapshot_print,
6027	};
6028
6029	static int
6030	ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6031	char glob, char cmd, char *param, int enable)
6032	{
6033	struct ftrace_probe_ops *ops;
6034	void count = (void )-1;
6035	char *number;
6036	int ret;
6037
6038	/* hash funcs only work with set_ftrace_filter */
6039	if (!enable)
6040	return -EINVAL;
6041
6042	ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6043
6044	if (glob[0] == '!') {
6045	unregister_ftrace_function_probe_func(glob+1, ops);
6046	return 0;
6047	}
6048
6049	if (!param)
6050	goto out_reg;
6051
6052	number = strsep(&param, ":");
6053
6054	if (!strlen(number))
6055	goto out_reg;
6056
6057	/*
6058	* We use the callback data field (which is a pointer)
6059	* as our counter.
6060	*/
6061	ret = kstrtoul(number, 0, (unsigned long *)&count);
6062	if (ret)
6063	return ret;
6064
6065	out_reg:
6066	ret = register_ftrace_function_probe(glob, ops, count);
6067
6068	if (ret >= 0)
6069	alloc_snapshot(&global_trace);
6070
6071	return ret < 0 ? ret : 0;
6072	}
6073
6074	static struct ftrace_func_command ftrace_snapshot_cmd = {
6075	.name = "snapshot",
6076	.func = ftrace_trace_snapshot_callback,
6077	};
6078
6079	static __init int register_snapshot_cmd(void)
6080	{
6081	return register_ftrace_command(&ftrace_snapshot_cmd);
6082	}
6083	#else
6084	static inline __init int register_snapshot_cmd(void) { return 0; }
6085	#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6086
6087	static struct dentry tracing_get_dentry(struct trace_array tr)
6088	{
6089	if (WARN_ON(!tr->dir))
6090	return ERR_PTR(-ENODEV);
6091
6092	/* Top directory uses NULL as the parent */
6093	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6094	return NULL;
6095
6096	/* All sub buffers have a descriptor */
6097	return tr->dir;
6098	}
6099
6100	static struct dentry tracing_dentry_percpu(struct trace_array tr, int cpu)
6101	{
6102	struct dentry *d_tracer;
6103
6104	if (tr->percpu_dir)
6105	return tr->percpu_dir;
6106
6107	d_tracer = tracing_get_dentry(tr);
6108	if (IS_ERR(d_tracer))
6109	return NULL;
6110
6111	tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6112
6113	WARN_ONCE(!tr->percpu_dir,
6114	"Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6115
6116	return tr->percpu_dir;
6117	}
6118
6119	static struct dentry *
6120	trace_create_cpu_file(const char name, umode_t mode, struct dentry parent,
6121	void data, long cpu, const struct file_operations fops)
6122	{
6123	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6124
6125	if (ret) /* See tracing_get_cpu() */
6126	d_inode(ret)->i_cdev = (void *)(cpu + 1);
6127	return ret;
6128	}
6129
6130	static void
6131	tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6132	{
6133	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6134	struct dentry *d_cpu;
6135	char cpu_dir[30]; /* 30 characters should be more than enough */
6136
6137	if (!d_percpu)
6138	return;
6139
6140	snprintf(cpu_dir, 30, "cpu%ld", cpu);
6141	d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6142	if (!d_cpu) {
6143	pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
6144	return;
6145	}
6146
6147	/* per cpu trace_pipe */
6148	trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6149	tr, cpu, &tracing_pipe_fops);
6150
6151	/* per cpu trace */
6152	trace_create_cpu_file("trace", 0644, d_cpu,
6153	tr, cpu, &tracing_fops);
6154
6155	trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6156	tr, cpu, &tracing_buffers_fops);
6157
6158	trace_create_cpu_file("stats", 0444, d_cpu,
6159	tr, cpu, &tracing_stats_fops);
6160
6161	trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6162	tr, cpu, &tracing_entries_fops);
6163
6164	#ifdef CONFIG_TRACER_SNAPSHOT
6165	trace_create_cpu_file("snapshot", 0644, d_cpu,
6166	tr, cpu, &snapshot_fops);
6167
6168	trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6169	tr, cpu, &snapshot_raw_fops);
6170	#endif
6171	}
6172
6173	#ifdef CONFIG_FTRACE_SELFTEST
6174	/* Let selftest have access to static functions in this file */
6175	#include "trace_selftest.c"
6176	#endif
6177
6178	static ssize_t
6179	trace_options_read(struct file filp, char __user ubuf, size_t cnt,
6180	loff_t *ppos)
6181	{
6182	struct trace_option_dentry *topt = filp->private_data;
6183	char *buf;
6184
6185	if (topt->flags->val & topt->opt->bit)
6186	buf = "1\n";
6187	else
6188	buf = "0\n";
6189
6190	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6191	}
6192
6193	static ssize_t
6194	trace_options_write(struct file filp, const char __user ubuf, size_t cnt,
6195	loff_t *ppos)
6196	{
6197	struct trace_option_dentry *topt = filp->private_data;
6198	unsigned long val;
6199	int ret;
6200
6201	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6202	if (ret)
6203	return ret;
6204
6205	if (val != 0 && val != 1)
6206	return -EINVAL;
6207
6208	if (!!(topt->flags->val & topt->opt->bit) != val) {
6209	mutex_lock(&trace_types_lock);
6210	ret = __set_tracer_option(topt->tr, topt->flags,
6211	topt->opt, !val);
6212	mutex_unlock(&trace_types_lock);
6213	if (ret)
6214	return ret;
6215	}
6216
6217	*ppos += cnt;
6218
6219	return cnt;
6220	}
6221
6222
6223	static const struct file_operations trace_options_fops = {
6224	.open = tracing_open_generic,
6225	.read = trace_options_read,
6226	.write = trace_options_write,
6227	.llseek = generic_file_llseek,
6228	};
6229
6230	/*
6231	* In order to pass in both the trace_array descriptor as well as the index
6232	* to the flag that the trace option file represents, the trace_array
6233	* has a character array of trace_flags_index[], which holds the index
6234	* of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6235	* The address of this character array is passed to the flag option file
6236	* read/write callbacks.
6237	*
6238	* In order to extract both the index and the trace_array descriptor,
6239	* get_tr_index() uses the following algorithm.
6240	*
6241	* idx = *ptr;
6242	*
6243	* As the pointer itself contains the address of the index (remember
6244	* index[1] == 1).
6245	*
6246	* Then to get the trace_array descriptor, by subtracting that index
6247	* from the ptr, we get to the start of the index itself.
6248	*
6249	* ptr - idx == &index[0]
6250	*
6251	* Then a simple container_of() from that pointer gets us to the
6252	* trace_array descriptor.
6253	*/
6254	static void get_tr_index(void data, struct trace_array *ptr,
6255	unsigned int *pindex)
6256	{
6257	pindex = (unsigned char *)data;
6258
6259	ptr = container_of(data - pindex, struct trace_array,
6260	trace_flags_index);
6261	}
6262
6263	static ssize_t
6264	trace_options_core_read(struct file filp, char __user ubuf, size_t cnt,
6265	loff_t *ppos)
6266	{
6267	void *tr_index = filp->private_data;
6268	struct trace_array *tr;
6269	unsigned int index;
6270	char *buf;
6271
6272	get_tr_index(tr_index, &tr, &index);
6273
6274	if (tr->trace_flags & (1 << index))
6275	buf = "1\n";
6276	else
6277	buf = "0\n";
6278
6279	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6280	}
6281
6282	static ssize_t
6283	trace_options_core_write(struct file filp, const char __user ubuf, size_t cnt,
6284	loff_t *ppos)
6285	{
6286	void *tr_index = filp->private_data;
6287	struct trace_array *tr;
6288	unsigned int index;
6289	unsigned long val;
6290	int ret;
6291
6292	get_tr_index(tr_index, &tr, &index);
6293
6294	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6295	if (ret)
6296	return ret;
6297
6298	if (val != 0 && val != 1)
6299	return -EINVAL;
6300
6301	mutex_lock(&trace_types_lock);
6302	ret = set_tracer_flag(tr, 1 << index, val);
6303	mutex_unlock(&trace_types_lock);
6304
6305	if (ret < 0)
6306	return ret;
6307
6308	*ppos += cnt;
6309
6310	return cnt;
6311	}
6312
6313	static const struct file_operations trace_options_core_fops = {
6314	.open = tracing_open_generic,
6315	.read = trace_options_core_read,
6316	.write = trace_options_core_write,
6317	.llseek = generic_file_llseek,
6318	};
6319
6320	struct dentry trace_create_file(const char name,
6321	umode_t mode,
6322	struct dentry *parent,
6323	void *data,
6324	const struct file_operations *fops)
6325	{
6326	struct dentry *ret;
6327
6328	ret = tracefs_create_file(name, mode, parent, data, fops);
6329	if (!ret)
6330	pr_warn("Could not create tracefs '%s' entry\n", name);
6331
6332	return ret;
6333	}
6334
6335
6336	static struct dentry trace_options_init_dentry(struct trace_array tr)
6337	{
6338	struct dentry *d_tracer;
6339
6340	if (tr->options)
6341	return tr->options;
6342
6343	d_tracer = tracing_get_dentry(tr);
6344	if (IS_ERR(d_tracer))
6345	return NULL;
6346
6347	tr->options = tracefs_create_dir("options", d_tracer);
6348	if (!tr->options) {
6349	pr_warn("Could not create tracefs directory 'options'\n");
6350	return NULL;
6351	}
6352
6353	return tr->options;
6354	}
6355
6356	static void
6357	create_trace_option_file(struct trace_array *tr,
6358	struct trace_option_dentry *topt,
6359	struct tracer_flags *flags,
6360	struct tracer_opt *opt)
6361	{
6362	struct dentry *t_options;
6363
6364	t_options = trace_options_init_dentry(tr);
6365	if (!t_options)
6366	return;
6367
6368	topt->flags = flags;
6369	topt->opt = opt;
6370	topt->tr = tr;
6371
6372	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6373	&trace_options_fops);
6374
6375	}
6376
6377	static void
6378	create_trace_option_files(struct trace_array tr, struct tracer tracer)
6379	{
6380	struct trace_option_dentry *topts;
6381	struct trace_options *tr_topts;
6382	struct tracer_flags *flags;
6383	struct tracer_opt *opts;
6384	int cnt;
6385	int i;
6386
6387	if (!tracer)
6388	return;
6389
6390	flags = tracer->flags;
6391
6392	if (!flags \|\| !flags->opts)
6393	return;
6394
6395	/*
6396	* If this is an instance, only create flags for tracers
6397	* the instance may have.
6398	*/
6399	if (!trace_ok_for_array(tracer, tr))
6400	return;
6401
6402	for (i = 0; i < tr->nr_topts; i++) {
6403	/* Make sure there's no duplicate flags. */
6404	if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
6405	return;
6406	}
6407
6408	opts = flags->opts;
6409
6410	for (cnt = 0; opts[cnt].name; cnt++)
6411	;
6412
6413	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6414	if (!topts)
6415	return;
6416
6417	tr_topts = krealloc(tr->topts, sizeof(tr->topts) (tr->nr_topts + 1),
6418	GFP_KERNEL);
6419	if (!tr_topts) {
6420	kfree(topts);
6421	return;
6422	}
6423
6424	tr->topts = tr_topts;
6425	tr->topts[tr->nr_topts].tracer = tracer;
6426	tr->topts[tr->nr_topts].topts = topts;
6427	tr->nr_topts++;
6428
6429	for (cnt = 0; opts[cnt].name; cnt++) {
6430	create_trace_option_file(tr, &topts[cnt], flags,
6431	&opts[cnt]);
6432	WARN_ONCE(topts[cnt].entry == NULL,
6433	"Failed to create trace option: %s",
6434	opts[cnt].name);
6435	}
6436	}
6437
6438	static struct dentry *
6439	create_trace_option_core_file(struct trace_array *tr,
6440	const char *option, long index)
6441	{
6442	struct dentry *t_options;
6443
6444	t_options = trace_options_init_dentry(tr);
6445	if (!t_options)
6446	return NULL;
6447
6448	return trace_create_file(option, 0644, t_options,
6449	(void *)&tr->trace_flags_index[index],
6450	&trace_options_core_fops);
6451	}
6452
6453	static void create_trace_options_dir(struct trace_array *tr)
6454	{
6455	struct dentry *t_options;
6456	bool top_level = tr == &global_trace;
6457	int i;
6458
6459	t_options = trace_options_init_dentry(tr);
6460	if (!t_options)
6461	return;
6462
6463	for (i = 0; trace_options[i]; i++) {
6464	if (top_level \|\|
6465	!((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6466	create_trace_option_core_file(tr, trace_options[i], i);
6467	}
6468	}
6469
6470	static ssize_t
6471	rb_simple_read(struct file filp, char __user ubuf,
6472	size_t cnt, loff_t *ppos)
6473	{
6474	struct trace_array *tr = filp->private_data;
6475	char buf[64];
6476	int r;
6477
6478	r = tracer_tracing_is_on(tr);
6479	r = sprintf(buf, "%d\n", r);
6480
6481	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6482	}
6483
6484	static ssize_t
6485	rb_simple_write(struct file filp, const char __user ubuf,
6486	size_t cnt, loff_t *ppos)
6487	{
6488	struct trace_array *tr = filp->private_data;
6489	struct ring_buffer *buffer = tr->trace_buffer.buffer;
6490	unsigned long val;
6491	int ret;
6492
6493	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6494	if (ret)
6495	return ret;
6496
6497	if (buffer) {
6498	mutex_lock(&trace_types_lock);
6499	if (val) {
6500	tracer_tracing_on(tr);
6501	if (tr->current_trace->start)
6502	tr->current_trace->start(tr);
6503	} else {
6504	tracer_tracing_off(tr);
6505	if (tr->current_trace->stop)
6506	tr->current_trace->stop(tr);
6507	}
6508	mutex_unlock(&trace_types_lock);
6509	}
6510
6511	(*ppos)++;
6512
6513	return cnt;
6514	}
6515
6516	static const struct file_operations rb_simple_fops = {
6517	.open = tracing_open_generic_tr,
6518	.read = rb_simple_read,
6519	.write = rb_simple_write,
6520	.release = tracing_release_generic_tr,
6521	.llseek = default_llseek,
6522	};
6523
6524	struct dentry *trace_instance_dir;
6525
6526	static void
6527	init_tracer_tracefs(struct trace_array tr, struct dentry d_tracer);
6528
6529	static int
6530	allocate_trace_buffer(struct trace_array tr, struct trace_buffer buf, int size)
6531	{
6532	enum ring_buffer_flags rb_flags;
6533
6534	rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6535
6536	buf->tr = tr;
6537
6538	buf->buffer = ring_buffer_alloc(size, rb_flags);
6539	if (!buf->buffer)
6540	return -ENOMEM;
6541
6542	buf->data = alloc_percpu(struct trace_array_cpu);
6543	if (!buf->data) {
6544	ring_buffer_free(buf->buffer);
6545	return -ENOMEM;
6546	}
6547
6548	/* Allocate the first page for all buffers */
6549	set_buffer_entries(&tr->trace_buffer,
6550	ring_buffer_size(tr->trace_buffer.buffer, 0));
6551
6552	return 0;
6553	}
6554
6555	static int allocate_trace_buffers(struct trace_array *tr, int size)
6556	{
6557	int ret;
6558
6559	ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6560	if (ret)
6561	return ret;
6562
6563	#ifdef CONFIG_TRACER_MAX_TRACE
6564	ret = allocate_trace_buffer(tr, &tr->max_buffer,
6565	allocate_snapshot ? size : 1);
6566	if (WARN_ON(ret)) {
6567	ring_buffer_free(tr->trace_buffer.buffer);
6568	free_percpu(tr->trace_buffer.data);
6569	return -ENOMEM;
6570	}
6571	tr->allocated_snapshot = allocate_snapshot;
6572
6573	/*
6574	* Only the top level trace array gets its snapshot allocated
6575	* from the kernel command line.
6576	*/
6577	allocate_snapshot = false;
6578	#endif
6579	return 0;
6580	}
6581
6582	static void free_trace_buffer(struct trace_buffer *buf)
6583	{
6584	if (buf->buffer) {
6585	ring_buffer_free(buf->buffer);
6586	buf->buffer = NULL;
6587	free_percpu(buf->data);
6588	buf->data = NULL;
6589	}
6590	}
6591
6592	static void free_trace_buffers(struct trace_array *tr)
6593	{
6594	if (!tr)
6595	return;
6596
6597	free_trace_buffer(&tr->trace_buffer);
6598
6599	#ifdef CONFIG_TRACER_MAX_TRACE
6600	free_trace_buffer(&tr->max_buffer);
6601	#endif
6602	}
6603
6604	static void init_trace_flags_index(struct trace_array *tr)
6605	{
6606	int i;
6607
6608	/* Used by the trace options files */
6609	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
6610	tr->trace_flags_index[i] = i;
6611	}
6612
6613	static void __update_tracer_options(struct trace_array *tr)
6614	{
6615	struct tracer *t;
6616
6617	for (t = trace_types; t; t = t->next)
6618	add_tracer_options(tr, t);
6619	}
6620
6621	static void update_tracer_options(struct trace_array *tr)
6622	{
6623	mutex_lock(&trace_types_lock);
6624	__update_tracer_options(tr);
6625	mutex_unlock(&trace_types_lock);
6626	}
6627
6628	static int instance_mkdir(const char *name)
6629	{
6630	struct trace_array *tr;
6631	int ret;
6632
6633	mutex_lock(&trace_types_lock);
6634
6635	ret = -EEXIST;
6636	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6637	if (tr->name && strcmp(tr->name, name) == 0)
6638	goto out_unlock;
6639	}
6640
6641	ret = -ENOMEM;
6642	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6643	if (!tr)
6644	goto out_unlock;
6645
6646	tr->name = kstrdup(name, GFP_KERNEL);
6647	if (!tr->name)
6648	goto out_free_tr;
6649
6650	if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6651	goto out_free_tr;
6652
6653	tr->trace_flags = global_trace.trace_flags;
6654
6655	cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6656
6657	raw_spin_lock_init(&tr->start_lock);
6658
6659	tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6660
6661	tr->current_trace = &nop_trace;
6662
6663	INIT_LIST_HEAD(&tr->systems);
6664	INIT_LIST_HEAD(&tr->events);
6665
6666	if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6667	goto out_free_tr;
6668
6669	tr->dir = tracefs_create_dir(name, trace_instance_dir);
6670	if (!tr->dir)
6671	goto out_free_tr;
6672
6673	ret = event_trace_add_tracer(tr->dir, tr);
6674	if (ret) {
6675	tracefs_remove_recursive(tr->dir);
6676	goto out_free_tr;
6677	}
6678
6679	init_tracer_tracefs(tr, tr->dir);
6680	init_trace_flags_index(tr);
6681	__update_tracer_options(tr);
6682
6683	list_add(&tr->list, &ftrace_trace_arrays);
6684
6685	mutex_unlock(&trace_types_lock);
6686
6687	return 0;
6688
6689	out_free_tr:
6690	free_trace_buffers(tr);
6691	free_cpumask_var(tr->tracing_cpumask);
6692	kfree(tr->name);
6693	kfree(tr);
6694
6695	out_unlock:
6696	mutex_unlock(&trace_types_lock);
6697
6698	return ret;
6699
6700	}
6701
6702	static int instance_rmdir(const char *name)
6703	{
6704	struct trace_array *tr;
6705	int found = 0;
6706	int ret;
6707	int i;
6708
6709	mutex_lock(&trace_types_lock);
6710
6711	ret = -ENODEV;
6712	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6713	if (tr->name && strcmp(tr->name, name) == 0) {
6714	found = 1;
6715	break;
6716	}
6717	}
6718	if (!found)
6719	goto out_unlock;
6720
6721	ret = -EBUSY;
6722	if (tr->ref \|\| (tr->current_trace && tr->current_trace->ref))
6723	goto out_unlock;
6724
6725	list_del(&tr->list);
6726
6727	tracing_set_nop(tr);
6728	event_trace_del_tracer(tr);
6729	ftrace_destroy_function_files(tr);
6730	tracefs_remove_recursive(tr->dir);
6731	free_trace_buffers(tr);
6732
6733	for (i = 0; i < tr->nr_topts; i++) {
6734	kfree(tr->topts[i].topts);
6735	}
6736	kfree(tr->topts);
6737
6738	kfree(tr->name);
6739	kfree(tr);
6740
6741	ret = 0;
6742
6743	out_unlock:
6744	mutex_unlock(&trace_types_lock);
6745
6746	return ret;
6747	}
6748
6749	static __init void create_trace_instances(struct dentry *d_tracer)
6750	{
6751	trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6752	instance_mkdir,
6753	instance_rmdir);
6754	if (WARN_ON(!trace_instance_dir))
6755	return;
6756	}
6757
6758	static void
6759	init_tracer_tracefs(struct trace_array tr, struct dentry d_tracer)
6760	{
6761	int cpu;
6762
6763	trace_create_file("available_tracers", 0444, d_tracer,
6764	tr, &show_traces_fops);
6765
6766	trace_create_file("current_tracer", 0644, d_tracer,
6767	tr, &set_tracer_fops);
6768
6769	trace_create_file("tracing_cpumask", 0644, d_tracer,
6770	tr, &tracing_cpumask_fops);
6771
6772	trace_create_file("trace_options", 0644, d_tracer,
6773	tr, &tracing_iter_fops);
6774
6775	trace_create_file("trace", 0644, d_tracer,
6776	tr, &tracing_fops);
6777
6778	trace_create_file("trace_pipe", 0444, d_tracer,
6779	tr, &tracing_pipe_fops);
6780
6781	trace_create_file("buffer_size_kb", 0644, d_tracer,
6782	tr, &tracing_entries_fops);
6783
6784	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6785	tr, &tracing_total_entries_fops);
6786
6787	trace_create_file("free_buffer", 0200, d_tracer,
6788	tr, &tracing_free_buffer_fops);
6789
6790	trace_create_file("trace_marker", 0220, d_tracer,
6791	tr, &tracing_mark_fops);
6792
6793	trace_create_file("trace_clock", 0644, d_tracer, tr,
6794	&trace_clock_fops);
6795
6796	trace_create_file("tracing_on", 0644, d_tracer,
6797	tr, &rb_simple_fops);
6798
6799	create_trace_options_dir(tr);
6800
6801	#ifdef CONFIG_TRACER_MAX_TRACE
6802	trace_create_file("tracing_max_latency", 0644, d_tracer,
6803	&tr->max_latency, &tracing_max_lat_fops);
6804	#endif
6805
6806	if (ftrace_create_function_files(tr, d_tracer))
6807	WARN(1, "Could not allocate function filter files");
6808
6809	#ifdef CONFIG_TRACER_SNAPSHOT
6810	trace_create_file("snapshot", 0644, d_tracer,
6811	tr, &snapshot_fops);
6812	#endif
6813
6814	for_each_tracing_cpu(cpu)
6815	tracing_init_tracefs_percpu(tr, cpu);
6816
6817	}
6818
6819	static struct vfsmount trace_automount(void ingore)
6820	{
6821	struct vfsmount *mnt;
6822	struct file_system_type *type;
6823
6824	/*
6825	* To maintain backward compatibility for tools that mount
6826	* debugfs to get to the tracing facility, tracefs is automatically
6827	* mounted to the debugfs/tracing directory.
6828	*/
6829	type = get_fs_type("tracefs");
6830	if (!type)
6831	return NULL;
6832	mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6833	put_filesystem(type);
6834	if (IS_ERR(mnt))
6835	return NULL;
6836	mntget(mnt);
6837
6838	return mnt;
6839	}
6840
6841	/**
6842	* tracing_init_dentry - initialize top level trace array
6843	*
6844	* This is called when creating files or directories in the tracing
6845	* directory. It is called via fs_initcall() by any of the boot up code
6846	* and expects to return the dentry of the top level tracing directory.
6847	*/
6848	struct dentry *tracing_init_dentry(void)
6849	{
6850	struct trace_array *tr = &global_trace;
6851
6852	/* The top level trace array uses NULL as parent */
6853	if (tr->dir)
6854	return NULL;
6855
6856	if (WARN_ON(!tracefs_initialized()) \|\|
6857	(IS_ENABLED(CONFIG_DEBUG_FS) &&
6858	WARN_ON(!debugfs_initialized())))
6859	return ERR_PTR(-ENODEV);
6860
6861	/*
6862	* As there may still be users that expect the tracing
6863	* files to exist in debugfs/tracing, we must automount
6864	* the tracefs file system there, so older tools still
6865	* work with the newer kerenl.
6866	*/
6867	tr->dir = debugfs_create_automount("tracing", NULL,
6868	trace_automount, NULL);
6869	if (!tr->dir) {
6870	pr_warn_once("Could not create debugfs directory 'tracing'\n");
6871	return ERR_PTR(-ENOMEM);
6872	}
6873
6874	return NULL;
6875	}
6876
6877	extern struct trace_enum_map *__start_ftrace_enum_maps[];
6878	extern struct trace_enum_map *__stop_ftrace_enum_maps[];
6879
6880	static void __init trace_enum_init(void)
6881	{
6882	int len;
6883
6884	len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
6885	trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
6886	}
6887
6888	#ifdef CONFIG_MODULES
6889	static void trace_module_add_enums(struct module *mod)
6890	{
6891	if (!mod->num_trace_enums)
6892	return;
6893
6894	/*
6895	* Modules with bad taint do not have events created, do
6896	* not bother with enums either.
6897	*/
6898	if (trace_module_has_bad_taint(mod))
6899	return;
6900
6901	trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
6902	}
6903
6904	#ifdef CONFIG_TRACE_ENUM_MAP_FILE
6905	static void trace_module_remove_enums(struct module *mod)
6906	{
6907	union trace_enum_map_item *map;
6908	union trace_enum_map_item **last = &trace_enum_maps;
6909
6910	if (!mod->num_trace_enums)
6911	return;
6912
6913	mutex_lock(&trace_enum_mutex);
6914
6915	map = trace_enum_maps;
6916
6917	while (map) {
6918	if (map->head.mod == mod)
6919	break;
6920	map = trace_enum_jmp_to_tail(map);
6921	last = &map->tail.next;
6922	map = map->tail.next;
6923	}
6924	if (!map)
6925	goto out;
6926
6927	*last = trace_enum_jmp_to_tail(map)->tail.next;
6928	kfree(map);
6929	out:
6930	mutex_unlock(&trace_enum_mutex);
6931	}
6932	#else
6933	static inline void trace_module_remove_enums(struct module *mod) { }
6934	#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
6935
6936	static int trace_module_notify(struct notifier_block *self,
6937	unsigned long val, void *data)
6938	{
6939	struct module *mod = data;
6940
6941	switch (val) {
6942	case MODULE_STATE_COMING:
6943	trace_module_add_enums(mod);
6944	break;
6945	case MODULE_STATE_GOING:
6946	trace_module_remove_enums(mod);
6947	break;
6948	}
6949
6950	return 0;
6951	}
6952
6953	static struct notifier_block trace_module_nb = {
6954	.notifier_call = trace_module_notify,
6955	.priority = 0,
6956	};
6957	#endif /* CONFIG_MODULES */
6958
6959	static __init int tracer_init_tracefs(void)
6960	{
6961	struct dentry *d_tracer;
6962
6963	trace_access_lock_init();
6964
6965	d_tracer = tracing_init_dentry();
6966	if (IS_ERR(d_tracer))
6967	return 0;
6968
6969	init_tracer_tracefs(&global_trace, d_tracer);
6970
6971	trace_create_file("tracing_thresh", 0644, d_tracer,
6972	&global_trace, &tracing_thresh_fops);
6973
6974	trace_create_file("README", 0444, d_tracer,
6975	NULL, &tracing_readme_fops);
6976
6977	trace_create_file("saved_cmdlines", 0444, d_tracer,
6978	NULL, &tracing_saved_cmdlines_fops);
6979
6980	trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6981	NULL, &tracing_saved_cmdlines_size_fops);
6982
6983	trace_enum_init();
6984
6985	trace_create_enum_file(d_tracer);
6986
6987	#ifdef CONFIG_MODULES
6988	register_module_notifier(&trace_module_nb);
6989	#endif
6990
6991	#ifdef CONFIG_DYNAMIC_FTRACE
6992	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6993	&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6994	#endif
6995
6996	create_trace_instances(d_tracer);
6997
6998	update_tracer_options(&global_trace);
6999
7000	return 0;
7001	}
7002
7003	static int trace_panic_handler(struct notifier_block *this,
7004	unsigned long event, void *unused)
7005	{
7006	if (ftrace_dump_on_oops)
7007	ftrace_dump(ftrace_dump_on_oops);
7008	return NOTIFY_OK;
7009	}
7010
7011	static struct notifier_block trace_panic_notifier = {
7012	.notifier_call = trace_panic_handler,
7013	.next = NULL,
7014	.priority = 150 /* priority: INT_MAX >= x >= 0 */
7015	};
7016
7017	static int trace_die_handler(struct notifier_block *self,
7018	unsigned long val,
7019	void *data)
7020	{
7021	switch (val) {
7022	case DIE_OOPS:
7023	if (ftrace_dump_on_oops)
7024	ftrace_dump(ftrace_dump_on_oops);
7025	break;
7026	default:
7027	break;
7028	}
7029	return NOTIFY_OK;
7030	}
7031
7032	static struct notifier_block trace_die_notifier = {
7033	.notifier_call = trace_die_handler,
7034	.priority = 200
7035	};
7036
7037	/*
7038	* printk is set to max of 1024, we really don't need it that big.
7039	* Nothing should be printing 1000 characters anyway.
7040	*/
7041	#define TRACE_MAX_PRINT 1000
7042
7043	/*
7044	* Define here KERN_TRACE so that we have one place to modify
7045	* it if we decide to change what log level the ftrace dump
7046	* should be at.
7047	*/
7048	#define KERN_TRACE KERN_EMERG
7049
7050	void
7051	trace_printk_seq(struct trace_seq *s)
7052	{
7053	/* Probably should print a warning here. */
7054	if (s->seq.len >= TRACE_MAX_PRINT)
7055	s->seq.len = TRACE_MAX_PRINT;
7056
7057	/*
7058	* More paranoid code. Although the buffer size is set to
7059	* PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7060	* an extra layer of protection.
7061	*/
7062	if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7063	s->seq.len = s->seq.size - 1;
7064
7065	/* should be zero ended, but we are paranoid. */
7066	s->buffer[s->seq.len] = 0;
7067
7068	printk(KERN_TRACE "%s", s->buffer);
7069
7070	trace_seq_init(s);
7071	}
7072
7073	void trace_init_global_iter(struct trace_iterator *iter)
7074	{
7075	iter->tr = &global_trace;
7076	iter->trace = iter->tr->current_trace;
7077	iter->cpu_file = RING_BUFFER_ALL_CPUS;
7078	iter->trace_buffer = &global_trace.trace_buffer;
7079
7080	if (iter->trace && iter->trace->open)
7081	iter->trace->open(iter);
7082
7083	/* Annotate start of buffers if we had overruns */
7084	if (ring_buffer_overruns(iter->trace_buffer->buffer))
7085	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
7086
7087	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
7088	if (trace_clocks[iter->tr->clock_id].in_ns)
7089	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
7090	}
7091
7092	void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
7093	{
7094	/* use static because iter can be a bit big for the stack */
7095	static struct trace_iterator iter;
7096	static atomic_t dump_running;
7097	struct trace_array *tr = &global_trace;
7098	unsigned int old_userobj;
7099	unsigned long flags;
7100	int cnt = 0, cpu;
7101
7102	/* Only allow one dump user at a time. */
7103	if (atomic_inc_return(&dump_running) != 1) {
7104	atomic_dec(&dump_running);
7105	return;
7106	}
7107
7108	/*
7109	* Always turn off tracing when we dump.
7110	* We don't need to show trace output of what happens
7111	* between multiple crashes.
7112	*
7113	* If the user does a sysrq-z, then they can re-enable
7114	* tracing with echo 1 > tracing_on.
7115	*/
7116	tracing_off();
7117
7118	local_irq_save(flags);
7119
7120	/* Simulate the iterator */
7121	trace_init_global_iter(&iter);
7122
7123	for_each_tracing_cpu(cpu) {
7124	atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7125	}
7126
7127	old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
7128
7129	/* don't look at user memory in panic mode */
7130	tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
7131
7132	switch (oops_dump_mode) {
7133	case DUMP_ALL:
7134	iter.cpu_file = RING_BUFFER_ALL_CPUS;
7135	break;
7136	case DUMP_ORIG:
7137	iter.cpu_file = raw_smp_processor_id();
7138	break;
7139	case DUMP_NONE:
7140	goto out_enable;
7141	default:
7142	printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7143	iter.cpu_file = RING_BUFFER_ALL_CPUS;
7144	}
7145
7146	printk(KERN_TRACE "Dumping ftrace buffer:\n");
7147
7148	/* Did function tracer already get disabled? */
7149	if (ftrace_is_dead()) {
7150	printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7151	printk("# MAY BE MISSING FUNCTION EVENTS\n");
7152	}
7153
7154	/*
7155	* We need to stop all tracing on all CPUS to read the
7156	* the next buffer. This is a bit expensive, but is
7157	* not done often. We fill all what we can read,
7158	* and then release the locks again.
7159	*/
7160
7161	while (!trace_empty(&iter)) {
7162
7163	if (!cnt)
7164	printk(KERN_TRACE "---------------------------------\n");
7165
7166	cnt++;
7167
7168	/* reset all but tr, trace, and overruns */
7169	memset(&iter.seq, 0,
7170	sizeof(struct trace_iterator) -
7171	offsetof(struct trace_iterator, seq));
7172	iter.iter_flags \|= TRACE_FILE_LAT_FMT;
7173	iter.pos = -1;
7174
7175	if (trace_find_next_entry_inc(&iter) != NULL) {
7176	int ret;
7177
7178	ret = print_trace_line(&iter);
7179	if (ret != TRACE_TYPE_NO_CONSUME)
7180	trace_consume(&iter);
7181	}
7182	touch_nmi_watchdog();
7183
7184	trace_printk_seq(&iter.seq);
7185	}
7186
7187	if (!cnt)
7188	printk(KERN_TRACE " (ftrace buffer empty)\n");
7189	else
7190	printk(KERN_TRACE "---------------------------------\n");
7191
7192	out_enable:
7193	tr->trace_flags \|= old_userobj;
7194
7195	for_each_tracing_cpu(cpu) {
7196	atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7197	}
7198	atomic_dec(&dump_running);
7199	local_irq_restore(flags);
7200	}
7201	EXPORT_SYMBOL_GPL(ftrace_dump);
7202
7203	__init static int tracer_alloc_buffers(void)
7204	{
7205	int ring_buf_size;
7206	int ret = -ENOMEM;
7207
7208	/*
7209	* Make sure we don't accidently add more trace options
7210	* than we have bits for.
7211	*/
7212	BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
7213
7214	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7215	goto out;
7216
7217	if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7218	goto out_free_buffer_mask;
7219
7220	/* Only allocate trace_printk buffers if a trace_printk exists */
7221	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
7222	/* Must be called before global_trace.buffer is allocated */
7223	trace_printk_init_buffers();
7224
7225	/* To save memory, keep the ring buffer size to its minimum */
7226	if (ring_buffer_expanded)
7227	ring_buf_size = trace_buf_size;
7228	else
7229	ring_buf_size = 1;
7230
7231	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7232	cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7233
7234	raw_spin_lock_init(&global_trace.start_lock);
7235
7236	/* Used for event triggers */
7237	temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7238	if (!temp_buffer)
7239	goto out_free_cpumask;
7240
7241	if (trace_create_savedcmd() < 0)
7242	goto out_free_temp_buffer;
7243
7244	/* TODO: make the number of buffers hot pluggable with CPUS */
7245	if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7246	printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7247	WARN_ON(1);
7248	goto out_free_savedcmd;
7249	}
7250
7251	if (global_trace.buffer_disabled)
7252	tracing_off();
7253
7254	if (trace_boot_clock) {
7255	ret = tracing_set_clock(&global_trace, trace_boot_clock);
7256	if (ret < 0)
7257	pr_warn("Trace clock %s not defined, going back to default\n",
7258	trace_boot_clock);
7259	}
7260
7261	/*
7262	* register_tracer() might reference current_trace, so it
7263	* needs to be set before we register anything. This is
7264	* just a bootstrap of current_trace anyway.
7265	*/
7266	global_trace.current_trace = &nop_trace;
7267
7268	global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7269
7270	ftrace_init_global_array_ops(&global_trace);
7271
7272	init_trace_flags_index(&global_trace);
7273
7274	register_tracer(&nop_trace);
7275
7276	/* All seems OK, enable tracing */
7277	tracing_disabled = 0;
7278
7279	atomic_notifier_chain_register(&panic_notifier_list,
7280	&trace_panic_notifier);
7281
7282	register_die_notifier(&trace_die_notifier);
7283
7284	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7285
7286	INIT_LIST_HEAD(&global_trace.systems);
7287	INIT_LIST_HEAD(&global_trace.events);
7288	list_add(&global_trace.list, &ftrace_trace_arrays);
7289
7290	apply_trace_boot_options();
7291
7292	register_snapshot_cmd();
7293
7294	return 0;
7295
7296	out_free_savedcmd:
7297	free_saved_cmdlines_buffer(savedcmd);
7298	out_free_temp_buffer:
7299	ring_buffer_free(temp_buffer);
7300	out_free_cpumask:
7301	free_cpumask_var(global_trace.tracing_cpumask);
7302	out_free_buffer_mask:
7303	free_cpumask_var(tracing_buffer_mask);
7304	out:
7305	return ret;
7306	}
7307
7308	void __init trace_init(void)
7309	{
7310	if (tracepoint_printk) {
7311	tracepoint_print_iter =
7312	kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7313	if (WARN_ON(!tracepoint_print_iter))
7314	tracepoint_printk = 0;
7315	}
7316	tracer_alloc_buffers();
7317	trace_event_init();
7318	}
7319
7320	__init static int clear_boot_tracer(void)
7321	{
7322	/*
7323	* The default tracer at boot buffer is an init section.
7324	* This function is called in lateinit. If we did not
7325	* find the boot tracer, then clear it out, to prevent
7326	* later registration from accessing the buffer that is
7327	* about to be freed.
7328	*/
7329	if (!default_bootup_tracer)
7330	return 0;
7331
7332	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7333	default_bootup_tracer);
7334	default_bootup_tracer = NULL;
7335
7336	return 0;
7337	}
7338
7339	fs_initcall(tracer_init_tracefs);
7340	late_initcall(clear_boot_tracer);