linux - kernel/trace/trace.c

10519 lines · 255.7 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* ring buffer based function tracer
4	*
5	* Copyright (C) 2007-2012 Steven Rostedt <[email protected]>
6	* Copyright (C) 2008 Ingo Molnar <[email protected]>
7	*
8	* Originally taken from the RT patch by:
9	* Arnaldo Carvalho de Melo <[email protected]>
10	*
11	* Based on code from the latency_tracer, that is:
12	* Copyright (C) 2004-2006 Ingo Molnar
13	* Copyright (C) 2004 Nadia Yvette Chambers
14	*/
15	#include <linux/ring_buffer.h>
16	#include <generated/utsrelease.h>
17	#include <linux/stacktrace.h>
18	#include <linux/writeback.h>
19	#include <linux/kallsyms.h>
20	#include <linux/security.h>
21	#include <linux/seq_file.h>
22	#include <linux/irqflags.h>
23	#include <linux/debugfs.h>
24	#include <linux/tracefs.h>
25	#include <linux/pagemap.h>
26	#include <linux/hardirq.h>
27	#include <linux/linkage.h>
28	#include <linux/uaccess.h>
29	#include <linux/vmalloc.h>
30	#include <linux/ftrace.h>
31	#include <linux/module.h>
32	#include <linux/percpu.h>
33	#include <linux/splice.h>
34	#include <linux/kdebug.h>
35	#include <linux/string.h>
36	#include <linux/mount.h>
37	#include <linux/rwsem.h>
38	#include <linux/slab.h>
39	#include <linux/ctype.h>
40	#include <linux/init.h>
41	#include <linux/panic_notifier.h>
42	#include <linux/poll.h>
43	#include <linux/nmi.h>
44	#include <linux/fs.h>
45	#include <linux/trace.h>
46	#include <linux/sched/clock.h>
47	#include <linux/sched/rt.h>
48	#include <linux/fsnotify.h>
49	#include <linux/irq_work.h>
50	#include <linux/workqueue.h>
51
52	#include <asm/setup.h> /* COMMAND_LINE_SIZE */
53
54	#include "trace.h"
55	#include "trace_output.h"
56
57	/*
58	* On boot up, the ring buffer is set to the minimum size, so that
59	* we do not waste memory on systems that are not using tracing.
60	*/
61	bool ring_buffer_expanded;
62
63	#ifdef CONFIG_FTRACE_STARTUP_TEST
64	/*
65	* We need to change this state when a selftest is running.
66	* A selftest will lurk into the ring-buffer to count the
67	* entries inserted during the selftest although some concurrent
68	* insertions into the ring-buffer such as trace_printk could occurred
69	* at the same time, giving false positive or negative results.
70	*/
71	static bool __read_mostly tracing_selftest_running;
72
73	/*
74	* If boot-time tracing including tracers/events via kernel cmdline
75	* is running, we do not want to run SELFTEST.
76	*/
77	bool __read_mostly tracing_selftest_disabled;
78
79	void __init disable_tracing_selftest(const char *reason)
80	{
81	if (!tracing_selftest_disabled) {
82	tracing_selftest_disabled = true;
83	pr_info("Ftrace startup test is disabled due to %s\n", reason);
84	}
85	}
86	#else
87	#define tracing_selftest_running 0
88	#define tracing_selftest_disabled 0
89	#endif
90
91	/* Pipe tracepoints to printk */
92	static struct trace_iterator *tracepoint_print_iter;
93	int tracepoint_printk;
94	static bool tracepoint_printk_stop_on_boot __initdata;
95	static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
96
97	/* For tracers that don't implement custom flags */
98	static struct tracer_opt dummy_tracer_opt[] = {
99	{ }
100	};
101
102	static int
103	dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
104	{
105	return 0;
106	}
107
108	/*
109	* To prevent the comm cache from being overwritten when no
110	* tracing is active, only save the comm when a trace event
111	* occurred.
112	*/
113	static DEFINE_PER_CPU(bool, trace_taskinfo_save);
114
115	/*
116	* Kill all tracing for good (never come back).
117	* It is initialized to 1 but will turn to zero if the initialization
118	* of the tracer is successful. But that is the only place that sets
119	* this back to zero.
120	*/
121	static int tracing_disabled = 1;
122
123	cpumask_var_t __read_mostly tracing_buffer_mask;
124
125	/*
126	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
127	*
128	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
129	* is set, then ftrace_dump is called. This will output the contents
130	* of the ftrace buffers to the console. This is very useful for
131	* capturing traces that lead to crashes and outputing it to a
132	* serial console.
133	*
134	* It is default off, but you can enable it with either specifying
135	* "ftrace_dump_on_oops" in the kernel command line, or setting
136	* /proc/sys/kernel/ftrace_dump_on_oops
137	* Set 1 if you want to dump buffers of all CPUs
138	* Set 2 if you want to dump the buffer of the CPU that triggered oops
139	*/
140
141	enum ftrace_dump_mode ftrace_dump_on_oops;
142
143	/* When set, tracing will stop when a WARN() is hit /
144	int __disable_trace_on_warning;
145
146	#ifdef CONFIG_TRACE_EVAL_MAP_FILE
147	/* Map of enums to their values, for "eval_map" file */
148	struct trace_eval_map_head {
149	struct module *mod;
150	unsigned long length;
151	};
152
153	union trace_eval_map_item;
154
155	struct trace_eval_map_tail {
156	/*
157	* "end" is first and points to NULL as it must be different
158	* than "mod" or "eval_string"
159	*/
160	union trace_eval_map_item *next;
161	const char end; / points to NULL */
162	};
163
164	static DEFINE_MUTEX(trace_eval_mutex);
165
166	/*
167	* The trace_eval_maps are saved in an array with two extra elements,
168	* one at the beginning, and one at the end. The beginning item contains
169	* the count of the saved maps (head.length), and the module they
170	* belong to if not built in (head.mod). The ending item contains a
171	* pointer to the next array of saved eval_map items.
172	*/
173	union trace_eval_map_item {
174	struct trace_eval_map map;
175	struct trace_eval_map_head head;
176	struct trace_eval_map_tail tail;
177	};
178
179	static union trace_eval_map_item *trace_eval_maps;
180	#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
181
182	int tracing_set_tracer(struct trace_array tr, const char buf);
183	static void ftrace_trace_userstack(struct trace_array *tr,
184	struct trace_buffer *buffer,
185	unsigned int trace_ctx);
186
187	#define MAX_TRACER_SIZE 100
188	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
189	static char *default_bootup_tracer;
190
191	static bool allocate_snapshot;
192	static bool snapshot_at_boot;
193
194	static char boot_instance_info[COMMAND_LINE_SIZE] __initdata;
195	static int boot_instance_index;
196
197	static char boot_snapshot_info[COMMAND_LINE_SIZE] __initdata;
198	static int boot_snapshot_index;
199
200	static int __init set_cmdline_ftrace(char *str)
201	{
202	strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
203	default_bootup_tracer = bootup_tracer_buf;
204	/* We are using ftrace early, expand it */
205	ring_buffer_expanded = true;
206	return 1;
207	}
208	__setup("ftrace=", set_cmdline_ftrace);
209
210	static int __init set_ftrace_dump_on_oops(char *str)
211	{
212	if (str++ != '=' \|\| !str \|\| !strcmp("1", str)) {
213	ftrace_dump_on_oops = DUMP_ALL;
214	return 1;
215	}
216
217	if (!strcmp("orig_cpu", str) \|\| !strcmp("2", str)) {
218	ftrace_dump_on_oops = DUMP_ORIG;
219	return 1;
220	}
221
222	return 0;
223	}
224	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
225
226	static int __init stop_trace_on_warning(char *str)
227	{
228	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
229	__disable_trace_on_warning = 1;
230	return 1;
231	}
232	__setup("traceoff_on_warning", stop_trace_on_warning);
233
234	static int __init boot_alloc_snapshot(char *str)
235	{
236	char *slot = boot_snapshot_info + boot_snapshot_index;
237	int left = sizeof(boot_snapshot_info) - boot_snapshot_index;
238	int ret;
239
240	if (str[0] == '=') {
241	str++;
242	if (strlen(str) >= left)
243	return -1;
244
245	ret = snprintf(slot, left, "%s\t", str);
246	boot_snapshot_index += ret;
247	} else {
248	allocate_snapshot = true;
249	/* We also need the main ring buffer expanded */
250	ring_buffer_expanded = true;
251	}
252	return 1;
253	}
254	__setup("alloc_snapshot", boot_alloc_snapshot);
255
256
257	static int __init boot_snapshot(char *str)
258	{
259	snapshot_at_boot = true;
260	boot_alloc_snapshot(str);
261	return 1;
262	}
263	__setup("ftrace_boot_snapshot", boot_snapshot);
264
265
266	static int __init boot_instance(char *str)
267	{
268	char *slot = boot_instance_info + boot_instance_index;
269	int left = sizeof(boot_instance_info) - boot_instance_index;
270	int ret;
271
272	if (strlen(str) >= left)
273	return -1;
274
275	ret = snprintf(slot, left, "%s\t", str);
276	boot_instance_index += ret;
277
278	return 1;
279	}
280	__setup("trace_instance=", boot_instance);
281
282
283	static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
284
285	static int __init set_trace_boot_options(char *str)
286	{
287	strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
288	return 1;
289	}
290	__setup("trace_options=", set_trace_boot_options);
291
292	static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
293	static char *trace_boot_clock __initdata;
294
295	static int __init set_trace_boot_clock(char *str)
296	{
297	strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
298	trace_boot_clock = trace_boot_clock_buf;
299	return 1;
300	}
301	__setup("trace_clock=", set_trace_boot_clock);
302
303	static int __init set_tracepoint_printk(char *str)
304	{
305	/* Ignore the "tp_printk_stop_on_boot" param */
306	if (*str == '_')
307	return 0;
308
309	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
310	tracepoint_printk = 1;
311	return 1;
312	}
313	__setup("tp_printk", set_tracepoint_printk);
314
315	static int __init set_tracepoint_printk_stop(char *str)
316	{
317	tracepoint_printk_stop_on_boot = true;
318	return 1;
319	}
320	__setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);
321
322	unsigned long long ns2usecs(u64 nsec)
323	{
324	nsec += 500;
325	do_div(nsec, 1000);
326	return nsec;
327	}
328
329	static void
330	trace_process_export(struct trace_export *export,
331	struct ring_buffer_event *event, int flag)
332	{
333	struct trace_entry *entry;
334	unsigned int size = 0;
335
336	if (export->flags & flag) {
337	entry = ring_buffer_event_data(event);
338	size = ring_buffer_event_length(event);
339	export->write(export, entry, size);
340	}
341	}
342
343	static DEFINE_MUTEX(ftrace_export_lock);
344
345	static struct trace_export __rcu *ftrace_exports_list __read_mostly;
346
347	static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
348	static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
349	static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
350
351	static inline void ftrace_exports_enable(struct trace_export *export)
352	{
353	if (export->flags & TRACE_EXPORT_FUNCTION)
354	static_branch_inc(&trace_function_exports_enabled);
355
356	if (export->flags & TRACE_EXPORT_EVENT)
357	static_branch_inc(&trace_event_exports_enabled);
358
359	if (export->flags & TRACE_EXPORT_MARKER)
360	static_branch_inc(&trace_marker_exports_enabled);
361	}
362
363	static inline void ftrace_exports_disable(struct trace_export *export)
364	{
365	if (export->flags & TRACE_EXPORT_FUNCTION)
366	static_branch_dec(&trace_function_exports_enabled);
367
368	if (export->flags & TRACE_EXPORT_EVENT)
369	static_branch_dec(&trace_event_exports_enabled);
370
371	if (export->flags & TRACE_EXPORT_MARKER)
372	static_branch_dec(&trace_marker_exports_enabled);
373	}
374
375	static void ftrace_exports(struct ring_buffer_event *event, int flag)
376	{
377	struct trace_export *export;
378
379	preempt_disable_notrace();
380
381	export = rcu_dereference_raw_check(ftrace_exports_list);
382	while (export) {
383	trace_process_export(export, event, flag);
384	export = rcu_dereference_raw_check(export->next);
385	}
386
387	preempt_enable_notrace();
388	}
389
390	static inline void
391	add_trace_export(struct trace_export *list, struct trace_export export)
392	{
393	rcu_assign_pointer(export->next, *list);
394	/*
395	* We are entering export into the list but another
396	* CPU might be walking that list. We need to make sure
397	* the export->next pointer is valid before another CPU sees
398	* the export pointer included into the list.
399	*/
400	rcu_assign_pointer(*list, export);
401	}
402
403	static inline int
404	rm_trace_export(struct trace_export *list, struct trace_export export)
405	{
406	struct trace_export **p;
407
408	for (p = list; p != NULL; p = &(p)->next)
409	if (*p == export)
410	break;
411
412	if (*p != export)
413	return -1;
414
415	rcu_assign_pointer(p, (p)->next);
416
417	return 0;
418	}
419
420	static inline void
421	add_ftrace_export(struct trace_export *list, struct trace_export export)
422	{
423	ftrace_exports_enable(export);
424
425	add_trace_export(list, export);
426	}
427
428	static inline int
429	rm_ftrace_export(struct trace_export *list, struct trace_export export)
430	{
431	int ret;
432
433	ret = rm_trace_export(list, export);
434	ftrace_exports_disable(export);
435
436	return ret;
437	}
438
439	int register_ftrace_export(struct trace_export *export)
440	{
441	if (WARN_ON_ONCE(!export->write))
442	return -1;
443
444	mutex_lock(&ftrace_export_lock);
445
446	add_ftrace_export(&ftrace_exports_list, export);
447
448	mutex_unlock(&ftrace_export_lock);
449
450	return 0;
451	}
452	EXPORT_SYMBOL_GPL(register_ftrace_export);
453
454	int unregister_ftrace_export(struct trace_export *export)
455	{
456	int ret;
457
458	mutex_lock(&ftrace_export_lock);
459
460	ret = rm_ftrace_export(&ftrace_exports_list, export);
461
462	mutex_unlock(&ftrace_export_lock);
463
464	return ret;
465	}
466	EXPORT_SYMBOL_GPL(unregister_ftrace_export);
467
468	/* trace_flags holds trace_options default values */
469	#define TRACE_DEFAULT_FLAGS \
470	(FUNCTION_DEFAULT_FLAGS \| \
471	TRACE_ITER_PRINT_PARENT \| TRACE_ITER_PRINTK \| \
472	TRACE_ITER_ANNOTATE \| TRACE_ITER_CONTEXT_INFO \| \
473	TRACE_ITER_RECORD_CMD \| TRACE_ITER_OVERWRITE \| \
474	TRACE_ITER_IRQ_INFO \| TRACE_ITER_MARKERS \| \
475	TRACE_ITER_HASH_PTR)
476
477	/* trace_options that are only supported by global_trace */
478	#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK \| \
479	TRACE_ITER_PRINTK_MSGONLY \| TRACE_ITER_RECORD_CMD)
480
481	/* trace_flags that are default zero for instances */
482	#define ZEROED_TRACE_FLAGS \
483	(TRACE_ITER_EVENT_FORK \| TRACE_ITER_FUNC_FORK)
484
485	/*
486	* The global_trace is the descriptor that holds the top-level tracing
487	* buffers for the live tracing.
488	*/
489	static struct trace_array global_trace = {
490	.trace_flags = TRACE_DEFAULT_FLAGS,
491	};
492
493	LIST_HEAD(ftrace_trace_arrays);
494
495	int trace_array_get(struct trace_array *this_tr)
496	{
497	struct trace_array *tr;
498	int ret = -ENODEV;
499
500	mutex_lock(&trace_types_lock);
501	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
502	if (tr == this_tr) {
503	tr->ref++;
504	ret = 0;
505	break;
506	}
507	}
508	mutex_unlock(&trace_types_lock);
509
510	return ret;
511	}
512
513	static void __trace_array_put(struct trace_array *this_tr)
514	{
515	WARN_ON(!this_tr->ref);
516	this_tr->ref--;
517	}
518
519	/**
520	* trace_array_put - Decrement the reference counter for this trace array.
521	* @this_tr : pointer to the trace array
522	*
523	* NOTE: Use this when we no longer need the trace array returned by
524	* trace_array_get_by_name(). This ensures the trace array can be later
525	* destroyed.
526	*
527	*/
528	void trace_array_put(struct trace_array *this_tr)
529	{
530	if (!this_tr)
531	return;
532
533	mutex_lock(&trace_types_lock);
534	__trace_array_put(this_tr);
535	mutex_unlock(&trace_types_lock);
536	}
537	EXPORT_SYMBOL_GPL(trace_array_put);
538
539	int tracing_check_open_get_tr(struct trace_array *tr)
540	{
541	int ret;
542
543	ret = security_locked_down(LOCKDOWN_TRACEFS);
544	if (ret)
545	return ret;
546
547	if (tracing_disabled)
548	return -ENODEV;
549
550	if (tr && trace_array_get(tr) < 0)
551	return -ENODEV;
552
553	return 0;
554	}
555
556	int call_filter_check_discard(struct trace_event_call call, void rec,
557	struct trace_buffer *buffer,
558	struct ring_buffer_event *event)
559	{
560	if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
561	!filter_match_preds(call->filter, rec)) {
562	__trace_event_discard_commit(buffer, event);
563	return 1;
564	}
565
566	return 0;
567	}
568
569	/**
570	* trace_find_filtered_pid - check if a pid exists in a filtered_pid list
571	* @filtered_pids: The list of pids to check
572	* @search_pid: The PID to find in @filtered_pids
573	*
574	* Returns true if @search_pid is found in @filtered_pids, and false otherwise.
575	*/
576	bool
577	trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
578	{
579	return trace_pid_list_is_set(filtered_pids, search_pid);
580	}
581
582	/**
583	* trace_ignore_this_task - should a task be ignored for tracing
584	* @filtered_pids: The list of pids to check
585	* @filtered_no_pids: The list of pids not to be traced
586	* @task: The task that should be ignored if not filtered
587	*
588	* Checks if @task should be traced or not from @filtered_pids.
589	* Returns true if @task should NOT be traced.
590	* Returns false if @task should be traced.
591	*/
592	bool
593	trace_ignore_this_task(struct trace_pid_list *filtered_pids,
594	struct trace_pid_list *filtered_no_pids,
595	struct task_struct *task)
596	{
597	/*
598	* If filtered_no_pids is not empty, and the task's pid is listed
599	* in filtered_no_pids, then return true.
600	* Otherwise, if filtered_pids is empty, that means we can
601	* trace all tasks. If it has content, then only trace pids
602	* within filtered_pids.
603	*/
604
605	return (filtered_pids &&
606	!trace_find_filtered_pid(filtered_pids, task->pid)) \|\|
607	(filtered_no_pids &&
608	trace_find_filtered_pid(filtered_no_pids, task->pid));
609	}
610
611	/**
612	* trace_filter_add_remove_task - Add or remove a task from a pid_list
613	* @pid_list: The list to modify
614	* @self: The current task for fork or NULL for exit
615	* @task: The task to add or remove
616	*
617	* If adding a task, if @self is defined, the task is only added if @self
618	* is also included in @pid_list. This happens on fork and tasks should
619	* only be added when the parent is listed. If @self is NULL, then the
620	* @task pid will be removed from the list, which would happen on exit
621	* of a task.
622	*/
623	void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
624	struct task_struct *self,
625	struct task_struct *task)
626	{
627	if (!pid_list)
628	return;
629
630	/* For forks, we only add if the forking task is listed */
631	if (self) {
632	if (!trace_find_filtered_pid(pid_list, self->pid))
633	return;
634	}
635
636	/* "self" is set for forks, and NULL for exits */
637	if (self)
638	trace_pid_list_set(pid_list, task->pid);
639	else
640	trace_pid_list_clear(pid_list, task->pid);
641	}
642
643	/**
644	* trace_pid_next - Used for seq_file to get to the next pid of a pid_list
645	* @pid_list: The pid list to show
646	* @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
647	* @pos: The position of the file
648	*
649	* This is used by the seq_file "next" operation to iterate the pids
650	* listed in a trace_pid_list structure.
651	*
652	* Returns the pid+1 as we want to display pid of zero, but NULL would
653	* stop the iteration.
654	*/
655	void trace_pid_next(struct trace_pid_list pid_list, void v, loff_t pos)
656	{
657	long pid = (unsigned long)v;
658	unsigned int next;
659
660	(*pos)++;
661
662	/* pid already is +1 of the actual previous bit */
663	if (trace_pid_list_next(pid_list, pid, &next) < 0)
664	return NULL;
665
666	pid = next;
667
668	/* Return pid + 1 to allow zero to be represented */
669	return (void *)(pid + 1);
670	}
671
672	/**
673	* trace_pid_start - Used for seq_file to start reading pid lists
674	* @pid_list: The pid list to show
675	* @pos: The position of the file
676	*
677	* This is used by seq_file "start" operation to start the iteration
678	* of listing pids.
679	*
680	* Returns the pid+1 as we want to display pid of zero, but NULL would
681	* stop the iteration.
682	*/
683	void trace_pid_start(struct trace_pid_list pid_list, loff_t *pos)
684	{
685	unsigned long pid;
686	unsigned int first;
687	loff_t l = 0;
688
689	if (trace_pid_list_first(pid_list, &first) < 0)
690	return NULL;
691
692	pid = first;
693
694	/* Return pid + 1 so that zero can be the exit value */
695	for (pid++; pid && l < *pos;
696	pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
697	;
698	return (void *)pid;
699	}
700
701	/**
702	* trace_pid_show - show the current pid in seq_file processing
703	* @m: The seq_file structure to write into
704	* @v: A void pointer of the pid (+1) value to display
705	*
706	* Can be directly used by seq_file operations to display the current
707	* pid value.
708	*/
709	int trace_pid_show(struct seq_file m, void v)
710	{
711	unsigned long pid = (unsigned long)v - 1;
712
713	seq_printf(m, "%lu\n", pid);
714	return 0;
715	}
716
717	/* 128 should be much more than enough */
718	#define PID_BUF_SIZE 127
719
720	int trace_pid_write(struct trace_pid_list *filtered_pids,
721	struct trace_pid_list **new_pid_list,
722	const char __user *ubuf, size_t cnt)
723	{
724	struct trace_pid_list *pid_list;
725	struct trace_parser parser;
726	unsigned long val;
727	int nr_pids = 0;
728	ssize_t read = 0;
729	ssize_t ret;
730	loff_t pos;
731	pid_t pid;
732
733	if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
734	return -ENOMEM;
735
736	/*
737	* Always recreate a new array. The write is an all or nothing
738	* operation. Always create a new array when adding new pids by
739	* the user. If the operation fails, then the current list is
740	* not modified.
741	*/
742	pid_list = trace_pid_list_alloc();
743	if (!pid_list) {
744	trace_parser_put(&parser);
745	return -ENOMEM;
746	}
747
748	if (filtered_pids) {
749	/* copy the current bits to the new max */
750	ret = trace_pid_list_first(filtered_pids, &pid);
751	while (!ret) {
752	trace_pid_list_set(pid_list, pid);
753	ret = trace_pid_list_next(filtered_pids, pid + 1, &pid);
754	nr_pids++;
755	}
756	}
757
758	ret = 0;
759	while (cnt > 0) {
760
761	pos = 0;
762
763	ret = trace_get_user(&parser, ubuf, cnt, &pos);
764	if (ret < 0)
765	break;
766
767	read += ret;
768	ubuf += ret;
769	cnt -= ret;
770
771	if (!trace_parser_loaded(&parser))
772	break;
773
774	ret = -EINVAL;
775	if (kstrtoul(parser.buffer, 0, &val))
776	break;
777
778	pid = (pid_t)val;
779
780	if (trace_pid_list_set(pid_list, pid) < 0) {
781	ret = -1;
782	break;
783	}
784	nr_pids++;
785
786	trace_parser_clear(&parser);
787	ret = 0;
788	}
789	trace_parser_put(&parser);
790
791	if (ret < 0) {
792	trace_pid_list_free(pid_list);
793	return ret;
794	}
795
796	if (!nr_pids) {
797	/* Cleared the list of pids */
798	trace_pid_list_free(pid_list);
799	pid_list = NULL;
800	}
801
802	*new_pid_list = pid_list;
803
804	return read;
805	}
806
807	static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
808	{
809	u64 ts;
810
811	/* Early boot up does not have a buffer yet */
812	if (!buf->buffer)
813	return trace_clock_local();
814
815	ts = ring_buffer_time_stamp(buf->buffer);
816	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
817
818	return ts;
819	}
820
821	u64 ftrace_now(int cpu)
822	{
823	return buffer_ftrace_now(&global_trace.array_buffer, cpu);
824	}
825
826	/**
827	* tracing_is_enabled - Show if global_trace has been enabled
828	*
829	* Shows if the global trace has been enabled or not. It uses the
830	* mirror flag "buffer_disabled" to be used in fast paths such as for
831	* the irqsoff tracer. But it may be inaccurate due to races. If you
832	* need to know the accurate state, use tracing_is_on() which is a little
833	* slower, but accurate.
834	*/
835	int tracing_is_enabled(void)
836	{
837	/*
838	* For quick access (irqsoff uses this in fast path), just
839	* return the mirror variable of the state of the ring buffer.
840	* It's a little racy, but we don't really care.
841	*/
842	smp_rmb();
843	return !global_trace.buffer_disabled;
844	}
845
846	/*
847	* trace_buf_size is the size in bytes that is allocated
848	* for a buffer. Note, the number of bytes is always rounded
849	* to page size.
850	*
851	* This number is purposely set to a low number of 16384.
852	* If the dump on oops happens, it will be much appreciated
853	* to not have to wait for all that output. Anyway this can be
854	* boot time and run time configurable.
855	*/
856	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
857
858	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
859
860	/* trace_types holds a link list of available tracers. */
861	static struct tracer *trace_types __read_mostly;
862
863	/*
864	* trace_types_lock is used to protect the trace_types list.
865	*/
866	DEFINE_MUTEX(trace_types_lock);
867
868	/*
869	* serialize the access of the ring buffer
870	*
871	* ring buffer serializes readers, but it is low level protection.
872	* The validity of the events (which returns by ring_buffer_peek() ..etc)
873	* are not protected by ring buffer.
874	*
875	* The content of events may become garbage if we allow other process consumes
876	* these events concurrently:
877	* A) the page of the consumed events may become a normal page
878	* (not reader page) in ring buffer, and this page will be rewritten
879	* by events producer.
880	* B) The page of the consumed events may become a page for splice_read,
881	* and this page will be returned to system.
882	*
883	* These primitives allow multi process access to different cpu ring buffer
884	* concurrently.
885	*
886	* These primitives don't distinguish read-only and read-consume access.
887	* Multi read-only access are also serialized.
888	*/
889
890	#ifdef CONFIG_SMP
891	static DECLARE_RWSEM(all_cpu_access_lock);
892	static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
893
894	static inline void trace_access_lock(int cpu)
895	{
896	if (cpu == RING_BUFFER_ALL_CPUS) {
897	/* gain it for accessing the whole ring buffer. */
898	down_write(&all_cpu_access_lock);
899	} else {
900	/* gain it for accessing a cpu ring buffer. */
901
902	/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
903	down_read(&all_cpu_access_lock);
904
905	/* Secondly block other access to this @cpu ring buffer. */
906	mutex_lock(&per_cpu(cpu_access_lock, cpu));
907	}
908	}
909
910	static inline void trace_access_unlock(int cpu)
911	{
912	if (cpu == RING_BUFFER_ALL_CPUS) {
913	up_write(&all_cpu_access_lock);
914	} else {
915	mutex_unlock(&per_cpu(cpu_access_lock, cpu));
916	up_read(&all_cpu_access_lock);
917	}
918	}
919
920	static inline void trace_access_lock_init(void)
921	{
922	int cpu;
923
924	for_each_possible_cpu(cpu)
925	mutex_init(&per_cpu(cpu_access_lock, cpu));
926	}
927
928	#else
929
930	static DEFINE_MUTEX(access_lock);
931
932	static inline void trace_access_lock(int cpu)
933	{
934	(void)cpu;
935	mutex_lock(&access_lock);
936	}
937
938	static inline void trace_access_unlock(int cpu)
939	{
940	(void)cpu;
941	mutex_unlock(&access_lock);
942	}
943
944	static inline void trace_access_lock_init(void)
945	{
946	}
947
948	#endif
949
950	#ifdef CONFIG_STACKTRACE
951	static void __ftrace_trace_stack(struct trace_buffer *buffer,
952	unsigned int trace_ctx,
953	int skip, struct pt_regs *regs);
954	static inline void ftrace_trace_stack(struct trace_array *tr,
955	struct trace_buffer *buffer,
956	unsigned int trace_ctx,
957	int skip, struct pt_regs *regs);
958
959	#else
960	static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
961	unsigned int trace_ctx,
962	int skip, struct pt_regs *regs)
963	{
964	}
965	static inline void ftrace_trace_stack(struct trace_array *tr,
966	struct trace_buffer *buffer,
967	unsigned long trace_ctx,
968	int skip, struct pt_regs *regs)
969	{
970	}
971
972	#endif
973
974	static __always_inline void
975	trace_event_setup(struct ring_buffer_event *event,
976	int type, unsigned int trace_ctx)
977	{
978	struct trace_entry *ent = ring_buffer_event_data(event);
979
980	tracing_generic_entry_update(ent, type, trace_ctx);
981	}
982
983	static __always_inline struct ring_buffer_event *
984	__trace_buffer_lock_reserve(struct trace_buffer *buffer,
985	int type,
986	unsigned long len,
987	unsigned int trace_ctx)
988	{
989	struct ring_buffer_event *event;
990
991	event = ring_buffer_lock_reserve(buffer, len);
992	if (event != NULL)
993	trace_event_setup(event, type, trace_ctx);
994
995	return event;
996	}
997
998	void tracer_tracing_on(struct trace_array *tr)
999	{
1000	if (tr->array_buffer.buffer)
1001	ring_buffer_record_on(tr->array_buffer.buffer);
1002	/*
1003	* This flag is looked at when buffers haven't been allocated
1004	* yet, or by some tracers (like irqsoff), that just want to
1005	* know if the ring buffer has been disabled, but it can handle
1006	* races of where it gets disabled but we still do a record.
1007	* As the check is in the fast path of the tracers, it is more
1008	* important to be fast than accurate.
1009	*/
1010	tr->buffer_disabled = 0;
1011	/* Make the flag seen by readers */
1012	smp_wmb();
1013	}
1014
1015	/**
1016	* tracing_on - enable tracing buffers
1017	*
1018	* This function enables tracing buffers that may have been
1019	* disabled with tracing_off.
1020	*/
1021	void tracing_on(void)
1022	{
1023	tracer_tracing_on(&global_trace);
1024	}
1025	EXPORT_SYMBOL_GPL(tracing_on);
1026
1027
1028	static __always_inline void
1029	__buffer_unlock_commit(struct trace_buffer buffer, struct ring_buffer_event event)
1030	{
1031	__this_cpu_write(trace_taskinfo_save, true);
1032
1033	/* If this is the temp buffer, we need to commit fully */
1034	if (this_cpu_read(trace_buffered_event) == event) {
1035	/* Length is in event->array[0] */
1036	ring_buffer_write(buffer, event->array[0], &event->array[1]);
1037	/* Release the temp buffer */
1038	this_cpu_dec(trace_buffered_event_cnt);
1039	/* ring_buffer_unlock_commit() enables preemption */
1040	preempt_enable_notrace();
1041	} else
1042	ring_buffer_unlock_commit(buffer);
1043	}
1044
1045	int __trace_array_puts(struct trace_array *tr, unsigned long ip,
1046	const char *str, int size)
1047	{
1048	struct ring_buffer_event *event;
1049	struct trace_buffer *buffer;
1050	struct print_entry *entry;
1051	unsigned int trace_ctx;
1052	int alloc;
1053
1054	if (!(tr->trace_flags & TRACE_ITER_PRINTK))
1055	return 0;
1056
1057	if (unlikely(tracing_selftest_running && tr == &global_trace))
1058	return 0;
1059
1060	if (unlikely(tracing_disabled))
1061	return 0;
1062
1063	alloc = sizeof(entry) + size + 2; / possible \n added */
1064
1065	trace_ctx = tracing_gen_ctx();
1066	buffer = tr->array_buffer.buffer;
1067	ring_buffer_nest_start(buffer);
1068	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
1069	trace_ctx);
1070	if (!event) {
1071	size = 0;
1072	goto out;
1073	}
1074
1075	entry = ring_buffer_event_data(event);
1076	entry->ip = ip;
1077
1078	memcpy(&entry->buf, str, size);
1079
1080	/* Add a newline if necessary */
1081	if (entry->buf[size - 1] != '\n') {
1082	entry->buf[size] = '\n';
1083	entry->buf[size + 1] = '\0';
1084	} else
1085	entry->buf[size] = '\0';
1086
1087	__buffer_unlock_commit(buffer, event);
1088	ftrace_trace_stack(tr, buffer, trace_ctx, 4, NULL);
1089	out:
1090	ring_buffer_nest_end(buffer);
1091	return size;
1092	}
1093	EXPORT_SYMBOL_GPL(__trace_array_puts);
1094
1095	/**
1096	* __trace_puts - write a constant string into the trace buffer.
1097	* @ip: The address of the caller
1098	* @str: The constant string to write
1099	* @size: The size of the string.
1100	*/
1101	int __trace_puts(unsigned long ip, const char *str, int size)
1102	{
1103	return __trace_array_puts(&global_trace, ip, str, size);
1104	}
1105	EXPORT_SYMBOL_GPL(__trace_puts);
1106
1107	/**
1108	* __trace_bputs - write the pointer to a constant string into trace buffer
1109	* @ip: The address of the caller
1110	* @str: The constant string to write to the buffer to
1111	*/
1112	int __trace_bputs(unsigned long ip, const char *str)
1113	{
1114	struct ring_buffer_event *event;
1115	struct trace_buffer *buffer;
1116	struct bputs_entry *entry;
1117	unsigned int trace_ctx;
1118	int size = sizeof(struct bputs_entry);
1119	int ret = 0;
1120
1121	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
1122	return 0;
1123
1124	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
1125	return 0;
1126
1127	trace_ctx = tracing_gen_ctx();
1128	buffer = global_trace.array_buffer.buffer;
1129
1130	ring_buffer_nest_start(buffer);
1131	event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
1132	trace_ctx);
1133	if (!event)
1134	goto out;
1135
1136	entry = ring_buffer_event_data(event);
1137	entry->ip = ip;
1138	entry->str = str;
1139
1140	__buffer_unlock_commit(buffer, event);
1141	ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
1142
1143	ret = 1;
1144	out:
1145	ring_buffer_nest_end(buffer);
1146	return ret;
1147	}
1148	EXPORT_SYMBOL_GPL(__trace_bputs);
1149
1150	#ifdef CONFIG_TRACER_SNAPSHOT
1151	static void tracing_snapshot_instance_cond(struct trace_array *tr,
1152	void *cond_data)
1153	{
1154	struct tracer *tracer = tr->current_trace;
1155	unsigned long flags;
1156
1157	if (in_nmi()) {
1158	trace_array_puts(tr, "* SNAPSHOT CALLED FROM NMI CONTEXT *\n");
1159	trace_array_puts(tr, "* snapshot is being ignored *\n");
1160	return;
1161	}
1162
1163	if (!tr->allocated_snapshot) {
1164	trace_array_puts(tr, "* SNAPSHOT NOT ALLOCATED *\n");
1165	trace_array_puts(tr, "* stopping trace here! *\n");
1166	tracer_tracing_off(tr);
1167	return;
1168	}
1169
1170	/* Note, snapshot can not be used when the tracer uses it */
1171	if (tracer->use_max_tr) {
1172	trace_array_puts(tr, "* LATENCY TRACER ACTIVE *\n");
1173	trace_array_puts(tr, "* Can not use snapshot (sorry) *\n");
1174	return;
1175	}
1176
1177	local_irq_save(flags);
1178	update_max_tr(tr, current, smp_processor_id(), cond_data);
1179	local_irq_restore(flags);
1180	}
1181
1182	void tracing_snapshot_instance(struct trace_array *tr)
1183	{
1184	tracing_snapshot_instance_cond(tr, NULL);
1185	}
1186
1187	/**
1188	* tracing_snapshot - take a snapshot of the current buffer.
1189	*
1190	* This causes a swap between the snapshot buffer and the current live
1191	* tracing buffer. You can use this to take snapshots of the live
1192	* trace when some condition is triggered, but continue to trace.
1193	*
1194	* Note, make sure to allocate the snapshot with either
1195	* a tracing_snapshot_alloc(), or by doing it manually
1196	* with: echo 1 > /sys/kernel/tracing/snapshot
1197	*
1198	* If the snapshot buffer is not allocated, it will stop tracing.
1199	* Basically making a permanent snapshot.
1200	*/
1201	void tracing_snapshot(void)
1202	{
1203	struct trace_array *tr = &global_trace;
1204
1205	tracing_snapshot_instance(tr);
1206	}
1207	EXPORT_SYMBOL_GPL(tracing_snapshot);
1208
1209	/**
1210	* tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1211	* @tr: The tracing instance to snapshot
1212	* @cond_data: The data to be tested conditionally, and possibly saved
1213	*
1214	* This is the same as tracing_snapshot() except that the snapshot is
1215	* conditional - the snapshot will only happen if the
1216	* cond_snapshot.update() implementation receiving the cond_data
1217	* returns true, which means that the trace array's cond_snapshot
1218	* update() operation used the cond_data to determine whether the
1219	* snapshot should be taken, and if it was, presumably saved it along
1220	* with the snapshot.
1221	*/
1222	void tracing_snapshot_cond(struct trace_array tr, void cond_data)
1223	{
1224	tracing_snapshot_instance_cond(tr, cond_data);
1225	}
1226	EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1227
1228	/**
1229	* tracing_cond_snapshot_data - get the user data associated with a snapshot
1230	* @tr: The tracing instance
1231	*
1232	* When the user enables a conditional snapshot using
1233	* tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1234	* with the snapshot. This accessor is used to retrieve it.
1235	*
1236	* Should not be called from cond_snapshot.update(), since it takes
1237	* the tr->max_lock lock, which the code calling
1238	* cond_snapshot.update() has already done.
1239	*
1240	* Returns the cond_data associated with the trace array's snapshot.
1241	*/
1242	void tracing_cond_snapshot_data(struct trace_array tr)
1243	{
1244	void *cond_data = NULL;
1245
1246	local_irq_disable();
1247	arch_spin_lock(&tr->max_lock);
1248
1249	if (tr->cond_snapshot)
1250	cond_data = tr->cond_snapshot->cond_data;
1251
1252	arch_spin_unlock(&tr->max_lock);
1253	local_irq_enable();
1254
1255	return cond_data;
1256	}
1257	EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1258
1259	static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1260	struct array_buffer *size_buf, int cpu_id);
1261	static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1262
1263	int tracing_alloc_snapshot_instance(struct trace_array *tr)
1264	{
1265	int ret;
1266
1267	if (!tr->allocated_snapshot) {
1268
1269	/* allocate spare buffer */
1270	ret = resize_buffer_duplicate_size(&tr->max_buffer,
1271	&tr->array_buffer, RING_BUFFER_ALL_CPUS);
1272	if (ret < 0)
1273	return ret;
1274
1275	tr->allocated_snapshot = true;
1276	}
1277
1278	return 0;
1279	}
1280
1281	static void free_snapshot(struct trace_array *tr)
1282	{
1283	/*
1284	* We don't free the ring buffer. instead, resize it because
1285	* The max_tr ring buffer has some state (e.g. ring->clock) and
1286	* we want preserve it.
1287	*/
1288	ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1289	set_buffer_entries(&tr->max_buffer, 1);
1290	tracing_reset_online_cpus(&tr->max_buffer);
1291	tr->allocated_snapshot = false;
1292	}
1293
1294	/**
1295	* tracing_alloc_snapshot - allocate snapshot buffer.
1296	*
1297	* This only allocates the snapshot buffer if it isn't already
1298	* allocated - it doesn't also take a snapshot.
1299	*
1300	* This is meant to be used in cases where the snapshot buffer needs
1301	* to be set up for events that can't sleep but need to be able to
1302	* trigger a snapshot.
1303	*/
1304	int tracing_alloc_snapshot(void)
1305	{
1306	struct trace_array *tr = &global_trace;
1307	int ret;
1308
1309	ret = tracing_alloc_snapshot_instance(tr);
1310	WARN_ON(ret < 0);
1311
1312	return ret;
1313	}
1314	EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1315
1316	/**
1317	* tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1318	*
1319	* This is similar to tracing_snapshot(), but it will allocate the
1320	* snapshot buffer if it isn't already allocated. Use this only
1321	* where it is safe to sleep, as the allocation may sleep.
1322	*
1323	* This causes a swap between the snapshot buffer and the current live
1324	* tracing buffer. You can use this to take snapshots of the live
1325	* trace when some condition is triggered, but continue to trace.
1326	*/
1327	void tracing_snapshot_alloc(void)
1328	{
1329	int ret;
1330
1331	ret = tracing_alloc_snapshot();
1332	if (ret < 0)
1333	return;
1334
1335	tracing_snapshot();
1336	}
1337	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1338
1339	/**
1340	* tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1341	* @tr: The tracing instance
1342	* @cond_data: User data to associate with the snapshot
1343	* @update: Implementation of the cond_snapshot update function
1344	*
1345	* Check whether the conditional snapshot for the given instance has
1346	* already been enabled, or if the current tracer is already using a
1347	* snapshot; if so, return -EBUSY, else create a cond_snapshot and
1348	* save the cond_data and update function inside.
1349	*
1350	* Returns 0 if successful, error otherwise.
1351	*/
1352	int tracing_snapshot_cond_enable(struct trace_array tr, void cond_data,
1353	cond_update_fn_t update)
1354	{
1355	struct cond_snapshot *cond_snapshot;
1356	int ret = 0;
1357
1358	cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1359	if (!cond_snapshot)
1360	return -ENOMEM;
1361
1362	cond_snapshot->cond_data = cond_data;
1363	cond_snapshot->update = update;
1364
1365	mutex_lock(&trace_types_lock);
1366
1367	ret = tracing_alloc_snapshot_instance(tr);
1368	if (ret)
1369	goto fail_unlock;
1370
1371	if (tr->current_trace->use_max_tr) {
1372	ret = -EBUSY;
1373	goto fail_unlock;
1374	}
1375
1376	/*
1377	* The cond_snapshot can only change to NULL without the
1378	* trace_types_lock. We don't care if we race with it going
1379	* to NULL, but we want to make sure that it's not set to
1380	* something other than NULL when we get here, which we can
1381	* do safely with only holding the trace_types_lock and not
1382	* having to take the max_lock.
1383	*/
1384	if (tr->cond_snapshot) {
1385	ret = -EBUSY;
1386	goto fail_unlock;
1387	}
1388
1389	local_irq_disable();
1390	arch_spin_lock(&tr->max_lock);
1391	tr->cond_snapshot = cond_snapshot;
1392	arch_spin_unlock(&tr->max_lock);
1393	local_irq_enable();
1394
1395	mutex_unlock(&trace_types_lock);
1396
1397	return ret;
1398
1399	fail_unlock:
1400	mutex_unlock(&trace_types_lock);
1401	kfree(cond_snapshot);
1402	return ret;
1403	}
1404	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1405
1406	/**
1407	* tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1408	* @tr: The tracing instance
1409	*
1410	* Check whether the conditional snapshot for the given instance is
1411	* enabled; if so, free the cond_snapshot associated with it,
1412	* otherwise return -EINVAL.
1413	*
1414	* Returns 0 if successful, error otherwise.
1415	*/
1416	int tracing_snapshot_cond_disable(struct trace_array *tr)
1417	{
1418	int ret = 0;
1419
1420	local_irq_disable();
1421	arch_spin_lock(&tr->max_lock);
1422
1423	if (!tr->cond_snapshot)
1424	ret = -EINVAL;
1425	else {
1426	kfree(tr->cond_snapshot);
1427	tr->cond_snapshot = NULL;
1428	}
1429
1430	arch_spin_unlock(&tr->max_lock);
1431	local_irq_enable();
1432
1433	return ret;
1434	}
1435	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1436	#else
1437	void tracing_snapshot(void)
1438	{
1439	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1440	}
1441	EXPORT_SYMBOL_GPL(tracing_snapshot);
1442	void tracing_snapshot_cond(struct trace_array tr, void cond_data)
1443	{
1444	WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1445	}
1446	EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1447	int tracing_alloc_snapshot(void)
1448	{
1449	WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1450	return -ENODEV;
1451	}
1452	EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1453	void tracing_snapshot_alloc(void)
1454	{
1455	/* Give warning */
1456	tracing_snapshot();
1457	}
1458	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1459	void tracing_cond_snapshot_data(struct trace_array tr)
1460	{
1461	return NULL;
1462	}
1463	EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1464	int tracing_snapshot_cond_enable(struct trace_array tr, void cond_data, cond_update_fn_t update)
1465	{
1466	return -ENODEV;
1467	}
1468	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1469	int tracing_snapshot_cond_disable(struct trace_array *tr)
1470	{
1471	return false;
1472	}
1473	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1474	#define free_snapshot(tr) do { } while (0)
1475	#endif /* CONFIG_TRACER_SNAPSHOT */
1476
1477	void tracer_tracing_off(struct trace_array *tr)
1478	{
1479	if (tr->array_buffer.buffer)
1480	ring_buffer_record_off(tr->array_buffer.buffer);
1481	/*
1482	* This flag is looked at when buffers haven't been allocated
1483	* yet, or by some tracers (like irqsoff), that just want to
1484	* know if the ring buffer has been disabled, but it can handle
1485	* races of where it gets disabled but we still do a record.
1486	* As the check is in the fast path of the tracers, it is more
1487	* important to be fast than accurate.
1488	*/
1489	tr->buffer_disabled = 1;
1490	/* Make the flag seen by readers */
1491	smp_wmb();
1492	}
1493
1494	/**
1495	* tracing_off - turn off tracing buffers
1496	*
1497	* This function stops the tracing buffers from recording data.
1498	* It does not disable any overhead the tracers themselves may
1499	* be causing. This function simply causes all recording to
1500	* the ring buffers to fail.
1501	*/
1502	void tracing_off(void)
1503	{
1504	tracer_tracing_off(&global_trace);
1505	}
1506	EXPORT_SYMBOL_GPL(tracing_off);
1507
1508	void disable_trace_on_warning(void)
1509	{
1510	if (__disable_trace_on_warning) {
1511	trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
1512	"Disabling tracing due to warning\n");
1513	tracing_off();
1514	}
1515	}
1516
1517	/**
1518	* tracer_tracing_is_on - show real state of ring buffer enabled
1519	* @tr : the trace array to know if ring buffer is enabled
1520	*
1521	* Shows real state of the ring buffer if it is enabled or not.
1522	*/
1523	bool tracer_tracing_is_on(struct trace_array *tr)
1524	{
1525	if (tr->array_buffer.buffer)
1526	return ring_buffer_record_is_on(tr->array_buffer.buffer);
1527	return !tr->buffer_disabled;
1528	}
1529
1530	/**
1531	* tracing_is_on - show state of ring buffers enabled
1532	*/
1533	int tracing_is_on(void)
1534	{
1535	return tracer_tracing_is_on(&global_trace);
1536	}
1537	EXPORT_SYMBOL_GPL(tracing_is_on);
1538
1539	static int __init set_buf_size(char *str)
1540	{
1541	unsigned long buf_size;
1542
1543	if (!str)
1544	return 0;
1545	buf_size = memparse(str, &str);
1546	/*
1547	* nr_entries can not be zero and the startup
1548	* tests require some buffer space. Therefore
1549	* ensure we have at least 4096 bytes of buffer.
1550	*/
1551	trace_buf_size = max(4096UL, buf_size);
1552	return 1;
1553	}
1554	__setup("trace_buf_size=", set_buf_size);
1555
1556	static int __init set_tracing_thresh(char *str)
1557	{
1558	unsigned long threshold;
1559	int ret;
1560
1561	if (!str)
1562	return 0;
1563	ret = kstrtoul(str, 0, &threshold);
1564	if (ret < 0)
1565	return 0;
1566	tracing_thresh = threshold * 1000;
1567	return 1;
1568	}
1569	__setup("tracing_thresh=", set_tracing_thresh);
1570
1571	unsigned long nsecs_to_usecs(unsigned long nsecs)
1572	{
1573	return nsecs / 1000;
1574	}
1575
1576	/*
1577	* TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1578	* It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1579	* matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1580	* of strings in the order that the evals (enum) were defined.
1581	*/
1582	#undef C
1583	#define C(a, b) b
1584
1585	/* These must match the bit positions in trace_iterator_flags */
1586	static const char *trace_options[] = {
1587	TRACE_FLAGS
1588	NULL
1589	};
1590
1591	static struct {
1592	u64 (*func)(void);
1593	const char *name;
1594	int in_ns; /* is this clock in nanoseconds? */
1595	} trace_clocks[] = {
1596	{ trace_clock_local, "local", 1 },
1597	{ trace_clock_global, "global", 1 },
1598	{ trace_clock_counter, "counter", 0 },
1599	{ trace_clock_jiffies, "uptime", 0 },
1600	{ trace_clock, "perf", 1 },
1601	{ ktime_get_mono_fast_ns, "mono", 1 },
1602	{ ktime_get_raw_fast_ns, "mono_raw", 1 },
1603	{ ktime_get_boot_fast_ns, "boot", 1 },
1604	{ ktime_get_tai_fast_ns, "tai", 1 },
1605	ARCH_TRACE_CLOCKS
1606	};
1607
1608	bool trace_clock_in_ns(struct trace_array *tr)
1609	{
1610	if (trace_clocks[tr->clock_id].in_ns)
1611	return true;
1612
1613	return false;
1614	}
1615
1616	/*
1617	* trace_parser_get_init - gets the buffer for trace parser
1618	*/
1619	int trace_parser_get_init(struct trace_parser *parser, int size)
1620	{
1621	memset(parser, 0, sizeof(*parser));
1622
1623	parser->buffer = kmalloc(size, GFP_KERNEL);
1624	if (!parser->buffer)
1625	return 1;
1626
1627	parser->size = size;
1628	return 0;
1629	}
1630
1631	/*
1632	* trace_parser_put - frees the buffer for trace parser
1633	*/
1634	void trace_parser_put(struct trace_parser *parser)
1635	{
1636	kfree(parser->buffer);
1637	parser->buffer = NULL;
1638	}
1639
1640	/*
1641	* trace_get_user - reads the user input string separated by space
1642	* (matched by isspace(ch))
1643	*
1644	* For each string found the 'struct trace_parser' is updated,
1645	* and the function returns.
1646	*
1647	* Returns number of bytes read.
1648	*
1649	* See kernel/trace/trace.h for 'struct trace_parser' details.
1650	*/
1651	int trace_get_user(struct trace_parser parser, const char __user ubuf,
1652	size_t cnt, loff_t *ppos)
1653	{
1654	char ch;
1655	size_t read = 0;
1656	ssize_t ret;
1657
1658	if (!*ppos)
1659	trace_parser_clear(parser);
1660
1661	ret = get_user(ch, ubuf++);
1662	if (ret)
1663	goto out;
1664
1665	read++;
1666	cnt--;
1667
1668	/*
1669	* The parser is not finished with the last write,
1670	* continue reading the user input without skipping spaces.
1671	*/
1672	if (!parser->cont) {
1673	/* skip white space */
1674	while (cnt && isspace(ch)) {
1675	ret = get_user(ch, ubuf++);
1676	if (ret)
1677	goto out;
1678	read++;
1679	cnt--;
1680	}
1681
1682	parser->idx = 0;
1683
1684	/* only spaces were written */
1685	if (isspace(ch) \|\| !ch) {
1686	*ppos += read;
1687	ret = read;
1688	goto out;
1689	}
1690	}
1691
1692	/* read the non-space input */
1693	while (cnt && !isspace(ch) && ch) {
1694	if (parser->idx < parser->size - 1)
1695	parser->buffer[parser->idx++] = ch;
1696	else {
1697	ret = -EINVAL;
1698	goto out;
1699	}
1700	ret = get_user(ch, ubuf++);
1701	if (ret)
1702	goto out;
1703	read++;
1704	cnt--;
1705	}
1706
1707	/* We either got finished input or we have to wait for another call. */
1708	if (isspace(ch) \|\| !ch) {
1709	parser->buffer[parser->idx] = 0;
1710	parser->cont = false;
1711	} else if (parser->idx < parser->size - 1) {
1712	parser->cont = true;
1713	parser->buffer[parser->idx++] = ch;
1714	/* Make sure the parsed string always terminates with '\0'. */
1715	parser->buffer[parser->idx] = 0;
1716	} else {
1717	ret = -EINVAL;
1718	goto out;
1719	}
1720
1721	*ppos += read;
1722	ret = read;
1723
1724	out:
1725	return ret;
1726	}
1727
1728	/* TODO add a seq_buf_to_buffer() */
1729	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
1730	{
1731	int len;
1732
1733	if (trace_seq_used(s) <= s->seq.readpos)
1734	return -EBUSY;
1735
1736	len = trace_seq_used(s) - s->seq.readpos;
1737	if (cnt > len)
1738	cnt = len;
1739	memcpy(buf, s->buffer + s->seq.readpos, cnt);
1740
1741	s->seq.readpos += cnt;
1742	return cnt;
1743	}
1744
1745	unsigned long __read_mostly tracing_thresh;
1746
1747	#ifdef CONFIG_TRACER_MAX_TRACE
1748	static const struct file_operations tracing_max_lat_fops;
1749
1750	#ifdef LATENCY_FS_NOTIFY
1751
1752	static struct workqueue_struct *fsnotify_wq;
1753
1754	static void latency_fsnotify_workfn(struct work_struct *work)
1755	{
1756	struct trace_array *tr = container_of(work, struct trace_array,
1757	fsnotify_work);
1758	fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);
1759	}
1760
1761	static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1762	{
1763	struct trace_array *tr = container_of(iwork, struct trace_array,
1764	fsnotify_irqwork);
1765	queue_work(fsnotify_wq, &tr->fsnotify_work);
1766	}
1767
1768	static void trace_create_maxlat_file(struct trace_array *tr,
1769	struct dentry *d_tracer)
1770	{
1771	INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1772	init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1773	tr->d_max_latency = trace_create_file("tracing_max_latency",
1774	TRACE_MODE_WRITE,
1775	d_tracer, &tr->max_latency,
1776	&tracing_max_lat_fops);
1777	}
1778
1779	__init static int latency_fsnotify_init(void)
1780	{
1781	fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1782	WQ_UNBOUND \| WQ_HIGHPRI, 0);
1783	if (!fsnotify_wq) {
1784	pr_err("Unable to allocate tr_max_lat_wq\n");
1785	return -ENOMEM;
1786	}
1787	return 0;
1788	}
1789
1790	late_initcall_sync(latency_fsnotify_init);
1791
1792	void latency_fsnotify(struct trace_array *tr)
1793	{
1794	if (!fsnotify_wq)
1795	return;
1796	/*
1797	* We cannot call queue_work(&tr->fsnotify_work) from here because it's
1798	* possible that we are called from __schedule() or do_idle(), which
1799	* could cause a deadlock.
1800	*/
1801	irq_work_queue(&tr->fsnotify_irqwork);
1802	}
1803
1804	#else /* !LATENCY_FS_NOTIFY */
1805
1806	#define trace_create_maxlat_file(tr, d_tracer) \
1807	trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \
1808	d_tracer, &tr->max_latency, &tracing_max_lat_fops)
1809
1810	#endif
1811
1812	/*
1813	* Copy the new maximum trace into the separate maximum-trace
1814	* structure. (this way the maximum trace is permanently saved,
1815	* for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1816	*/
1817	static void
1818	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
1819	{
1820	struct array_buffer *trace_buf = &tr->array_buffer;
1821	struct array_buffer *max_buf = &tr->max_buffer;
1822	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1823	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1824
1825	max_buf->cpu = cpu;
1826	max_buf->time_start = data->preempt_timestamp;
1827
1828	max_data->saved_latency = tr->max_latency;
1829	max_data->critical_start = data->critical_start;
1830	max_data->critical_end = data->critical_end;
1831
1832	strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1833	max_data->pid = tsk->pid;
1834	/*
1835	* If tsk == current, then use current_uid(), as that does not use
1836	* RCU. The irq tracer can be called out of RCU scope.
1837	*/
1838	if (tsk == current)
1839	max_data->uid = current_uid();
1840	else
1841	max_data->uid = task_uid(tsk);
1842
1843	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1844	max_data->policy = tsk->policy;
1845	max_data->rt_priority = tsk->rt_priority;
1846
1847	/* record this tasks comm */
1848	tracing_record_cmdline(tsk);
1849	latency_fsnotify(tr);
1850	}
1851
1852	/**
1853	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
1854	* @tr: tracer
1855	* @tsk: the task with the latency
1856	* @cpu: The cpu that initiated the trace.
1857	* @cond_data: User data associated with a conditional snapshot
1858	*
1859	* Flip the buffers between the @tr and the max_tr and record information
1860	* about which task was the cause of this latency.
1861	*/
1862	void
1863	update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu,
1864	void *cond_data)
1865	{
1866	if (tr->stop_count)
1867	return;
1868
1869	WARN_ON_ONCE(!irqs_disabled());
1870
1871	if (!tr->allocated_snapshot) {
1872	/* Only the nop tracer should hit this when disabling */
1873	WARN_ON_ONCE(tr->current_trace != &nop_trace);
1874	return;
1875	}
1876
1877	arch_spin_lock(&tr->max_lock);
1878
1879	/* Inherit the recordable setting from array_buffer */
1880	if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1881	ring_buffer_record_on(tr->max_buffer.buffer);
1882	else
1883	ring_buffer_record_off(tr->max_buffer.buffer);
1884
1885	#ifdef CONFIG_TRACER_SNAPSHOT
1886	if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) {
1887	arch_spin_unlock(&tr->max_lock);
1888	return;
1889	}
1890	#endif
1891	swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1892
1893	__update_max_tr(tr, tsk, cpu);
1894
1895	arch_spin_unlock(&tr->max_lock);
1896	}
1897
1898	/**
1899	* update_max_tr_single - only copy one trace over, and reset the rest
1900	* @tr: tracer
1901	* @tsk: task with the latency
1902	* @cpu: the cpu of the buffer to copy.
1903	*
1904	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
1905	*/
1906	void
1907	update_max_tr_single(struct trace_array tr, struct task_struct tsk, int cpu)
1908	{
1909	int ret;
1910
1911	if (tr->stop_count)
1912	return;
1913
1914	WARN_ON_ONCE(!irqs_disabled());
1915	if (!tr->allocated_snapshot) {
1916	/* Only the nop tracer should hit this when disabling */
1917	WARN_ON_ONCE(tr->current_trace != &nop_trace);
1918	return;
1919	}
1920
1921	arch_spin_lock(&tr->max_lock);
1922
1923	ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
1924
1925	if (ret == -EBUSY) {
1926	/*
1927	* We failed to swap the buffer due to a commit taking
1928	* place on this CPU. We fail to record, but we reset
1929	* the max trace buffer (no one writes directly to it)
1930	* and flag that it failed.
1931	*/
1932	trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1933	"Failed to swap buffers due to commit in progress\n");
1934	}
1935
1936	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1937
1938	__update_max_tr(tr, tsk, cpu);
1939	arch_spin_unlock(&tr->max_lock);
1940	}
1941
1942	#endif /* CONFIG_TRACER_MAX_TRACE */
1943
1944	static int wait_on_pipe(struct trace_iterator *iter, int full)
1945	{
1946	/* Iterators are static, they should be filled or empty */
1947	if (trace_buffer_iter(iter, iter->cpu_file))
1948	return 0;
1949
1950	return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file,
1951	full);
1952	}
1953
1954	#ifdef CONFIG_FTRACE_STARTUP_TEST
1955	static bool selftests_can_run;
1956
1957	struct trace_selftests {
1958	struct list_head list;
1959	struct tracer *type;
1960	};
1961
1962	static LIST_HEAD(postponed_selftests);
1963
1964	static int save_selftest(struct tracer *type)
1965	{
1966	struct trace_selftests *selftest;
1967
1968	selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1969	if (!selftest)
1970	return -ENOMEM;
1971
1972	selftest->type = type;
1973	list_add(&selftest->list, &postponed_selftests);
1974	return 0;
1975	}
1976
1977	static int run_tracer_selftest(struct tracer *type)
1978	{
1979	struct trace_array *tr = &global_trace;
1980	struct tracer *saved_tracer = tr->current_trace;
1981	int ret;
1982
1983	if (!type->selftest \|\| tracing_selftest_disabled)
1984	return 0;
1985
1986	/*
1987	* If a tracer registers early in boot up (before scheduling is
1988	* initialized and such), then do not run its selftests yet.
1989	* Instead, run it a little later in the boot process.
1990	*/
1991	if (!selftests_can_run)
1992	return save_selftest(type);
1993
1994	if (!tracing_is_on()) {
1995	pr_warn("Selftest for tracer %s skipped due to tracing disabled\n",
1996	type->name);
1997	return 0;
1998	}
1999
2000	/*
2001	* Run a selftest on this tracer.
2002	* Here we reset the trace buffer, and set the current
2003	* tracer to be this tracer. The tracer can then run some
2004	* internal tracing to verify that everything is in order.
2005	* If we fail, we do not register this tracer.
2006	*/
2007	tracing_reset_online_cpus(&tr->array_buffer);
2008
2009	tr->current_trace = type;
2010
2011	#ifdef CONFIG_TRACER_MAX_TRACE
2012	if (type->use_max_tr) {
2013	/* If we expanded the buffers, make sure the max is expanded too */
2014	if (ring_buffer_expanded)
2015	ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
2016	RING_BUFFER_ALL_CPUS);
2017	tr->allocated_snapshot = true;
2018	}
2019	#endif
2020
2021	/* the test is responsible for initializing and enabling */
2022	pr_info("Testing tracer %s: ", type->name);
2023	ret = type->selftest(type, tr);
2024	/* the test is responsible for resetting too */
2025	tr->current_trace = saved_tracer;
2026	if (ret) {
2027	printk(KERN_CONT "FAILED!\n");
2028	/* Add the warning after printing 'FAILED' */
2029	WARN_ON(1);
2030	return -1;
2031	}
2032	/* Only reset on passing, to avoid touching corrupted buffers */
2033	tracing_reset_online_cpus(&tr->array_buffer);
2034
2035	#ifdef CONFIG_TRACER_MAX_TRACE
2036	if (type->use_max_tr) {
2037	tr->allocated_snapshot = false;
2038
2039	/* Shrink the max buffer again */
2040	if (ring_buffer_expanded)
2041	ring_buffer_resize(tr->max_buffer.buffer, 1,
2042	RING_BUFFER_ALL_CPUS);
2043	}
2044	#endif
2045
2046	printk(KERN_CONT "PASSED\n");
2047	return 0;
2048	}
2049
2050	static int do_run_tracer_selftest(struct tracer *type)
2051	{
2052	int ret;
2053
2054	/*
2055	* Tests can take a long time, especially if they are run one after the
2056	* other, as does happen during bootup when all the tracers are
2057	* registered. This could cause the soft lockup watchdog to trigger.
2058	*/
2059	cond_resched();
2060
2061	tracing_selftest_running = true;
2062	ret = run_tracer_selftest(type);
2063	tracing_selftest_running = false;
2064
2065	return ret;
2066	}
2067
2068	static __init int init_trace_selftests(void)
2069	{
2070	struct trace_selftests p, n;
2071	struct tracer t, *last;
2072	int ret;
2073
2074	selftests_can_run = true;
2075
2076	mutex_lock(&trace_types_lock);
2077
2078	if (list_empty(&postponed_selftests))
2079	goto out;
2080
2081	pr_info("Running postponed tracer tests:\n");
2082
2083	tracing_selftest_running = true;
2084	list_for_each_entry_safe(p, n, &postponed_selftests, list) {
2085	/* This loop can take minutes when sanitizers are enabled, so
2086	* lets make sure we allow RCU processing.
2087	*/
2088	cond_resched();
2089	ret = run_tracer_selftest(p->type);
2090	/* If the test fails, then warn and remove from available_tracers */
2091	if (ret < 0) {
2092	WARN(1, "tracer: %s failed selftest, disabling\n",
2093	p->type->name);
2094	last = &trace_types;
2095	for (t = trace_types; t; t = t->next) {
2096	if (t == p->type) {
2097	*last = t->next;
2098	break;
2099	}
2100	last = &t->next;
2101	}
2102	}
2103	list_del(&p->list);
2104	kfree(p);
2105	}
2106	tracing_selftest_running = false;
2107
2108	out:
2109	mutex_unlock(&trace_types_lock);
2110
2111	return 0;
2112	}
2113	core_initcall(init_trace_selftests);
2114	#else
2115	static inline int run_tracer_selftest(struct tracer *type)
2116	{
2117	return 0;
2118	}
2119	static inline int do_run_tracer_selftest(struct tracer *type)
2120	{
2121	return 0;
2122	}
2123	#endif /* CONFIG_FTRACE_STARTUP_TEST */
2124
2125	static void add_tracer_options(struct trace_array tr, struct tracer t);
2126
2127	static void __init apply_trace_boot_options(void);
2128
2129	/**
2130	* register_tracer - register a tracer with the ftrace system.
2131	* @type: the plugin for the tracer
2132	*
2133	* Register a new plugin tracer.
2134	*/
2135	int __init register_tracer(struct tracer *type)
2136	{
2137	struct tracer *t;
2138	int ret = 0;
2139
2140	if (!type->name) {
2141	pr_info("Tracer must have a name\n");
2142	return -1;
2143	}
2144
2145	if (strlen(type->name) >= MAX_TRACER_SIZE) {
2146	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
2147	return -1;
2148	}
2149
2150	if (security_locked_down(LOCKDOWN_TRACEFS)) {
2151	pr_warn("Can not register tracer %s due to lockdown\n",
2152	type->name);
2153	return -EPERM;
2154	}
2155
2156	mutex_lock(&trace_types_lock);
2157
2158	for (t = trace_types; t; t = t->next) {
2159	if (strcmp(type->name, t->name) == 0) {
2160	/* already found */
2161	pr_info("Tracer %s already registered\n",
2162	type->name);
2163	ret = -1;
2164	goto out;
2165	}
2166	}
2167
2168	if (!type->set_flag)
2169	type->set_flag = &dummy_set_flag;
2170	if (!type->flags) {
2171	/allocate a dummy tracer_flags/
2172	type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
2173	if (!type->flags) {
2174	ret = -ENOMEM;
2175	goto out;
2176	}
2177	type->flags->val = 0;
2178	type->flags->opts = dummy_tracer_opt;
2179	} else
2180	if (!type->flags->opts)
2181	type->flags->opts = dummy_tracer_opt;
2182
2183	/* store the tracer for __set_tracer_option */
2184	type->flags->trace = type;
2185
2186	ret = do_run_tracer_selftest(type);
2187	if (ret < 0)
2188	goto out;
2189
2190	type->next = trace_types;
2191	trace_types = type;
2192	add_tracer_options(&global_trace, type);
2193
2194	out:
2195	mutex_unlock(&trace_types_lock);
2196
2197	if (ret \|\| !default_bootup_tracer)
2198	goto out_unlock;
2199
2200	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
2201	goto out_unlock;
2202
2203	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
2204	/* Do we want this tracer to start on bootup? */
2205	tracing_set_tracer(&global_trace, type->name);
2206	default_bootup_tracer = NULL;
2207
2208	apply_trace_boot_options();
2209
2210	/* disable other selftests, since this will break it. */
2211	disable_tracing_selftest("running a tracer");
2212
2213	out_unlock:
2214	return ret;
2215	}
2216
2217	static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
2218	{
2219	struct trace_buffer *buffer = buf->buffer;
2220
2221	if (!buffer)
2222	return;
2223
2224	ring_buffer_record_disable(buffer);
2225
2226	/* Make sure all commits have finished */
2227	synchronize_rcu();
2228	ring_buffer_reset_cpu(buffer, cpu);
2229
2230	ring_buffer_record_enable(buffer);
2231	}
2232
2233	void tracing_reset_online_cpus(struct array_buffer *buf)
2234	{
2235	struct trace_buffer *buffer = buf->buffer;
2236
2237	if (!buffer)
2238	return;
2239
2240	ring_buffer_record_disable(buffer);
2241
2242	/* Make sure all commits have finished */
2243	synchronize_rcu();
2244
2245	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2246
2247	ring_buffer_reset_online_cpus(buffer);
2248
2249	ring_buffer_record_enable(buffer);
2250	}
2251
2252	/* Must have trace_types_lock held */
2253	void tracing_reset_all_online_cpus_unlocked(void)
2254	{
2255	struct trace_array *tr;
2256
2257	lockdep_assert_held(&trace_types_lock);
2258
2259	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2260	if (!tr->clear_trace)
2261	continue;
2262	tr->clear_trace = false;
2263	tracing_reset_online_cpus(&tr->array_buffer);
2264	#ifdef CONFIG_TRACER_MAX_TRACE
2265	tracing_reset_online_cpus(&tr->max_buffer);
2266	#endif
2267	}
2268	}
2269
2270	void tracing_reset_all_online_cpus(void)
2271	{
2272	mutex_lock(&trace_types_lock);
2273	tracing_reset_all_online_cpus_unlocked();
2274	mutex_unlock(&trace_types_lock);
2275	}
2276
2277	/*
2278	* The tgid_map array maps from pid to tgid; i.e. the value stored at index i
2279	* is the tgid last observed corresponding to pid=i.
2280	*/
2281	static int *tgid_map;
2282
2283	/* The maximum valid index into tgid_map. */
2284	static size_t tgid_map_max;
2285
2286	#define SAVED_CMDLINES_DEFAULT 128
2287	#define NO_CMDLINE_MAP UINT_MAX
2288	/*
2289	* Preemption must be disabled before acquiring trace_cmdline_lock.
2290	* The various trace_arrays' max_lock must be acquired in a context
2291	* where interrupt is disabled.
2292	*/
2293	static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2294	struct saved_cmdlines_buffer {
2295	unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
2296	unsigned *map_cmdline_to_pid;
2297	unsigned cmdline_num;
2298	int cmdline_idx;
2299	char *saved_cmdlines;
2300	};
2301	static struct saved_cmdlines_buffer *savedcmd;
2302
2303	static inline char *get_saved_cmdlines(int idx)
2304	{
2305	return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
2306	}
2307
2308	static inline void set_cmdline(int idx, const char *cmdline)
2309	{
2310	strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
2311	}
2312
2313	static int allocate_cmdlines_buffer(unsigned int val,
2314	struct saved_cmdlines_buffer *s)
2315	{
2316	s->map_cmdline_to_pid = kmalloc_array(val,
2317	sizeof(*s->map_cmdline_to_pid),
2318	GFP_KERNEL);
2319	if (!s->map_cmdline_to_pid)
2320	return -ENOMEM;
2321
2322	s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
2323	if (!s->saved_cmdlines) {
2324	kfree(s->map_cmdline_to_pid);
2325	return -ENOMEM;
2326	}
2327
2328	s->cmdline_idx = 0;
2329	s->cmdline_num = val;
2330	memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
2331	sizeof(s->map_pid_to_cmdline));
2332	memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
2333	val * sizeof(*s->map_cmdline_to_pid));
2334
2335	return 0;
2336	}
2337
2338	static int trace_create_savedcmd(void)
2339	{
2340	int ret;
2341
2342	savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
2343	if (!savedcmd)
2344	return -ENOMEM;
2345
2346	ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
2347	if (ret < 0) {
2348	kfree(savedcmd);
2349	savedcmd = NULL;
2350	return -ENOMEM;
2351	}
2352
2353	return 0;
2354	}
2355
2356	int is_tracing_stopped(void)
2357	{
2358	return global_trace.stop_count;
2359	}
2360
2361	/**
2362	* tracing_start - quick start of the tracer
2363	*
2364	* If tracing is enabled but was stopped by tracing_stop,
2365	* this will start the tracer back up.
2366	*/
2367	void tracing_start(void)
2368	{
2369	struct trace_buffer *buffer;
2370	unsigned long flags;
2371
2372	if (tracing_disabled)
2373	return;
2374
2375	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2376	if (--global_trace.stop_count) {
2377	if (global_trace.stop_count < 0) {
2378	/* Someone screwed up their debugging */
2379	WARN_ON_ONCE(1);
2380	global_trace.stop_count = 0;
2381	}
2382	goto out;
2383	}
2384
2385	/* Prevent the buffers from switching */
2386	arch_spin_lock(&global_trace.max_lock);
2387
2388	buffer = global_trace.array_buffer.buffer;
2389	if (buffer)
2390	ring_buffer_record_enable(buffer);
2391
2392	#ifdef CONFIG_TRACER_MAX_TRACE
2393	buffer = global_trace.max_buffer.buffer;
2394	if (buffer)
2395	ring_buffer_record_enable(buffer);
2396	#endif
2397
2398	arch_spin_unlock(&global_trace.max_lock);
2399
2400	out:
2401	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2402	}
2403
2404	static void tracing_start_tr(struct trace_array *tr)
2405	{
2406	struct trace_buffer *buffer;
2407	unsigned long flags;
2408
2409	if (tracing_disabled)
2410	return;
2411
2412	/* If global, we need to also start the max tracer */
2413	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2414	return tracing_start();
2415
2416	raw_spin_lock_irqsave(&tr->start_lock, flags);
2417
2418	if (--tr->stop_count) {
2419	if (tr->stop_count < 0) {
2420	/* Someone screwed up their debugging */
2421	WARN_ON_ONCE(1);
2422	tr->stop_count = 0;
2423	}
2424	goto out;
2425	}
2426
2427	buffer = tr->array_buffer.buffer;
2428	if (buffer)
2429	ring_buffer_record_enable(buffer);
2430
2431	out:
2432	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2433	}
2434
2435	/**
2436	* tracing_stop - quick stop of the tracer
2437	*
2438	* Light weight way to stop tracing. Use in conjunction with
2439	* tracing_start.
2440	*/
2441	void tracing_stop(void)
2442	{
2443	struct trace_buffer *buffer;
2444	unsigned long flags;
2445
2446	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2447	if (global_trace.stop_count++)
2448	goto out;
2449
2450	/* Prevent the buffers from switching */
2451	arch_spin_lock(&global_trace.max_lock);
2452
2453	buffer = global_trace.array_buffer.buffer;
2454	if (buffer)
2455	ring_buffer_record_disable(buffer);
2456
2457	#ifdef CONFIG_TRACER_MAX_TRACE
2458	buffer = global_trace.max_buffer.buffer;
2459	if (buffer)
2460	ring_buffer_record_disable(buffer);
2461	#endif
2462
2463	arch_spin_unlock(&global_trace.max_lock);
2464
2465	out:
2466	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2467	}
2468
2469	static void tracing_stop_tr(struct trace_array *tr)
2470	{
2471	struct trace_buffer *buffer;
2472	unsigned long flags;
2473
2474	/* If global, we need to also stop the max tracer */
2475	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2476	return tracing_stop();
2477
2478	raw_spin_lock_irqsave(&tr->start_lock, flags);
2479	if (tr->stop_count++)
2480	goto out;
2481
2482	buffer = tr->array_buffer.buffer;
2483	if (buffer)
2484	ring_buffer_record_disable(buffer);
2485
2486	out:
2487	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2488	}
2489
2490	static int trace_save_cmdline(struct task_struct *tsk)
2491	{
2492	unsigned tpid, idx;
2493
2494	/* treat recording of idle task as a success */
2495	if (!tsk->pid)
2496	return 1;
2497
2498	tpid = tsk->pid & (PID_MAX_DEFAULT - 1);
2499
2500	/*
2501	* It's not the end of the world if we don't get
2502	* the lock, but we also don't want to spin
2503	* nor do we want to disable interrupts,
2504	* so if we miss here, then better luck next time.
2505	*
2506	* This is called within the scheduler and wake up, so interrupts
2507	* had better been disabled and run queue lock been held.
2508	*/
2509	lockdep_assert_preemption_disabled();
2510	if (!arch_spin_trylock(&trace_cmdline_lock))
2511	return 0;
2512
2513	idx = savedcmd->map_pid_to_cmdline[tpid];
2514	if (idx == NO_CMDLINE_MAP) {
2515	idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2516
2517	savedcmd->map_pid_to_cmdline[tpid] = idx;
2518	savedcmd->cmdline_idx = idx;
2519	}
2520
2521	savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2522	set_cmdline(idx, tsk->comm);
2523
2524	arch_spin_unlock(&trace_cmdline_lock);
2525
2526	return 1;
2527	}
2528
2529	static void __trace_find_cmdline(int pid, char comm[])
2530	{
2531	unsigned map;
2532	int tpid;
2533
2534	if (!pid) {
2535	strcpy(comm, "<idle>");
2536	return;
2537	}
2538
2539	if (WARN_ON_ONCE(pid < 0)) {
2540	strcpy(comm, "<XXX>");
2541	return;
2542	}
2543
2544	tpid = pid & (PID_MAX_DEFAULT - 1);
2545	map = savedcmd->map_pid_to_cmdline[tpid];
2546	if (map != NO_CMDLINE_MAP) {
2547	tpid = savedcmd->map_cmdline_to_pid[map];
2548	if (tpid == pid) {
2549	strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2550	return;
2551	}
2552	}
2553	strcpy(comm, "<...>");
2554	}
2555
2556	void trace_find_cmdline(int pid, char comm[])
2557	{
2558	preempt_disable();
2559	arch_spin_lock(&trace_cmdline_lock);
2560
2561	__trace_find_cmdline(pid, comm);
2562
2563	arch_spin_unlock(&trace_cmdline_lock);
2564	preempt_enable();
2565	}
2566
2567	static int *trace_find_tgid_ptr(int pid)
2568	{
2569	/*
2570	* Pairs with the smp_store_release in set_tracer_flag() to ensure that
2571	* if we observe a non-NULL tgid_map then we also observe the correct
2572	* tgid_map_max.
2573	*/
2574	int *map = smp_load_acquire(&tgid_map);
2575
2576	if (unlikely(!map \|\| pid > tgid_map_max))
2577	return NULL;
2578
2579	return &map[pid];
2580	}
2581
2582	int trace_find_tgid(int pid)
2583	{
2584	int *ptr = trace_find_tgid_ptr(pid);
2585
2586	return ptr ? *ptr : 0;
2587	}
2588
2589	static int trace_save_tgid(struct task_struct *tsk)
2590	{
2591	int *ptr;
2592
2593	/* treat recording of idle task as a success */
2594	if (!tsk->pid)
2595	return 1;
2596
2597	ptr = trace_find_tgid_ptr(tsk->pid);
2598	if (!ptr)
2599	return 0;
2600
2601	*ptr = tsk->tgid;
2602	return 1;
2603	}
2604
2605	static bool tracing_record_taskinfo_skip(int flags)
2606	{
2607	if (unlikely(!(flags & (TRACE_RECORD_CMDLINE \| TRACE_RECORD_TGID))))
2608	return true;
2609	if (!__this_cpu_read(trace_taskinfo_save))
2610	return true;
2611	return false;
2612	}
2613
2614	/**
2615	* tracing_record_taskinfo - record the task info of a task
2616	*
2617	* @task: task to record
2618	* @flags: TRACE_RECORD_CMDLINE for recording comm
2619	* TRACE_RECORD_TGID for recording tgid
2620	*/
2621	void tracing_record_taskinfo(struct task_struct *task, int flags)
2622	{
2623	bool done;
2624
2625	if (tracing_record_taskinfo_skip(flags))
2626	return;
2627
2628	/*
2629	* Record as much task information as possible. If some fail, continue
2630	* to try to record the others.
2631	*/
2632	done = !(flags & TRACE_RECORD_CMDLINE) \|\| trace_save_cmdline(task);
2633	done &= !(flags & TRACE_RECORD_TGID) \|\| trace_save_tgid(task);
2634
2635	/* If recording any information failed, retry again soon. */
2636	if (!done)
2637	return;
2638
2639	__this_cpu_write(trace_taskinfo_save, false);
2640	}
2641
2642	/**
2643	* tracing_record_taskinfo_sched_switch - record task info for sched_switch
2644	*
2645	* @prev: previous task during sched_switch
2646	* @next: next task during sched_switch
2647	* @flags: TRACE_RECORD_CMDLINE for recording comm
2648	* TRACE_RECORD_TGID for recording tgid
2649	*/
2650	void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2651	struct task_struct *next, int flags)
2652	{
2653	bool done;
2654
2655	if (tracing_record_taskinfo_skip(flags))
2656	return;
2657
2658	/*
2659	* Record as much task information as possible. If some fail, continue
2660	* to try to record the others.
2661	*/
2662	done = !(flags & TRACE_RECORD_CMDLINE) \|\| trace_save_cmdline(prev);
2663	done &= !(flags & TRACE_RECORD_CMDLINE) \|\| trace_save_cmdline(next);
2664	done &= !(flags & TRACE_RECORD_TGID) \|\| trace_save_tgid(prev);
2665	done &= !(flags & TRACE_RECORD_TGID) \|\| trace_save_tgid(next);
2666
2667	/* If recording any information failed, retry again soon. */
2668	if (!done)
2669	return;
2670
2671	__this_cpu_write(trace_taskinfo_save, false);
2672	}
2673
2674	/* Helpers to record a specific task information */
2675	void tracing_record_cmdline(struct task_struct *task)
2676	{
2677	tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2678	}
2679
2680	void tracing_record_tgid(struct task_struct *task)
2681	{
2682	tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2683	}
2684
2685	/*
2686	* Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2687	* overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2688	* simplifies those functions and keeps them in sync.
2689	*/
2690	enum print_line_t trace_handle_return(struct trace_seq *s)
2691	{
2692	return trace_seq_has_overflowed(s) ?
2693	TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2694	}
2695	EXPORT_SYMBOL_GPL(trace_handle_return);
2696
2697	static unsigned short migration_disable_value(void)
2698	{
2699	#if defined(CONFIG_SMP)
2700	return current->migration_disabled;
2701	#else
2702	return 0;
2703	#endif
2704	}
2705
2706	unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
2707	{
2708	unsigned int trace_flags = irqs_status;
2709	unsigned int pc;
2710
2711	pc = preempt_count();
2712
2713	if (pc & NMI_MASK)
2714	trace_flags \|= TRACE_FLAG_NMI;
2715	if (pc & HARDIRQ_MASK)
2716	trace_flags \|= TRACE_FLAG_HARDIRQ;
2717	if (in_serving_softirq())
2718	trace_flags \|= TRACE_FLAG_SOFTIRQ;
2719	if (softirq_count() >> (SOFTIRQ_SHIFT + 1))
2720	trace_flags \|= TRACE_FLAG_BH_OFF;
2721
2722	if (tif_need_resched())
2723	trace_flags \|= TRACE_FLAG_NEED_RESCHED;
2724	if (test_preempt_need_resched())
2725	trace_flags \|= TRACE_FLAG_PREEMPT_RESCHED;
2726	return (trace_flags << 16) \| (min_t(unsigned int, pc & 0xff, 0xf)) \|
2727	(min_t(unsigned int, migration_disable_value(), 0xf)) << 4;
2728	}
2729
2730	struct ring_buffer_event *
2731	trace_buffer_lock_reserve(struct trace_buffer *buffer,
2732	int type,
2733	unsigned long len,
2734	unsigned int trace_ctx)
2735	{
2736	return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
2737	}
2738
2739	DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2740	DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2741	static int trace_buffered_event_ref;
2742
2743	/**
2744	* trace_buffered_event_enable - enable buffering events
2745	*
2746	* When events are being filtered, it is quicker to use a temporary
2747	* buffer to write the event data into if there's a likely chance
2748	* that it will not be committed. The discard of the ring buffer
2749	* is not as fast as committing, and is much slower than copying
2750	* a commit.
2751	*
2752	* When an event is to be filtered, allocate per cpu buffers to
2753	* write the event data into, and if the event is filtered and discarded
2754	* it is simply dropped, otherwise, the entire data is to be committed
2755	* in one shot.
2756	*/
2757	void trace_buffered_event_enable(void)
2758	{
2759	struct ring_buffer_event *event;
2760	struct page *page;
2761	int cpu;
2762
2763	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2764
2765	if (trace_buffered_event_ref++)
2766	return;
2767
2768	for_each_tracing_cpu(cpu) {
2769	page = alloc_pages_node(cpu_to_node(cpu),
2770	GFP_KERNEL \| __GFP_NORETRY, 0);
2771	if (!page)
2772	goto failed;
2773
2774	event = page_address(page);
2775	memset(event, 0, sizeof(*event));
2776
2777	per_cpu(trace_buffered_event, cpu) = event;
2778
2779	preempt_disable();
2780	if (cpu == smp_processor_id() &&
2781	__this_cpu_read(trace_buffered_event) !=
2782	per_cpu(trace_buffered_event, cpu))
2783	WARN_ON_ONCE(1);
2784	preempt_enable();
2785	}
2786
2787	return;
2788	failed:
2789	trace_buffered_event_disable();
2790	}
2791
2792	static void enable_trace_buffered_event(void *data)
2793	{
2794	/* Probably not needed, but do it anyway */
2795	smp_rmb();
2796	this_cpu_dec(trace_buffered_event_cnt);
2797	}
2798
2799	static void disable_trace_buffered_event(void *data)
2800	{
2801	this_cpu_inc(trace_buffered_event_cnt);
2802	}
2803
2804	/**
2805	* trace_buffered_event_disable - disable buffering events
2806	*
2807	* When a filter is removed, it is faster to not use the buffered
2808	* events, and to commit directly into the ring buffer. Free up
2809	* the temp buffers when there are no more users. This requires
2810	* special synchronization with current events.
2811	*/
2812	void trace_buffered_event_disable(void)
2813	{
2814	int cpu;
2815
2816	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2817
2818	if (WARN_ON_ONCE(!trace_buffered_event_ref))
2819	return;
2820
2821	if (--trace_buffered_event_ref)
2822	return;
2823
2824	preempt_disable();
2825	/* For each CPU, set the buffer as used. */
2826	smp_call_function_many(tracing_buffer_mask,
2827	disable_trace_buffered_event, NULL, 1);
2828	preempt_enable();
2829
2830	/* Wait for all current users to finish */
2831	synchronize_rcu();
2832
2833	for_each_tracing_cpu(cpu) {
2834	free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2835	per_cpu(trace_buffered_event, cpu) = NULL;
2836	}
2837	/*
2838	* Make sure trace_buffered_event is NULL before clearing
2839	* trace_buffered_event_cnt.
2840	*/
2841	smp_wmb();
2842
2843	preempt_disable();
2844	/* Do the work on each cpu */
2845	smp_call_function_many(tracing_buffer_mask,
2846	enable_trace_buffered_event, NULL, 1);
2847	preempt_enable();
2848	}
2849
2850	static struct trace_buffer *temp_buffer;
2851
2852	struct ring_buffer_event *
2853	trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2854	struct trace_event_file *trace_file,
2855	int type, unsigned long len,
2856	unsigned int trace_ctx)
2857	{
2858	struct ring_buffer_event *entry;
2859	struct trace_array *tr = trace_file->tr;
2860	int val;
2861
2862	*current_rb = tr->array_buffer.buffer;
2863
2864	if (!tr->no_filter_buffering_ref &&
2865	(trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED \| EVENT_FILE_FL_FILTERED))) {
2866	preempt_disable_notrace();
2867	/*
2868	* Filtering is on, so try to use the per cpu buffer first.
2869	* This buffer will simulate a ring_buffer_event,
2870	* where the type_len is zero and the array[0] will
2871	* hold the full length.
2872	* (see include/linux/ring-buffer.h for details on
2873	* how the ring_buffer_event is structured).
2874	*
2875	* Using a temp buffer during filtering and copying it
2876	* on a matched filter is quicker than writing directly
2877	* into the ring buffer and then discarding it when
2878	* it doesn't match. That is because the discard
2879	* requires several atomic operations to get right.
2880	* Copying on match and doing nothing on a failed match
2881	* is still quicker than no copy on match, but having
2882	* to discard out of the ring buffer on a failed match.
2883	*/
2884	if ((entry = __this_cpu_read(trace_buffered_event))) {
2885	int max_len = PAGE_SIZE - struct_size(entry, array, 1);
2886
2887	val = this_cpu_inc_return(trace_buffered_event_cnt);
2888
2889	/*
2890	* Preemption is disabled, but interrupts and NMIs
2891	* can still come in now. If that happens after
2892	* the above increment, then it will have to go
2893	* back to the old method of allocating the event
2894	* on the ring buffer, and if the filter fails, it
2895	* will have to call ring_buffer_discard_commit()
2896	* to remove it.
2897	*
2898	* Need to also check the unlikely case that the
2899	* length is bigger than the temp buffer size.
2900	* If that happens, then the reserve is pretty much
2901	* guaranteed to fail, as the ring buffer currently
2902	* only allows events less than a page. But that may
2903	* change in the future, so let the ring buffer reserve
2904	* handle the failure in that case.
2905	*/
2906	if (val == 1 && likely(len <= max_len)) {
2907	trace_event_setup(entry, type, trace_ctx);
2908	entry->array[0] = len;
2909	/* Return with preemption disabled */
2910	return entry;
2911	}
2912	this_cpu_dec(trace_buffered_event_cnt);
2913	}
2914	/* __trace_buffer_lock_reserve() disables preemption */
2915	preempt_enable_notrace();
2916	}
2917
2918	entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2919	trace_ctx);
2920	/*
2921	* If tracing is off, but we have triggers enabled
2922	* we still need to look at the event data. Use the temp_buffer
2923	* to store the trace event for the trigger to use. It's recursive
2924	* safe and will not be recorded anywhere.
2925	*/
2926	if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2927	*current_rb = temp_buffer;
2928	entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2929	trace_ctx);
2930	}
2931	return entry;
2932	}
2933	EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2934
2935	static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock);
2936	static DEFINE_MUTEX(tracepoint_printk_mutex);
2937
2938	static void output_printk(struct trace_event_buffer *fbuffer)
2939	{
2940	struct trace_event_call *event_call;
2941	struct trace_event_file *file;
2942	struct trace_event *event;
2943	unsigned long flags;
2944	struct trace_iterator *iter = tracepoint_print_iter;
2945
2946	/* We should never get here if iter is NULL */
2947	if (WARN_ON_ONCE(!iter))
2948	return;
2949
2950	event_call = fbuffer->trace_file->event_call;
2951	if (!event_call \|\| !event_call->event.funcs \|\|
2952	!event_call->event.funcs->trace)
2953	return;
2954
2955	file = fbuffer->trace_file;
2956	if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) \|\|
2957	(unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2958	!filter_match_preds(file->filter, fbuffer->entry)))
2959	return;
2960
2961	event = &fbuffer->trace_file->event_call->event;
2962
2963	raw_spin_lock_irqsave(&tracepoint_iter_lock, flags);
2964	trace_seq_init(&iter->seq);
2965	iter->ent = fbuffer->entry;
2966	event_call->event.funcs->trace(iter, 0, event);
2967	trace_seq_putc(&iter->seq, 0);
2968	printk("%s", iter->seq.buffer);
2969
2970	raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2971	}
2972
2973	int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2974	void buffer, size_t lenp,
2975	loff_t *ppos)
2976	{
2977	int save_tracepoint_printk;
2978	int ret;
2979
2980	mutex_lock(&tracepoint_printk_mutex);
2981	save_tracepoint_printk = tracepoint_printk;
2982
2983	ret = proc_dointvec(table, write, buffer, lenp, ppos);
2984
2985	/*
2986	* This will force exiting early, as tracepoint_printk
2987	* is always zero when tracepoint_printk_iter is not allocated
2988	*/
2989	if (!tracepoint_print_iter)
2990	tracepoint_printk = 0;
2991
2992	if (save_tracepoint_printk == tracepoint_printk)
2993	goto out;
2994
2995	if (tracepoint_printk)
2996	static_key_enable(&tracepoint_printk_key.key);
2997	else
2998	static_key_disable(&tracepoint_printk_key.key);
2999
3000	out:
3001	mutex_unlock(&tracepoint_printk_mutex);
3002
3003	return ret;
3004	}
3005
3006	void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
3007	{
3008	enum event_trigger_type tt = ETT_NONE;
3009	struct trace_event_file *file = fbuffer->trace_file;
3010
3011	if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event,
3012	fbuffer->entry, &tt))
3013	goto discard;
3014
3015	if (static_key_false(&tracepoint_printk_key.key))
3016	output_printk(fbuffer);
3017
3018	if (static_branch_unlikely(&trace_event_exports_enabled))
3019	ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
3020
3021	trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer,
3022	fbuffer->event, fbuffer->trace_ctx, fbuffer->regs);
3023
3024	discard:
3025	if (tt)
3026	event_triggers_post_call(file, tt);
3027
3028	}
3029	EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
3030
3031	/*
3032	* Skip 3:
3033	*
3034	* trace_buffer_unlock_commit_regs()
3035	* trace_event_buffer_commit()
3036	* trace_event_raw_event_xxx()
3037	*/
3038	# define STACK_SKIP 3
3039
3040	void trace_buffer_unlock_commit_regs(struct trace_array *tr,
3041	struct trace_buffer *buffer,
3042	struct ring_buffer_event *event,
3043	unsigned int trace_ctx,
3044	struct pt_regs *regs)
3045	{
3046	__buffer_unlock_commit(buffer, event);
3047
3048	/*
3049	* If regs is not set, then skip the necessary functions.
3050	* Note, we can still get here via blktrace, wakeup tracer
3051	* and mmiotrace, but that's ok if they lose a function or
3052	* two. They are not that meaningful.
3053	*/
3054	ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs);
3055	ftrace_trace_userstack(tr, buffer, trace_ctx);
3056	}
3057
3058	/*
3059	* Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
3060	*/
3061	void
3062	trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
3063	struct ring_buffer_event *event)
3064	{
3065	__buffer_unlock_commit(buffer, event);
3066	}
3067
3068	void
3069	trace_function(struct trace_array *tr, unsigned long ip, unsigned long
3070	parent_ip, unsigned int trace_ctx)
3071	{
3072	struct trace_event_call *call = &event_function;
3073	struct trace_buffer *buffer = tr->array_buffer.buffer;
3074	struct ring_buffer_event *event;
3075	struct ftrace_entry *entry;
3076
3077	event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
3078	trace_ctx);
3079	if (!event)
3080	return;
3081	entry = ring_buffer_event_data(event);
3082	entry->ip = ip;
3083	entry->parent_ip = parent_ip;
3084
3085	if (!call_filter_check_discard(call, entry, buffer, event)) {
3086	if (static_branch_unlikely(&trace_function_exports_enabled))
3087	ftrace_exports(event, TRACE_EXPORT_FUNCTION);
3088	__buffer_unlock_commit(buffer, event);
3089	}
3090	}
3091
3092	#ifdef CONFIG_STACKTRACE
3093
3094	/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
3095	#define FTRACE_KSTACK_NESTING 4
3096
3097	#define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
3098
3099	struct ftrace_stack {
3100	unsigned long calls[FTRACE_KSTACK_ENTRIES];
3101	};
3102
3103
3104	struct ftrace_stacks {
3105	struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
3106	};
3107
3108	static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
3109	static DEFINE_PER_CPU(int, ftrace_stack_reserve);
3110
3111	static void __ftrace_trace_stack(struct trace_buffer *buffer,
3112	unsigned int trace_ctx,
3113	int skip, struct pt_regs *regs)
3114	{
3115	struct trace_event_call *call = &event_kernel_stack;
3116	struct ring_buffer_event *event;
3117	unsigned int size, nr_entries;
3118	struct ftrace_stack *fstack;
3119	struct stack_entry *entry;
3120	int stackidx;
3121
3122	/*
3123	* Add one, for this function and the call to save_stack_trace()
3124	* If regs is set, then these functions will not be in the way.
3125	*/
3126	#ifndef CONFIG_UNWINDER_ORC
3127	if (!regs)
3128	skip++;
3129	#endif
3130
3131	preempt_disable_notrace();
3132
3133	stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
3134
3135	/* This should never happen. If it does, yell once and skip */
3136	if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
3137	goto out;
3138
3139	/*
3140	* The above __this_cpu_inc_return() is 'atomic' cpu local. An
3141	* interrupt will either see the value pre increment or post
3142	* increment. If the interrupt happens pre increment it will have
3143	* restored the counter when it returns. We just need a barrier to
3144	* keep gcc from moving things around.
3145	*/
3146	barrier();
3147
3148	fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
3149	size = ARRAY_SIZE(fstack->calls);
3150
3151	if (regs) {
3152	nr_entries = stack_trace_save_regs(regs, fstack->calls,
3153	size, skip);
3154	} else {
3155	nr_entries = stack_trace_save(fstack->calls, size, skip);
3156	}
3157
3158	size = nr_entries * sizeof(unsigned long);
3159	event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
3160	(sizeof(*entry) - sizeof(entry->caller)) + size,
3161	trace_ctx);
3162	if (!event)
3163	goto out;
3164	entry = ring_buffer_event_data(event);
3165
3166	memcpy(&entry->caller, fstack->calls, size);
3167	entry->size = nr_entries;
3168
3169	if (!call_filter_check_discard(call, entry, buffer, event))
3170	__buffer_unlock_commit(buffer, event);
3171
3172	out:
3173	/* Again, don't let gcc optimize things here */
3174	barrier();
3175	__this_cpu_dec(ftrace_stack_reserve);
3176	preempt_enable_notrace();
3177
3178	}
3179
3180	static inline void ftrace_trace_stack(struct trace_array *tr,
3181	struct trace_buffer *buffer,
3182	unsigned int trace_ctx,
3183	int skip, struct pt_regs *regs)
3184	{
3185	if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
3186	return;
3187
3188	__ftrace_trace_stack(buffer, trace_ctx, skip, regs);
3189	}
3190
3191	void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
3192	int skip)
3193	{
3194	struct trace_buffer *buffer = tr->array_buffer.buffer;
3195
3196	if (rcu_is_watching()) {
3197	__ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3198	return;
3199	}
3200
3201	if (WARN_ON_ONCE(IS_ENABLED(CONFIG_GENERIC_ENTRY)))
3202	return;
3203
3204	/*
3205	* When an NMI triggers, RCU is enabled via ct_nmi_enter(),
3206	* but if the above rcu_is_watching() failed, then the NMI
3207	* triggered someplace critical, and ct_irq_enter() should
3208	* not be called from NMI.
3209	*/
3210	if (unlikely(in_nmi()))
3211	return;
3212
3213	ct_irq_enter_irqson();
3214	__ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3215	ct_irq_exit_irqson();
3216	}
3217
3218	/**
3219	* trace_dump_stack - record a stack back trace in the trace buffer
3220	* @skip: Number of functions to skip (helper handlers)
3221	*/
3222	void trace_dump_stack(int skip)
3223	{
3224	if (tracing_disabled \|\| tracing_selftest_running)
3225	return;
3226
3227	#ifndef CONFIG_UNWINDER_ORC
3228	/* Skip 1 to skip this function. */
3229	skip++;
3230	#endif
3231	__ftrace_trace_stack(global_trace.array_buffer.buffer,
3232	tracing_gen_ctx(), skip, NULL);
3233	}
3234	EXPORT_SYMBOL_GPL(trace_dump_stack);
3235
3236	#ifdef CONFIG_USER_STACKTRACE_SUPPORT
3237	static DEFINE_PER_CPU(int, user_stack_count);
3238
3239	static void
3240	ftrace_trace_userstack(struct trace_array *tr,
3241	struct trace_buffer *buffer, unsigned int trace_ctx)
3242	{
3243	struct trace_event_call *call = &event_user_stack;
3244	struct ring_buffer_event *event;
3245	struct userstack_entry *entry;
3246
3247	if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
3248	return;
3249
3250	/*
3251	* NMIs can not handle page faults, even with fix ups.
3252	* The save user stack can (and often does) fault.
3253	*/
3254	if (unlikely(in_nmi()))
3255	return;
3256
3257	/*
3258	* prevent recursion, since the user stack tracing may
3259	* trigger other kernel events.
3260	*/
3261	preempt_disable();
3262	if (__this_cpu_read(user_stack_count))
3263	goto out;
3264
3265	__this_cpu_inc(user_stack_count);
3266
3267	event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3268	sizeof(*entry), trace_ctx);
3269	if (!event)
3270	goto out_drop_count;
3271	entry = ring_buffer_event_data(event);
3272
3273	entry->tgid = current->tgid;
3274	memset(&entry->caller, 0, sizeof(entry->caller));
3275
3276	stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3277	if (!call_filter_check_discard(call, entry, buffer, event))
3278	__buffer_unlock_commit(buffer, event);
3279
3280	out_drop_count:
3281	__this_cpu_dec(user_stack_count);
3282	out:
3283	preempt_enable();
3284	}
3285	#else /* CONFIG_USER_STACKTRACE_SUPPORT */
3286	static void ftrace_trace_userstack(struct trace_array *tr,
3287	struct trace_buffer *buffer,
3288	unsigned int trace_ctx)
3289	{
3290	}
3291	#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3292
3293	#endif /* CONFIG_STACKTRACE */
3294
3295	static inline void
3296	func_repeats_set_delta_ts(struct func_repeats_entry *entry,
3297	unsigned long long delta)
3298	{
3299	entry->bottom_delta_ts = delta & U32_MAX;
3300	entry->top_delta_ts = (delta >> 32);
3301	}
3302
3303	void trace_last_func_repeats(struct trace_array *tr,
3304	struct trace_func_repeats *last_info,
3305	unsigned int trace_ctx)
3306	{
3307	struct trace_buffer *buffer = tr->array_buffer.buffer;
3308	struct func_repeats_entry *entry;
3309	struct ring_buffer_event *event;
3310	u64 delta;
3311
3312	event = __trace_buffer_lock_reserve(buffer, TRACE_FUNC_REPEATS,
3313	sizeof(*entry), trace_ctx);
3314	if (!event)
3315	return;
3316
3317	delta = ring_buffer_event_time_stamp(buffer, event) -
3318	last_info->ts_last_call;
3319
3320	entry = ring_buffer_event_data(event);
3321	entry->ip = last_info->ip;
3322	entry->parent_ip = last_info->parent_ip;
3323	entry->count = last_info->count;
3324	func_repeats_set_delta_ts(entry, delta);
3325
3326	__buffer_unlock_commit(buffer, event);
3327	}
3328
3329	/* created for use with alloc_percpu */
3330	struct trace_buffer_struct {
3331	int nesting;
3332	char buffer[4][TRACE_BUF_SIZE];
3333	};
3334
3335	static struct trace_buffer_struct __percpu *trace_percpu_buffer;
3336
3337	/*
3338	* This allows for lockless recording. If we're nested too deeply, then
3339	* this returns NULL.
3340	*/
3341	static char *get_trace_buf(void)
3342	{
3343	struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3344
3345	if (!trace_percpu_buffer \|\| buffer->nesting >= 4)
3346	return NULL;
3347
3348	buffer->nesting++;
3349
3350	/* Interrupts must see nesting incremented before we use the buffer */
3351	barrier();
3352	return &buffer->buffer[buffer->nesting - 1][0];
3353	}
3354
3355	static void put_trace_buf(void)
3356	{
3357	/* Don't let the decrement of nesting leak before this */
3358	barrier();
3359	this_cpu_dec(trace_percpu_buffer->nesting);
3360	}
3361
3362	static int alloc_percpu_trace_buffer(void)
3363	{
3364	struct trace_buffer_struct __percpu *buffers;
3365
3366	if (trace_percpu_buffer)
3367	return 0;
3368
3369	buffers = alloc_percpu(struct trace_buffer_struct);
3370	if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3371	return -ENOMEM;
3372
3373	trace_percpu_buffer = buffers;
3374	return 0;
3375	}
3376
3377	static int buffers_allocated;
3378
3379	void trace_printk_init_buffers(void)
3380	{
3381	if (buffers_allocated)
3382	return;
3383
3384	if (alloc_percpu_trace_buffer())
3385	return;
3386
3387	/* trace_printk() is for debug use only. Don't use it in production. */
3388
3389	pr_warn("\n");
3390	pr_warn("**********************************************************\n");
3391	pr_warn(" NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE \n");
3392	pr_warn(" \n");
3393	pr_warn(" trace_printk() being used. Allocating extra memory. \n");
3394	pr_warn(" \n");
3395	pr_warn(" This means that this is a DEBUG kernel and it is \n");
3396	pr_warn(" unsafe for production use. \n");
3397	pr_warn(" \n");
3398	pr_warn(" If you see this message and you are not debugging \n");
3399	pr_warn(" the kernel, report this immediately to your vendor! \n");
3400	pr_warn(" \n");
3401	pr_warn(" NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE \n");
3402	pr_warn("**********************************************************\n");
3403
3404	/* Expand the buffers to set size */
3405	tracing_update_buffers();
3406
3407	buffers_allocated = 1;
3408
3409	/*
3410	* trace_printk_init_buffers() can be called by modules.
3411	* If that happens, then we need to start cmdline recording
3412	* directly here. If the global_trace.buffer is already
3413	* allocated here, then this was called by module code.
3414	*/
3415	if (global_trace.array_buffer.buffer)
3416	tracing_start_cmdline_record();
3417	}
3418	EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3419
3420	void trace_printk_start_comm(void)
3421	{
3422	/* Start tracing comms if trace printk is set */
3423	if (!buffers_allocated)
3424	return;
3425	tracing_start_cmdline_record();
3426	}
3427
3428	static void trace_printk_start_stop_comm(int enabled)
3429	{
3430	if (!buffers_allocated)
3431	return;
3432
3433	if (enabled)
3434	tracing_start_cmdline_record();
3435	else
3436	tracing_stop_cmdline_record();
3437	}
3438
3439	/**
3440	* trace_vbprintk - write binary msg to tracing buffer
3441	* @ip: The address of the caller
3442	* @fmt: The string format to write to the buffer
3443	* @args: Arguments for @fmt
3444	*/
3445	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3446	{
3447	struct trace_event_call *call = &event_bprint;
3448	struct ring_buffer_event *event;
3449	struct trace_buffer *buffer;
3450	struct trace_array *tr = &global_trace;
3451	struct bprint_entry *entry;
3452	unsigned int trace_ctx;
3453	char *tbuffer;
3454	int len = 0, size;
3455
3456	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
3457	return 0;
3458
3459	/* Don't pollute graph traces with trace_vprintk internals */
3460	pause_graph_tracing();
3461
3462	trace_ctx = tracing_gen_ctx();
3463	preempt_disable_notrace();
3464
3465	tbuffer = get_trace_buf();
3466	if (!tbuffer) {
3467	len = 0;
3468	goto out_nobuffer;
3469	}
3470
3471	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3472
3473	if (len > TRACE_BUF_SIZE/sizeof(int) \|\| len < 0)
3474	goto out_put;
3475
3476	size = sizeof(entry) + sizeof(u32) len;
3477	buffer = tr->array_buffer.buffer;
3478	ring_buffer_nest_start(buffer);
3479	event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3480	trace_ctx);
3481	if (!event)
3482	goto out;
3483	entry = ring_buffer_event_data(event);
3484	entry->ip = ip;
3485	entry->fmt = fmt;
3486
3487	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3488	if (!call_filter_check_discard(call, entry, buffer, event)) {
3489	__buffer_unlock_commit(buffer, event);
3490	ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL);
3491	}
3492
3493	out:
3494	ring_buffer_nest_end(buffer);
3495	out_put:
3496	put_trace_buf();
3497
3498	out_nobuffer:
3499	preempt_enable_notrace();
3500	unpause_graph_tracing();
3501
3502	return len;
3503	}
3504	EXPORT_SYMBOL_GPL(trace_vbprintk);
3505
3506	__printf(3, 0)
3507	static int
3508	__trace_array_vprintk(struct trace_buffer *buffer,
3509	unsigned long ip, const char *fmt, va_list args)
3510	{
3511	struct trace_event_call *call = &event_print;
3512	struct ring_buffer_event *event;
3513	int len = 0, size;
3514	struct print_entry *entry;
3515	unsigned int trace_ctx;
3516	char *tbuffer;
3517
3518	if (tracing_disabled)
3519	return 0;
3520
3521	/* Don't pollute graph traces with trace_vprintk internals */
3522	pause_graph_tracing();
3523
3524	trace_ctx = tracing_gen_ctx();
3525	preempt_disable_notrace();
3526
3527
3528	tbuffer = get_trace_buf();
3529	if (!tbuffer) {
3530	len = 0;
3531	goto out_nobuffer;
3532	}
3533
3534	len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3535
3536	size = sizeof(*entry) + len + 1;
3537	ring_buffer_nest_start(buffer);
3538	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3539	trace_ctx);
3540	if (!event)
3541	goto out;
3542	entry = ring_buffer_event_data(event);
3543	entry->ip = ip;
3544
3545	memcpy(&entry->buf, tbuffer, len + 1);
3546	if (!call_filter_check_discard(call, entry, buffer, event)) {
3547	__buffer_unlock_commit(buffer, event);
3548	ftrace_trace_stack(&global_trace, buffer, trace_ctx, 6, NULL);
3549	}
3550
3551	out:
3552	ring_buffer_nest_end(buffer);
3553	put_trace_buf();
3554
3555	out_nobuffer:
3556	preempt_enable_notrace();
3557	unpause_graph_tracing();
3558
3559	return len;
3560	}
3561
3562	__printf(3, 0)
3563	int trace_array_vprintk(struct trace_array *tr,
3564	unsigned long ip, const char *fmt, va_list args)
3565	{
3566	if (tracing_selftest_running && tr == &global_trace)
3567	return 0;
3568
3569	return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3570	}
3571
3572	/**
3573	* trace_array_printk - Print a message to a specific instance
3574	* @tr: The instance trace_array descriptor
3575	* @ip: The instruction pointer that this is called from.
3576	* @fmt: The format to print (printf format)
3577	*
3578	* If a subsystem sets up its own instance, they have the right to
3579	* printk strings into their tracing instance buffer using this
3580	* function. Note, this function will not write into the top level
3581	* buffer (use trace_printk() for that), as writing into the top level
3582	* buffer should only have events that can be individually disabled.
3583	* trace_printk() is only used for debugging a kernel, and should not
3584	* be ever incorporated in normal use.
3585	*
3586	* trace_array_printk() can be used, as it will not add noise to the
3587	* top level tracing buffer.
3588	*
3589	* Note, trace_array_init_printk() must be called on @tr before this
3590	* can be used.
3591	*/
3592	__printf(3, 0)
3593	int trace_array_printk(struct trace_array *tr,
3594	unsigned long ip, const char *fmt, ...)
3595	{
3596	int ret;
3597	va_list ap;
3598
3599	if (!tr)
3600	return -ENOENT;
3601
3602	/* This is only allowed for created instances */
3603	if (tr == &global_trace)
3604	return 0;
3605
3606	if (!(tr->trace_flags & TRACE_ITER_PRINTK))
3607	return 0;
3608
3609	va_start(ap, fmt);
3610	ret = trace_array_vprintk(tr, ip, fmt, ap);
3611	va_end(ap);
3612	return ret;
3613	}
3614	EXPORT_SYMBOL_GPL(trace_array_printk);
3615
3616	/**
3617	* trace_array_init_printk - Initialize buffers for trace_array_printk()
3618	* @tr: The trace array to initialize the buffers for
3619	*
3620	* As trace_array_printk() only writes into instances, they are OK to
3621	* have in the kernel (unlike trace_printk()). This needs to be called
3622	* before trace_array_printk() can be used on a trace_array.
3623	*/
3624	int trace_array_init_printk(struct trace_array *tr)
3625	{
3626	if (!tr)
3627	return -ENOENT;
3628
3629	/* This is only allowed for created instances */
3630	if (tr == &global_trace)
3631	return -EINVAL;
3632
3633	return alloc_percpu_trace_buffer();
3634	}
3635	EXPORT_SYMBOL_GPL(trace_array_init_printk);
3636
3637	__printf(3, 4)
3638	int trace_array_printk_buf(struct trace_buffer *buffer,
3639	unsigned long ip, const char *fmt, ...)
3640	{
3641	int ret;
3642	va_list ap;
3643
3644	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3645	return 0;
3646
3647	va_start(ap, fmt);
3648	ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3649	va_end(ap);
3650	return ret;
3651	}
3652
3653	__printf(2, 0)
3654	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3655	{
3656	return trace_array_vprintk(&global_trace, ip, fmt, args);
3657	}
3658	EXPORT_SYMBOL_GPL(trace_vprintk);
3659
3660	static void trace_iterator_increment(struct trace_iterator *iter)
3661	{
3662	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3663
3664	iter->idx++;
3665	if (buf_iter)
3666	ring_buffer_iter_advance(buf_iter);
3667	}
3668
3669	static struct trace_entry *
3670	peek_next_entry(struct trace_iterator iter, int cpu, u64 ts,
3671	unsigned long *lost_events)
3672	{
3673	struct ring_buffer_event *event;
3674	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3675
3676	if (buf_iter) {
3677	event = ring_buffer_iter_peek(buf_iter, ts);
3678	if (lost_events)
3679	*lost_events = ring_buffer_iter_dropped(buf_iter) ?
3680	(unsigned long)-1 : 0;
3681	} else {
3682	event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3683	lost_events);
3684	}
3685
3686	if (event) {
3687	iter->ent_size = ring_buffer_event_length(event);
3688	return ring_buffer_event_data(event);
3689	}
3690	iter->ent_size = 0;
3691	return NULL;
3692	}
3693
3694	static struct trace_entry *
3695	__find_next_entry(struct trace_iterator iter, int ent_cpu,
3696	unsigned long missing_events, u64 ent_ts)
3697	{
3698	struct trace_buffer *buffer = iter->array_buffer->buffer;
3699	struct trace_entry ent, next = NULL;
3700	unsigned long lost_events = 0, next_lost = 0;
3701	int cpu_file = iter->cpu_file;
3702	u64 next_ts = 0, ts;
3703	int next_cpu = -1;
3704	int next_size = 0;
3705	int cpu;
3706
3707	/*
3708	* If we are in a per_cpu trace file, don't bother by iterating over
3709	* all cpu and peek directly.
3710	*/
3711	if (cpu_file > RING_BUFFER_ALL_CPUS) {
3712	if (ring_buffer_empty_cpu(buffer, cpu_file))
3713	return NULL;
3714	ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3715	if (ent_cpu)
3716	*ent_cpu = cpu_file;
3717
3718	return ent;
3719	}
3720
3721	for_each_tracing_cpu(cpu) {
3722
3723	if (ring_buffer_empty_cpu(buffer, cpu))
3724	continue;
3725
3726	ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3727
3728	/*
3729	* Pick the entry with the smallest timestamp:
3730	*/
3731	if (ent && (!next \|\| ts < next_ts)) {
3732	next = ent;
3733	next_cpu = cpu;
3734	next_ts = ts;
3735	next_lost = lost_events;
3736	next_size = iter->ent_size;
3737	}
3738	}
3739
3740	iter->ent_size = next_size;
3741
3742	if (ent_cpu)
3743	*ent_cpu = next_cpu;
3744
3745	if (ent_ts)
3746	*ent_ts = next_ts;
3747
3748	if (missing_events)
3749	*missing_events = next_lost;
3750
3751	return next;
3752	}
3753
3754	#define STATIC_FMT_BUF_SIZE 128
3755	static char static_fmt_buf[STATIC_FMT_BUF_SIZE];
3756
3757	char trace_iter_expand_format(struct trace_iterator iter)
3758	{
3759	char *tmp;
3760
3761	/*
3762	* iter->tr is NULL when used with tp_printk, which makes
3763	* this get called where it is not safe to call krealloc().
3764	*/
3765	if (!iter->tr \|\| iter->fmt == static_fmt_buf)
3766	return NULL;
3767
3768	tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE,
3769	GFP_KERNEL);
3770	if (tmp) {
3771	iter->fmt_size += STATIC_FMT_BUF_SIZE;
3772	iter->fmt = tmp;
3773	}
3774
3775	return tmp;
3776	}
3777
3778	/* Returns true if the string is safe to dereference from an event */
3779	static bool trace_safe_str(struct trace_iterator iter, const char str,
3780	bool star, int len)
3781	{
3782	unsigned long addr = (unsigned long)str;
3783	struct trace_event *trace_event;
3784	struct trace_event_call *event;
3785
3786	/* Ignore strings with no length */
3787	if (star && !len)
3788	return true;
3789
3790	/* OK if part of the event data */
3791	if ((addr >= (unsigned long)iter->ent) &&
3792	(addr < (unsigned long)iter->ent + iter->ent_size))
3793	return true;
3794
3795	/* OK if part of the temp seq buffer */
3796	if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&
3797	(addr < (unsigned long)iter->tmp_seq.buffer + PAGE_SIZE))
3798	return true;
3799
3800	/* Core rodata can not be freed */
3801	if (is_kernel_rodata(addr))
3802	return true;
3803
3804	if (trace_is_tracepoint_string(str))
3805	return true;
3806
3807	/*
3808	* Now this could be a module event, referencing core module
3809	* data, which is OK.
3810	*/
3811	if (!iter->ent)
3812	return false;
3813
3814	trace_event = ftrace_find_event(iter->ent->type);
3815	if (!trace_event)
3816	return false;
3817
3818	event = container_of(trace_event, struct trace_event_call, event);
3819	if ((event->flags & TRACE_EVENT_FL_DYNAMIC) \|\| !event->module)
3820	return false;
3821
3822	/* Would rather have rodata, but this will suffice */
3823	if (within_module_core(addr, event->module))
3824	return true;
3825
3826	return false;
3827	}
3828
3829	static const char show_buffer(struct trace_seq s)
3830	{
3831	struct seq_buf *seq = &s->seq;
3832
3833	seq_buf_terminate(seq);
3834
3835	return seq->buffer;
3836	}
3837
3838	static DEFINE_STATIC_KEY_FALSE(trace_no_verify);
3839
3840	static int test_can_verify_check(const char *fmt, ...)
3841	{
3842	char buf[16];
3843	va_list ap;
3844	int ret;
3845
3846	/*
3847	* The verifier is dependent on vsnprintf() modifies the va_list
3848	* passed to it, where it is sent as a reference. Some architectures
3849	* (like x86_32) passes it by value, which means that vsnprintf()
3850	* does not modify the va_list passed to it, and the verifier
3851	* would then need to be able to understand all the values that
3852	* vsnprintf can use. If it is passed by value, then the verifier
3853	* is disabled.
3854	*/
3855	va_start(ap, fmt);
3856	vsnprintf(buf, 16, "%d", ap);
3857	ret = va_arg(ap, int);
3858	va_end(ap);
3859
3860	return ret;
3861	}
3862
3863	static void test_can_verify(void)
3864	{
3865	if (!test_can_verify_check("%d %d", 0, 1)) {
3866	pr_info("trace event string verifier disabled\n");
3867	static_branch_inc(&trace_no_verify);
3868	}
3869	}
3870
3871	/**
3872	* trace_check_vprintf - Check dereferenced strings while writing to the seq buffer
3873	* @iter: The iterator that holds the seq buffer and the event being printed
3874	* @fmt: The format used to print the event
3875	* @ap: The va_list holding the data to print from @fmt.
3876	*
3877	* This writes the data into the @iter->seq buffer using the data from
3878	* @fmt and @ap. If the format has a %s, then the source of the string
3879	* is examined to make sure it is safe to print, otherwise it will
3880	* warn and print "[UNSAFE MEMORY]" in place of the dereferenced string
3881	* pointer.
3882	*/
3883	void trace_check_vprintf(struct trace_iterator iter, const char fmt,
3884	va_list ap)
3885	{
3886	const char *p = fmt;
3887	const char *str;
3888	int i, j;
3889
3890	if (WARN_ON_ONCE(!fmt))
3891	return;
3892
3893	if (static_branch_unlikely(&trace_no_verify))
3894	goto print;
3895
3896	/* Don't bother checking when doing a ftrace_dump() */
3897	if (iter->fmt == static_fmt_buf)
3898	goto print;
3899
3900	while (*p) {
3901	bool star = false;
3902	int len = 0;
3903
3904	j = 0;
3905
3906	/* We only care about %s and variants */
3907	for (i = 0; p[i]; i++) {
3908	if (i + 1 >= iter->fmt_size) {
3909	/*
3910	* If we can't expand the copy buffer,
3911	* just print it.
3912	*/
3913	if (!trace_iter_expand_format(iter))
3914	goto print;
3915	}
3916
3917	if (p[i] == '\\' && p[i+1]) {
3918	i++;
3919	continue;
3920	}
3921	if (p[i] == '%') {
3922	/* Need to test cases like %08.s /
3923	for (j = 1; p[i+j]; j++) {
3924	if (isdigit(p[i+j]) \|\|
3925	p[i+j] == '.')
3926	continue;
3927	if (p[i+j] == '*') {
3928	star = true;
3929	continue;
3930	}
3931	break;
3932	}
3933	if (p[i+j] == 's')
3934	break;
3935	star = false;
3936	}
3937	j = 0;
3938	}
3939	/* If no %s found then just print normally */
3940	if (!p[i])
3941	break;
3942
3943	/* Copy up to the %s, and print that */
3944	strncpy(iter->fmt, p, i);
3945	iter->fmt[i] = '\0';
3946	trace_seq_vprintf(&iter->seq, iter->fmt, ap);
3947
3948	/*
3949	* If iter->seq is full, the above call no longer guarantees
3950	* that ap is in sync with fmt processing, and further calls
3951	* to va_arg() can return wrong positional arguments.
3952	*
3953	* Ensure that ap is no longer used in this case.
3954	*/
3955	if (iter->seq.full) {
3956	p = "";
3957	break;
3958	}
3959
3960	if (star)
3961	len = va_arg(ap, int);
3962
3963	/* The ap now points to the string data of the %s */
3964	str = va_arg(ap, const char *);
3965
3966	/*
3967	* If you hit this warning, it is likely that the
3968	* trace event in question used %s on a string that
3969	* was saved at the time of the event, but may not be
3970	* around when the trace is read. Use __string(),
3971	* __assign_str() and __get_str() helpers in the TRACE_EVENT()
3972	* instead. See samples/trace_events/trace-events-sample.h
3973	* for reference.
3974	*/
3975	if (WARN_ONCE(!trace_safe_str(iter, str, star, len),
3976	"fmt: '%s' current_buffer: '%s'",
3977	fmt, show_buffer(&iter->seq))) {
3978	int ret;
3979
3980	/* Try to safely read the string */
3981	if (star) {
3982	if (len + 1 > iter->fmt_size)
3983	len = iter->fmt_size - 1;
3984	if (len < 0)
3985	len = 0;
3986	ret = copy_from_kernel_nofault(iter->fmt, str, len);
3987	iter->fmt[len] = 0;
3988	star = false;
3989	} else {
3990	ret = strncpy_from_kernel_nofault(iter->fmt, str,
3991	iter->fmt_size);
3992	}
3993	if (ret < 0)
3994	trace_seq_printf(&iter->seq, "(0x%px)", str);
3995	else
3996	trace_seq_printf(&iter->seq, "(0x%px:%s)",
3997	str, iter->fmt);
3998	str = "[UNSAFE-MEMORY]";
3999	strcpy(iter->fmt, "%s");
4000	} else {
4001	strncpy(iter->fmt, p + i, j + 1);
4002	iter->fmt[j+1] = '\0';
4003	}
4004	if (star)
4005	trace_seq_printf(&iter->seq, iter->fmt, len, str);
4006	else
4007	trace_seq_printf(&iter->seq, iter->fmt, str);
4008
4009	p += i + j + 1;
4010	}
4011	print:
4012	if (*p)
4013	trace_seq_vprintf(&iter->seq, p, ap);
4014	}
4015
4016	const char trace_event_format(struct trace_iterator iter, const char *fmt)
4017	{
4018	const char p, new_fmt;
4019	char *q;
4020
4021	if (WARN_ON_ONCE(!fmt))
4022	return fmt;
4023
4024	if (!iter->tr \|\| iter->tr->trace_flags & TRACE_ITER_HASH_PTR)
4025	return fmt;
4026
4027	p = fmt;
4028	new_fmt = q = iter->fmt;
4029	while (*p) {
4030	if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {
4031	if (!trace_iter_expand_format(iter))
4032	return fmt;
4033
4034	q += iter->fmt - new_fmt;
4035	new_fmt = iter->fmt;
4036	}
4037
4038	q++ = p++;
4039
4040	/* Replace %p with %px */
4041	if (p[-1] == '%') {
4042	if (p[0] == '%') {
4043	q++ = p++;
4044	} else if (p[0] == 'p' && !isalnum(p[1])) {
4045	q++ = p++;
4046	*q++ = 'x';
4047	}
4048	}
4049	}
4050	*q = '\0';
4051
4052	return new_fmt;
4053	}
4054
4055	#define STATIC_TEMP_BUF_SIZE 128
4056	static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
4057
4058	/* Find the next real entry, without updating the iterator itself */
4059	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
4060	int ent_cpu, u64 ent_ts)
4061	{
4062	/* __find_next_entry will reset ent_size */
4063	int ent_size = iter->ent_size;
4064	struct trace_entry *entry;
4065
4066	/*
4067	* If called from ftrace_dump(), then the iter->temp buffer
4068	* will be the static_temp_buf and not created from kmalloc.
4069	* If the entry size is greater than the buffer, we can
4070	* not save it. Just return NULL in that case. This is only
4071	* used to add markers when two consecutive events' time
4072	* stamps have a large delta. See trace_print_lat_context()
4073	*/
4074	if (iter->temp == static_temp_buf &&
4075	STATIC_TEMP_BUF_SIZE < ent_size)
4076	return NULL;
4077
4078	/*
4079	* The __find_next_entry() may call peek_next_entry(), which may
4080	* call ring_buffer_peek() that may make the contents of iter->ent
4081	* undefined. Need to copy iter->ent now.
4082	*/
4083	if (iter->ent && iter->ent != iter->temp) {
4084	if ((!iter->temp \|\| iter->temp_size < iter->ent_size) &&
4085	!WARN_ON_ONCE(iter->temp == static_temp_buf)) {
4086	void *temp;
4087	temp = kmalloc(iter->ent_size, GFP_KERNEL);
4088	if (!temp)
4089	return NULL;
4090	kfree(iter->temp);
4091	iter->temp = temp;
4092	iter->temp_size = iter->ent_size;
4093	}
4094	memcpy(iter->temp, iter->ent, iter->ent_size);
4095	iter->ent = iter->temp;
4096	}
4097	entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
4098	/* Put back the original ent_size */
4099	iter->ent_size = ent_size;
4100
4101	return entry;
4102	}
4103
4104	/* Find the next real entry, and increment the iterator to the next entry */
4105	void trace_find_next_entry_inc(struct trace_iterator iter)
4106	{
4107	iter->ent = __find_next_entry(iter, &iter->cpu,
4108	&iter->lost_events, &iter->ts);
4109
4110	if (iter->ent)
4111	trace_iterator_increment(iter);
4112
4113	return iter->ent ? iter : NULL;
4114	}
4115
4116	static void trace_consume(struct trace_iterator *iter)
4117	{
4118	ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
4119	&iter->lost_events);
4120	}
4121
4122	static void s_next(struct seq_file m, void v, loff_t pos)
4123	{
4124	struct trace_iterator *iter = m->private;
4125	int i = (int)*pos;
4126	void *ent;
4127
4128	WARN_ON_ONCE(iter->leftover);
4129
4130	(*pos)++;
4131
4132	/* can't go backwards */
4133	if (iter->idx > i)
4134	return NULL;
4135
4136	if (iter->idx < 0)
4137	ent = trace_find_next_entry_inc(iter);
4138	else
4139	ent = iter;
4140
4141	while (ent && iter->idx < i)
4142	ent = trace_find_next_entry_inc(iter);
4143
4144	iter->pos = *pos;
4145
4146	return ent;
4147	}
4148
4149	void tracing_iter_reset(struct trace_iterator *iter, int cpu)
4150	{
4151	struct ring_buffer_iter *buf_iter;
4152	unsigned long entries = 0;
4153	u64 ts;
4154
4155	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
4156
4157	buf_iter = trace_buffer_iter(iter, cpu);
4158	if (!buf_iter)
4159	return;
4160
4161	ring_buffer_iter_reset(buf_iter);
4162
4163	/*
4164	* We could have the case with the max latency tracers
4165	* that a reset never took place on a cpu. This is evident
4166	* by the timestamp being before the start of the buffer.
4167	*/
4168	while (ring_buffer_iter_peek(buf_iter, &ts)) {
4169	if (ts >= iter->array_buffer->time_start)
4170	break;
4171	entries++;
4172	ring_buffer_iter_advance(buf_iter);
4173	}
4174
4175	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
4176	}
4177
4178	/*
4179	* The current tracer is copied to avoid a global locking
4180	* all around.
4181	*/
4182	static void s_start(struct seq_file m, loff_t *pos)
4183	{
4184	struct trace_iterator *iter = m->private;
4185	struct trace_array *tr = iter->tr;
4186	int cpu_file = iter->cpu_file;
4187	void *p = NULL;
4188	loff_t l = 0;
4189	int cpu;
4190
4191	/*
4192	* copy the tracer to avoid using a global lock all around.
4193	* iter->trace is a copy of current_trace, the pointer to the
4194	* name may be used instead of a strcmp(), as iter->trace->name
4195	* will point to the same string as current_trace->name.
4196	*/
4197	mutex_lock(&trace_types_lock);
4198	if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
4199	iter->trace = tr->current_trace;
4200	mutex_unlock(&trace_types_lock);
4201
4202	#ifdef CONFIG_TRACER_MAX_TRACE
4203	if (iter->snapshot && iter->trace->use_max_tr)
4204	return ERR_PTR(-EBUSY);
4205	#endif
4206
4207	if (*pos != iter->pos) {
4208	iter->ent = NULL;
4209	iter->cpu = 0;
4210	iter->idx = -1;
4211
4212	if (cpu_file == RING_BUFFER_ALL_CPUS) {
4213	for_each_tracing_cpu(cpu)
4214	tracing_iter_reset(iter, cpu);
4215	} else
4216	tracing_iter_reset(iter, cpu_file);
4217
4218	iter->leftover = 0;
4219	for (p = iter; p && l < *pos; p = s_next(m, p, &l))
4220	;
4221
4222	} else {
4223	/*
4224	* If we overflowed the seq_file before, then we want
4225	* to just reuse the trace_seq buffer again.
4226	*/
4227	if (iter->leftover)
4228	p = iter;
4229	else {
4230	l = *pos - 1;
4231	p = s_next(m, p, &l);
4232	}
4233	}
4234
4235	trace_event_read_lock();
4236	trace_access_lock(cpu_file);
4237	return p;
4238	}
4239
4240	static void s_stop(struct seq_file m, void p)
4241	{
4242	struct trace_iterator *iter = m->private;
4243
4244	#ifdef CONFIG_TRACER_MAX_TRACE
4245	if (iter->snapshot && iter->trace->use_max_tr)
4246	return;
4247	#endif
4248
4249	trace_access_unlock(iter->cpu_file);
4250	trace_event_read_unlock();
4251	}
4252
4253	static void
4254	get_total_entries_cpu(struct array_buffer buf, unsigned long total,
4255	unsigned long *entries, int cpu)
4256	{
4257	unsigned long count;
4258
4259	count = ring_buffer_entries_cpu(buf->buffer, cpu);
4260	/*
4261	* If this buffer has skipped entries, then we hold all
4262	* entries for the trace and we need to ignore the
4263	* ones before the time stamp.
4264	*/
4265	if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
4266	count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
4267	/* total is the same as the entries */
4268	*total = count;
4269	} else
4270	*total = count +
4271	ring_buffer_overrun_cpu(buf->buffer, cpu);
4272	*entries = count;
4273	}
4274
4275	static void
4276	get_total_entries(struct array_buffer *buf,
4277	unsigned long total, unsigned long entries)
4278	{
4279	unsigned long t, e;
4280	int cpu;
4281
4282	*total = 0;
4283	*entries = 0;
4284
4285	for_each_tracing_cpu(cpu) {
4286	get_total_entries_cpu(buf, &t, &e, cpu);
4287	*total += t;
4288	*entries += e;
4289	}
4290	}
4291
4292	unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
4293	{
4294	unsigned long total, entries;
4295
4296	if (!tr)
4297	tr = &global_trace;
4298
4299	get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
4300
4301	return entries;
4302	}
4303
4304	unsigned long trace_total_entries(struct trace_array *tr)
4305	{
4306	unsigned long total, entries;
4307
4308	if (!tr)
4309	tr = &global_trace;
4310
4311	get_total_entries(&tr->array_buffer, &total, &entries);
4312
4313	return entries;
4314	}
4315
4316	static void print_lat_help_header(struct seq_file *m)
4317	{
4318	seq_puts(m, "# _------=> CPU# \n"
4319	"# / _-----=> irqs-off/BH-disabled\n"
4320	"# \| / _----=> need-resched \n"
4321	"# \|\| / _---=> hardirq/softirq \n"
4322	"# \|\|\| / _--=> preempt-depth \n"
4323	"# \|\|\|\| / _-=> migrate-disable \n"
4324	"# \|\|\|\|\| / delay \n"
4325	"# cmd pid \|\|\|\|\|\| time \| caller \n"
4326	"# \\ / \|\|\|\|\|\| \\ \| / \n");
4327	}
4328
4329	static void print_event_info(struct array_buffer buf, struct seq_file m)
4330	{
4331	unsigned long total;
4332	unsigned long entries;
4333
4334	get_total_entries(buf, &total, &entries);
4335	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
4336	entries, total, num_online_cpus());
4337	seq_puts(m, "#\n");
4338	}
4339
4340	static void print_func_help_header(struct array_buffer buf, struct seq_file m,
4341	unsigned int flags)
4342	{
4343	bool tgid = flags & TRACE_ITER_RECORD_TGID;
4344
4345	print_event_info(buf, m);
4346
4347	seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
4348	seq_printf(m, "# \| \| %s \| \| \|\n", tgid ? " \| " : "");
4349	}
4350
4351	static void print_func_help_header_irq(struct array_buffer buf, struct seq_file m,
4352	unsigned int flags)
4353	{
4354	bool tgid = flags & TRACE_ITER_RECORD_TGID;
4355	static const char space[] = " ";
4356	int prec = tgid ? 12 : 2;
4357
4358	print_event_info(buf, m);
4359
4360	seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space);
4361	seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
4362	seq_printf(m, "# %.*s\| / _---=> hardirq/softirq\n", prec, space);
4363	seq_printf(m, "# %.*s\|\| / _--=> preempt-depth\n", prec, space);
4364	seq_printf(m, "# %.*s\|\|\| / _-=> migrate-disable\n", prec, space);
4365	seq_printf(m, "# %.*s\|\|\|\| / delay\n", prec, space);
4366	seq_printf(m, "# TASK-PID %.*s CPU# \|\|\|\|\| TIMESTAMP FUNCTION\n", prec, " TGID ");
4367	seq_printf(m, "# \| \| %.*s \| \|\|\|\|\| \| \|\n", prec, " \| ");
4368	}
4369
4370	void
4371	print_trace_header(struct seq_file m, struct trace_iterator iter)
4372	{
4373	unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
4374	struct array_buffer *buf = iter->array_buffer;
4375	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
4376	struct tracer *type = iter->trace;
4377	unsigned long entries;
4378	unsigned long total;
4379	const char *name = type->name;
4380
4381	get_total_entries(buf, &total, &entries);
4382
4383	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
4384	name, UTS_RELEASE);
4385	seq_puts(m, "# -----------------------------------"
4386	"---------------------------------\n");
4387	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d \|"
4388	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
4389	nsecs_to_usecs(data->saved_latency),
4390	entries,
4391	total,
4392	buf->cpu,
4393	preempt_model_none() ? "server" :
4394	preempt_model_voluntary() ? "desktop" :
4395	preempt_model_full() ? "preempt" :
4396	preempt_model_rt() ? "preempt_rt" :
4397	"unknown",
4398	/* These are reserved for later use */
4399	0, 0, 0, 0);
4400	#ifdef CONFIG_SMP
4401	seq_printf(m, " #P:%d)\n", num_online_cpus());
4402	#else
4403	seq_puts(m, ")\n");
4404	#endif
4405	seq_puts(m, "# -----------------\n");
4406	seq_printf(m, "# \| task: %.16s-%d "
4407	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
4408	data->comm, data->pid,
4409	from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
4410	data->policy, data->rt_priority);
4411	seq_puts(m, "# -----------------\n");
4412
4413	if (data->critical_start) {
4414	seq_puts(m, "# => started at: ");
4415	seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
4416	trace_print_seq(m, &iter->seq);
4417	seq_puts(m, "\n# => ended at: ");
4418	seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
4419	trace_print_seq(m, &iter->seq);
4420	seq_puts(m, "\n#\n");
4421	}
4422
4423	seq_puts(m, "#\n");
4424	}
4425
4426	static void test_cpu_buff_start(struct trace_iterator *iter)
4427	{
4428	struct trace_seq *s = &iter->seq;
4429	struct trace_array *tr = iter->tr;
4430
4431	if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
4432	return;
4433
4434	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
4435	return;
4436
4437	if (cpumask_available(iter->started) &&
4438	cpumask_test_cpu(iter->cpu, iter->started))
4439	return;
4440
4441	if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
4442	return;
4443
4444	if (cpumask_available(iter->started))
4445	cpumask_set_cpu(iter->cpu, iter->started);
4446
4447	/* Don't print started cpu buffer for the first entry of the trace */
4448	if (iter->idx > 1)
4449	trace_seq_printf(s, "##### CPU %u buffer started ####\n",
4450	iter->cpu);
4451	}
4452
4453	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
4454	{
4455	struct trace_array *tr = iter->tr;
4456	struct trace_seq *s = &iter->seq;
4457	unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4458	struct trace_entry *entry;
4459	struct trace_event *event;
4460
4461	entry = iter->ent;
4462
4463	test_cpu_buff_start(iter);
4464
4465	event = ftrace_find_event(entry->type);
4466
4467	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4468	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4469	trace_print_lat_context(iter);
4470	else
4471	trace_print_context(iter);
4472	}
4473
4474	if (trace_seq_has_overflowed(s))
4475	return TRACE_TYPE_PARTIAL_LINE;
4476
4477	if (event) {
4478	if (tr->trace_flags & TRACE_ITER_FIELDS)
4479	return print_event_fields(iter, event);
4480	return event->funcs->trace(iter, sym_flags, event);
4481	}
4482
4483	trace_seq_printf(s, "Unknown type %d\n", entry->type);
4484
4485	return trace_handle_return(s);
4486	}
4487
4488	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
4489	{
4490	struct trace_array *tr = iter->tr;
4491	struct trace_seq *s = &iter->seq;
4492	struct trace_entry *entry;
4493	struct trace_event *event;
4494
4495	entry = iter->ent;
4496
4497	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
4498	trace_seq_printf(s, "%d %d %llu ",
4499	entry->pid, iter->cpu, iter->ts);
4500
4501	if (trace_seq_has_overflowed(s))
4502	return TRACE_TYPE_PARTIAL_LINE;
4503
4504	event = ftrace_find_event(entry->type);
4505	if (event)
4506	return event->funcs->raw(iter, 0, event);
4507
4508	trace_seq_printf(s, "%d ?\n", entry->type);
4509
4510	return trace_handle_return(s);
4511	}
4512
4513	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
4514	{
4515	struct trace_array *tr = iter->tr;
4516	struct trace_seq *s = &iter->seq;
4517	unsigned char newline = '\n';
4518	struct trace_entry *entry;
4519	struct trace_event *event;
4520
4521	entry = iter->ent;
4522
4523	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4524	SEQ_PUT_HEX_FIELD(s, entry->pid);
4525	SEQ_PUT_HEX_FIELD(s, iter->cpu);
4526	SEQ_PUT_HEX_FIELD(s, iter->ts);
4527	if (trace_seq_has_overflowed(s))
4528	return TRACE_TYPE_PARTIAL_LINE;
4529	}
4530
4531	event = ftrace_find_event(entry->type);
4532	if (event) {
4533	enum print_line_t ret = event->funcs->hex(iter, 0, event);
4534	if (ret != TRACE_TYPE_HANDLED)
4535	return ret;
4536	}
4537
4538	SEQ_PUT_FIELD(s, newline);
4539
4540	return trace_handle_return(s);
4541	}
4542
4543	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
4544	{
4545	struct trace_array *tr = iter->tr;
4546	struct trace_seq *s = &iter->seq;
4547	struct trace_entry *entry;
4548	struct trace_event *event;
4549
4550	entry = iter->ent;
4551
4552	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4553	SEQ_PUT_FIELD(s, entry->pid);
4554	SEQ_PUT_FIELD(s, iter->cpu);
4555	SEQ_PUT_FIELD(s, iter->ts);
4556	if (trace_seq_has_overflowed(s))
4557	return TRACE_TYPE_PARTIAL_LINE;
4558	}
4559
4560	event = ftrace_find_event(entry->type);
4561	return event ? event->funcs->binary(iter, 0, event) :
4562	TRACE_TYPE_HANDLED;
4563	}
4564
4565	int trace_empty(struct trace_iterator *iter)
4566	{
4567	struct ring_buffer_iter *buf_iter;
4568	int cpu;
4569
4570	/* If we are looking at one CPU buffer, only check that one */
4571	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4572	cpu = iter->cpu_file;
4573	buf_iter = trace_buffer_iter(iter, cpu);
4574	if (buf_iter) {
4575	if (!ring_buffer_iter_empty(buf_iter))
4576	return 0;
4577	} else {
4578	if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4579	return 0;
4580	}
4581	return 1;
4582	}
4583
4584	for_each_tracing_cpu(cpu) {
4585	buf_iter = trace_buffer_iter(iter, cpu);
4586	if (buf_iter) {
4587	if (!ring_buffer_iter_empty(buf_iter))
4588	return 0;
4589	} else {
4590	if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4591	return 0;
4592	}
4593	}
4594
4595	return 1;
4596	}
4597
4598	/* Called with trace_event_read_lock() held. */
4599	enum print_line_t print_trace_line(struct trace_iterator *iter)
4600	{
4601	struct trace_array *tr = iter->tr;
4602	unsigned long trace_flags = tr->trace_flags;
4603	enum print_line_t ret;
4604
4605	if (iter->lost_events) {
4606	if (iter->lost_events == (unsigned long)-1)
4607	trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
4608	iter->cpu);
4609	else
4610	trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
4611	iter->cpu, iter->lost_events);
4612	if (trace_seq_has_overflowed(&iter->seq))
4613	return TRACE_TYPE_PARTIAL_LINE;
4614	}
4615
4616	if (iter->trace && iter->trace->print_line) {
4617	ret = iter->trace->print_line(iter);
4618	if (ret != TRACE_TYPE_UNHANDLED)
4619	return ret;
4620	}
4621
4622	if (iter->ent->type == TRACE_BPUTS &&
4623	trace_flags & TRACE_ITER_PRINTK &&
4624	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4625	return trace_print_bputs_msg_only(iter);
4626
4627	if (iter->ent->type == TRACE_BPRINT &&
4628	trace_flags & TRACE_ITER_PRINTK &&
4629	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4630	return trace_print_bprintk_msg_only(iter);
4631
4632	if (iter->ent->type == TRACE_PRINT &&
4633	trace_flags & TRACE_ITER_PRINTK &&
4634	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4635	return trace_print_printk_msg_only(iter);
4636
4637	if (trace_flags & TRACE_ITER_BIN)
4638	return print_bin_fmt(iter);
4639
4640	if (trace_flags & TRACE_ITER_HEX)
4641	return print_hex_fmt(iter);
4642
4643	if (trace_flags & TRACE_ITER_RAW)
4644	return print_raw_fmt(iter);
4645
4646	return print_trace_fmt(iter);
4647	}
4648
4649	void trace_latency_header(struct seq_file *m)
4650	{
4651	struct trace_iterator *iter = m->private;
4652	struct trace_array *tr = iter->tr;
4653
4654	/* print nothing if the buffers are empty */
4655	if (trace_empty(iter))
4656	return;
4657
4658	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4659	print_trace_header(m, iter);
4660
4661	if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4662	print_lat_help_header(m);
4663	}
4664
4665	void trace_default_header(struct seq_file *m)
4666	{
4667	struct trace_iterator *iter = m->private;
4668	struct trace_array *tr = iter->tr;
4669	unsigned long trace_flags = tr->trace_flags;
4670
4671	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4672	return;
4673
4674	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4675	/* print nothing if the buffers are empty */
4676	if (trace_empty(iter))
4677	return;
4678	print_trace_header(m, iter);
4679	if (!(trace_flags & TRACE_ITER_VERBOSE))
4680	print_lat_help_header(m);
4681	} else {
4682	if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4683	if (trace_flags & TRACE_ITER_IRQ_INFO)
4684	print_func_help_header_irq(iter->array_buffer,
4685	m, trace_flags);
4686	else
4687	print_func_help_header(iter->array_buffer, m,
4688	trace_flags);
4689	}
4690	}
4691	}
4692
4693	static void test_ftrace_alive(struct seq_file *m)
4694	{
4695	if (!ftrace_is_dead())
4696	return;
4697	seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4698	"# MAY BE MISSING FUNCTION EVENTS\n");
4699	}
4700
4701	#ifdef CONFIG_TRACER_MAX_TRACE
4702	static void show_snapshot_main_help(struct seq_file *m)
4703	{
4704	seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4705	"# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4706	"# Takes a snapshot of the main buffer.\n"
4707	"# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4708	"# (Doesn't have to be '2' works with any number that\n"
4709	"# is not a '0' or '1')\n");
4710	}
4711
4712	static void show_snapshot_percpu_help(struct seq_file *m)
4713	{
4714	seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4715	#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4716	seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4717	"# Takes a snapshot of the main buffer for this cpu.\n");
4718	#else
4719	seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4720	"# Must use main snapshot file to allocate.\n");
4721	#endif
4722	seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4723	"# (Doesn't have to be '2' works with any number that\n"
4724	"# is not a '0' or '1')\n");
4725	}
4726
4727	static void print_snapshot_help(struct seq_file m, struct trace_iterator iter)
4728	{
4729	if (iter->tr->allocated_snapshot)
4730	seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4731	else
4732	seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4733
4734	seq_puts(m, "# Snapshot commands:\n");
4735	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4736	show_snapshot_main_help(m);
4737	else
4738	show_snapshot_percpu_help(m);
4739	}
4740	#else
4741	/* Should never be called */
4742	static inline void print_snapshot_help(struct seq_file m, struct trace_iterator iter) { }
4743	#endif
4744
4745	static int s_show(struct seq_file m, void v)
4746	{
4747	struct trace_iterator *iter = v;
4748	int ret;
4749
4750	if (iter->ent == NULL) {
4751	if (iter->tr) {
4752	seq_printf(m, "# tracer: %s\n", iter->trace->name);
4753	seq_puts(m, "#\n");
4754	test_ftrace_alive(m);
4755	}
4756	if (iter->snapshot && trace_empty(iter))
4757	print_snapshot_help(m, iter);
4758	else if (iter->trace && iter->trace->print_header)
4759	iter->trace->print_header(m);
4760	else
4761	trace_default_header(m);
4762
4763	} else if (iter->leftover) {
4764	/*
4765	* If we filled the seq_file buffer earlier, we
4766	* want to just show it now.
4767	*/
4768	ret = trace_print_seq(m, &iter->seq);
4769
4770	/* ret should this time be zero, but you never know */
4771	iter->leftover = ret;
4772
4773	} else {
4774	print_trace_line(iter);
4775	ret = trace_print_seq(m, &iter->seq);
4776	/*
4777	* If we overflow the seq_file buffer, then it will
4778	* ask us for this data again at start up.
4779	* Use that instead.
4780	* ret is 0 if seq_file write succeeded.
4781	* -1 otherwise.
4782	*/
4783	iter->leftover = ret;
4784	}
4785
4786	return 0;
4787	}
4788
4789	/*
4790	* Should be used after trace_array_get(), trace_types_lock
4791	* ensures that i_cdev was already initialized.
4792	*/
4793	static inline int tracing_get_cpu(struct inode *inode)
4794	{
4795	if (inode->i_cdev) /* See trace_create_cpu_file() */
4796	return (long)inode->i_cdev - 1;
4797	return RING_BUFFER_ALL_CPUS;
4798	}
4799
4800	static const struct seq_operations tracer_seq_ops = {
4801	.start = s_start,
4802	.next = s_next,
4803	.stop = s_stop,
4804	.show = s_show,
4805	};
4806
4807	static struct trace_iterator *
4808	__tracing_open(struct inode inode, struct file file, bool snapshot)
4809	{
4810	struct trace_array *tr = inode->i_private;
4811	struct trace_iterator *iter;
4812	int cpu;
4813
4814	if (tracing_disabled)
4815	return ERR_PTR(-ENODEV);
4816
4817	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4818	if (!iter)
4819	return ERR_PTR(-ENOMEM);
4820
4821	iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4822	GFP_KERNEL);
4823	if (!iter->buffer_iter)
4824	goto release;
4825
4826	/*
4827	* trace_find_next_entry() may need to save off iter->ent.
4828	* It will place it into the iter->temp buffer. As most
4829	* events are less than 128, allocate a buffer of that size.
4830	* If one is greater, then trace_find_next_entry() will
4831	* allocate a new buffer to adjust for the bigger iter->ent.
4832	* It's not critical if it fails to get allocated here.
4833	*/
4834	iter->temp = kmalloc(128, GFP_KERNEL);
4835	if (iter->temp)
4836	iter->temp_size = 128;
4837
4838	/*
4839	* trace_event_printf() may need to modify given format
4840	* string to replace %p with %px so that it shows real address
4841	* instead of hash value. However, that is only for the event
4842	* tracing, other tracer may not need. Defer the allocation
4843	* until it is needed.
4844	*/
4845	iter->fmt = NULL;
4846	iter->fmt_size = 0;
4847
4848	/*
4849	* We make a copy of the current tracer to avoid concurrent
4850	* changes on it while we are reading.
4851	*/
4852	mutex_lock(&trace_types_lock);
4853	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4854	if (!iter->trace)
4855	goto fail;
4856
4857	iter->trace = tr->current_trace;
4858
4859	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4860	goto fail;
4861
4862	iter->tr = tr;
4863
4864	#ifdef CONFIG_TRACER_MAX_TRACE
4865	/* Currently only the top directory has a snapshot */
4866	if (tr->current_trace->print_max \|\| snapshot)
4867	iter->array_buffer = &tr->max_buffer;
4868	else
4869	#endif
4870	iter->array_buffer = &tr->array_buffer;
4871	iter->snapshot = snapshot;
4872	iter->pos = -1;
4873	iter->cpu_file = tracing_get_cpu(inode);
4874	mutex_init(&iter->mutex);
4875
4876	/* Notify the tracer early; before we stop tracing. */
4877	if (iter->trace->open)
4878	iter->trace->open(iter);
4879
4880	/* Annotate start of buffers if we had overruns */
4881	if (ring_buffer_overruns(iter->array_buffer->buffer))
4882	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
4883
4884	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
4885	if (trace_clocks[tr->clock_id].in_ns)
4886	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
4887
4888	/*
4889	* If pause-on-trace is enabled, then stop the trace while
4890	* dumping, unless this is the "snapshot" file
4891	*/
4892	if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4893	tracing_stop_tr(tr);
4894
4895	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4896	for_each_tracing_cpu(cpu) {
4897	iter->buffer_iter[cpu] =
4898	ring_buffer_read_prepare(iter->array_buffer->buffer,
4899	cpu, GFP_KERNEL);
4900	}
4901	ring_buffer_read_prepare_sync();
4902	for_each_tracing_cpu(cpu) {
4903	ring_buffer_read_start(iter->buffer_iter[cpu]);
4904	tracing_iter_reset(iter, cpu);
4905	}
4906	} else {
4907	cpu = iter->cpu_file;
4908	iter->buffer_iter[cpu] =
4909	ring_buffer_read_prepare(iter->array_buffer->buffer,
4910	cpu, GFP_KERNEL);
4911	ring_buffer_read_prepare_sync();
4912	ring_buffer_read_start(iter->buffer_iter[cpu]);
4913	tracing_iter_reset(iter, cpu);
4914	}
4915
4916	mutex_unlock(&trace_types_lock);
4917
4918	return iter;
4919
4920	fail:
4921	mutex_unlock(&trace_types_lock);
4922	kfree(iter->trace);
4923	kfree(iter->temp);
4924	kfree(iter->buffer_iter);
4925	release:
4926	seq_release_private(inode, file);
4927	return ERR_PTR(-ENOMEM);
4928	}
4929
4930	int tracing_open_generic(struct inode inode, struct file filp)
4931	{
4932	int ret;
4933
4934	ret = tracing_check_open_get_tr(NULL);
4935	if (ret)
4936	return ret;
4937
4938	filp->private_data = inode->i_private;
4939	return 0;
4940	}
4941
4942	bool tracing_is_disabled(void)
4943	{
4944	return (tracing_disabled) ? true: false;
4945	}
4946
4947	/*
4948	* Open and update trace_array ref count.
4949	* Must have the current trace_array passed to it.
4950	*/
4951	int tracing_open_generic_tr(struct inode inode, struct file filp)
4952	{
4953	struct trace_array *tr = inode->i_private;
4954	int ret;
4955
4956	ret = tracing_check_open_get_tr(tr);
4957	if (ret)
4958	return ret;
4959
4960	filp->private_data = inode->i_private;
4961
4962	return 0;
4963	}
4964
4965	static int tracing_mark_open(struct inode inode, struct file filp)
4966	{
4967	stream_open(inode, filp);
4968	return tracing_open_generic_tr(inode, filp);
4969	}
4970
4971	static int tracing_release(struct inode inode, struct file file)
4972	{
4973	struct trace_array *tr = inode->i_private;
4974	struct seq_file *m = file->private_data;
4975	struct trace_iterator *iter;
4976	int cpu;
4977
4978	if (!(file->f_mode & FMODE_READ)) {
4979	trace_array_put(tr);
4980	return 0;
4981	}
4982
4983	/* Writes do not use seq_file */
4984	iter = m->private;
4985	mutex_lock(&trace_types_lock);
4986
4987	for_each_tracing_cpu(cpu) {
4988	if (iter->buffer_iter[cpu])
4989	ring_buffer_read_finish(iter->buffer_iter[cpu]);
4990	}
4991
4992	if (iter->trace && iter->trace->close)
4993	iter->trace->close(iter);
4994
4995	if (!iter->snapshot && tr->stop_count)
4996	/* reenable tracing if it was previously enabled */
4997	tracing_start_tr(tr);
4998
4999	__trace_array_put(tr);
5000
5001	mutex_unlock(&trace_types_lock);
5002
5003	mutex_destroy(&iter->mutex);
5004	free_cpumask_var(iter->started);
5005	kfree(iter->fmt);
5006	kfree(iter->temp);
5007	kfree(iter->trace);
5008	kfree(iter->buffer_iter);
5009	seq_release_private(inode, file);
5010
5011	return 0;
5012	}
5013
5014	static int tracing_release_generic_tr(struct inode inode, struct file file)
5015	{
5016	struct trace_array *tr = inode->i_private;
5017
5018	trace_array_put(tr);
5019	return 0;
5020	}
5021
5022	static int tracing_single_release_tr(struct inode inode, struct file file)
5023	{
5024	struct trace_array *tr = inode->i_private;
5025
5026	trace_array_put(tr);
5027
5028	return single_release(inode, file);
5029	}
5030
5031	static int tracing_open(struct inode inode, struct file file)
5032	{
5033	struct trace_array *tr = inode->i_private;
5034	struct trace_iterator *iter;
5035	int ret;
5036
5037	ret = tracing_check_open_get_tr(tr);
5038	if (ret)
5039	return ret;
5040
5041	/* If this file was open for write, then erase contents */
5042	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
5043	int cpu = tracing_get_cpu(inode);
5044	struct array_buffer *trace_buf = &tr->array_buffer;
5045
5046	#ifdef CONFIG_TRACER_MAX_TRACE
5047	if (tr->current_trace->print_max)
5048	trace_buf = &tr->max_buffer;
5049	#endif
5050
5051	if (cpu == RING_BUFFER_ALL_CPUS)
5052	tracing_reset_online_cpus(trace_buf);
5053	else
5054	tracing_reset_cpu(trace_buf, cpu);
5055	}
5056
5057	if (file->f_mode & FMODE_READ) {
5058	iter = __tracing_open(inode, file, false);
5059	if (IS_ERR(iter))
5060	ret = PTR_ERR(iter);
5061	else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5062	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
5063	}
5064
5065	if (ret < 0)
5066	trace_array_put(tr);
5067
5068	return ret;
5069	}
5070
5071	/*
5072	* Some tracers are not suitable for instance buffers.
5073	* A tracer is always available for the global array (toplevel)
5074	* or if it explicitly states that it is.
5075	*/
5076	static bool
5077	trace_ok_for_array(struct tracer t, struct trace_array tr)
5078	{
5079	return (tr->flags & TRACE_ARRAY_FL_GLOBAL) \|\| t->allow_instances;
5080	}
5081
5082	/* Find the next tracer that this trace array may use */
5083	static struct tracer *
5084	get_tracer_for_array(struct trace_array tr, struct tracer t)
5085	{
5086	while (t && !trace_ok_for_array(t, tr))
5087	t = t->next;
5088
5089	return t;
5090	}
5091
5092	static void *
5093	t_next(struct seq_file m, void v, loff_t *pos)
5094	{
5095	struct trace_array *tr = m->private;
5096	struct tracer *t = v;
5097
5098	(*pos)++;
5099
5100	if (t)
5101	t = get_tracer_for_array(tr, t->next);
5102
5103	return t;
5104	}
5105
5106	static void t_start(struct seq_file m, loff_t *pos)
5107	{
5108	struct trace_array *tr = m->private;
5109	struct tracer *t;
5110	loff_t l = 0;
5111
5112	mutex_lock(&trace_types_lock);
5113
5114	t = get_tracer_for_array(tr, trace_types);
5115	for (; t && l < *pos; t = t_next(m, t, &l))
5116	;
5117
5118	return t;
5119	}
5120
5121	static void t_stop(struct seq_file m, void p)
5122	{
5123	mutex_unlock(&trace_types_lock);
5124	}
5125
5126	static int t_show(struct seq_file m, void v)
5127	{
5128	struct tracer *t = v;
5129
5130	if (!t)
5131	return 0;
5132
5133	seq_puts(m, t->name);
5134	if (t->next)
5135	seq_putc(m, ' ');
5136	else
5137	seq_putc(m, '\n');
5138
5139	return 0;
5140	}
5141
5142	static const struct seq_operations show_traces_seq_ops = {
5143	.start = t_start,
5144	.next = t_next,
5145	.stop = t_stop,
5146	.show = t_show,
5147	};
5148
5149	static int show_traces_open(struct inode inode, struct file file)
5150	{
5151	struct trace_array *tr = inode->i_private;
5152	struct seq_file *m;
5153	int ret;
5154
5155	ret = tracing_check_open_get_tr(tr);
5156	if (ret)
5157	return ret;
5158
5159	ret = seq_open(file, &show_traces_seq_ops);
5160	if (ret) {
5161	trace_array_put(tr);
5162	return ret;
5163	}
5164
5165	m = file->private_data;
5166	m->private = tr;
5167
5168	return 0;
5169	}
5170
5171	static int show_traces_release(struct inode inode, struct file file)
5172	{
5173	struct trace_array *tr = inode->i_private;
5174
5175	trace_array_put(tr);
5176	return seq_release(inode, file);
5177	}
5178
5179	static ssize_t
5180	tracing_write_stub(struct file filp, const char __user ubuf,
5181	size_t count, loff_t *ppos)
5182	{
5183	return count;
5184	}
5185
5186	loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
5187	{
5188	int ret;
5189
5190	if (file->f_mode & FMODE_READ)
5191	ret = seq_lseek(file, offset, whence);
5192	else
5193	file->f_pos = ret = 0;
5194
5195	return ret;
5196	}
5197
5198	static const struct file_operations tracing_fops = {
5199	.open = tracing_open,
5200	.read = seq_read,
5201	.read_iter = seq_read_iter,
5202	.splice_read = generic_file_splice_read,
5203	.write = tracing_write_stub,
5204	.llseek = tracing_lseek,
5205	.release = tracing_release,
5206	};
5207
5208	static const struct file_operations show_traces_fops = {
5209	.open = show_traces_open,
5210	.read = seq_read,
5211	.llseek = seq_lseek,
5212	.release = show_traces_release,
5213	};
5214
5215	static ssize_t
5216	tracing_cpumask_read(struct file filp, char __user ubuf,
5217	size_t count, loff_t *ppos)
5218	{
5219	struct trace_array *tr = file_inode(filp)->i_private;
5220	char *mask_str;
5221	int len;
5222
5223	len = snprintf(NULL, 0, "%*pb\n",
5224	cpumask_pr_args(tr->tracing_cpumask)) + 1;
5225	mask_str = kmalloc(len, GFP_KERNEL);
5226	if (!mask_str)
5227	return -ENOMEM;
5228
5229	len = snprintf(mask_str, len, "%*pb\n",
5230	cpumask_pr_args(tr->tracing_cpumask));
5231	if (len >= count) {
5232	count = -EINVAL;
5233	goto out_err;
5234	}
5235	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
5236
5237	out_err:
5238	kfree(mask_str);
5239
5240	return count;
5241	}
5242
5243	int tracing_set_cpumask(struct trace_array *tr,
5244	cpumask_var_t tracing_cpumask_new)
5245	{
5246	int cpu;
5247
5248	if (!tr)
5249	return -EINVAL;
5250
5251	local_irq_disable();
5252	arch_spin_lock(&tr->max_lock);
5253	for_each_tracing_cpu(cpu) {
5254	/*
5255	* Increase/decrease the disabled counter if we are
5256	* about to flip a bit in the cpumask:
5257	*/
5258	if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5259	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5260	atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5261	ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
5262	}
5263	if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5264	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5265	atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5266	ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
5267	}
5268	}
5269	arch_spin_unlock(&tr->max_lock);
5270	local_irq_enable();
5271
5272	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
5273
5274	return 0;
5275	}
5276
5277	static ssize_t
5278	tracing_cpumask_write(struct file filp, const char __user ubuf,
5279	size_t count, loff_t *ppos)
5280	{
5281	struct trace_array *tr = file_inode(filp)->i_private;
5282	cpumask_var_t tracing_cpumask_new;
5283	int err;
5284
5285	if (!zalloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
5286	return -ENOMEM;
5287
5288	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
5289	if (err)
5290	goto err_free;
5291
5292	err = tracing_set_cpumask(tr, tracing_cpumask_new);
5293	if (err)
5294	goto err_free;
5295
5296	free_cpumask_var(tracing_cpumask_new);
5297
5298	return count;
5299
5300	err_free:
5301	free_cpumask_var(tracing_cpumask_new);
5302
5303	return err;
5304	}
5305
5306	static const struct file_operations tracing_cpumask_fops = {
5307	.open = tracing_open_generic_tr,
5308	.read = tracing_cpumask_read,
5309	.write = tracing_cpumask_write,
5310	.release = tracing_release_generic_tr,
5311	.llseek = generic_file_llseek,
5312	};
5313
5314	static int tracing_trace_options_show(struct seq_file m, void v)
5315	{
5316	struct tracer_opt *trace_opts;
5317	struct trace_array *tr = m->private;
5318	u32 tracer_flags;
5319	int i;
5320
5321	mutex_lock(&trace_types_lock);
5322	tracer_flags = tr->current_trace->flags->val;
5323	trace_opts = tr->current_trace->flags->opts;
5324
5325	for (i = 0; trace_options[i]; i++) {
5326	if (tr->trace_flags & (1 << i))
5327	seq_printf(m, "%s\n", trace_options[i]);
5328	else
5329	seq_printf(m, "no%s\n", trace_options[i]);
5330	}
5331
5332	for (i = 0; trace_opts[i].name; i++) {
5333	if (tracer_flags & trace_opts[i].bit)
5334	seq_printf(m, "%s\n", trace_opts[i].name);
5335	else
5336	seq_printf(m, "no%s\n", trace_opts[i].name);
5337	}
5338	mutex_unlock(&trace_types_lock);
5339
5340	return 0;
5341	}
5342
5343	static int __set_tracer_option(struct trace_array *tr,
5344	struct tracer_flags *tracer_flags,
5345	struct tracer_opt *opts, int neg)
5346	{
5347	struct tracer *trace = tracer_flags->trace;
5348	int ret;
5349
5350	ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
5351	if (ret)
5352	return ret;
5353
5354	if (neg)
5355	tracer_flags->val &= ~opts->bit;
5356	else
5357	tracer_flags->val \|= opts->bit;
5358	return 0;
5359	}
5360
5361	/* Try to assign a tracer specific option */
5362	static int set_tracer_option(struct trace_array tr, char cmp, int neg)
5363	{
5364	struct tracer *trace = tr->current_trace;
5365	struct tracer_flags *tracer_flags = trace->flags;
5366	struct tracer_opt *opts = NULL;
5367	int i;
5368
5369	for (i = 0; tracer_flags->opts[i].name; i++) {
5370	opts = &tracer_flags->opts[i];
5371
5372	if (strcmp(cmp, opts->name) == 0)
5373	return __set_tracer_option(tr, trace->flags, opts, neg);
5374	}
5375
5376	return -EINVAL;
5377	}
5378
5379	/* Some tracers require overwrite to stay enabled */
5380	int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
5381	{
5382	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
5383	return -1;
5384
5385	return 0;
5386	}
5387
5388	int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
5389	{
5390	int *map;
5391
5392	if ((mask == TRACE_ITER_RECORD_TGID) \|\|
5393	(mask == TRACE_ITER_RECORD_CMD))
5394	lockdep_assert_held(&event_mutex);
5395
5396	/* do nothing if flag is already set */
5397	if (!!(tr->trace_flags & mask) == !!enabled)
5398	return 0;
5399
5400	/* Give the tracer a chance to approve the change */
5401	if (tr->current_trace->flag_changed)
5402	if (tr->current_trace->flag_changed(tr, mask, !!enabled))
5403	return -EINVAL;
5404
5405	if (enabled)
5406	tr->trace_flags \|= mask;
5407	else
5408	tr->trace_flags &= ~mask;
5409
5410	if (mask == TRACE_ITER_RECORD_CMD)
5411	trace_event_enable_cmd_record(enabled);
5412
5413	if (mask == TRACE_ITER_RECORD_TGID) {
5414	if (!tgid_map) {
5415	tgid_map_max = pid_max;
5416	map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map),
5417	GFP_KERNEL);
5418
5419	/*
5420	* Pairs with smp_load_acquire() in
5421	* trace_find_tgid_ptr() to ensure that if it observes
5422	* the tgid_map we just allocated then it also observes
5423	* the corresponding tgid_map_max value.
5424	*/
5425	smp_store_release(&tgid_map, map);
5426	}
5427	if (!tgid_map) {
5428	tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
5429	return -ENOMEM;
5430	}
5431
5432	trace_event_enable_tgid_record(enabled);
5433	}
5434
5435	if (mask == TRACE_ITER_EVENT_FORK)
5436	trace_event_follow_fork(tr, enabled);
5437
5438	if (mask == TRACE_ITER_FUNC_FORK)
5439	ftrace_pid_follow_fork(tr, enabled);
5440
5441	if (mask == TRACE_ITER_OVERWRITE) {
5442	ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
5443	#ifdef CONFIG_TRACER_MAX_TRACE
5444	ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
5445	#endif
5446	}
5447
5448	if (mask == TRACE_ITER_PRINTK) {
5449	trace_printk_start_stop_comm(enabled);
5450	trace_printk_control(enabled);
5451	}
5452
5453	return 0;
5454	}
5455
5456	int trace_set_options(struct trace_array tr, char option)
5457	{
5458	char *cmp;
5459	int neg = 0;
5460	int ret;
5461	size_t orig_len = strlen(option);
5462	int len;
5463
5464	cmp = strstrip(option);
5465
5466	len = str_has_prefix(cmp, "no");
5467	if (len)
5468	neg = 1;
5469
5470	cmp += len;
5471
5472	mutex_lock(&event_mutex);
5473	mutex_lock(&trace_types_lock);
5474
5475	ret = match_string(trace_options, -1, cmp);
5476	/* If no option could be set, test the specific tracer options */
5477	if (ret < 0)
5478	ret = set_tracer_option(tr, cmp, neg);
5479	else
5480	ret = set_tracer_flag(tr, 1 << ret, !neg);
5481
5482	mutex_unlock(&trace_types_lock);
5483	mutex_unlock(&event_mutex);
5484
5485	/*
5486	* If the first trailing whitespace is replaced with '\0' by strstrip,
5487	* turn it back into a space.
5488	*/
5489	if (orig_len > strlen(option))
5490	option[strlen(option)] = ' ';
5491
5492	return ret;
5493	}
5494
5495	static void __init apply_trace_boot_options(void)
5496	{
5497	char *buf = trace_boot_options_buf;
5498	char *option;
5499
5500	while (true) {
5501	option = strsep(&buf, ",");
5502
5503	if (!option)
5504	break;
5505
5506	if (*option)
5507	trace_set_options(&global_trace, option);
5508
5509	/* Put back the comma to allow this to be called again */
5510	if (buf)
5511	*(buf - 1) = ',';
5512	}
5513	}
5514
5515	static ssize_t
5516	tracing_trace_options_write(struct file filp, const char __user ubuf,
5517	size_t cnt, loff_t *ppos)
5518	{
5519	struct seq_file *m = filp->private_data;
5520	struct trace_array *tr = m->private;
5521	char buf[64];
5522	int ret;
5523
5524	if (cnt >= sizeof(buf))
5525	return -EINVAL;
5526
5527	if (copy_from_user(buf, ubuf, cnt))
5528	return -EFAULT;
5529
5530	buf[cnt] = 0;
5531
5532	ret = trace_set_options(tr, buf);
5533	if (ret < 0)
5534	return ret;
5535
5536	*ppos += cnt;
5537
5538	return cnt;
5539	}
5540
5541	static int tracing_trace_options_open(struct inode inode, struct file file)
5542	{
5543	struct trace_array *tr = inode->i_private;
5544	int ret;
5545
5546	ret = tracing_check_open_get_tr(tr);
5547	if (ret)
5548	return ret;
5549
5550	ret = single_open(file, tracing_trace_options_show, inode->i_private);
5551	if (ret < 0)
5552	trace_array_put(tr);
5553
5554	return ret;
5555	}
5556
5557	static const struct file_operations tracing_iter_fops = {
5558	.open = tracing_trace_options_open,
5559	.read = seq_read,
5560	.llseek = seq_lseek,
5561	.release = tracing_single_release_tr,
5562	.write = tracing_trace_options_write,
5563	};
5564
5565	static const char readme_msg[] =
5566	"tracing mini-HOWTO:\n\n"
5567	"# echo 0 > tracing_on : quick way to disable tracing\n"
5568	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5569	" Important files:\n"
5570	" trace\t\t\t- The static contents of the buffer\n"
5571	"\t\t\t To clear the buffer write into this file: echo > trace\n"
5572	" trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5573	" current_tracer\t- function and latency tracers\n"
5574	" available_tracers\t- list of configured tracers for current_tracer\n"
5575	" error_log\t- error log for failed commands (that support it)\n"
5576	" buffer_size_kb\t- view and modify size of per cpu buffer\n"
5577	" buffer_total_size_kb - view total size of all cpu buffers\n\n"
5578	" trace_clock\t\t- change the clock used to order events\n"
5579	" local: Per cpu clock but may not be synced across CPUs\n"
5580	" global: Synced across CPUs but slows tracing down.\n"
5581	" counter: Not a clock, but just an increment\n"
5582	" uptime: Jiffy counter from time of boot\n"
5583	" perf: Same clock that perf events use\n"
5584	#ifdef CONFIG_X86_64
5585	" x86-tsc: TSC cycle counter\n"
5586	#endif
5587	"\n timestamp_mode\t- view the mode used to timestamp events\n"
5588	" delta: Delta difference against a buffer-wide timestamp\n"
5589	" absolute: Absolute (standalone) timestamp\n"
5590	"\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5591	"\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5592	" tracing_cpumask\t- Limit which CPUs to trace\n"
5593	" instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5594	"\t\t\t Remove sub-buffer with rmdir\n"
5595	" trace_options\t\t- Set format or modify how tracing happens\n"
5596	"\t\t\t Disable an option by prefixing 'no' to the\n"
5597	"\t\t\t option name\n"
5598	" saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5599	#ifdef CONFIG_DYNAMIC_FTRACE
5600	"\n available_filter_functions - list of functions that can be filtered on\n"
5601	" set_ftrace_filter\t- echo function name in here to only trace these\n"
5602	"\t\t\t functions\n"
5603	"\t accepts: func_full_name or glob-matching-pattern\n"
5604	"\t modules: Can select a group via module\n"
5605	"\t Format: :mod:<module-name>\n"
5606	"\t example: echo :mod:ext3 > set_ftrace_filter\n"
5607	"\t triggers: a command to perform when function is hit\n"
5608	"\t Format: <function>:<trigger>[:count]\n"
5609	"\t trigger: traceon, traceoff\n"
5610	"\t\t enable_event:<system>:<event>\n"
5611	"\t\t disable_event:<system>:<event>\n"
5612	#ifdef CONFIG_STACKTRACE
5613	"\t\t stacktrace\n"
5614	#endif
5615	#ifdef CONFIG_TRACER_SNAPSHOT
5616	"\t\t snapshot\n"
5617	#endif
5618	"\t\t dump\n"
5619	"\t\t cpudump\n"
5620	"\t example: echo do_fault:traceoff > set_ftrace_filter\n"
5621	"\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
5622	"\t The first one will disable tracing every time do_fault is hit\n"
5623	"\t The second will disable tracing at most 3 times when do_trap is hit\n"
5624	"\t The first time do trap is hit and it disables tracing, the\n"
5625	"\t counter will decrement to 2. If tracing is already disabled,\n"
5626	"\t the counter will not decrement. It only decrements when the\n"
5627	"\t trigger did work\n"
5628	"\t To remove trigger without count:\n"
5629	"\t echo '!<function>:<trigger> > set_ftrace_filter\n"
5630	"\t To remove trigger with a count:\n"
5631	"\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5632	" set_ftrace_notrace\t- echo function name in here to never trace.\n"
5633	"\t accepts: func_full_name, func_end, func_begin, func_middle\n"
5634	"\t modules: Can select a group via module command :mod:\n"
5635	"\t Does not accept triggers\n"
5636	#endif /* CONFIG_DYNAMIC_FTRACE */
5637	#ifdef CONFIG_FUNCTION_TRACER
5638	" set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5639	"\t\t (function)\n"
5640	" set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5641	"\t\t (function)\n"
5642	#endif
5643	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5644	" set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5645	" set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5646	" max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5647	#endif
5648	#ifdef CONFIG_TRACER_SNAPSHOT
5649	"\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5650	"\t\t\t snapshot buffer. Read the contents for more\n"
5651	"\t\t\t information\n"
5652	#endif
5653	#ifdef CONFIG_STACK_TRACER
5654	" stack_trace\t\t- Shows the max stack trace when active\n"
5655	" stack_max_size\t- Shows current max stack size that was traced\n"
5656	"\t\t\t Write into this file to reset the max size (trigger a\n"
5657	"\t\t\t new trace)\n"
5658	#ifdef CONFIG_DYNAMIC_FTRACE
5659	" stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5660	"\t\t\t traces\n"
5661	#endif
5662	#endif /* CONFIG_STACK_TRACER */
5663	#ifdef CONFIG_DYNAMIC_EVENTS
5664	" dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5665	"\t\t\t Write into this file to define/undefine new trace events.\n"
5666	#endif
5667	#ifdef CONFIG_KPROBE_EVENTS
5668	" kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5669	"\t\t\t Write into this file to define/undefine new trace events.\n"
5670	#endif
5671	#ifdef CONFIG_UPROBE_EVENTS
5672	" uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5673	"\t\t\t Write into this file to define/undefine new trace events.\n"
5674	#endif
5675	#if defined(CONFIG_KPROBE_EVENTS) \|\| defined(CONFIG_UPROBE_EVENTS)
5676	"\t accepts: event-definitions (one definition per line)\n"
5677	"\t Format: p[:[<group>/][<event>]] <place> [<args>]\n"
5678	"\t r[maxactive][:[<group>/][<event>]] <place> [<args>]\n"
5679	#ifdef CONFIG_HIST_TRIGGERS
5680	"\t s:[synthetic/]<event> <field> [<field>]\n"
5681	#endif
5682	"\t e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n"
5683	"\t -:[<group>/][<event>]\n"
5684	#ifdef CONFIG_KPROBE_EVENTS
5685	"\t place: [<module>:]<symbol>[+<offset>]\|<memaddr>\n"
5686	"place (kretprobe): [<module>:]<symbol>[+<offset>]%return\|<memaddr>\n"
5687	#endif
5688	#ifdef CONFIG_UPROBE_EVENTS
5689	" place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
5690	#endif
5691	"\t args: <name>=fetcharg[:type]\n"
5692	"\t fetcharg: (%<register>\|$<efield>), @<address>, @<symbol>[+\|-<offset>],\n"
5693	#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5694	"\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5695	#else
5696	"\t $stack<index>, $stack, $retval, $comm,\n"
5697	#endif
5698	"\t +\|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5699	"\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n"
5700	"\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5701	"\t symstr, <type>\\[<array-size>\\]\n"
5702	#ifdef CONFIG_HIST_TRIGGERS
5703	"\t field: <stype> <name>;\n"
5704	"\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5705	"\t [unsigned] char/int/long\n"
5706	#endif
5707	"\t efield: For event probes ('e' types), the field is on of the fields\n"
5708	"\t of the <attached-group>/<attached-event>.\n"
5709	#endif
5710	" events/\t\t- Directory containing all trace event subsystems:\n"
5711	" enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5712	" events/<system>/\t- Directory containing all trace events for <system>:\n"
5713	" enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5714	"\t\t\t events\n"
5715	" filter\t\t- If set, only events passing filter are traced\n"
5716	" events/<system>/<event>/\t- Directory containing control files for\n"
5717	"\t\t\t <event>:\n"
5718	" enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5719	" filter\t\t- If set, only events passing filter are traced\n"
5720	" trigger\t\t- If set, a command to perform when event is hit\n"
5721	"\t Format: <trigger>[:count][if <filter>]\n"
5722	"\t trigger: traceon, traceoff\n"
5723	"\t enable_event:<system>:<event>\n"
5724	"\t disable_event:<system>:<event>\n"
5725	#ifdef CONFIG_HIST_TRIGGERS
5726	"\t enable_hist:<system>:<event>\n"
5727	"\t disable_hist:<system>:<event>\n"
5728	#endif
5729	#ifdef CONFIG_STACKTRACE
5730	"\t\t stacktrace\n"
5731	#endif
5732	#ifdef CONFIG_TRACER_SNAPSHOT
5733	"\t\t snapshot\n"
5734	#endif
5735	#ifdef CONFIG_HIST_TRIGGERS
5736	"\t\t hist (see below)\n"
5737	#endif
5738	"\t example: echo traceoff > events/block/block_unplug/trigger\n"
5739	"\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5740	"\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5741	"\t events/block/block_unplug/trigger\n"
5742	"\t The first disables tracing every time block_unplug is hit.\n"
5743	"\t The second disables tracing the first 3 times block_unplug is hit.\n"
5744	"\t The third enables the kmalloc event the first 3 times block_unplug\n"
5745	"\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5746	"\t Like function triggers, the counter is only decremented if it\n"
5747	"\t enabled or disabled tracing.\n"
5748	"\t To remove a trigger without a count:\n"
5749	"\t echo '!<trigger> > <system>/<event>/trigger\n"
5750	"\t To remove a trigger with a count:\n"
5751	"\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5752	"\t Filters can be ignored when removing a trigger.\n"
5753	#ifdef CONFIG_HIST_TRIGGERS
5754	" hist trigger\t- If set, event hits are aggregated into a hash table\n"
5755	"\t Format: hist:keys=<field1[,field2,...]>\n"
5756	"\t [:<var1>=<field\|var_ref\|numeric_literal>[,<var2>=...]]\n"
5757	"\t [:values=<field1[,field2,...]>]\n"
5758	"\t [:sort=<field1[,field2,...]>]\n"
5759	"\t [:size=#entries]\n"
5760	"\t [:pause][:continue][:clear]\n"
5761	"\t [:name=histname1]\n"
5762	"\t [:nohitcount]\n"
5763	"\t [:<handler>.<action>]\n"
5764	"\t [if <filter>]\n\n"
5765	"\t Note, special fields can be used as well:\n"
5766	"\t common_timestamp - to record current timestamp\n"
5767	"\t common_cpu - to record the CPU the event happened on\n"
5768	"\n"
5769	"\t A hist trigger variable can be:\n"
5770	"\t - a reference to a field e.g. x=current_timestamp,\n"
5771	"\t - a reference to another variable e.g. y=$x,\n"
5772	"\t - a numeric literal: e.g. ms_per_sec=1000,\n"
5773	"\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
5774	"\n"
5775	"\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
5776	"\t multiplication(*) and division(/) operators. An operand can be either a\n"
5777	"\t variable reference, field or numeric literal.\n"
5778	"\n"
5779	"\t When a matching event is hit, an entry is added to a hash\n"
5780	"\t table using the key(s) and value(s) named, and the value of a\n"
5781	"\t sum called 'hitcount' is incremented. Keys and values\n"
5782	"\t correspond to fields in the event's format description. Keys\n"
5783	"\t can be any field, or the special string 'common_stacktrace'.\n"
5784	"\t Compound keys consisting of up to two fields can be specified\n"
5785	"\t by the 'keys' keyword. Values must correspond to numeric\n"
5786	"\t fields. Sort keys consisting of up to two fields can be\n"
5787	"\t specified using the 'sort' keyword. The sort direction can\n"
5788	"\t be modified by appending '.descending' or '.ascending' to a\n"
5789	"\t sort field. The 'size' parameter can be used to specify more\n"
5790	"\t or fewer than the default 2048 entries for the hashtable size.\n"
5791	"\t If a hist trigger is given a name using the 'name' parameter,\n"
5792	"\t its histogram data will be shared with other triggers of the\n"
5793	"\t same name, and trigger hits will update this common data.\n\n"
5794	"\t Reading the 'hist' file for the event will dump the hash\n"
5795	"\t table in its entirety to stdout. If there are multiple hist\n"
5796	"\t triggers attached to an event, there will be a table for each\n"
5797	"\t trigger in the output. The table displayed for a named\n"
5798	"\t trigger will be the same as any other instance having the\n"
5799	"\t same name. The default format used to display a given field\n"
5800	"\t can be modified by appending any of the following modifiers\n"
5801	"\t to the field name, as applicable:\n\n"
5802	"\t .hex display a number as a hex value\n"
5803	"\t .sym display an address as a symbol\n"
5804	"\t .sym-offset display an address as a symbol and offset\n"
5805	"\t .execname display a common_pid as a program name\n"
5806	"\t .syscall display a syscall id as a syscall name\n"
5807	"\t .log2 display log2 value rather than raw number\n"
5808	"\t .buckets=size display values in groups of size rather than raw number\n"
5809	"\t .usecs display a common_timestamp in microseconds\n"
5810	"\t .percent display a number of percentage value\n"
5811	"\t .graph display a bar-graph of a value\n\n"
5812	"\t The 'pause' parameter can be used to pause an existing hist\n"
5813	"\t trigger or to start a hist trigger but not log any events\n"
5814	"\t until told to do so. 'continue' can be used to start or\n"
5815	"\t restart a paused hist trigger.\n\n"
5816	"\t The 'clear' parameter will clear the contents of a running\n"
5817	"\t hist trigger and leave its current paused/active state\n"
5818	"\t unchanged.\n\n"
5819	"\t The 'nohitcount' (or NOHC) parameter will suppress display of\n"
5820	"\t raw hitcount in the histogram.\n\n"
5821	"\t The enable_hist and disable_hist triggers can be used to\n"
5822	"\t have one event conditionally start and stop another event's\n"
5823	"\t already-attached hist trigger. The syntax is analogous to\n"
5824	"\t the enable_event and disable_event triggers.\n\n"
5825	"\t Hist trigger handlers and actions are executed whenever a\n"
5826	"\t a histogram entry is added or updated. They take the form:\n\n"
5827	"\t <handler>.<action>\n\n"
5828	"\t The available handlers are:\n\n"
5829	"\t onmatch(matching.event) - invoke on addition or update\n"
5830	"\t onmax(var) - invoke if var exceeds current max\n"
5831	"\t onchange(var) - invoke action if var changes\n\n"
5832	"\t The available actions are:\n\n"
5833	"\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5834	"\t save(field,...) - save current event fields\n"
5835	#ifdef CONFIG_TRACER_SNAPSHOT
5836	"\t snapshot() - snapshot the trace buffer\n\n"
5837	#endif
5838	#ifdef CONFIG_SYNTH_EVENTS
5839	" events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5840	"\t Write into this file to define/undefine new synthetic events.\n"
5841	"\t example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n"
5842	#endif
5843	#endif
5844	;
5845
5846	static ssize_t
5847	tracing_readme_read(struct file filp, char __user ubuf,
5848	size_t cnt, loff_t *ppos)
5849	{
5850	return simple_read_from_buffer(ubuf, cnt, ppos,
5851	readme_msg, strlen(readme_msg));
5852	}
5853
5854	static const struct file_operations tracing_readme_fops = {
5855	.open = tracing_open_generic,
5856	.read = tracing_readme_read,
5857	.llseek = generic_file_llseek,
5858	};
5859
5860	static void saved_tgids_next(struct seq_file m, void v, loff_t pos)
5861	{
5862	int pid = ++(*pos);
5863
5864	return trace_find_tgid_ptr(pid);
5865	}
5866
5867	static void saved_tgids_start(struct seq_file m, loff_t *pos)
5868	{
5869	int pid = *pos;
5870
5871	return trace_find_tgid_ptr(pid);
5872	}
5873
5874	static void saved_tgids_stop(struct seq_file m, void v)
5875	{
5876	}
5877
5878	static int saved_tgids_show(struct seq_file m, void v)
5879	{
5880	int entry = (int )v;
5881	int pid = entry - tgid_map;
5882	int tgid = *entry;
5883
5884	if (tgid == 0)
5885	return SEQ_SKIP;
5886
5887	seq_printf(m, "%d %d\n", pid, tgid);
5888	return 0;
5889	}
5890
5891	static const struct seq_operations tracing_saved_tgids_seq_ops = {
5892	.start = saved_tgids_start,
5893	.stop = saved_tgids_stop,
5894	.next = saved_tgids_next,
5895	.show = saved_tgids_show,
5896	};
5897
5898	static int tracing_saved_tgids_open(struct inode inode, struct file filp)
5899	{
5900	int ret;
5901
5902	ret = tracing_check_open_get_tr(NULL);
5903	if (ret)
5904	return ret;
5905
5906	return seq_open(filp, &tracing_saved_tgids_seq_ops);
5907	}
5908
5909
5910	static const struct file_operations tracing_saved_tgids_fops = {
5911	.open = tracing_saved_tgids_open,
5912	.read = seq_read,
5913	.llseek = seq_lseek,
5914	.release = seq_release,
5915	};
5916
5917	static void saved_cmdlines_next(struct seq_file m, void v, loff_t pos)
5918	{
5919	unsigned int *ptr = v;
5920
5921	if (*pos \|\| m->count)
5922	ptr++;
5923
5924	(*pos)++;
5925
5926	for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5927	ptr++) {
5928	if (ptr == -1 \|\| ptr == NO_CMDLINE_MAP)
5929	continue;
5930
5931	return ptr;
5932	}
5933
5934	return NULL;
5935	}
5936
5937	static void saved_cmdlines_start(struct seq_file m, loff_t *pos)
5938	{
5939	void *v;
5940	loff_t l = 0;
5941
5942	preempt_disable();
5943	arch_spin_lock(&trace_cmdline_lock);
5944
5945	v = &savedcmd->map_cmdline_to_pid[0];
5946	while (l <= *pos) {
5947	v = saved_cmdlines_next(m, v, &l);
5948	if (!v)
5949	return NULL;
5950	}
5951
5952	return v;
5953	}
5954
5955	static void saved_cmdlines_stop(struct seq_file m, void v)
5956	{
5957	arch_spin_unlock(&trace_cmdline_lock);
5958	preempt_enable();
5959	}
5960
5961	static int saved_cmdlines_show(struct seq_file m, void v)
5962	{
5963	char buf[TASK_COMM_LEN];
5964	unsigned int *pid = v;
5965
5966	__trace_find_cmdline(*pid, buf);
5967	seq_printf(m, "%d %s\n", *pid, buf);
5968	return 0;
5969	}
5970
5971	static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5972	.start = saved_cmdlines_start,
5973	.next = saved_cmdlines_next,
5974	.stop = saved_cmdlines_stop,
5975	.show = saved_cmdlines_show,
5976	};
5977
5978	static int tracing_saved_cmdlines_open(struct inode inode, struct file filp)
5979	{
5980	int ret;
5981
5982	ret = tracing_check_open_get_tr(NULL);
5983	if (ret)
5984	return ret;
5985
5986	return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5987	}
5988
5989	static const struct file_operations tracing_saved_cmdlines_fops = {
5990	.open = tracing_saved_cmdlines_open,
5991	.read = seq_read,
5992	.llseek = seq_lseek,
5993	.release = seq_release,
5994	};
5995
5996	static ssize_t
5997	tracing_saved_cmdlines_size_read(struct file filp, char __user ubuf,
5998	size_t cnt, loff_t *ppos)
5999	{
6000	char buf[64];
6001	int r;
6002
6003	preempt_disable();
6004	arch_spin_lock(&trace_cmdline_lock);
6005	r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
6006	arch_spin_unlock(&trace_cmdline_lock);
6007	preempt_enable();
6008
6009	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6010	}
6011
6012	static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
6013	{
6014	kfree(s->saved_cmdlines);
6015	kfree(s->map_cmdline_to_pid);
6016	kfree(s);
6017	}
6018
6019	static int tracing_resize_saved_cmdlines(unsigned int val)
6020	{
6021	struct saved_cmdlines_buffer s, savedcmd_temp;
6022
6023	s = kmalloc(sizeof(*s), GFP_KERNEL);
6024	if (!s)
6025	return -ENOMEM;
6026
6027	if (allocate_cmdlines_buffer(val, s) < 0) {
6028	kfree(s);
6029	return -ENOMEM;
6030	}
6031
6032	preempt_disable();
6033	arch_spin_lock(&trace_cmdline_lock);
6034	savedcmd_temp = savedcmd;
6035	savedcmd = s;
6036	arch_spin_unlock(&trace_cmdline_lock);
6037	preempt_enable();
6038	free_saved_cmdlines_buffer(savedcmd_temp);
6039
6040	return 0;
6041	}
6042
6043	static ssize_t
6044	tracing_saved_cmdlines_size_write(struct file filp, const char __user ubuf,
6045	size_t cnt, loff_t *ppos)
6046	{
6047	unsigned long val;
6048	int ret;
6049
6050	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6051	if (ret)
6052	return ret;
6053
6054	/* must have at least 1 entry or less than PID_MAX_DEFAULT */
6055	if (!val \|\| val > PID_MAX_DEFAULT)
6056	return -EINVAL;
6057
6058	ret = tracing_resize_saved_cmdlines((unsigned int)val);
6059	if (ret < 0)
6060	return ret;
6061
6062	*ppos += cnt;
6063
6064	return cnt;
6065	}
6066
6067	static const struct file_operations tracing_saved_cmdlines_size_fops = {
6068	.open = tracing_open_generic,
6069	.read = tracing_saved_cmdlines_size_read,
6070	.write = tracing_saved_cmdlines_size_write,
6071	};
6072
6073	#ifdef CONFIG_TRACE_EVAL_MAP_FILE
6074	static union trace_eval_map_item *
6075	update_eval_map(union trace_eval_map_item *ptr)
6076	{
6077	if (!ptr->map.eval_string) {
6078	if (ptr->tail.next) {
6079	ptr = ptr->tail.next;
6080	/* Set ptr to the next real item (skip head) */
6081	ptr++;
6082	} else
6083	return NULL;
6084	}
6085	return ptr;
6086	}
6087
6088	static void eval_map_next(struct seq_file m, void v, loff_t pos)
6089	{
6090	union trace_eval_map_item *ptr = v;
6091
6092	/*
6093	* Paranoid! If ptr points to end, we don't want to increment past it.
6094	* This really should never happen.
6095	*/
6096	(*pos)++;
6097	ptr = update_eval_map(ptr);
6098	if (WARN_ON_ONCE(!ptr))
6099	return NULL;
6100
6101	ptr++;
6102	ptr = update_eval_map(ptr);
6103
6104	return ptr;
6105	}
6106
6107	static void eval_map_start(struct seq_file m, loff_t *pos)
6108	{
6109	union trace_eval_map_item *v;
6110	loff_t l = 0;
6111
6112	mutex_lock(&trace_eval_mutex);
6113
6114	v = trace_eval_maps;
6115	if (v)
6116	v++;
6117
6118	while (v && l < *pos) {
6119	v = eval_map_next(m, v, &l);
6120	}
6121
6122	return v;
6123	}
6124
6125	static void eval_map_stop(struct seq_file m, void v)
6126	{
6127	mutex_unlock(&trace_eval_mutex);
6128	}
6129
6130	static int eval_map_show(struct seq_file m, void v)
6131	{
6132	union trace_eval_map_item *ptr = v;
6133
6134	seq_printf(m, "%s %ld (%s)\n",
6135	ptr->map.eval_string, ptr->map.eval_value,
6136	ptr->map.system);
6137
6138	return 0;
6139	}
6140
6141	static const struct seq_operations tracing_eval_map_seq_ops = {
6142	.start = eval_map_start,
6143	.next = eval_map_next,
6144	.stop = eval_map_stop,
6145	.show = eval_map_show,
6146	};
6147
6148	static int tracing_eval_map_open(struct inode inode, struct file filp)
6149	{
6150	int ret;
6151
6152	ret = tracing_check_open_get_tr(NULL);
6153	if (ret)
6154	return ret;
6155
6156	return seq_open(filp, &tracing_eval_map_seq_ops);
6157	}
6158
6159	static const struct file_operations tracing_eval_map_fops = {
6160	.open = tracing_eval_map_open,
6161	.read = seq_read,
6162	.llseek = seq_lseek,
6163	.release = seq_release,
6164	};
6165
6166	static inline union trace_eval_map_item *
6167	trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
6168	{
6169	/* Return tail of array given the head */
6170	return ptr + ptr->head.length + 1;
6171	}
6172
6173	static void
6174	trace_insert_eval_map_file(struct module mod, struct trace_eval_map *start,
6175	int len)
6176	{
6177	struct trace_eval_map **stop;
6178	struct trace_eval_map **map;
6179	union trace_eval_map_item *map_array;
6180	union trace_eval_map_item *ptr;
6181
6182	stop = start + len;
6183
6184	/*
6185	* The trace_eval_maps contains the map plus a head and tail item,
6186	* where the head holds the module and length of array, and the
6187	* tail holds a pointer to the next list.
6188	*/
6189	map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
6190	if (!map_array) {
6191	pr_warn("Unable to allocate trace eval mapping\n");
6192	return;
6193	}
6194
6195	mutex_lock(&trace_eval_mutex);
6196
6197	if (!trace_eval_maps)
6198	trace_eval_maps = map_array;
6199	else {
6200	ptr = trace_eval_maps;
6201	for (;;) {
6202	ptr = trace_eval_jmp_to_tail(ptr);
6203	if (!ptr->tail.next)
6204	break;
6205	ptr = ptr->tail.next;
6206
6207	}
6208	ptr->tail.next = map_array;
6209	}
6210	map_array->head.mod = mod;
6211	map_array->head.length = len;
6212	map_array++;
6213
6214	for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
6215	map_array->map = **map;
6216	map_array++;
6217	}
6218	memset(map_array, 0, sizeof(*map_array));
6219
6220	mutex_unlock(&trace_eval_mutex);
6221	}
6222
6223	static void trace_create_eval_file(struct dentry *d_tracer)
6224	{
6225	trace_create_file("eval_map", TRACE_MODE_READ, d_tracer,
6226	NULL, &tracing_eval_map_fops);
6227	}
6228
6229	#else /* CONFIG_TRACE_EVAL_MAP_FILE */
6230	static inline void trace_create_eval_file(struct dentry *d_tracer) { }
6231	static inline void trace_insert_eval_map_file(struct module *mod,
6232	struct trace_eval_map **start, int len) { }
6233	#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
6234
6235	static void trace_insert_eval_map(struct module *mod,
6236	struct trace_eval_map **start, int len)
6237	{
6238	struct trace_eval_map **map;
6239
6240	if (len <= 0)
6241	return;
6242
6243	map = start;
6244
6245	trace_event_eval_update(map, len);
6246
6247	trace_insert_eval_map_file(mod, start, len);
6248	}
6249
6250	static ssize_t
6251	tracing_set_trace_read(struct file filp, char __user ubuf,
6252	size_t cnt, loff_t *ppos)
6253	{
6254	struct trace_array *tr = filp->private_data;
6255	char buf[MAX_TRACER_SIZE+2];
6256	int r;
6257
6258	mutex_lock(&trace_types_lock);
6259	r = sprintf(buf, "%s\n", tr->current_trace->name);
6260	mutex_unlock(&trace_types_lock);
6261
6262	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6263	}
6264
6265	int tracer_init(struct tracer t, struct trace_array tr)
6266	{
6267	tracing_reset_online_cpus(&tr->array_buffer);
6268	return t->init(tr);
6269	}
6270
6271	static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
6272	{
6273	int cpu;
6274
6275	for_each_tracing_cpu(cpu)
6276	per_cpu_ptr(buf->data, cpu)->entries = val;
6277	}
6278
6279	#ifdef CONFIG_TRACER_MAX_TRACE
6280	/* resize @tr's buffer to the size of @size_tr's entries */
6281	static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
6282	struct array_buffer *size_buf, int cpu_id)
6283	{
6284	int cpu, ret = 0;
6285
6286	if (cpu_id == RING_BUFFER_ALL_CPUS) {
6287	for_each_tracing_cpu(cpu) {
6288	ret = ring_buffer_resize(trace_buf->buffer,
6289	per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
6290	if (ret < 0)
6291	break;
6292	per_cpu_ptr(trace_buf->data, cpu)->entries =
6293	per_cpu_ptr(size_buf->data, cpu)->entries;
6294	}
6295	} else {
6296	ret = ring_buffer_resize(trace_buf->buffer,
6297	per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
6298	if (ret == 0)
6299	per_cpu_ptr(trace_buf->data, cpu_id)->entries =
6300	per_cpu_ptr(size_buf->data, cpu_id)->entries;
6301	}
6302
6303	return ret;
6304	}
6305	#endif /* CONFIG_TRACER_MAX_TRACE */
6306
6307	static int __tracing_resize_ring_buffer(struct trace_array *tr,
6308	unsigned long size, int cpu)
6309	{
6310	int ret;
6311
6312	/*
6313	* If kernel or user changes the size of the ring buffer
6314	* we use the size that was given, and we can forget about
6315	* expanding it later.
6316	*/
6317	ring_buffer_expanded = true;
6318
6319	/* May be called before buffers are initialized */
6320	if (!tr->array_buffer.buffer)
6321	return 0;
6322
6323	ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
6324	if (ret < 0)
6325	return ret;
6326
6327	#ifdef CONFIG_TRACER_MAX_TRACE
6328	if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) \|\|
6329	!tr->current_trace->use_max_tr)
6330	goto out;
6331
6332	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
6333	if (ret < 0) {
6334	int r = resize_buffer_duplicate_size(&tr->array_buffer,
6335	&tr->array_buffer, cpu);
6336	if (r < 0) {
6337	/*
6338	* AARGH! We are left with different
6339	* size max buffer!!!!
6340	* The max buffer is our "snapshot" buffer.
6341	* When a tracer needs a snapshot (one of the
6342	* latency tracers), it swaps the max buffer
6343	* with the saved snap shot. We succeeded to
6344	* update the size of the main buffer, but failed to
6345	* update the size of the max buffer. But when we tried
6346	* to reset the main buffer to the original size, we
6347	* failed there too. This is very unlikely to
6348	* happen, but if it does, warn and kill all
6349	* tracing.
6350	*/
6351	WARN_ON(1);
6352	tracing_disabled = 1;
6353	}
6354	return ret;
6355	}
6356
6357	if (cpu == RING_BUFFER_ALL_CPUS)
6358	set_buffer_entries(&tr->max_buffer, size);
6359	else
6360	per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
6361
6362	out:
6363	#endif /* CONFIG_TRACER_MAX_TRACE */
6364
6365	if (cpu == RING_BUFFER_ALL_CPUS)
6366	set_buffer_entries(&tr->array_buffer, size);
6367	else
6368	per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size;
6369
6370	return ret;
6371	}
6372
6373	ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
6374	unsigned long size, int cpu_id)
6375	{
6376	int ret;
6377
6378	mutex_lock(&trace_types_lock);
6379
6380	if (cpu_id != RING_BUFFER_ALL_CPUS) {
6381	/* make sure, this cpu is enabled in the mask */
6382	if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
6383	ret = -EINVAL;
6384	goto out;
6385	}
6386	}
6387
6388	ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
6389	if (ret < 0)
6390	ret = -ENOMEM;
6391
6392	out:
6393	mutex_unlock(&trace_types_lock);
6394
6395	return ret;
6396	}
6397
6398
6399	/**
6400	* tracing_update_buffers - used by tracing facility to expand ring buffers
6401	*
6402	* To save on memory when the tracing is never used on a system with it
6403	* configured in. The ring buffers are set to a minimum size. But once
6404	* a user starts to use the tracing facility, then they need to grow
6405	* to their default size.
6406	*
6407	* This function is to be called when a tracer is about to be used.
6408	*/
6409	int tracing_update_buffers(void)
6410	{
6411	int ret = 0;
6412
6413	mutex_lock(&trace_types_lock);
6414	if (!ring_buffer_expanded)
6415	ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
6416	RING_BUFFER_ALL_CPUS);
6417	mutex_unlock(&trace_types_lock);
6418
6419	return ret;
6420	}
6421
6422	struct trace_option_dentry;
6423
6424	static void
6425	create_trace_option_files(struct trace_array tr, struct tracer tracer);
6426
6427	/*
6428	* Used to clear out the tracer before deletion of an instance.
6429	* Must have trace_types_lock held.
6430	*/
6431	static void tracing_set_nop(struct trace_array *tr)
6432	{
6433	if (tr->current_trace == &nop_trace)
6434	return;
6435
6436	tr->current_trace->enabled--;
6437
6438	if (tr->current_trace->reset)
6439	tr->current_trace->reset(tr);
6440
6441	tr->current_trace = &nop_trace;
6442	}
6443
6444	static bool tracer_options_updated;
6445
6446	static void add_tracer_options(struct trace_array tr, struct tracer t)
6447	{
6448	/* Only enable if the directory has been created already. */
6449	if (!tr->dir)
6450	return;
6451
6452	/* Only create trace option files after update_tracer_options finish */
6453	if (!tracer_options_updated)
6454	return;
6455
6456	create_trace_option_files(tr, t);
6457	}
6458
6459	int tracing_set_tracer(struct trace_array tr, const char buf)
6460	{
6461	struct tracer *t;
6462	#ifdef CONFIG_TRACER_MAX_TRACE
6463	bool had_max_tr;
6464	#endif
6465	int ret = 0;
6466
6467	mutex_lock(&trace_types_lock);
6468
6469	if (!ring_buffer_expanded) {
6470	ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
6471	RING_BUFFER_ALL_CPUS);
6472	if (ret < 0)
6473	goto out;
6474	ret = 0;
6475	}
6476
6477	for (t = trace_types; t; t = t->next) {
6478	if (strcmp(t->name, buf) == 0)
6479	break;
6480	}
6481	if (!t) {
6482	ret = -EINVAL;
6483	goto out;
6484	}
6485	if (t == tr->current_trace)
6486	goto out;
6487
6488	#ifdef CONFIG_TRACER_SNAPSHOT
6489	if (t->use_max_tr) {
6490	local_irq_disable();
6491	arch_spin_lock(&tr->max_lock);
6492	if (tr->cond_snapshot)
6493	ret = -EBUSY;
6494	arch_spin_unlock(&tr->max_lock);
6495	local_irq_enable();
6496	if (ret)
6497	goto out;
6498	}
6499	#endif
6500	/* Some tracers won't work on kernel command line */
6501	if (system_state < SYSTEM_RUNNING && t->noboot) {
6502	pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
6503	t->name);
6504	goto out;
6505	}
6506
6507	/* Some tracers are only allowed for the top level buffer */
6508	if (!trace_ok_for_array(t, tr)) {
6509	ret = -EINVAL;
6510	goto out;
6511	}
6512
6513	/* If trace pipe files are being read, we can't change the tracer */
6514	if (tr->trace_ref) {
6515	ret = -EBUSY;
6516	goto out;
6517	}
6518
6519	trace_branch_disable();
6520
6521	tr->current_trace->enabled--;
6522
6523	if (tr->current_trace->reset)
6524	tr->current_trace->reset(tr);
6525
6526	#ifdef CONFIG_TRACER_MAX_TRACE
6527	had_max_tr = tr->current_trace->use_max_tr;
6528
6529	/* Current trace needs to be nop_trace before synchronize_rcu */
6530	tr->current_trace = &nop_trace;
6531
6532	if (had_max_tr && !t->use_max_tr) {
6533	/*
6534	* We need to make sure that the update_max_tr sees that
6535	* current_trace changed to nop_trace to keep it from
6536	* swapping the buffers after we resize it.
6537	* The update_max_tr is called from interrupts disabled
6538	* so a synchronized_sched() is sufficient.
6539	*/
6540	synchronize_rcu();
6541	free_snapshot(tr);
6542	}
6543
6544	if (t->use_max_tr && !tr->allocated_snapshot) {
6545	ret = tracing_alloc_snapshot_instance(tr);
6546	if (ret < 0)
6547	goto out;
6548	}
6549	#else
6550	tr->current_trace = &nop_trace;
6551	#endif
6552
6553	if (t->init) {
6554	ret = tracer_init(t, tr);
6555	if (ret)
6556	goto out;
6557	}
6558
6559	tr->current_trace = t;
6560	tr->current_trace->enabled++;
6561	trace_branch_enable(tr);
6562	out:
6563	mutex_unlock(&trace_types_lock);
6564
6565	return ret;
6566	}
6567
6568	static ssize_t
6569	tracing_set_trace_write(struct file filp, const char __user ubuf,
6570	size_t cnt, loff_t *ppos)
6571	{
6572	struct trace_array *tr = filp->private_data;
6573	char buf[MAX_TRACER_SIZE+1];
6574	char *name;
6575	size_t ret;
6576	int err;
6577
6578	ret = cnt;
6579
6580	if (cnt > MAX_TRACER_SIZE)
6581	cnt = MAX_TRACER_SIZE;
6582
6583	if (copy_from_user(buf, ubuf, cnt))
6584	return -EFAULT;
6585
6586	buf[cnt] = 0;
6587
6588	name = strim(buf);
6589
6590	err = tracing_set_tracer(tr, name);
6591	if (err)
6592	return err;
6593
6594	*ppos += ret;
6595
6596	return ret;
6597	}
6598
6599	static ssize_t
6600	tracing_nsecs_read(unsigned long ptr, char __user ubuf,
6601	size_t cnt, loff_t *ppos)
6602	{
6603	char buf[64];
6604	int r;
6605
6606	r = snprintf(buf, sizeof(buf), "%ld\n",
6607	ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(ptr));
6608	if (r > sizeof(buf))
6609	r = sizeof(buf);
6610	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6611	}
6612
6613	static ssize_t
6614	tracing_nsecs_write(unsigned long ptr, const char __user ubuf,
6615	size_t cnt, loff_t *ppos)
6616	{
6617	unsigned long val;
6618	int ret;
6619
6620	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6621	if (ret)
6622	return ret;
6623
6624	ptr = val 1000;
6625
6626	return cnt;
6627	}
6628
6629	static ssize_t
6630	tracing_thresh_read(struct file filp, char __user ubuf,
6631	size_t cnt, loff_t *ppos)
6632	{
6633	return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
6634	}
6635
6636	static ssize_t
6637	tracing_thresh_write(struct file filp, const char __user ubuf,
6638	size_t cnt, loff_t *ppos)
6639	{
6640	struct trace_array *tr = filp->private_data;
6641	int ret;
6642
6643	mutex_lock(&trace_types_lock);
6644	ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
6645	if (ret < 0)
6646	goto out;
6647
6648	if (tr->current_trace->update_thresh) {
6649	ret = tr->current_trace->update_thresh(tr);
6650	if (ret < 0)
6651	goto out;
6652	}
6653
6654	ret = cnt;
6655	out:
6656	mutex_unlock(&trace_types_lock);
6657
6658	return ret;
6659	}
6660
6661	#ifdef CONFIG_TRACER_MAX_TRACE
6662
6663	static ssize_t
6664	tracing_max_lat_read(struct file filp, char __user ubuf,
6665	size_t cnt, loff_t *ppos)
6666	{
6667	return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
6668	}
6669
6670	static ssize_t
6671	tracing_max_lat_write(struct file filp, const char __user ubuf,
6672	size_t cnt, loff_t *ppos)
6673	{
6674	return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
6675	}
6676
6677	#endif
6678
6679	static int tracing_open_pipe(struct inode inode, struct file filp)
6680	{
6681	struct trace_array *tr = inode->i_private;
6682	struct trace_iterator *iter;
6683	int ret;
6684
6685	ret = tracing_check_open_get_tr(tr);
6686	if (ret)
6687	return ret;
6688
6689	mutex_lock(&trace_types_lock);
6690
6691	/* create a buffer to store the information to pass to userspace */
6692	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6693	if (!iter) {
6694	ret = -ENOMEM;
6695	__trace_array_put(tr);
6696	goto out;
6697	}
6698
6699	trace_seq_init(&iter->seq);
6700	iter->trace = tr->current_trace;
6701
6702	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
6703	ret = -ENOMEM;
6704	goto fail;
6705	}
6706
6707	/* trace pipe does not show start of buffer */
6708	cpumask_setall(iter->started);
6709
6710	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6711	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
6712
6713	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
6714	if (trace_clocks[tr->clock_id].in_ns)
6715	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
6716
6717	iter->tr = tr;
6718	iter->array_buffer = &tr->array_buffer;
6719	iter->cpu_file = tracing_get_cpu(inode);
6720	mutex_init(&iter->mutex);
6721	filp->private_data = iter;
6722
6723	if (iter->trace->pipe_open)
6724	iter->trace->pipe_open(iter);
6725
6726	nonseekable_open(inode, filp);
6727
6728	tr->trace_ref++;
6729	out:
6730	mutex_unlock(&trace_types_lock);
6731	return ret;
6732
6733	fail:
6734	kfree(iter);
6735	__trace_array_put(tr);
6736	mutex_unlock(&trace_types_lock);
6737	return ret;
6738	}
6739
6740	static int tracing_release_pipe(struct inode inode, struct file file)
6741	{
6742	struct trace_iterator *iter = file->private_data;
6743	struct trace_array *tr = inode->i_private;
6744
6745	mutex_lock(&trace_types_lock);
6746
6747	tr->trace_ref--;
6748
6749	if (iter->trace->pipe_close)
6750	iter->trace->pipe_close(iter);
6751
6752	mutex_unlock(&trace_types_lock);
6753
6754	free_cpumask_var(iter->started);
6755	kfree(iter->fmt);
6756	mutex_destroy(&iter->mutex);
6757	kfree(iter);
6758
6759	trace_array_put(tr);
6760
6761	return 0;
6762	}
6763
6764	static __poll_t
6765	trace_poll(struct trace_iterator iter, struct file filp, poll_table *poll_table)
6766	{
6767	struct trace_array *tr = iter->tr;
6768
6769	/* Iterators are static, they should be filled or empty */
6770	if (trace_buffer_iter(iter, iter->cpu_file))
6771	return EPOLLIN \| EPOLLRDNORM;
6772
6773	if (tr->trace_flags & TRACE_ITER_BLOCK)
6774	/*
6775	* Always select as readable when in blocking mode
6776	*/
6777	return EPOLLIN \| EPOLLRDNORM;
6778	else
6779	return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
6780	filp, poll_table, iter->tr->buffer_percent);
6781	}
6782
6783	static __poll_t
6784	tracing_poll_pipe(struct file filp, poll_table poll_table)
6785	{
6786	struct trace_iterator *iter = filp->private_data;
6787
6788	return trace_poll(iter, filp, poll_table);
6789	}
6790
6791	/* Must be called with iter->mutex held. */
6792	static int tracing_wait_pipe(struct file *filp)
6793	{
6794	struct trace_iterator *iter = filp->private_data;
6795	int ret;
6796
6797	while (trace_empty(iter)) {
6798
6799	if ((filp->f_flags & O_NONBLOCK)) {
6800	return -EAGAIN;
6801	}
6802
6803	/*
6804	* We block until we read something and tracing is disabled.
6805	* We still block if tracing is disabled, but we have never
6806	* read anything. This allows a user to cat this file, and
6807	* then enable tracing. But after we have read something,
6808	* we give an EOF when tracing is again disabled.
6809	*
6810	* iter->pos will be 0 if we haven't read anything.
6811	*/
6812	if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6813	break;
6814
6815	mutex_unlock(&iter->mutex);
6816
6817	ret = wait_on_pipe(iter, 0);
6818
6819	mutex_lock(&iter->mutex);
6820
6821	if (ret)
6822	return ret;
6823	}
6824
6825	return 1;
6826	}
6827
6828	/*
6829	* Consumer reader.
6830	*/
6831	static ssize_t
6832	tracing_read_pipe(struct file filp, char __user ubuf,
6833	size_t cnt, loff_t *ppos)
6834	{
6835	struct trace_iterator *iter = filp->private_data;
6836	ssize_t sret;
6837
6838	/*
6839	* Avoid more than one consumer on a single file descriptor
6840	* This is just a matter of traces coherency, the ring buffer itself
6841	* is protected.
6842	*/
6843	mutex_lock(&iter->mutex);
6844
6845	/* return any leftover data */
6846	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6847	if (sret != -EBUSY)
6848	goto out;
6849
6850	trace_seq_init(&iter->seq);
6851
6852	if (iter->trace->read) {
6853	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6854	if (sret)
6855	goto out;
6856	}
6857
6858	waitagain:
6859	sret = tracing_wait_pipe(filp);
6860	if (sret <= 0)
6861	goto out;
6862
6863	/* stop when tracing is finished */
6864	if (trace_empty(iter)) {
6865	sret = 0;
6866	goto out;
6867	}
6868
6869	if (cnt >= PAGE_SIZE)
6870	cnt = PAGE_SIZE - 1;
6871
6872	/* reset all but tr, trace, and overruns */
6873	trace_iterator_reset(iter);
6874	cpumask_clear(iter->started);
6875	trace_seq_init(&iter->seq);
6876
6877	trace_event_read_lock();
6878	trace_access_lock(iter->cpu_file);
6879	while (trace_find_next_entry_inc(iter) != NULL) {
6880	enum print_line_t ret;
6881	int save_len = iter->seq.seq.len;
6882
6883	ret = print_trace_line(iter);
6884	if (ret == TRACE_TYPE_PARTIAL_LINE) {
6885	/*
6886	* If one print_trace_line() fills entire trace_seq in one shot,
6887	* trace_seq_to_user() will returns -EBUSY because save_len == 0,
6888	* In this case, we need to consume it, otherwise, loop will peek
6889	* this event next time, resulting in an infinite loop.
6890	*/
6891	if (save_len == 0) {
6892	iter->seq.full = 0;
6893	trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
6894	trace_consume(iter);
6895	break;
6896	}
6897
6898	/* In other cases, don't print partial lines */
6899	iter->seq.seq.len = save_len;
6900	break;
6901	}
6902	if (ret != TRACE_TYPE_NO_CONSUME)
6903	trace_consume(iter);
6904
6905	if (trace_seq_used(&iter->seq) >= cnt)
6906	break;
6907
6908	/*
6909	* Setting the full flag means we reached the trace_seq buffer
6910	* size and we should leave by partial output condition above.
6911	* One of the trace_seq_* functions is not used properly.
6912	*/
6913	WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6914	iter->ent->type);
6915	}
6916	trace_access_unlock(iter->cpu_file);
6917	trace_event_read_unlock();
6918
6919	/* Now copy what we have to the user */
6920	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6921	if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6922	trace_seq_init(&iter->seq);
6923
6924	/*
6925	* If there was nothing to send to user, in spite of consuming trace
6926	* entries, go back to wait for more entries.
6927	*/
6928	if (sret == -EBUSY)
6929	goto waitagain;
6930
6931	out:
6932	mutex_unlock(&iter->mutex);
6933
6934	return sret;
6935	}
6936
6937	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6938	unsigned int idx)
6939	{
6940	__free_page(spd->pages[idx]);
6941	}
6942
6943	static size_t
6944	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6945	{
6946	size_t count;
6947	int save_len;
6948	int ret;
6949
6950	/* Seq buffer is page-sized, exactly what we need. */
6951	for (;;) {
6952	save_len = iter->seq.seq.len;
6953	ret = print_trace_line(iter);
6954
6955	if (trace_seq_has_overflowed(&iter->seq)) {
6956	iter->seq.seq.len = save_len;
6957	break;
6958	}
6959
6960	/*
6961	* This should not be hit, because it should only
6962	* be set if the iter->seq overflowed. But check it
6963	* anyway to be safe.
6964	*/
6965	if (ret == TRACE_TYPE_PARTIAL_LINE) {
6966	iter->seq.seq.len = save_len;
6967	break;
6968	}
6969
6970	count = trace_seq_used(&iter->seq) - save_len;
6971	if (rem < count) {
6972	rem = 0;
6973	iter->seq.seq.len = save_len;
6974	break;
6975	}
6976
6977	if (ret != TRACE_TYPE_NO_CONSUME)
6978	trace_consume(iter);
6979	rem -= count;
6980	if (!trace_find_next_entry_inc(iter)) {
6981	rem = 0;
6982	iter->ent = NULL;
6983	break;
6984	}
6985	}
6986
6987	return rem;
6988	}
6989
6990	static ssize_t tracing_splice_read_pipe(struct file *filp,
6991	loff_t *ppos,
6992	struct pipe_inode_info *pipe,
6993	size_t len,
6994	unsigned int flags)
6995	{
6996	struct page *pages_def[PIPE_DEF_BUFFERS];
6997	struct partial_page partial_def[PIPE_DEF_BUFFERS];
6998	struct trace_iterator *iter = filp->private_data;
6999	struct splice_pipe_desc spd = {
7000	.pages = pages_def,
7001	.partial = partial_def,
7002	.nr_pages = 0, /* This gets updated below. */
7003	.nr_pages_max = PIPE_DEF_BUFFERS,
7004	.ops = &default_pipe_buf_ops,
7005	.spd_release = tracing_spd_release_pipe,
7006	};
7007	ssize_t ret;
7008	size_t rem;
7009	unsigned int i;
7010
7011	if (splice_grow_spd(pipe, &spd))
7012	return -ENOMEM;
7013
7014	mutex_lock(&iter->mutex);
7015
7016	if (iter->trace->splice_read) {
7017	ret = iter->trace->splice_read(iter, filp,
7018	ppos, pipe, len, flags);
7019	if (ret)
7020	goto out_err;
7021	}
7022
7023	ret = tracing_wait_pipe(filp);
7024	if (ret <= 0)
7025	goto out_err;
7026
7027	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
7028	ret = -EFAULT;
7029	goto out_err;
7030	}
7031
7032	trace_event_read_lock();
7033	trace_access_lock(iter->cpu_file);
7034
7035	/* Fill as many pages as possible. */
7036	for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
7037	spd.pages[i] = alloc_page(GFP_KERNEL);
7038	if (!spd.pages[i])
7039	break;
7040
7041	rem = tracing_fill_pipe_page(rem, iter);
7042
7043	/* Copy the data into the page, so we can start over. */
7044	ret = trace_seq_to_buffer(&iter->seq,
7045	page_address(spd.pages[i]),
7046	trace_seq_used(&iter->seq));
7047	if (ret < 0) {
7048	__free_page(spd.pages[i]);
7049	break;
7050	}
7051	spd.partial[i].offset = 0;
7052	spd.partial[i].len = trace_seq_used(&iter->seq);
7053
7054	trace_seq_init(&iter->seq);
7055	}
7056
7057	trace_access_unlock(iter->cpu_file);
7058	trace_event_read_unlock();
7059	mutex_unlock(&iter->mutex);
7060
7061	spd.nr_pages = i;
7062
7063	if (i)
7064	ret = splice_to_pipe(pipe, &spd);
7065	else
7066	ret = 0;
7067	out:
7068	splice_shrink_spd(&spd);
7069	return ret;
7070
7071	out_err:
7072	mutex_unlock(&iter->mutex);
7073	goto out;
7074	}
7075
7076	static ssize_t
7077	tracing_entries_read(struct file filp, char __user ubuf,
7078	size_t cnt, loff_t *ppos)
7079	{
7080	struct inode *inode = file_inode(filp);
7081	struct trace_array *tr = inode->i_private;
7082	int cpu = tracing_get_cpu(inode);
7083	char buf[64];
7084	int r = 0;
7085	ssize_t ret;
7086
7087	mutex_lock(&trace_types_lock);
7088
7089	if (cpu == RING_BUFFER_ALL_CPUS) {
7090	int cpu, buf_size_same;
7091	unsigned long size;
7092
7093	size = 0;
7094	buf_size_same = 1;
7095	/* check if all cpu sizes are same */
7096	for_each_tracing_cpu(cpu) {
7097	/* fill in the size from first enabled cpu */
7098	if (size == 0)
7099	size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
7100	if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
7101	buf_size_same = 0;
7102	break;
7103	}
7104	}
7105
7106	if (buf_size_same) {
7107	if (!ring_buffer_expanded)
7108	r = sprintf(buf, "%lu (expanded: %lu)\n",
7109	size >> 10,
7110	trace_buf_size >> 10);
7111	else
7112	r = sprintf(buf, "%lu\n", size >> 10);
7113	} else
7114	r = sprintf(buf, "X\n");
7115	} else
7116	r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
7117
7118	mutex_unlock(&trace_types_lock);
7119
7120	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7121	return ret;
7122	}
7123
7124	static ssize_t
7125	tracing_entries_write(struct file filp, const char __user ubuf,
7126	size_t cnt, loff_t *ppos)
7127	{
7128	struct inode *inode = file_inode(filp);
7129	struct trace_array *tr = inode->i_private;
7130	unsigned long val;
7131	int ret;
7132
7133	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7134	if (ret)
7135	return ret;
7136
7137	/* must have at least 1 entry */
7138	if (!val)
7139	return -EINVAL;
7140
7141	/* value is in KB */
7142	val <<= 10;
7143	ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
7144	if (ret < 0)
7145	return ret;
7146
7147	*ppos += cnt;
7148
7149	return cnt;
7150	}
7151
7152	static ssize_t
7153	tracing_total_entries_read(struct file filp, char __user ubuf,
7154	size_t cnt, loff_t *ppos)
7155	{
7156	struct trace_array *tr = filp->private_data;
7157	char buf[64];
7158	int r, cpu;
7159	unsigned long size = 0, expanded_size = 0;
7160
7161	mutex_lock(&trace_types_lock);
7162	for_each_tracing_cpu(cpu) {
7163	size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
7164	if (!ring_buffer_expanded)
7165	expanded_size += trace_buf_size >> 10;
7166	}
7167	if (ring_buffer_expanded)
7168	r = sprintf(buf, "%lu\n", size);
7169	else
7170	r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
7171	mutex_unlock(&trace_types_lock);
7172
7173	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7174	}
7175
7176	static ssize_t
7177	tracing_free_buffer_write(struct file filp, const char __user ubuf,
7178	size_t cnt, loff_t *ppos)
7179	{
7180	/*
7181	* There is no need to read what the user has written, this function
7182	* is just to make sure that there is no error when "echo" is used
7183	*/
7184
7185	*ppos += cnt;
7186
7187	return cnt;
7188	}
7189
7190	static int
7191	tracing_free_buffer_release(struct inode inode, struct file filp)
7192	{
7193	struct trace_array *tr = inode->i_private;
7194
7195	/* disable tracing ? */
7196	if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
7197	tracer_tracing_off(tr);
7198	/* resize the ring buffer to 0 */
7199	tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
7200
7201	trace_array_put(tr);
7202
7203	return 0;
7204	}
7205
7206	static ssize_t
7207	tracing_mark_write(struct file filp, const char __user ubuf,
7208	size_t cnt, loff_t *fpos)
7209	{
7210	struct trace_array *tr = filp->private_data;
7211	struct ring_buffer_event *event;
7212	enum event_trigger_type tt = ETT_NONE;
7213	struct trace_buffer *buffer;
7214	struct print_entry *entry;
7215	ssize_t written;
7216	int size;
7217	int len;
7218
7219	/* Used in tracing_mark_raw_write() as well */
7220	#define FAULTED_STR "<faulted>"
7221	#define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
7222
7223	if (tracing_disabled)
7224	return -EINVAL;
7225
7226	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7227	return -EINVAL;
7228
7229	if (cnt > TRACE_BUF_SIZE)
7230	cnt = TRACE_BUF_SIZE;
7231
7232	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7233
7234	size = sizeof(entry) + cnt + 2; / add '\0' and possible '\n' */
7235
7236	/* If less than "<faulted>", then make sure we can still add that */
7237	if (cnt < FAULTED_SIZE)
7238	size += FAULTED_SIZE - cnt;
7239
7240	buffer = tr->array_buffer.buffer;
7241	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
7242	tracing_gen_ctx());
7243	if (unlikely(!event))
7244	/* Ring buffer disabled, return as if not open for write */
7245	return -EBADF;
7246
7247	entry = ring_buffer_event_data(event);
7248	entry->ip = _THIS_IP_;
7249
7250	len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
7251	if (len) {
7252	memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7253	cnt = FAULTED_SIZE;
7254	written = -EFAULT;
7255	} else
7256	written = cnt;
7257
7258	if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
7259	/* do not add \n before testing triggers, but add \0 */
7260	entry->buf[cnt] = '\0';
7261	tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event);
7262	}
7263
7264	if (entry->buf[cnt - 1] != '\n') {
7265	entry->buf[cnt] = '\n';
7266	entry->buf[cnt + 1] = '\0';
7267	} else
7268	entry->buf[cnt] = '\0';
7269
7270	if (static_branch_unlikely(&trace_marker_exports_enabled))
7271	ftrace_exports(event, TRACE_EXPORT_MARKER);
7272	__buffer_unlock_commit(buffer, event);
7273
7274	if (tt)
7275	event_triggers_post_call(tr->trace_marker_file, tt);
7276
7277	return written;
7278	}
7279
7280	/* Limit it for now to 3K (including tag) */
7281	#define RAW_DATA_MAX_SIZE (1024*3)
7282
7283	static ssize_t
7284	tracing_mark_raw_write(struct file filp, const char __user ubuf,
7285	size_t cnt, loff_t *fpos)
7286	{
7287	struct trace_array *tr = filp->private_data;
7288	struct ring_buffer_event *event;
7289	struct trace_buffer *buffer;
7290	struct raw_data_entry *entry;
7291	ssize_t written;
7292	int size;
7293	int len;
7294
7295	#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
7296
7297	if (tracing_disabled)
7298	return -EINVAL;
7299
7300	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7301	return -EINVAL;
7302
7303	/* The marker must at least have a tag id */
7304	if (cnt < sizeof(unsigned int) \|\| cnt > RAW_DATA_MAX_SIZE)
7305	return -EINVAL;
7306
7307	if (cnt > TRACE_BUF_SIZE)
7308	cnt = TRACE_BUF_SIZE;
7309
7310	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7311
7312	size = sizeof(*entry) + cnt;
7313	if (cnt < FAULT_SIZE_ID)
7314	size += FAULT_SIZE_ID - cnt;
7315
7316	buffer = tr->array_buffer.buffer;
7317	event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
7318	tracing_gen_ctx());
7319	if (!event)
7320	/* Ring buffer disabled, return as if not open for write */
7321	return -EBADF;
7322
7323	entry = ring_buffer_event_data(event);
7324
7325	len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
7326	if (len) {
7327	entry->id = -1;
7328	memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7329	written = -EFAULT;
7330	} else
7331	written = cnt;
7332
7333	__buffer_unlock_commit(buffer, event);
7334
7335	return written;
7336	}
7337
7338	static int tracing_clock_show(struct seq_file m, void v)
7339	{
7340	struct trace_array *tr = m->private;
7341	int i;
7342
7343	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
7344	seq_printf(m,
7345	"%s%s%s%s", i ? " " : "",
7346	i == tr->clock_id ? "[" : "", trace_clocks[i].name,
7347	i == tr->clock_id ? "]" : "");
7348	seq_putc(m, '\n');
7349
7350	return 0;
7351	}
7352
7353	int tracing_set_clock(struct trace_array tr, const char clockstr)
7354	{
7355	int i;
7356
7357	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
7358	if (strcmp(trace_clocks[i].name, clockstr) == 0)
7359	break;
7360	}
7361	if (i == ARRAY_SIZE(trace_clocks))
7362	return -EINVAL;
7363
7364	mutex_lock(&trace_types_lock);
7365
7366	tr->clock_id = i;
7367
7368	ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
7369
7370	/*
7371	* New clock may not be consistent with the previous clock.
7372	* Reset the buffer so that it doesn't have incomparable timestamps.
7373	*/
7374	tracing_reset_online_cpus(&tr->array_buffer);
7375
7376	#ifdef CONFIG_TRACER_MAX_TRACE
7377	if (tr->max_buffer.buffer)
7378	ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
7379	tracing_reset_online_cpus(&tr->max_buffer);
7380	#endif
7381
7382	mutex_unlock(&trace_types_lock);
7383
7384	return 0;
7385	}
7386
7387	static ssize_t tracing_clock_write(struct file filp, const char __user ubuf,
7388	size_t cnt, loff_t *fpos)
7389	{
7390	struct seq_file *m = filp->private_data;
7391	struct trace_array *tr = m->private;
7392	char buf[64];
7393	const char *clockstr;
7394	int ret;
7395
7396	if (cnt >= sizeof(buf))
7397	return -EINVAL;
7398
7399	if (copy_from_user(buf, ubuf, cnt))
7400	return -EFAULT;
7401
7402	buf[cnt] = 0;
7403
7404	clockstr = strstrip(buf);
7405
7406	ret = tracing_set_clock(tr, clockstr);
7407	if (ret)
7408	return ret;
7409
7410	*fpos += cnt;
7411
7412	return cnt;
7413	}
7414
7415	static int tracing_clock_open(struct inode inode, struct file file)
7416	{
7417	struct trace_array *tr = inode->i_private;
7418	int ret;
7419
7420	ret = tracing_check_open_get_tr(tr);
7421	if (ret)
7422	return ret;
7423
7424	ret = single_open(file, tracing_clock_show, inode->i_private);
7425	if (ret < 0)
7426	trace_array_put(tr);
7427
7428	return ret;
7429	}
7430
7431	static int tracing_time_stamp_mode_show(struct seq_file m, void v)
7432	{
7433	struct trace_array *tr = m->private;
7434
7435	mutex_lock(&trace_types_lock);
7436
7437	if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
7438	seq_puts(m, "delta [absolute]\n");
7439	else
7440	seq_puts(m, "[delta] absolute\n");
7441
7442	mutex_unlock(&trace_types_lock);
7443
7444	return 0;
7445	}
7446
7447	static int tracing_time_stamp_mode_open(struct inode inode, struct file file)
7448	{
7449	struct trace_array *tr = inode->i_private;
7450	int ret;
7451
7452	ret = tracing_check_open_get_tr(tr);
7453	if (ret)
7454	return ret;
7455
7456	ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
7457	if (ret < 0)
7458	trace_array_put(tr);
7459
7460	return ret;
7461	}
7462
7463	u64 tracing_event_time_stamp(struct trace_buffer buffer, struct ring_buffer_event rbe)
7464	{
7465	if (rbe == this_cpu_read(trace_buffered_event))
7466	return ring_buffer_time_stamp(buffer);
7467
7468	return ring_buffer_event_time_stamp(buffer, rbe);
7469	}
7470
7471	/*
7472	* Set or disable using the per CPU trace_buffer_event when possible.
7473	*/
7474	int tracing_set_filter_buffering(struct trace_array *tr, bool set)
7475	{
7476	int ret = 0;
7477
7478	mutex_lock(&trace_types_lock);
7479
7480	if (set && tr->no_filter_buffering_ref++)
7481	goto out;
7482
7483	if (!set) {
7484	if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) {
7485	ret = -EINVAL;
7486	goto out;
7487	}
7488
7489	--tr->no_filter_buffering_ref;
7490	}
7491	out:
7492	mutex_unlock(&trace_types_lock);
7493
7494	return ret;
7495	}
7496
7497	struct ftrace_buffer_info {
7498	struct trace_iterator iter;
7499	void *spare;
7500	unsigned int spare_cpu;
7501	unsigned int read;
7502	};
7503
7504	#ifdef CONFIG_TRACER_SNAPSHOT
7505	static int tracing_snapshot_open(struct inode inode, struct file file)
7506	{
7507	struct trace_array *tr = inode->i_private;
7508	struct trace_iterator *iter;
7509	struct seq_file *m;
7510	int ret;
7511
7512	ret = tracing_check_open_get_tr(tr);
7513	if (ret)
7514	return ret;
7515
7516	if (file->f_mode & FMODE_READ) {
7517	iter = __tracing_open(inode, file, true);
7518	if (IS_ERR(iter))
7519	ret = PTR_ERR(iter);
7520	} else {
7521	/* Writes still need the seq_file to hold the private data */
7522	ret = -ENOMEM;
7523	m = kzalloc(sizeof(*m), GFP_KERNEL);
7524	if (!m)
7525	goto out;
7526	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
7527	if (!iter) {
7528	kfree(m);
7529	goto out;
7530	}
7531	ret = 0;
7532
7533	iter->tr = tr;
7534	iter->array_buffer = &tr->max_buffer;
7535	iter->cpu_file = tracing_get_cpu(inode);
7536	m->private = iter;
7537	file->private_data = m;
7538	}
7539	out:
7540	if (ret < 0)
7541	trace_array_put(tr);
7542
7543	return ret;
7544	}
7545
7546	static ssize_t
7547	tracing_snapshot_write(struct file filp, const char __user ubuf, size_t cnt,
7548	loff_t *ppos)
7549	{
7550	struct seq_file *m = filp->private_data;
7551	struct trace_iterator *iter = m->private;
7552	struct trace_array *tr = iter->tr;
7553	unsigned long val;
7554	int ret;
7555
7556	ret = tracing_update_buffers();
7557	if (ret < 0)
7558	return ret;
7559
7560	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7561	if (ret)
7562	return ret;
7563
7564	mutex_lock(&trace_types_lock);
7565
7566	if (tr->current_trace->use_max_tr) {
7567	ret = -EBUSY;
7568	goto out;
7569	}
7570
7571	local_irq_disable();
7572	arch_spin_lock(&tr->max_lock);
7573	if (tr->cond_snapshot)
7574	ret = -EBUSY;
7575	arch_spin_unlock(&tr->max_lock);
7576	local_irq_enable();
7577	if (ret)
7578	goto out;
7579
7580	switch (val) {
7581	case 0:
7582	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7583	ret = -EINVAL;
7584	break;
7585	}
7586	if (tr->allocated_snapshot)
7587	free_snapshot(tr);
7588	break;
7589	case 1:
7590	/* Only allow per-cpu swap if the ring buffer supports it */
7591	#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
7592	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7593	ret = -EINVAL;
7594	break;
7595	}
7596	#endif
7597	if (tr->allocated_snapshot)
7598	ret = resize_buffer_duplicate_size(&tr->max_buffer,
7599	&tr->array_buffer, iter->cpu_file);
7600	else
7601	ret = tracing_alloc_snapshot_instance(tr);
7602	if (ret < 0)
7603	break;
7604	local_irq_disable();
7605	/* Now, we're going to swap */
7606	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7607	update_max_tr(tr, current, smp_processor_id(), NULL);
7608	else
7609	update_max_tr_single(tr, current, iter->cpu_file);
7610	local_irq_enable();
7611	break;
7612	default:
7613	if (tr->allocated_snapshot) {
7614	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7615	tracing_reset_online_cpus(&tr->max_buffer);
7616	else
7617	tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
7618	}
7619	break;
7620	}
7621
7622	if (ret >= 0) {
7623	*ppos += cnt;
7624	ret = cnt;
7625	}
7626	out:
7627	mutex_unlock(&trace_types_lock);
7628	return ret;
7629	}
7630
7631	static int tracing_snapshot_release(struct inode inode, struct file file)
7632	{
7633	struct seq_file *m = file->private_data;
7634	int ret;
7635
7636	ret = tracing_release(inode, file);
7637
7638	if (file->f_mode & FMODE_READ)
7639	return ret;
7640
7641	/* If write only, the seq_file is just a stub */
7642	if (m)
7643	kfree(m->private);
7644	kfree(m);
7645
7646	return 0;
7647	}
7648
7649	static int tracing_buffers_open(struct inode inode, struct file filp);
7650	static ssize_t tracing_buffers_read(struct file filp, char __user ubuf,
7651	size_t count, loff_t *ppos);
7652	static int tracing_buffers_release(struct inode inode, struct file file);
7653	static ssize_t tracing_buffers_splice_read(struct file file, loff_t ppos,
7654	struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7655
7656	static int snapshot_raw_open(struct inode inode, struct file filp)
7657	{
7658	struct ftrace_buffer_info *info;
7659	int ret;
7660
7661	/* The following checks for tracefs lockdown */
7662	ret = tracing_buffers_open(inode, filp);
7663	if (ret < 0)
7664	return ret;
7665
7666	info = filp->private_data;
7667
7668	if (info->iter.trace->use_max_tr) {
7669	tracing_buffers_release(inode, filp);
7670	return -EBUSY;
7671	}
7672
7673	info->iter.snapshot = true;
7674	info->iter.array_buffer = &info->iter.tr->max_buffer;
7675
7676	return ret;
7677	}
7678
7679	#endif /* CONFIG_TRACER_SNAPSHOT */
7680
7681
7682	static const struct file_operations tracing_thresh_fops = {
7683	.open = tracing_open_generic,
7684	.read = tracing_thresh_read,
7685	.write = tracing_thresh_write,
7686	.llseek = generic_file_llseek,
7687	};
7688
7689	#ifdef CONFIG_TRACER_MAX_TRACE
7690	static const struct file_operations tracing_max_lat_fops = {
7691	.open = tracing_open_generic,
7692	.read = tracing_max_lat_read,
7693	.write = tracing_max_lat_write,
7694	.llseek = generic_file_llseek,
7695	};
7696	#endif
7697
7698	static const struct file_operations set_tracer_fops = {
7699	.open = tracing_open_generic,
7700	.read = tracing_set_trace_read,
7701	.write = tracing_set_trace_write,
7702	.llseek = generic_file_llseek,
7703	};
7704
7705	static const struct file_operations tracing_pipe_fops = {
7706	.open = tracing_open_pipe,
7707	.poll = tracing_poll_pipe,
7708	.read = tracing_read_pipe,
7709	.splice_read = tracing_splice_read_pipe,
7710	.release = tracing_release_pipe,
7711	.llseek = no_llseek,
7712	};
7713
7714	static const struct file_operations tracing_entries_fops = {
7715	.open = tracing_open_generic_tr,
7716	.read = tracing_entries_read,
7717	.write = tracing_entries_write,
7718	.llseek = generic_file_llseek,
7719	.release = tracing_release_generic_tr,
7720	};
7721
7722	static const struct file_operations tracing_total_entries_fops = {
7723	.open = tracing_open_generic_tr,
7724	.read = tracing_total_entries_read,
7725	.llseek = generic_file_llseek,
7726	.release = tracing_release_generic_tr,
7727	};
7728
7729	static const struct file_operations tracing_free_buffer_fops = {
7730	.open = tracing_open_generic_tr,
7731	.write = tracing_free_buffer_write,
7732	.release = tracing_free_buffer_release,
7733	};
7734
7735	static const struct file_operations tracing_mark_fops = {
7736	.open = tracing_mark_open,
7737	.write = tracing_mark_write,
7738	.release = tracing_release_generic_tr,
7739	};
7740
7741	static const struct file_operations tracing_mark_raw_fops = {
7742	.open = tracing_mark_open,
7743	.write = tracing_mark_raw_write,
7744	.release = tracing_release_generic_tr,
7745	};
7746
7747	static const struct file_operations trace_clock_fops = {
7748	.open = tracing_clock_open,
7749	.read = seq_read,
7750	.llseek = seq_lseek,
7751	.release = tracing_single_release_tr,
7752	.write = tracing_clock_write,
7753	};
7754
7755	static const struct file_operations trace_time_stamp_mode_fops = {
7756	.open = tracing_time_stamp_mode_open,
7757	.read = seq_read,
7758	.llseek = seq_lseek,
7759	.release = tracing_single_release_tr,
7760	};
7761
7762	#ifdef CONFIG_TRACER_SNAPSHOT
7763	static const struct file_operations snapshot_fops = {
7764	.open = tracing_snapshot_open,
7765	.read = seq_read,
7766	.write = tracing_snapshot_write,
7767	.llseek = tracing_lseek,
7768	.release = tracing_snapshot_release,
7769	};
7770
7771	static const struct file_operations snapshot_raw_fops = {
7772	.open = snapshot_raw_open,
7773	.read = tracing_buffers_read,
7774	.release = tracing_buffers_release,
7775	.splice_read = tracing_buffers_splice_read,
7776	.llseek = no_llseek,
7777	};
7778
7779	#endif /* CONFIG_TRACER_SNAPSHOT */
7780
7781	/*
7782	* trace_min_max_write - Write a u64 value to a trace_min_max_param struct
7783	* @filp: The active open file structure
7784	* @ubuf: The userspace provided buffer to read value into
7785	* @cnt: The maximum number of bytes to read
7786	* @ppos: The current "file" position
7787	*
7788	* This function implements the write interface for a struct trace_min_max_param.
7789	* The filp->private_data must point to a trace_min_max_param structure that
7790	* defines where to write the value, the min and the max acceptable values,
7791	* and a lock to protect the write.
7792	*/
7793	static ssize_t
7794	trace_min_max_write(struct file filp, const char __user ubuf, size_t cnt, loff_t *ppos)
7795	{
7796	struct trace_min_max_param *param = filp->private_data;
7797	u64 val;
7798	int err;
7799
7800	if (!param)
7801	return -EFAULT;
7802
7803	err = kstrtoull_from_user(ubuf, cnt, 10, &val);
7804	if (err)
7805	return err;
7806
7807	if (param->lock)
7808	mutex_lock(param->lock);
7809
7810	if (param->min && val < *param->min)
7811	err = -EINVAL;
7812
7813	if (param->max && val > *param->max)
7814	err = -EINVAL;
7815
7816	if (!err)
7817	*param->val = val;
7818
7819	if (param->lock)
7820	mutex_unlock(param->lock);
7821
7822	if (err)
7823	return err;
7824
7825	return cnt;
7826	}
7827
7828	/*
7829	* trace_min_max_read - Read a u64 value from a trace_min_max_param struct
7830	* @filp: The active open file structure
7831	* @ubuf: The userspace provided buffer to read value into
7832	* @cnt: The maximum number of bytes to read
7833	* @ppos: The current "file" position
7834	*
7835	* This function implements the read interface for a struct trace_min_max_param.
7836	* The filp->private_data must point to a trace_min_max_param struct with valid
7837	* data.
7838	*/
7839	static ssize_t
7840	trace_min_max_read(struct file filp, char __user ubuf, size_t cnt, loff_t *ppos)
7841	{
7842	struct trace_min_max_param *param = filp->private_data;
7843	char buf[U64_STR_SIZE];
7844	int len;
7845	u64 val;
7846
7847	if (!param)
7848	return -EFAULT;
7849
7850	val = *param->val;
7851
7852	if (cnt > sizeof(buf))
7853	cnt = sizeof(buf);
7854
7855	len = snprintf(buf, sizeof(buf), "%llu\n", val);
7856
7857	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
7858	}
7859
7860	const struct file_operations trace_min_max_fops = {
7861	.open = tracing_open_generic,
7862	.read = trace_min_max_read,
7863	.write = trace_min_max_write,
7864	};
7865
7866	#define TRACING_LOG_ERRS_MAX 8
7867	#define TRACING_LOG_LOC_MAX 128
7868
7869	#define CMD_PREFIX " Command: "
7870
7871	struct err_info {
7872	const char *errs; / ptr to loc-specific array of err strings */
7873	u8 type; /* index into errs -> specific err string */
7874	u16 pos; /* caret position */
7875	u64 ts;
7876	};
7877
7878	struct tracing_log_err {
7879	struct list_head list;
7880	struct err_info info;
7881	char loc[TRACING_LOG_LOC_MAX]; /* err location */
7882	char cmd; / what caused err */
7883	};
7884
7885	static DEFINE_MUTEX(tracing_err_log_lock);
7886
7887	static struct tracing_log_err *alloc_tracing_log_err(int len)
7888	{
7889	struct tracing_log_err *err;
7890
7891	err = kzalloc(sizeof(*err), GFP_KERNEL);
7892	if (!err)
7893	return ERR_PTR(-ENOMEM);
7894
7895	err->cmd = kzalloc(len, GFP_KERNEL);
7896	if (!err->cmd) {
7897	kfree(err);
7898	return ERR_PTR(-ENOMEM);
7899	}
7900
7901	return err;
7902	}
7903
7904	static void free_tracing_log_err(struct tracing_log_err *err)
7905	{
7906	kfree(err->cmd);
7907	kfree(err);
7908	}
7909
7910	static struct tracing_log_err get_tracing_log_err(struct trace_array tr,
7911	int len)
7912	{
7913	struct tracing_log_err *err;
7914	char *cmd;
7915
7916	if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7917	err = alloc_tracing_log_err(len);
7918	if (PTR_ERR(err) != -ENOMEM)
7919	tr->n_err_log_entries++;
7920
7921	return err;
7922	}
7923	cmd = kzalloc(len, GFP_KERNEL);
7924	if (!cmd)
7925	return ERR_PTR(-ENOMEM);
7926	err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7927	kfree(err->cmd);
7928	err->cmd = cmd;
7929	list_del(&err->list);
7930
7931	return err;
7932	}
7933
7934	/**
7935	* err_pos - find the position of a string within a command for error careting
7936	* @cmd: The tracing command that caused the error
7937	* @str: The string to position the caret at within @cmd
7938	*
7939	* Finds the position of the first occurrence of @str within @cmd. The
7940	* return value can be passed to tracing_log_err() for caret placement
7941	* within @cmd.
7942	*
7943	* Returns the index within @cmd of the first occurrence of @str or 0
7944	* if @str was not found.
7945	*/
7946	unsigned int err_pos(char cmd, const char str)
7947	{
7948	char *found;
7949
7950	if (WARN_ON(!strlen(cmd)))
7951	return 0;
7952
7953	found = strstr(cmd, str);
7954	if (found)
7955	return found - cmd;
7956
7957	return 0;
7958	}
7959
7960	/**
7961	* tracing_log_err - write an error to the tracing error log
7962	* @tr: The associated trace array for the error (NULL for top level array)
7963	* @loc: A string describing where the error occurred
7964	* @cmd: The tracing command that caused the error
7965	* @errs: The array of loc-specific static error strings
7966	* @type: The index into errs[], which produces the specific static err string
7967	* @pos: The position the caret should be placed in the cmd
7968	*
7969	* Writes an error into tracing/error_log of the form:
7970	*
7971	* <loc>: error: <text>
7972	* Command: <cmd>
7973	* ^
7974	*
7975	* tracing/error_log is a small log file containing the last
7976	* TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7977	* unless there has been a tracing error, and the error log can be
7978	* cleared and have its memory freed by writing the empty string in
7979	* truncation mode to it i.e. echo > tracing/error_log.
7980	*
7981	* NOTE: the @errs array along with the @type param are used to
7982	* produce a static error string - this string is not copied and saved
7983	* when the error is logged - only a pointer to it is saved. See
7984	* existing callers for examples of how static strings are typically
7985	* defined for use with tracing_log_err().
7986	*/
7987	void tracing_log_err(struct trace_array *tr,
7988	const char loc, const char cmd,
7989	const char **errs, u8 type, u16 pos)
7990	{
7991	struct tracing_log_err *err;
7992	int len = 0;
7993
7994	if (!tr)
7995	tr = &global_trace;
7996
7997	len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1;
7998
7999	mutex_lock(&tracing_err_log_lock);
8000	err = get_tracing_log_err(tr, len);
8001	if (PTR_ERR(err) == -ENOMEM) {
8002	mutex_unlock(&tracing_err_log_lock);
8003	return;
8004	}
8005
8006	snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
8007	snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd);
8008
8009	err->info.errs = errs;
8010	err->info.type = type;
8011	err->info.pos = pos;
8012	err->info.ts = local_clock();
8013
8014	list_add_tail(&err->list, &tr->err_log);
8015	mutex_unlock(&tracing_err_log_lock);
8016	}
8017
8018	static void clear_tracing_err_log(struct trace_array *tr)
8019	{
8020	struct tracing_log_err err, next;
8021
8022	mutex_lock(&tracing_err_log_lock);
8023	list_for_each_entry_safe(err, next, &tr->err_log, list) {
8024	list_del(&err->list);
8025	free_tracing_log_err(err);
8026	}
8027
8028	tr->n_err_log_entries = 0;
8029	mutex_unlock(&tracing_err_log_lock);
8030	}
8031
8032	static void tracing_err_log_seq_start(struct seq_file m, loff_t *pos)
8033	{
8034	struct trace_array *tr = m->private;
8035
8036	mutex_lock(&tracing_err_log_lock);
8037
8038	return seq_list_start(&tr->err_log, *pos);
8039	}
8040
8041	static void tracing_err_log_seq_next(struct seq_file m, void v, loff_t pos)
8042	{
8043	struct trace_array *tr = m->private;
8044
8045	return seq_list_next(v, &tr->err_log, pos);
8046	}
8047
8048	static void tracing_err_log_seq_stop(struct seq_file m, void v)
8049	{
8050	mutex_unlock(&tracing_err_log_lock);
8051	}
8052
8053	static void tracing_err_log_show_pos(struct seq_file *m, u16 pos)
8054	{
8055	u16 i;
8056
8057	for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
8058	seq_putc(m, ' ');
8059	for (i = 0; i < pos; i++)
8060	seq_putc(m, ' ');
8061	seq_puts(m, "^\n");
8062	}
8063
8064	static int tracing_err_log_seq_show(struct seq_file m, void v)
8065	{
8066	struct tracing_log_err *err = v;
8067
8068	if (err) {
8069	const char *err_text = err->info.errs[err->info.type];
8070	u64 sec = err->info.ts;
8071	u32 nsec;
8072
8073	nsec = do_div(sec, NSEC_PER_SEC);
8074	seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
8075	err->loc, err_text);
8076	seq_printf(m, "%s", err->cmd);
8077	tracing_err_log_show_pos(m, err->info.pos);
8078	}
8079
8080	return 0;
8081	}
8082
8083	static const struct seq_operations tracing_err_log_seq_ops = {
8084	.start = tracing_err_log_seq_start,
8085	.next = tracing_err_log_seq_next,
8086	.stop = tracing_err_log_seq_stop,
8087	.show = tracing_err_log_seq_show
8088	};
8089
8090	static int tracing_err_log_open(struct inode inode, struct file file)
8091	{
8092	struct trace_array *tr = inode->i_private;
8093	int ret = 0;
8094
8095	ret = tracing_check_open_get_tr(tr);
8096	if (ret)
8097	return ret;
8098
8099	/* If this file was opened for write, then erase contents */
8100	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
8101	clear_tracing_err_log(tr);
8102
8103	if (file->f_mode & FMODE_READ) {
8104	ret = seq_open(file, &tracing_err_log_seq_ops);
8105	if (!ret) {
8106	struct seq_file *m = file->private_data;
8107	m->private = tr;
8108	} else {
8109	trace_array_put(tr);
8110	}
8111	}
8112	return ret;
8113	}
8114
8115	static ssize_t tracing_err_log_write(struct file *file,
8116	const char __user *buffer,
8117	size_t count, loff_t *ppos)
8118	{
8119	return count;
8120	}
8121
8122	static int tracing_err_log_release(struct inode inode, struct file file)
8123	{
8124	struct trace_array *tr = inode->i_private;
8125
8126	trace_array_put(tr);
8127
8128	if (file->f_mode & FMODE_READ)
8129	seq_release(inode, file);
8130
8131	return 0;
8132	}
8133
8134	static const struct file_operations tracing_err_log_fops = {
8135	.open = tracing_err_log_open,
8136	.write = tracing_err_log_write,
8137	.read = seq_read,
8138	.llseek = seq_lseek,
8139	.release = tracing_err_log_release,
8140	};
8141
8142	static int tracing_buffers_open(struct inode inode, struct file filp)
8143	{
8144	struct trace_array *tr = inode->i_private;
8145	struct ftrace_buffer_info *info;
8146	int ret;
8147
8148	ret = tracing_check_open_get_tr(tr);
8149	if (ret)
8150	return ret;
8151
8152	info = kvzalloc(sizeof(*info), GFP_KERNEL);
8153	if (!info) {
8154	trace_array_put(tr);
8155	return -ENOMEM;
8156	}
8157
8158	mutex_lock(&trace_types_lock);
8159
8160	info->iter.tr = tr;
8161	info->iter.cpu_file = tracing_get_cpu(inode);
8162	info->iter.trace = tr->current_trace;
8163	info->iter.array_buffer = &tr->array_buffer;
8164	info->spare = NULL;
8165	/* Force reading ring buffer for first read */
8166	info->read = (unsigned int)-1;
8167
8168	filp->private_data = info;
8169
8170	tr->trace_ref++;
8171
8172	mutex_unlock(&trace_types_lock);
8173
8174	ret = nonseekable_open(inode, filp);
8175	if (ret < 0)
8176	trace_array_put(tr);
8177
8178	return ret;
8179	}
8180
8181	static __poll_t
8182	tracing_buffers_poll(struct file filp, poll_table poll_table)
8183	{
8184	struct ftrace_buffer_info *info = filp->private_data;
8185	struct trace_iterator *iter = &info->iter;
8186
8187	return trace_poll(iter, filp, poll_table);
8188	}
8189
8190	static ssize_t
8191	tracing_buffers_read(struct file filp, char __user ubuf,
8192	size_t count, loff_t *ppos)
8193	{
8194	struct ftrace_buffer_info *info = filp->private_data;
8195	struct trace_iterator *iter = &info->iter;
8196	ssize_t ret = 0;
8197	ssize_t size;
8198
8199	if (!count)
8200	return 0;
8201
8202	#ifdef CONFIG_TRACER_MAX_TRACE
8203	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8204	return -EBUSY;
8205	#endif
8206
8207	if (!info->spare) {
8208	info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
8209	iter->cpu_file);
8210	if (IS_ERR(info->spare)) {
8211	ret = PTR_ERR(info->spare);
8212	info->spare = NULL;
8213	} else {
8214	info->spare_cpu = iter->cpu_file;
8215	}
8216	}
8217	if (!info->spare)
8218	return ret;
8219
8220	/* Do we have previous read data to read? */
8221	if (info->read < PAGE_SIZE)
8222	goto read;
8223
8224	again:
8225	trace_access_lock(iter->cpu_file);
8226	ret = ring_buffer_read_page(iter->array_buffer->buffer,
8227	&info->spare,
8228	count,
8229	iter->cpu_file, 0);
8230	trace_access_unlock(iter->cpu_file);
8231
8232	if (ret < 0) {
8233	if (trace_empty(iter)) {
8234	if ((filp->f_flags & O_NONBLOCK))
8235	return -EAGAIN;
8236
8237	ret = wait_on_pipe(iter, 0);
8238	if (ret)
8239	return ret;
8240
8241	goto again;
8242	}
8243	return 0;
8244	}
8245
8246	info->read = 0;
8247	read:
8248	size = PAGE_SIZE - info->read;
8249	if (size > count)
8250	size = count;
8251
8252	ret = copy_to_user(ubuf, info->spare + info->read, size);
8253	if (ret == size)
8254	return -EFAULT;
8255
8256	size -= ret;
8257
8258	*ppos += size;
8259	info->read += size;
8260
8261	return size;
8262	}
8263
8264	static int tracing_buffers_release(struct inode inode, struct file file)
8265	{
8266	struct ftrace_buffer_info *info = file->private_data;
8267	struct trace_iterator *iter = &info->iter;
8268
8269	mutex_lock(&trace_types_lock);
8270
8271	iter->tr->trace_ref--;
8272
8273	__trace_array_put(iter->tr);
8274
8275	iter->wait_index++;
8276	/* Make sure the waiters see the new wait_index */
8277	smp_wmb();
8278
8279	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
8280
8281	if (info->spare)
8282	ring_buffer_free_read_page(iter->array_buffer->buffer,
8283	info->spare_cpu, info->spare);
8284	kvfree(info);
8285
8286	mutex_unlock(&trace_types_lock);
8287
8288	return 0;
8289	}
8290
8291	struct buffer_ref {
8292	struct trace_buffer *buffer;
8293	void *page;
8294	int cpu;
8295	refcount_t refcount;
8296	};
8297
8298	static void buffer_ref_release(struct buffer_ref *ref)
8299	{
8300	if (!refcount_dec_and_test(&ref->refcount))
8301	return;
8302	ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
8303	kfree(ref);
8304	}
8305
8306	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
8307	struct pipe_buffer *buf)
8308	{
8309	struct buffer_ref ref = (struct buffer_ref )buf->private;
8310
8311	buffer_ref_release(ref);
8312	buf->private = 0;
8313	}
8314
8315	static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
8316	struct pipe_buffer *buf)
8317	{
8318	struct buffer_ref ref = (struct buffer_ref )buf->private;
8319
8320	if (refcount_read(&ref->refcount) > INT_MAX/2)
8321	return false;
8322
8323	refcount_inc(&ref->refcount);
8324	return true;
8325	}
8326
8327	/* Pipe buffer operations for a buffer. */
8328	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
8329	.release = buffer_pipe_buf_release,
8330	.get = buffer_pipe_buf_get,
8331	};
8332
8333	/*
8334	* Callback from splice_to_pipe(), if we need to release some pages
8335	* at the end of the spd in case we error'ed out in filling the pipe.
8336	*/
8337	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
8338	{
8339	struct buffer_ref *ref =
8340	(struct buffer_ref *)spd->partial[i].private;
8341
8342	buffer_ref_release(ref);
8343	spd->partial[i].private = 0;
8344	}
8345
8346	static ssize_t
8347	tracing_buffers_splice_read(struct file file, loff_t ppos,
8348	struct pipe_inode_info *pipe, size_t len,
8349	unsigned int flags)
8350	{
8351	struct ftrace_buffer_info *info = file->private_data;
8352	struct trace_iterator *iter = &info->iter;
8353	struct partial_page partial_def[PIPE_DEF_BUFFERS];
8354	struct page *pages_def[PIPE_DEF_BUFFERS];
8355	struct splice_pipe_desc spd = {
8356	.pages = pages_def,
8357	.partial = partial_def,
8358	.nr_pages_max = PIPE_DEF_BUFFERS,
8359	.ops = &buffer_pipe_buf_ops,
8360	.spd_release = buffer_spd_release,
8361	};
8362	struct buffer_ref *ref;
8363	int entries, i;
8364	ssize_t ret = 0;
8365
8366	#ifdef CONFIG_TRACER_MAX_TRACE
8367	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8368	return -EBUSY;
8369	#endif
8370
8371	if (*ppos & (PAGE_SIZE - 1))
8372	return -EINVAL;
8373
8374	if (len & (PAGE_SIZE - 1)) {
8375	if (len < PAGE_SIZE)
8376	return -EINVAL;
8377	len &= PAGE_MASK;
8378	}
8379
8380	if (splice_grow_spd(pipe, &spd))
8381	return -ENOMEM;
8382
8383	again:
8384	trace_access_lock(iter->cpu_file);
8385	entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8386
8387	for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
8388	struct page *page;
8389	int r;
8390
8391	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
8392	if (!ref) {
8393	ret = -ENOMEM;
8394	break;
8395	}
8396
8397	refcount_set(&ref->refcount, 1);
8398	ref->buffer = iter->array_buffer->buffer;
8399	ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
8400	if (IS_ERR(ref->page)) {
8401	ret = PTR_ERR(ref->page);
8402	ref->page = NULL;
8403	kfree(ref);
8404	break;
8405	}
8406	ref->cpu = iter->cpu_file;
8407
8408	r = ring_buffer_read_page(ref->buffer, &ref->page,
8409	len, iter->cpu_file, 1);
8410	if (r < 0) {
8411	ring_buffer_free_read_page(ref->buffer, ref->cpu,
8412	ref->page);
8413	kfree(ref);
8414	break;
8415	}
8416
8417	page = virt_to_page(ref->page);
8418
8419	spd.pages[i] = page;
8420	spd.partial[i].len = PAGE_SIZE;
8421	spd.partial[i].offset = 0;
8422	spd.partial[i].private = (unsigned long)ref;
8423	spd.nr_pages++;
8424	*ppos += PAGE_SIZE;
8425
8426	entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8427	}
8428
8429	trace_access_unlock(iter->cpu_file);
8430	spd.nr_pages = i;
8431
8432	/* did we read anything? */
8433	if (!spd.nr_pages) {
8434	long wait_index;
8435
8436	if (ret)
8437	goto out;
8438
8439	ret = -EAGAIN;
8440	if ((file->f_flags & O_NONBLOCK) \|\| (flags & SPLICE_F_NONBLOCK))
8441	goto out;
8442
8443	wait_index = READ_ONCE(iter->wait_index);
8444
8445	ret = wait_on_pipe(iter, iter->tr->buffer_percent);
8446	if (ret)
8447	goto out;
8448
8449	/* No need to wait after waking up when tracing is off */
8450	if (!tracer_tracing_is_on(iter->tr))
8451	goto out;
8452
8453	/* Make sure we see the new wait_index */
8454	smp_rmb();
8455	if (wait_index != iter->wait_index)
8456	goto out;
8457
8458	goto again;
8459	}
8460
8461	ret = splice_to_pipe(pipe, &spd);
8462	out:
8463	splice_shrink_spd(&spd);
8464
8465	return ret;
8466	}
8467
8468	/* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */
8469	static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
8470	{
8471	struct ftrace_buffer_info *info = file->private_data;
8472	struct trace_iterator *iter = &info->iter;
8473
8474	if (cmd)
8475	return -ENOIOCTLCMD;
8476
8477	mutex_lock(&trace_types_lock);
8478
8479	iter->wait_index++;
8480	/* Make sure the waiters see the new wait_index */
8481	smp_wmb();
8482
8483	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
8484
8485	mutex_unlock(&trace_types_lock);
8486	return 0;
8487	}
8488
8489	static const struct file_operations tracing_buffers_fops = {
8490	.open = tracing_buffers_open,
8491	.read = tracing_buffers_read,
8492	.poll = tracing_buffers_poll,
8493	.release = tracing_buffers_release,
8494	.splice_read = tracing_buffers_splice_read,
8495	.unlocked_ioctl = tracing_buffers_ioctl,
8496	.llseek = no_llseek,
8497	};
8498
8499	static ssize_t
8500	tracing_stats_read(struct file filp, char __user ubuf,
8501	size_t count, loff_t *ppos)
8502	{
8503	struct inode *inode = file_inode(filp);
8504	struct trace_array *tr = inode->i_private;
8505	struct array_buffer *trace_buf = &tr->array_buffer;
8506	int cpu = tracing_get_cpu(inode);
8507	struct trace_seq *s;
8508	unsigned long cnt;
8509	unsigned long long t;
8510	unsigned long usec_rem;
8511
8512	s = kmalloc(sizeof(*s), GFP_KERNEL);
8513	if (!s)
8514	return -ENOMEM;
8515
8516	trace_seq_init(s);
8517
8518	cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
8519	trace_seq_printf(s, "entries: %ld\n", cnt);
8520
8521	cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
8522	trace_seq_printf(s, "overrun: %ld\n", cnt);
8523
8524	cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
8525	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
8526
8527	cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
8528	trace_seq_printf(s, "bytes: %ld\n", cnt);
8529
8530	if (trace_clocks[tr->clock_id].in_ns) {
8531	/* local or global for trace_clock */
8532	t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8533	usec_rem = do_div(t, USEC_PER_SEC);
8534	trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
8535	t, usec_rem);
8536
8537	t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer));
8538	usec_rem = do_div(t, USEC_PER_SEC);
8539	trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
8540	} else {
8541	/* counter or tsc mode for trace_clock */
8542	trace_seq_printf(s, "oldest event ts: %llu\n",
8543	ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8544
8545	trace_seq_printf(s, "now ts: %llu\n",
8546	ring_buffer_time_stamp(trace_buf->buffer));
8547	}
8548
8549	cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
8550	trace_seq_printf(s, "dropped events: %ld\n", cnt);
8551
8552	cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
8553	trace_seq_printf(s, "read events: %ld\n", cnt);
8554
8555	count = simple_read_from_buffer(ubuf, count, ppos,
8556	s->buffer, trace_seq_used(s));
8557
8558	kfree(s);
8559
8560	return count;
8561	}
8562
8563	static const struct file_operations tracing_stats_fops = {
8564	.open = tracing_open_generic_tr,
8565	.read = tracing_stats_read,
8566	.llseek = generic_file_llseek,
8567	.release = tracing_release_generic_tr,
8568	};
8569
8570	#ifdef CONFIG_DYNAMIC_FTRACE
8571
8572	static ssize_t
8573	tracing_read_dyn_info(struct file filp, char __user ubuf,
8574	size_t cnt, loff_t *ppos)
8575	{
8576	ssize_t ret;
8577	char *buf;
8578	int r;
8579
8580	/* 256 should be plenty to hold the amount needed */
8581	buf = kmalloc(256, GFP_KERNEL);
8582	if (!buf)
8583	return -ENOMEM;
8584
8585	r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
8586	ftrace_update_tot_cnt,
8587	ftrace_number_of_pages,
8588	ftrace_number_of_groups);
8589
8590	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8591	kfree(buf);
8592	return ret;
8593	}
8594
8595	static const struct file_operations tracing_dyn_info_fops = {
8596	.open = tracing_open_generic,
8597	.read = tracing_read_dyn_info,
8598	.llseek = generic_file_llseek,
8599	};
8600	#endif /* CONFIG_DYNAMIC_FTRACE */
8601
8602	#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
8603	static void
8604	ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
8605	struct trace_array tr, struct ftrace_probe_ops ops,
8606	void *data)
8607	{
8608	tracing_snapshot_instance(tr);
8609	}
8610
8611	static void
8612	ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
8613	struct trace_array tr, struct ftrace_probe_ops ops,
8614	void *data)
8615	{
8616	struct ftrace_func_mapper *mapper = data;
8617	long *count = NULL;
8618
8619	if (mapper)
8620	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8621
8622	if (count) {
8623
8624	if (*count <= 0)
8625	return;
8626
8627	(*count)--;
8628	}
8629
8630	tracing_snapshot_instance(tr);
8631	}
8632
8633	static int
8634	ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
8635	struct ftrace_probe_ops ops, void data)
8636	{
8637	struct ftrace_func_mapper *mapper = data;
8638	long *count = NULL;
8639
8640	seq_printf(m, "%ps:", (void *)ip);
8641
8642	seq_puts(m, "snapshot");
8643
8644	if (mapper)
8645	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8646
8647	if (count)
8648	seq_printf(m, ":count=%ld\n", *count);
8649	else
8650	seq_puts(m, ":unlimited\n");
8651
8652	return 0;
8653	}
8654
8655	static int
8656	ftrace_snapshot_init(struct ftrace_probe_ops ops, struct trace_array tr,
8657	unsigned long ip, void init_data, void *data)
8658	{
8659	struct ftrace_func_mapper mapper = data;
8660
8661	if (!mapper) {
8662	mapper = allocate_ftrace_func_mapper();
8663	if (!mapper)
8664	return -ENOMEM;
8665	*data = mapper;
8666	}
8667
8668	return ftrace_func_mapper_add_ip(mapper, ip, init_data);
8669	}
8670
8671	static void
8672	ftrace_snapshot_free(struct ftrace_probe_ops ops, struct trace_array tr,
8673	unsigned long ip, void *data)
8674	{
8675	struct ftrace_func_mapper *mapper = data;
8676
8677	if (!ip) {
8678	if (!mapper)
8679	return;
8680	free_ftrace_func_mapper(mapper, NULL);
8681	return;
8682	}
8683
8684	ftrace_func_mapper_remove_ip(mapper, ip);
8685	}
8686
8687	static struct ftrace_probe_ops snapshot_probe_ops = {
8688	.func = ftrace_snapshot,
8689	.print = ftrace_snapshot_print,
8690	};
8691
8692	static struct ftrace_probe_ops snapshot_count_probe_ops = {
8693	.func = ftrace_count_snapshot,
8694	.print = ftrace_snapshot_print,
8695	.init = ftrace_snapshot_init,
8696	.free = ftrace_snapshot_free,
8697	};
8698
8699	static int
8700	ftrace_trace_snapshot_callback(struct trace_array tr, struct ftrace_hash hash,
8701	char glob, char cmd, char *param, int enable)
8702	{
8703	struct ftrace_probe_ops *ops;
8704	void count = (void )-1;
8705	char *number;
8706	int ret;
8707
8708	if (!tr)
8709	return -ENODEV;
8710
8711	/* hash funcs only work with set_ftrace_filter */
8712	if (!enable)
8713	return -EINVAL;
8714
8715	ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
8716
8717	if (glob[0] == '!')
8718	return unregister_ftrace_function_probe_func(glob+1, tr, ops);
8719
8720	if (!param)
8721	goto out_reg;
8722
8723	number = strsep(&param, ":");
8724
8725	if (!strlen(number))
8726	goto out_reg;
8727
8728	/*
8729	* We use the callback data field (which is a pointer)
8730	* as our counter.
8731	*/
8732	ret = kstrtoul(number, 0, (unsigned long *)&count);
8733	if (ret)
8734	return ret;
8735
8736	out_reg:
8737	ret = tracing_alloc_snapshot_instance(tr);
8738	if (ret < 0)
8739	goto out;
8740
8741	ret = register_ftrace_function_probe(glob, tr, ops, count);
8742
8743	out:
8744	return ret < 0 ? ret : 0;
8745	}
8746
8747	static struct ftrace_func_command ftrace_snapshot_cmd = {
8748	.name = "snapshot",
8749	.func = ftrace_trace_snapshot_callback,
8750	};
8751
8752	static __init int register_snapshot_cmd(void)
8753	{
8754	return register_ftrace_command(&ftrace_snapshot_cmd);
8755	}
8756	#else
8757	static inline __init int register_snapshot_cmd(void) { return 0; }
8758	#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
8759
8760	static struct dentry tracing_get_dentry(struct trace_array tr)
8761	{
8762	if (WARN_ON(!tr->dir))
8763	return ERR_PTR(-ENODEV);
8764
8765	/* Top directory uses NULL as the parent */
8766	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
8767	return NULL;
8768
8769	/* All sub buffers have a descriptor */
8770	return tr->dir;
8771	}
8772
8773	static struct dentry tracing_dentry_percpu(struct trace_array tr, int cpu)
8774	{
8775	struct dentry *d_tracer;
8776
8777	if (tr->percpu_dir)
8778	return tr->percpu_dir;
8779
8780	d_tracer = tracing_get_dentry(tr);
8781	if (IS_ERR(d_tracer))
8782	return NULL;
8783
8784	tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
8785
8786	MEM_FAIL(!tr->percpu_dir,
8787	"Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8788
8789	return tr->percpu_dir;
8790	}
8791
8792	static struct dentry *
8793	trace_create_cpu_file(const char name, umode_t mode, struct dentry parent,
8794	void data, long cpu, const struct file_operations fops)
8795	{
8796	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8797
8798	if (ret) /* See tracing_get_cpu() */
8799	d_inode(ret)->i_cdev = (void *)(cpu + 1);
8800	return ret;
8801	}
8802
8803	static void
8804	tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8805	{
8806	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8807	struct dentry *d_cpu;
8808	char cpu_dir[30]; /* 30 characters should be more than enough */
8809
8810	if (!d_percpu)
8811	return;
8812
8813	snprintf(cpu_dir, 30, "cpu%ld", cpu);
8814	d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8815	if (!d_cpu) {
8816	pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8817	return;
8818	}
8819
8820	/* per cpu trace_pipe */
8821	trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu,
8822	tr, cpu, &tracing_pipe_fops);
8823
8824	/* per cpu trace */
8825	trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu,
8826	tr, cpu, &tracing_fops);
8827
8828	trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu,
8829	tr, cpu, &tracing_buffers_fops);
8830
8831	trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu,
8832	tr, cpu, &tracing_stats_fops);
8833
8834	trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu,
8835	tr, cpu, &tracing_entries_fops);
8836
8837	#ifdef CONFIG_TRACER_SNAPSHOT
8838	trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu,
8839	tr, cpu, &snapshot_fops);
8840
8841	trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu,
8842	tr, cpu, &snapshot_raw_fops);
8843	#endif
8844	}
8845
8846	#ifdef CONFIG_FTRACE_SELFTEST
8847	/* Let selftest have access to static functions in this file */
8848	#include "trace_selftest.c"
8849	#endif
8850
8851	static ssize_t
8852	trace_options_read(struct file filp, char __user ubuf, size_t cnt,
8853	loff_t *ppos)
8854	{
8855	struct trace_option_dentry *topt = filp->private_data;
8856	char *buf;
8857
8858	if (topt->flags->val & topt->opt->bit)
8859	buf = "1\n";
8860	else
8861	buf = "0\n";
8862
8863	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8864	}
8865
8866	static ssize_t
8867	trace_options_write(struct file filp, const char __user ubuf, size_t cnt,
8868	loff_t *ppos)
8869	{
8870	struct trace_option_dentry *topt = filp->private_data;
8871	unsigned long val;
8872	int ret;
8873
8874	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8875	if (ret)
8876	return ret;
8877
8878	if (val != 0 && val != 1)
8879	return -EINVAL;
8880
8881	if (!!(topt->flags->val & topt->opt->bit) != val) {
8882	mutex_lock(&trace_types_lock);
8883	ret = __set_tracer_option(topt->tr, topt->flags,
8884	topt->opt, !val);
8885	mutex_unlock(&trace_types_lock);
8886	if (ret)
8887	return ret;
8888	}
8889
8890	*ppos += cnt;
8891
8892	return cnt;
8893	}
8894
8895
8896	static const struct file_operations trace_options_fops = {
8897	.open = tracing_open_generic,
8898	.read = trace_options_read,
8899	.write = trace_options_write,
8900	.llseek = generic_file_llseek,
8901	};
8902
8903	/*
8904	* In order to pass in both the trace_array descriptor as well as the index
8905	* to the flag that the trace option file represents, the trace_array
8906	* has a character array of trace_flags_index[], which holds the index
8907	* of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8908	* The address of this character array is passed to the flag option file
8909	* read/write callbacks.
8910	*
8911	* In order to extract both the index and the trace_array descriptor,
8912	* get_tr_index() uses the following algorithm.
8913	*
8914	* idx = *ptr;
8915	*
8916	* As the pointer itself contains the address of the index (remember
8917	* index[1] == 1).
8918	*
8919	* Then to get the trace_array descriptor, by subtracting that index
8920	* from the ptr, we get to the start of the index itself.
8921	*
8922	* ptr - idx == &index[0]
8923	*
8924	* Then a simple container_of() from that pointer gets us to the
8925	* trace_array descriptor.
8926	*/
8927	static void get_tr_index(void data, struct trace_array *ptr,
8928	unsigned int *pindex)
8929	{
8930	pindex = (unsigned char *)data;
8931
8932	ptr = container_of(data - pindex, struct trace_array,
8933	trace_flags_index);
8934	}
8935
8936	static ssize_t
8937	trace_options_core_read(struct file filp, char __user ubuf, size_t cnt,
8938	loff_t *ppos)
8939	{
8940	void *tr_index = filp->private_data;
8941	struct trace_array *tr;
8942	unsigned int index;
8943	char *buf;
8944
8945	get_tr_index(tr_index, &tr, &index);
8946
8947	if (tr->trace_flags & (1 << index))
8948	buf = "1\n";
8949	else
8950	buf = "0\n";
8951
8952	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8953	}
8954
8955	static ssize_t
8956	trace_options_core_write(struct file filp, const char __user ubuf, size_t cnt,
8957	loff_t *ppos)
8958	{
8959	void *tr_index = filp->private_data;
8960	struct trace_array *tr;
8961	unsigned int index;
8962	unsigned long val;
8963	int ret;
8964
8965	get_tr_index(tr_index, &tr, &index);
8966
8967	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8968	if (ret)
8969	return ret;
8970
8971	if (val != 0 && val != 1)
8972	return -EINVAL;
8973
8974	mutex_lock(&event_mutex);
8975	mutex_lock(&trace_types_lock);
8976	ret = set_tracer_flag(tr, 1 << index, val);
8977	mutex_unlock(&trace_types_lock);
8978	mutex_unlock(&event_mutex);
8979
8980	if (ret < 0)
8981	return ret;
8982
8983	*ppos += cnt;
8984
8985	return cnt;
8986	}
8987
8988	static const struct file_operations trace_options_core_fops = {
8989	.open = tracing_open_generic,
8990	.read = trace_options_core_read,
8991	.write = trace_options_core_write,
8992	.llseek = generic_file_llseek,
8993	};
8994
8995	struct dentry trace_create_file(const char name,
8996	umode_t mode,
8997	struct dentry *parent,
8998	void *data,
8999	const struct file_operations *fops)
9000	{
9001	struct dentry *ret;
9002
9003	ret = tracefs_create_file(name, mode, parent, data, fops);
9004	if (!ret)
9005	pr_warn("Could not create tracefs '%s' entry\n", name);
9006
9007	return ret;
9008	}
9009
9010
9011	static struct dentry trace_options_init_dentry(struct trace_array tr)
9012	{
9013	struct dentry *d_tracer;
9014
9015	if (tr->options)
9016	return tr->options;
9017
9018	d_tracer = tracing_get_dentry(tr);
9019	if (IS_ERR(d_tracer))
9020	return NULL;
9021
9022	tr->options = tracefs_create_dir("options", d_tracer);
9023	if (!tr->options) {
9024	pr_warn("Could not create tracefs directory 'options'\n");
9025	return NULL;
9026	}
9027
9028	return tr->options;
9029	}
9030
9031	static void
9032	create_trace_option_file(struct trace_array *tr,
9033	struct trace_option_dentry *topt,
9034	struct tracer_flags *flags,
9035	struct tracer_opt *opt)
9036	{
9037	struct dentry *t_options;
9038
9039	t_options = trace_options_init_dentry(tr);
9040	if (!t_options)
9041	return;
9042
9043	topt->flags = flags;
9044	topt->opt = opt;
9045	topt->tr = tr;
9046
9047	topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,
9048	t_options, topt, &trace_options_fops);
9049
9050	}
9051
9052	static void
9053	create_trace_option_files(struct trace_array tr, struct tracer tracer)
9054	{
9055	struct trace_option_dentry *topts;
9056	struct trace_options *tr_topts;
9057	struct tracer_flags *flags;
9058	struct tracer_opt *opts;
9059	int cnt;
9060	int i;
9061
9062	if (!tracer)
9063	return;
9064
9065	flags = tracer->flags;
9066
9067	if (!flags \|\| !flags->opts)
9068	return;
9069
9070	/*
9071	* If this is an instance, only create flags for tracers
9072	* the instance may have.
9073	*/
9074	if (!trace_ok_for_array(tracer, tr))
9075	return;
9076
9077	for (i = 0; i < tr->nr_topts; i++) {
9078	/* Make sure there's no duplicate flags. */
9079	if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
9080	return;
9081	}
9082
9083	opts = flags->opts;
9084
9085	for (cnt = 0; opts[cnt].name; cnt++)
9086	;
9087
9088	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
9089	if (!topts)
9090	return;
9091
9092	tr_topts = krealloc(tr->topts, sizeof(tr->topts) (tr->nr_topts + 1),
9093	GFP_KERNEL);
9094	if (!tr_topts) {
9095	kfree(topts);
9096	return;
9097	}
9098
9099	tr->topts = tr_topts;
9100	tr->topts[tr->nr_topts].tracer = tracer;
9101	tr->topts[tr->nr_topts].topts = topts;
9102	tr->nr_topts++;
9103
9104	for (cnt = 0; opts[cnt].name; cnt++) {
9105	create_trace_option_file(tr, &topts[cnt], flags,
9106	&opts[cnt]);
9107	MEM_FAIL(topts[cnt].entry == NULL,
9108	"Failed to create trace option: %s",
9109	opts[cnt].name);
9110	}
9111	}
9112
9113	static struct dentry *
9114	create_trace_option_core_file(struct trace_array *tr,
9115	const char *option, long index)
9116	{
9117	struct dentry *t_options;
9118
9119	t_options = trace_options_init_dentry(tr);
9120	if (!t_options)
9121	return NULL;
9122
9123	return trace_create_file(option, TRACE_MODE_WRITE, t_options,
9124	(void *)&tr->trace_flags_index[index],
9125	&trace_options_core_fops);
9126	}
9127
9128	static void create_trace_options_dir(struct trace_array *tr)
9129	{
9130	struct dentry *t_options;
9131	bool top_level = tr == &global_trace;
9132	int i;
9133
9134	t_options = trace_options_init_dentry(tr);
9135	if (!t_options)
9136	return;
9137
9138	for (i = 0; trace_options[i]; i++) {
9139	if (top_level \|\|
9140	!((1 << i) & TOP_LEVEL_TRACE_FLAGS))
9141	create_trace_option_core_file(tr, trace_options[i], i);
9142	}
9143	}
9144
9145	static ssize_t
9146	rb_simple_read(struct file filp, char __user ubuf,
9147	size_t cnt, loff_t *ppos)
9148	{
9149	struct trace_array *tr = filp->private_data;
9150	char buf[64];
9151	int r;
9152
9153	r = tracer_tracing_is_on(tr);
9154	r = sprintf(buf, "%d\n", r);
9155
9156	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
9157	}
9158
9159	static ssize_t
9160	rb_simple_write(struct file filp, const char __user ubuf,
9161	size_t cnt, loff_t *ppos)
9162	{
9163	struct trace_array *tr = filp->private_data;
9164	struct trace_buffer *buffer = tr->array_buffer.buffer;
9165	unsigned long val;
9166	int ret;
9167
9168	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9169	if (ret)
9170	return ret;
9171
9172	if (buffer) {
9173	mutex_lock(&trace_types_lock);
9174	if (!!val == tracer_tracing_is_on(tr)) {
9175	val = 0; /* do nothing */
9176	} else if (val) {
9177	tracer_tracing_on(tr);
9178	if (tr->current_trace->start)
9179	tr->current_trace->start(tr);
9180	} else {
9181	tracer_tracing_off(tr);
9182	if (tr->current_trace->stop)
9183	tr->current_trace->stop(tr);
9184	/* Wake up any waiters */
9185	ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS);
9186	}
9187	mutex_unlock(&trace_types_lock);
9188	}
9189
9190	(*ppos)++;
9191
9192	return cnt;
9193	}
9194
9195	static const struct file_operations rb_simple_fops = {
9196	.open = tracing_open_generic_tr,
9197	.read = rb_simple_read,
9198	.write = rb_simple_write,
9199	.release = tracing_release_generic_tr,
9200	.llseek = default_llseek,
9201	};
9202
9203	static ssize_t
9204	buffer_percent_read(struct file filp, char __user ubuf,
9205	size_t cnt, loff_t *ppos)
9206	{
9207	struct trace_array *tr = filp->private_data;
9208	char buf[64];
9209	int r;
9210
9211	r = tr->buffer_percent;
9212	r = sprintf(buf, "%d\n", r);
9213
9214	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
9215	}
9216
9217	static ssize_t
9218	buffer_percent_write(struct file filp, const char __user ubuf,
9219	size_t cnt, loff_t *ppos)
9220	{
9221	struct trace_array *tr = filp->private_data;
9222	unsigned long val;
9223	int ret;
9224
9225	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9226	if (ret)
9227	return ret;
9228
9229	if (val > 100)
9230	return -EINVAL;
9231
9232	tr->buffer_percent = val;
9233
9234	(*ppos)++;
9235
9236	return cnt;
9237	}
9238
9239	static const struct file_operations buffer_percent_fops = {
9240	.open = tracing_open_generic_tr,
9241	.read = buffer_percent_read,
9242	.write = buffer_percent_write,
9243	.release = tracing_release_generic_tr,
9244	.llseek = default_llseek,
9245	};
9246
9247	static struct dentry *trace_instance_dir;
9248
9249	static void
9250	init_tracer_tracefs(struct trace_array tr, struct dentry d_tracer);
9251
9252	static int
9253	allocate_trace_buffer(struct trace_array tr, struct array_buffer buf, int size)
9254	{
9255	enum ring_buffer_flags rb_flags;
9256
9257	rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
9258
9259	buf->tr = tr;
9260
9261	buf->buffer = ring_buffer_alloc(size, rb_flags);
9262	if (!buf->buffer)
9263	return -ENOMEM;
9264
9265	buf->data = alloc_percpu(struct trace_array_cpu);
9266	if (!buf->data) {
9267	ring_buffer_free(buf->buffer);
9268	buf->buffer = NULL;
9269	return -ENOMEM;
9270	}
9271
9272	/* Allocate the first page for all buffers */
9273	set_buffer_entries(&tr->array_buffer,
9274	ring_buffer_size(tr->array_buffer.buffer, 0));
9275
9276	return 0;
9277	}
9278
9279	static void free_trace_buffer(struct array_buffer *buf)
9280	{
9281	if (buf->buffer) {
9282	ring_buffer_free(buf->buffer);
9283	buf->buffer = NULL;
9284	free_percpu(buf->data);
9285	buf->data = NULL;
9286	}
9287	}
9288
9289	static int allocate_trace_buffers(struct trace_array *tr, int size)
9290	{
9291	int ret;
9292
9293	ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
9294	if (ret)
9295	return ret;
9296
9297	#ifdef CONFIG_TRACER_MAX_TRACE
9298	ret = allocate_trace_buffer(tr, &tr->max_buffer,
9299	allocate_snapshot ? size : 1);
9300	if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
9301	free_trace_buffer(&tr->array_buffer);
9302	return -ENOMEM;
9303	}
9304	tr->allocated_snapshot = allocate_snapshot;
9305
9306	allocate_snapshot = false;
9307	#endif
9308
9309	return 0;
9310	}
9311
9312	static void free_trace_buffers(struct trace_array *tr)
9313	{
9314	if (!tr)
9315	return;
9316
9317	free_trace_buffer(&tr->array_buffer);
9318
9319	#ifdef CONFIG_TRACER_MAX_TRACE
9320	free_trace_buffer(&tr->max_buffer);
9321	#endif
9322	}
9323
9324	static void init_trace_flags_index(struct trace_array *tr)
9325	{
9326	int i;
9327
9328	/* Used by the trace options files */
9329	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
9330	tr->trace_flags_index[i] = i;
9331	}
9332
9333	static void __update_tracer_options(struct trace_array *tr)
9334	{
9335	struct tracer *t;
9336
9337	for (t = trace_types; t; t = t->next)
9338	add_tracer_options(tr, t);
9339	}
9340
9341	static void update_tracer_options(struct trace_array *tr)
9342	{
9343	mutex_lock(&trace_types_lock);
9344	tracer_options_updated = true;
9345	__update_tracer_options(tr);
9346	mutex_unlock(&trace_types_lock);
9347	}
9348
9349	/* Must have trace_types_lock held */
9350	struct trace_array trace_array_find(const char instance)
9351	{
9352	struct trace_array tr, found = NULL;
9353
9354	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9355	if (tr->name && strcmp(tr->name, instance) == 0) {
9356	found = tr;
9357	break;
9358	}
9359	}
9360
9361	return found;
9362	}
9363
9364	struct trace_array trace_array_find_get(const char instance)
9365	{
9366	struct trace_array *tr;
9367
9368	mutex_lock(&trace_types_lock);
9369	tr = trace_array_find(instance);
9370	if (tr)
9371	tr->ref++;
9372	mutex_unlock(&trace_types_lock);
9373
9374	return tr;
9375	}
9376
9377	static int trace_array_create_dir(struct trace_array *tr)
9378	{
9379	int ret;
9380
9381	tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
9382	if (!tr->dir)
9383	return -EINVAL;
9384
9385	ret = event_trace_add_tracer(tr->dir, tr);
9386	if (ret) {
9387	tracefs_remove(tr->dir);
9388	return ret;
9389	}
9390
9391	init_tracer_tracefs(tr, tr->dir);
9392	__update_tracer_options(tr);
9393
9394	return ret;
9395	}
9396
9397	static struct trace_array trace_array_create(const char name)
9398	{
9399	struct trace_array *tr;
9400	int ret;
9401
9402	ret = -ENOMEM;
9403	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
9404	if (!tr)
9405	return ERR_PTR(ret);
9406
9407	tr->name = kstrdup(name, GFP_KERNEL);
9408	if (!tr->name)
9409	goto out_free_tr;
9410
9411	if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
9412	goto out_free_tr;
9413
9414	tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
9415
9416	cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
9417
9418	raw_spin_lock_init(&tr->start_lock);
9419
9420	tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9421
9422	tr->current_trace = &nop_trace;
9423
9424	INIT_LIST_HEAD(&tr->systems);
9425	INIT_LIST_HEAD(&tr->events);
9426	INIT_LIST_HEAD(&tr->hist_vars);
9427	INIT_LIST_HEAD(&tr->err_log);
9428
9429	if (allocate_trace_buffers(tr, trace_buf_size) < 0)
9430	goto out_free_tr;
9431
9432	if (ftrace_allocate_ftrace_ops(tr) < 0)
9433	goto out_free_tr;
9434
9435	ftrace_init_trace_array(tr);
9436
9437	init_trace_flags_index(tr);
9438
9439	if (trace_instance_dir) {
9440	ret = trace_array_create_dir(tr);
9441	if (ret)
9442	goto out_free_tr;
9443	} else
9444	__trace_early_add_events(tr);
9445
9446	list_add(&tr->list, &ftrace_trace_arrays);
9447
9448	tr->ref++;
9449
9450	return tr;
9451
9452	out_free_tr:
9453	ftrace_free_ftrace_ops(tr);
9454	free_trace_buffers(tr);
9455	free_cpumask_var(tr->tracing_cpumask);
9456	kfree(tr->name);
9457	kfree(tr);
9458
9459	return ERR_PTR(ret);
9460	}
9461
9462	static int instance_mkdir(const char *name)
9463	{
9464	struct trace_array *tr;
9465	int ret;
9466
9467	mutex_lock(&event_mutex);
9468	mutex_lock(&trace_types_lock);
9469
9470	ret = -EEXIST;
9471	if (trace_array_find(name))
9472	goto out_unlock;
9473
9474	tr = trace_array_create(name);
9475
9476	ret = PTR_ERR_OR_ZERO(tr);
9477
9478	out_unlock:
9479	mutex_unlock(&trace_types_lock);
9480	mutex_unlock(&event_mutex);
9481	return ret;
9482	}
9483
9484	/**
9485	* trace_array_get_by_name - Create/Lookup a trace array, given its name.
9486	* @name: The name of the trace array to be looked up/created.
9487	*
9488	* Returns pointer to trace array with given name.
9489	* NULL, if it cannot be created.
9490	*
9491	* NOTE: This function increments the reference counter associated with the
9492	* trace array returned. This makes sure it cannot be freed while in use.
9493	* Use trace_array_put() once the trace array is no longer needed.
9494	* If the trace_array is to be freed, trace_array_destroy() needs to
9495	* be called after the trace_array_put(), or simply let user space delete
9496	* it from the tracefs instances directory. But until the
9497	* trace_array_put() is called, user space can not delete it.
9498	*
9499	*/
9500	struct trace_array trace_array_get_by_name(const char name)
9501	{
9502	struct trace_array *tr;
9503
9504	mutex_lock(&event_mutex);
9505	mutex_lock(&trace_types_lock);
9506
9507	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9508	if (tr->name && strcmp(tr->name, name) == 0)
9509	goto out_unlock;
9510	}
9511
9512	tr = trace_array_create(name);
9513
9514	if (IS_ERR(tr))
9515	tr = NULL;
9516	out_unlock:
9517	if (tr)
9518	tr->ref++;
9519
9520	mutex_unlock(&trace_types_lock);
9521	mutex_unlock(&event_mutex);
9522	return tr;
9523	}
9524	EXPORT_SYMBOL_GPL(trace_array_get_by_name);
9525
9526	static int __remove_instance(struct trace_array *tr)
9527	{
9528	int i;
9529
9530	/* Reference counter for a newly created trace array = 1. */
9531	if (tr->ref > 1 \|\| (tr->current_trace && tr->trace_ref))
9532	return -EBUSY;
9533
9534	list_del(&tr->list);
9535
9536	/* Disable all the flags that were enabled coming in */
9537	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
9538	if ((1 << i) & ZEROED_TRACE_FLAGS)
9539	set_tracer_flag(tr, 1 << i, 0);
9540	}
9541
9542	tracing_set_nop(tr);
9543	clear_ftrace_function_probes(tr);
9544	event_trace_del_tracer(tr);
9545	ftrace_clear_pids(tr);
9546	ftrace_destroy_function_files(tr);
9547	tracefs_remove(tr->dir);
9548	free_percpu(tr->last_func_repeats);
9549	free_trace_buffers(tr);
9550	clear_tracing_err_log(tr);
9551
9552	for (i = 0; i < tr->nr_topts; i++) {
9553	kfree(tr->topts[i].topts);
9554	}
9555	kfree(tr->topts);
9556
9557	free_cpumask_var(tr->tracing_cpumask);
9558	kfree(tr->name);
9559	kfree(tr);
9560
9561	return 0;
9562	}
9563
9564	int trace_array_destroy(struct trace_array *this_tr)
9565	{
9566	struct trace_array *tr;
9567	int ret;
9568
9569	if (!this_tr)
9570	return -EINVAL;
9571
9572	mutex_lock(&event_mutex);
9573	mutex_lock(&trace_types_lock);
9574
9575	ret = -ENODEV;
9576
9577	/* Making sure trace array exists before destroying it. */
9578	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9579	if (tr == this_tr) {
9580	ret = __remove_instance(tr);
9581	break;
9582	}
9583	}
9584
9585	mutex_unlock(&trace_types_lock);
9586	mutex_unlock(&event_mutex);
9587
9588	return ret;
9589	}
9590	EXPORT_SYMBOL_GPL(trace_array_destroy);
9591
9592	static int instance_rmdir(const char *name)
9593	{
9594	struct trace_array *tr;
9595	int ret;
9596
9597	mutex_lock(&event_mutex);
9598	mutex_lock(&trace_types_lock);
9599
9600	ret = -ENODEV;
9601	tr = trace_array_find(name);
9602	if (tr)
9603	ret = __remove_instance(tr);
9604
9605	mutex_unlock(&trace_types_lock);
9606	mutex_unlock(&event_mutex);
9607
9608	return ret;
9609	}
9610
9611	static __init void create_trace_instances(struct dentry *d_tracer)
9612	{
9613	struct trace_array *tr;
9614
9615	trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
9616	instance_mkdir,
9617	instance_rmdir);
9618	if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
9619	return;
9620
9621	mutex_lock(&event_mutex);
9622	mutex_lock(&trace_types_lock);
9623
9624	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9625	if (!tr->name)
9626	continue;
9627	if (MEM_FAIL(trace_array_create_dir(tr) < 0,
9628	"Failed to create instance directory\n"))
9629	break;
9630	}
9631
9632	mutex_unlock(&trace_types_lock);
9633	mutex_unlock(&event_mutex);
9634	}
9635
9636	static void
9637	init_tracer_tracefs(struct trace_array tr, struct dentry d_tracer)
9638	{
9639	struct trace_event_file *file;
9640	int cpu;
9641
9642	trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer,
9643	tr, &show_traces_fops);
9644
9645	trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer,
9646	tr, &set_tracer_fops);
9647
9648	trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer,
9649	tr, &tracing_cpumask_fops);
9650
9651	trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer,
9652	tr, &tracing_iter_fops);
9653
9654	trace_create_file("trace", TRACE_MODE_WRITE, d_tracer,
9655	tr, &tracing_fops);
9656
9657	trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer,
9658	tr, &tracing_pipe_fops);
9659
9660	trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer,
9661	tr, &tracing_entries_fops);
9662
9663	trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer,
9664	tr, &tracing_total_entries_fops);
9665
9666	trace_create_file("free_buffer", 0200, d_tracer,
9667	tr, &tracing_free_buffer_fops);
9668
9669	trace_create_file("trace_marker", 0220, d_tracer,
9670	tr, &tracing_mark_fops);
9671
9672	file = __find_event_file(tr, "ftrace", "print");
9673	if (file && file->dir)
9674	trace_create_file("trigger", TRACE_MODE_WRITE, file->dir,
9675	file, &event_trigger_fops);
9676	tr->trace_marker_file = file;
9677
9678	trace_create_file("trace_marker_raw", 0220, d_tracer,
9679	tr, &tracing_mark_raw_fops);
9680
9681	trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr,
9682	&trace_clock_fops);
9683
9684	trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer,
9685	tr, &rb_simple_fops);
9686
9687	trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr,
9688	&trace_time_stamp_mode_fops);
9689
9690	tr->buffer_percent = 50;
9691
9692	trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer,
9693	tr, &buffer_percent_fops);
9694
9695	create_trace_options_dir(tr);
9696
9697	#ifdef CONFIG_TRACER_MAX_TRACE
9698	trace_create_maxlat_file(tr, d_tracer);
9699	#endif
9700
9701	if (ftrace_create_function_files(tr, d_tracer))
9702	MEM_FAIL(1, "Could not allocate function filter files");
9703
9704	#ifdef CONFIG_TRACER_SNAPSHOT
9705	trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer,
9706	tr, &snapshot_fops);
9707	#endif
9708
9709	trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer,
9710	tr, &tracing_err_log_fops);
9711
9712	for_each_tracing_cpu(cpu)
9713	tracing_init_tracefs_percpu(tr, cpu);
9714
9715	ftrace_init_tracefs(tr, d_tracer);
9716	}
9717
9718	static struct vfsmount trace_automount(struct dentry mntpt, void *ingore)
9719	{
9720	struct vfsmount *mnt;
9721	struct file_system_type *type;
9722
9723	/*
9724	* To maintain backward compatibility for tools that mount
9725	* debugfs to get to the tracing facility, tracefs is automatically
9726	* mounted to the debugfs/tracing directory.
9727	*/
9728	type = get_fs_type("tracefs");
9729	if (!type)
9730	return NULL;
9731	mnt = vfs_submount(mntpt, type, "tracefs", NULL);
9732	put_filesystem(type);
9733	if (IS_ERR(mnt))
9734	return NULL;
9735	mntget(mnt);
9736
9737	return mnt;
9738	}
9739
9740	/**
9741	* tracing_init_dentry - initialize top level trace array
9742	*
9743	* This is called when creating files or directories in the tracing
9744	* directory. It is called via fs_initcall() by any of the boot up code
9745	* and expects to return the dentry of the top level tracing directory.
9746	*/
9747	int tracing_init_dentry(void)
9748	{
9749	struct trace_array *tr = &global_trace;
9750
9751	if (security_locked_down(LOCKDOWN_TRACEFS)) {
9752	pr_warn("Tracing disabled due to lockdown\n");
9753	return -EPERM;
9754	}
9755
9756	/* The top level trace array uses NULL as parent */
9757	if (tr->dir)
9758	return 0;
9759
9760	if (WARN_ON(!tracefs_initialized()))
9761	return -ENODEV;
9762
9763	/*
9764	* As there may still be users that expect the tracing
9765	* files to exist in debugfs/tracing, we must automount
9766	* the tracefs file system there, so older tools still
9767	* work with the newer kernel.
9768	*/
9769	tr->dir = debugfs_create_automount("tracing", NULL,
9770	trace_automount, NULL);
9771
9772	return 0;
9773	}
9774
9775	extern struct trace_eval_map *__start_ftrace_eval_maps[];
9776	extern struct trace_eval_map *__stop_ftrace_eval_maps[];
9777
9778	static struct workqueue_struct *eval_map_wq __initdata;
9779	static struct work_struct eval_map_work __initdata;
9780	static struct work_struct tracerfs_init_work __initdata;
9781
9782	static void __init eval_map_work_func(struct work_struct *work)
9783	{
9784	int len;
9785
9786	len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
9787	trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
9788	}
9789
9790	static int __init trace_eval_init(void)
9791	{
9792	INIT_WORK(&eval_map_work, eval_map_work_func);
9793
9794	eval_map_wq = alloc_workqueue("eval_map_wq", WQ_UNBOUND, 0);
9795	if (!eval_map_wq) {
9796	pr_err("Unable to allocate eval_map_wq\n");
9797	/* Do work here */
9798	eval_map_work_func(&eval_map_work);
9799	return -ENOMEM;
9800	}
9801
9802	queue_work(eval_map_wq, &eval_map_work);
9803	return 0;
9804	}
9805
9806	subsys_initcall(trace_eval_init);
9807
9808	static int __init trace_eval_sync(void)
9809	{
9810	/* Make sure the eval map updates are finished */
9811	if (eval_map_wq)
9812	destroy_workqueue(eval_map_wq);
9813	return 0;
9814	}
9815
9816	late_initcall_sync(trace_eval_sync);
9817
9818
9819	#ifdef CONFIG_MODULES
9820	static void trace_module_add_evals(struct module *mod)
9821	{
9822	if (!mod->num_trace_evals)
9823	return;
9824
9825	/*
9826	* Modules with bad taint do not have events created, do
9827	* not bother with enums either.
9828	*/
9829	if (trace_module_has_bad_taint(mod))
9830	return;
9831
9832	trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
9833	}
9834
9835	#ifdef CONFIG_TRACE_EVAL_MAP_FILE
9836	static void trace_module_remove_evals(struct module *mod)
9837	{
9838	union trace_eval_map_item *map;
9839	union trace_eval_map_item **last = &trace_eval_maps;
9840
9841	if (!mod->num_trace_evals)
9842	return;
9843
9844	mutex_lock(&trace_eval_mutex);
9845
9846	map = trace_eval_maps;
9847
9848	while (map) {
9849	if (map->head.mod == mod)
9850	break;
9851	map = trace_eval_jmp_to_tail(map);
9852	last = &map->tail.next;
9853	map = map->tail.next;
9854	}
9855	if (!map)
9856	goto out;
9857
9858	*last = trace_eval_jmp_to_tail(map)->tail.next;
9859	kfree(map);
9860	out:
9861	mutex_unlock(&trace_eval_mutex);
9862	}
9863	#else
9864	static inline void trace_module_remove_evals(struct module *mod) { }
9865	#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9866
9867	static int trace_module_notify(struct notifier_block *self,
9868	unsigned long val, void *data)
9869	{
9870	struct module *mod = data;
9871
9872	switch (val) {
9873	case MODULE_STATE_COMING:
9874	trace_module_add_evals(mod);
9875	break;
9876	case MODULE_STATE_GOING:
9877	trace_module_remove_evals(mod);
9878	break;
9879	}
9880
9881	return NOTIFY_OK;
9882	}
9883
9884	static struct notifier_block trace_module_nb = {
9885	.notifier_call = trace_module_notify,
9886	.priority = 0,
9887	};
9888	#endif /* CONFIG_MODULES */
9889
9890	static __init void tracer_init_tracefs_work_func(struct work_struct *work)
9891	{
9892
9893	event_trace_init();
9894
9895	init_tracer_tracefs(&global_trace, NULL);
9896	ftrace_init_tracefs_toplevel(&global_trace, NULL);
9897
9898	trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL,
9899	&global_trace, &tracing_thresh_fops);
9900
9901	trace_create_file("README", TRACE_MODE_READ, NULL,
9902	NULL, &tracing_readme_fops);
9903
9904	trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL,
9905	NULL, &tracing_saved_cmdlines_fops);
9906
9907	trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
9908	NULL, &tracing_saved_cmdlines_size_fops);
9909
9910	trace_create_file("saved_tgids", TRACE_MODE_READ, NULL,
9911	NULL, &tracing_saved_tgids_fops);
9912
9913	trace_create_eval_file(NULL);
9914
9915	#ifdef CONFIG_MODULES
9916	register_module_notifier(&trace_module_nb);
9917	#endif
9918
9919	#ifdef CONFIG_DYNAMIC_FTRACE
9920	trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
9921	NULL, &tracing_dyn_info_fops);
9922	#endif
9923
9924	create_trace_instances(NULL);
9925
9926	update_tracer_options(&global_trace);
9927	}
9928
9929	static __init int tracer_init_tracefs(void)
9930	{
9931	int ret;
9932
9933	trace_access_lock_init();
9934
9935	ret = tracing_init_dentry();
9936	if (ret)
9937	return 0;
9938
9939	if (eval_map_wq) {
9940	INIT_WORK(&tracerfs_init_work, tracer_init_tracefs_work_func);
9941	queue_work(eval_map_wq, &tracerfs_init_work);
9942	} else {
9943	tracer_init_tracefs_work_func(NULL);
9944	}
9945
9946	rv_init_interface();
9947
9948	return 0;
9949	}
9950
9951	fs_initcall(tracer_init_tracefs);
9952
9953	static int trace_die_panic_handler(struct notifier_block *self,
9954	unsigned long ev, void *unused);
9955
9956	static struct notifier_block trace_panic_notifier = {
9957	.notifier_call = trace_die_panic_handler,
9958	.priority = INT_MAX - 1,
9959	};
9960
9961	static struct notifier_block trace_die_notifier = {
9962	.notifier_call = trace_die_panic_handler,
9963	.priority = INT_MAX - 1,
9964	};
9965
9966	/*
9967	* The idea is to execute the following die/panic callback early, in order
9968	* to avoid showing irrelevant information in the trace (like other panic
9969	* notifier functions); we are the 2nd to run, after hung_task/rcu_stall
9970	* warnings get disabled (to prevent potential log flooding).
9971	*/
9972	static int trace_die_panic_handler(struct notifier_block *self,
9973	unsigned long ev, void *unused)
9974	{
9975	if (!ftrace_dump_on_oops)
9976	return NOTIFY_DONE;
9977
9978	/* The die notifier requires DIE_OOPS to trigger */
9979	if (self == &trace_die_notifier && ev != DIE_OOPS)
9980	return NOTIFY_DONE;
9981
9982	ftrace_dump(ftrace_dump_on_oops);
9983
9984	return NOTIFY_DONE;
9985	}
9986
9987	/*
9988	* printk is set to max of 1024, we really don't need it that big.
9989	* Nothing should be printing 1000 characters anyway.
9990	*/
9991	#define TRACE_MAX_PRINT 1000
9992
9993	/*
9994	* Define here KERN_TRACE so that we have one place to modify
9995	* it if we decide to change what log level the ftrace dump
9996	* should be at.
9997	*/
9998	#define KERN_TRACE KERN_EMERG
9999
10000	void
10001	trace_printk_seq(struct trace_seq *s)
10002	{
10003	/* Probably should print a warning here. */
10004	if (s->seq.len >= TRACE_MAX_PRINT)
10005	s->seq.len = TRACE_MAX_PRINT;
10006
10007	/*
10008	* More paranoid code. Although the buffer size is set to
10009	* PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
10010	* an extra layer of protection.
10011	*/
10012	if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
10013	s->seq.len = s->seq.size - 1;
10014
10015	/* should be zero ended, but we are paranoid. */
10016	s->buffer[s->seq.len] = 0;
10017
10018	printk(KERN_TRACE "%s", s->buffer);
10019
10020	trace_seq_init(s);
10021	}
10022
10023	void trace_init_global_iter(struct trace_iterator *iter)
10024	{
10025	iter->tr = &global_trace;
10026	iter->trace = iter->tr->current_trace;
10027	iter->cpu_file = RING_BUFFER_ALL_CPUS;
10028	iter->array_buffer = &global_trace.array_buffer;
10029
10030	if (iter->trace && iter->trace->open)
10031	iter->trace->open(iter);
10032
10033	/* Annotate start of buffers if we had overruns */
10034	if (ring_buffer_overruns(iter->array_buffer->buffer))
10035	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
10036
10037	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
10038	if (trace_clocks[iter->tr->clock_id].in_ns)
10039	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
10040
10041	/* Can not use kmalloc for iter.temp and iter.fmt */
10042	iter->temp = static_temp_buf;
10043	iter->temp_size = STATIC_TEMP_BUF_SIZE;
10044	iter->fmt = static_fmt_buf;
10045	iter->fmt_size = STATIC_FMT_BUF_SIZE;
10046	}
10047
10048	void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
10049	{
10050	/* use static because iter can be a bit big for the stack */
10051	static struct trace_iterator iter;
10052	static atomic_t dump_running;
10053	struct trace_array *tr = &global_trace;
10054	unsigned int old_userobj;
10055	unsigned long flags;
10056	int cnt = 0, cpu;
10057
10058	/* Only allow one dump user at a time. */
10059	if (atomic_inc_return(&dump_running) != 1) {
10060	atomic_dec(&dump_running);
10061	return;
10062	}
10063
10064	/*
10065	* Always turn off tracing when we dump.
10066	* We don't need to show trace output of what happens
10067	* between multiple crashes.
10068	*
10069	* If the user does a sysrq-z, then they can re-enable
10070	* tracing with echo 1 > tracing_on.
10071	*/
10072	tracing_off();
10073
10074	local_irq_save(flags);
10075
10076	/* Simulate the iterator */
10077	trace_init_global_iter(&iter);
10078
10079	for_each_tracing_cpu(cpu) {
10080	atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
10081	}
10082
10083	old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
10084
10085	/* don't look at user memory in panic mode */
10086	tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
10087
10088	switch (oops_dump_mode) {
10089	case DUMP_ALL:
10090	iter.cpu_file = RING_BUFFER_ALL_CPUS;
10091	break;
10092	case DUMP_ORIG:
10093	iter.cpu_file = raw_smp_processor_id();
10094	break;
10095	case DUMP_NONE:
10096	goto out_enable;
10097	default:
10098	printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
10099	iter.cpu_file = RING_BUFFER_ALL_CPUS;
10100	}
10101
10102	printk(KERN_TRACE "Dumping ftrace buffer:\n");
10103
10104	/* Did function tracer already get disabled? */
10105	if (ftrace_is_dead()) {
10106	printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
10107	printk("# MAY BE MISSING FUNCTION EVENTS\n");
10108	}
10109
10110	/*
10111	* We need to stop all tracing on all CPUS to read
10112	* the next buffer. This is a bit expensive, but is
10113	* not done often. We fill all what we can read,
10114	* and then release the locks again.
10115	*/
10116
10117	while (!trace_empty(&iter)) {
10118
10119	if (!cnt)
10120	printk(KERN_TRACE "---------------------------------\n");
10121
10122	cnt++;
10123
10124	trace_iterator_reset(&iter);
10125	iter.iter_flags \|= TRACE_FILE_LAT_FMT;
10126
10127	if (trace_find_next_entry_inc(&iter) != NULL) {
10128	int ret;
10129
10130	ret = print_trace_line(&iter);
10131	if (ret != TRACE_TYPE_NO_CONSUME)
10132	trace_consume(&iter);
10133	}
10134	touch_nmi_watchdog();
10135
10136	trace_printk_seq(&iter.seq);
10137	}
10138
10139	if (!cnt)
10140	printk(KERN_TRACE " (ftrace buffer empty)\n");
10141	else
10142	printk(KERN_TRACE "---------------------------------\n");
10143
10144	out_enable:
10145	tr->trace_flags \|= old_userobj;
10146
10147	for_each_tracing_cpu(cpu) {
10148	atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
10149	}
10150	atomic_dec(&dump_running);
10151	local_irq_restore(flags);
10152	}
10153	EXPORT_SYMBOL_GPL(ftrace_dump);
10154
10155	#define WRITE_BUFSIZE 4096
10156
10157	ssize_t trace_parse_run_command(struct file file, const char __user buffer,
10158	size_t count, loff_t *ppos,
10159	int (createfn)(const char ))
10160	{
10161	char kbuf, buf, *tmp;
10162	int ret = 0;
10163	size_t done = 0;
10164	size_t size;
10165
10166	kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
10167	if (!kbuf)
10168	return -ENOMEM;
10169
10170	while (done < count) {
10171	size = count - done;
10172
10173	if (size >= WRITE_BUFSIZE)
10174	size = WRITE_BUFSIZE - 1;
10175
10176	if (copy_from_user(kbuf, buffer + done, size)) {
10177	ret = -EFAULT;
10178	goto out;
10179	}
10180	kbuf[size] = '\0';
10181	buf = kbuf;
10182	do {
10183	tmp = strchr(buf, '\n');
10184	if (tmp) {
10185	*tmp = '\0';
10186	size = tmp - buf + 1;
10187	} else {
10188	size = strlen(buf);
10189	if (done + size < count) {
10190	if (buf != kbuf)
10191	break;
10192	/* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
10193	pr_warn("Line length is too long: Should be less than %d\n",
10194	WRITE_BUFSIZE - 2);
10195	ret = -EINVAL;
10196	goto out;
10197	}
10198	}
10199	done += size;
10200
10201	/* Remove comments */
10202	tmp = strchr(buf, '#');
10203
10204	if (tmp)
10205	*tmp = '\0';
10206
10207	ret = createfn(buf);
10208	if (ret)
10209	goto out;
10210	buf += size;
10211
10212	} while (done < count);
10213	}
10214	ret = done;
10215
10216	out:
10217	kfree(kbuf);
10218
10219	return ret;
10220	}
10221
10222	#ifdef CONFIG_TRACER_MAX_TRACE
10223	__init static bool tr_needs_alloc_snapshot(const char *name)
10224	{
10225	char *test;
10226	int len = strlen(name);
10227	bool ret;
10228
10229	if (!boot_snapshot_index)
10230	return false;
10231
10232	if (strncmp(name, boot_snapshot_info, len) == 0 &&
10233	boot_snapshot_info[len] == '\t')
10234	return true;
10235
10236	test = kmalloc(strlen(name) + 3, GFP_KERNEL);
10237	if (!test)
10238	return false;
10239
10240	sprintf(test, "\t%s\t", name);
10241	ret = strstr(boot_snapshot_info, test) == NULL;
10242	kfree(test);
10243	return ret;
10244	}
10245
10246	__init static void do_allocate_snapshot(const char *name)
10247	{
10248	if (!tr_needs_alloc_snapshot(name))
10249	return;
10250
10251	/*
10252	* When allocate_snapshot is set, the next call to
10253	* allocate_trace_buffers() (called by trace_array_get_by_name())
10254	* will allocate the snapshot buffer. That will alse clear
10255	* this flag.
10256	*/
10257	allocate_snapshot = true;
10258	}
10259	#else
10260	static inline void do_allocate_snapshot(const char *name) { }
10261	#endif
10262
10263	__init static void enable_instances(void)
10264	{
10265	struct trace_array *tr;
10266	char *curr_str;
10267	char *str;
10268	char *tok;
10269
10270	/* A tab is always appended */
10271	boot_instance_info[boot_instance_index - 1] = '\0';
10272	str = boot_instance_info;
10273
10274	while ((curr_str = strsep(&str, "\t"))) {
10275
10276	tok = strsep(&curr_str, ",");
10277
10278	if (IS_ENABLED(CONFIG_TRACER_MAX_TRACE))
10279	do_allocate_snapshot(tok);
10280
10281	tr = trace_array_get_by_name(tok);
10282	if (!tr) {
10283	pr_warn("Failed to create instance buffer %s\n", curr_str);
10284	continue;
10285	}
10286	/* Allow user space to delete it */
10287	trace_array_put(tr);
10288
10289	while ((tok = strsep(&curr_str, ","))) {
10290	early_enable_events(tr, tok, true);
10291	}
10292	}
10293	}
10294
10295	__init static int tracer_alloc_buffers(void)
10296	{
10297	int ring_buf_size;
10298	int ret = -ENOMEM;
10299
10300
10301	if (security_locked_down(LOCKDOWN_TRACEFS)) {
10302	pr_warn("Tracing disabled due to lockdown\n");
10303	return -EPERM;
10304	}
10305
10306	/*
10307	* Make sure we don't accidentally add more trace options
10308	* than we have bits for.
10309	*/
10310	BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
10311
10312	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
10313	goto out;
10314
10315	if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
10316	goto out_free_buffer_mask;
10317
10318	/* Only allocate trace_printk buffers if a trace_printk exists */
10319	if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
10320	/* Must be called before global_trace.buffer is allocated */
10321	trace_printk_init_buffers();
10322
10323	/* To save memory, keep the ring buffer size to its minimum */
10324	if (ring_buffer_expanded)
10325	ring_buf_size = trace_buf_size;
10326	else
10327	ring_buf_size = 1;
10328
10329	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
10330	cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
10331
10332	raw_spin_lock_init(&global_trace.start_lock);
10333
10334	/*
10335	* The prepare callbacks allocates some memory for the ring buffer. We
10336	* don't free the buffer if the CPU goes down. If we were to free
10337	* the buffer, then the user would lose any trace that was in the
10338	* buffer. The memory will be removed once the "instance" is removed.
10339	*/
10340	ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
10341	"trace/RB:prepare", trace_rb_cpu_prepare,
10342	NULL);
10343	if (ret < 0)
10344	goto out_free_cpumask;
10345	/* Used for event triggers */
10346	ret = -ENOMEM;
10347	temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
10348	if (!temp_buffer)
10349	goto out_rm_hp_state;
10350
10351	if (trace_create_savedcmd() < 0)
10352	goto out_free_temp_buffer;
10353
10354	/* TODO: make the number of buffers hot pluggable with CPUS */
10355	if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
10356	MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
10357	goto out_free_savedcmd;
10358	}
10359
10360	if (global_trace.buffer_disabled)
10361	tracing_off();
10362
10363	if (trace_boot_clock) {
10364	ret = tracing_set_clock(&global_trace, trace_boot_clock);
10365	if (ret < 0)
10366	pr_warn("Trace clock %s not defined, going back to default\n",
10367	trace_boot_clock);
10368	}
10369
10370	/*
10371	* register_tracer() might reference current_trace, so it
10372	* needs to be set before we register anything. This is
10373	* just a bootstrap of current_trace anyway.
10374	*/
10375	global_trace.current_trace = &nop_trace;
10376
10377	global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
10378
10379	ftrace_init_global_array_ops(&global_trace);
10380
10381	init_trace_flags_index(&global_trace);
10382
10383	register_tracer(&nop_trace);
10384
10385	/* Function tracing may start here (via kernel command line) */
10386	init_function_trace();
10387
10388	/* All seems OK, enable tracing */
10389	tracing_disabled = 0;
10390
10391	atomic_notifier_chain_register(&panic_notifier_list,
10392	&trace_panic_notifier);
10393
10394	register_die_notifier(&trace_die_notifier);
10395
10396	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
10397
10398	INIT_LIST_HEAD(&global_trace.systems);
10399	INIT_LIST_HEAD(&global_trace.events);
10400	INIT_LIST_HEAD(&global_trace.hist_vars);
10401	INIT_LIST_HEAD(&global_trace.err_log);
10402	list_add(&global_trace.list, &ftrace_trace_arrays);
10403
10404	apply_trace_boot_options();
10405
10406	register_snapshot_cmd();
10407
10408	test_can_verify();
10409
10410	return 0;
10411
10412	out_free_savedcmd:
10413	free_saved_cmdlines_buffer(savedcmd);
10414	out_free_temp_buffer:
10415	ring_buffer_free(temp_buffer);
10416	out_rm_hp_state:
10417	cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
10418	out_free_cpumask:
10419	free_cpumask_var(global_trace.tracing_cpumask);
10420	out_free_buffer_mask:
10421	free_cpumask_var(tracing_buffer_mask);
10422	out:
10423	return ret;
10424	}
10425
10426	void __init ftrace_boot_snapshot(void)
10427	{
10428	#ifdef CONFIG_TRACER_MAX_TRACE
10429	struct trace_array *tr;
10430
10431	if (!snapshot_at_boot)
10432	return;
10433
10434	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
10435	if (!tr->allocated_snapshot)
10436	continue;
10437
10438	tracing_snapshot_instance(tr);
10439	trace_array_puts(tr, " Boot snapshot taken \n");
10440	}
10441	#endif
10442	}
10443
10444	void __init early_trace_init(void)
10445	{
10446	if (tracepoint_printk) {
10447	tracepoint_print_iter =
10448	kzalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
10449	if (MEM_FAIL(!tracepoint_print_iter,
10450	"Failed to allocate trace iterator\n"))
10451	tracepoint_printk = 0;
10452	else
10453	static_key_enable(&tracepoint_printk_key.key);
10454	}
10455	tracer_alloc_buffers();
10456
10457	init_events();
10458	}
10459
10460	void __init trace_init(void)
10461	{
10462	trace_event_init();
10463
10464	if (boot_instance_index)
10465	enable_instances();
10466	}
10467
10468	__init static void clear_boot_tracer(void)
10469	{
10470	/*
10471	* The default tracer at boot buffer is an init section.
10472	* This function is called in lateinit. If we did not
10473	* find the boot tracer, then clear it out, to prevent
10474	* later registration from accessing the buffer that is
10475	* about to be freed.
10476	*/
10477	if (!default_bootup_tracer)
10478	return;
10479
10480	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
10481	default_bootup_tracer);
10482	default_bootup_tracer = NULL;
10483	}
10484
10485	#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
10486	__init static void tracing_set_default_clock(void)
10487	{
10488	/* sched_clock_stable() is determined in late_initcall */
10489	if (!trace_boot_clock && !sched_clock_stable()) {
10490	if (security_locked_down(LOCKDOWN_TRACEFS)) {
10491	pr_warn("Can not set tracing clock due to lockdown\n");
10492	return;
10493	}
10494
10495	printk(KERN_WARNING
10496	"Unstable clock detected, switching default tracing clock to \"global\"\n"
10497	"If you want to keep using the local clock, then add:\n"
10498	" \"trace_clock=local\"\n"
10499	"on the kernel command line\n");
10500	tracing_set_clock(&global_trace, "global");
10501	}
10502	}
10503	#else
10504	static inline void tracing_set_default_clock(void) { }
10505	#endif
10506
10507	__init static int late_trace_init(void)
10508	{
10509	if (tracepoint_printk && tracepoint_printk_stop_on_boot) {
10510	static_key_disable(&tracepoint_printk_key.key);
10511	tracepoint_printk = 0;
10512	}
10513
10514	tracing_set_default_clock();
10515	clear_boot_tracer();
10516	return 0;
10517	}
10518
10519	late_initcall_sync(late_trace_init);