linux - kernel/trace/trace.c

10757 lines · 264.0 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* ring buffer based function tracer
4	*
5	* Copyright (C) 2007-2012 Steven Rostedt <[email protected]>
6	* Copyright (C) 2008 Ingo Molnar <[email protected]>
7	*
8	* Originally taken from the RT patch by:
9	* Arnaldo Carvalho de Melo <[email protected]>
10	*
11	* Based on code from the latency_tracer, that is:
12	* Copyright (C) 2004-2006 Ingo Molnar
13	* Copyright (C) 2004 Nadia Yvette Chambers
14	*/
15	#include <linux/ring_buffer.h>
16	#include <linux/utsname.h>
17	#include <linux/stacktrace.h>
18	#include <linux/writeback.h>
19	#include <linux/kallsyms.h>
20	#include <linux/security.h>
21	#include <linux/seq_file.h>
22	#include <linux/irqflags.h>
23	#include <linux/debugfs.h>
24	#include <linux/tracefs.h>
25	#include <linux/pagemap.h>
26	#include <linux/hardirq.h>
27	#include <linux/linkage.h>
28	#include <linux/uaccess.h>
29	#include <linux/vmalloc.h>
30	#include <linux/ftrace.h>
31	#include <linux/module.h>
32	#include <linux/percpu.h>
33	#include <linux/splice.h>
34	#include <linux/kdebug.h>
35	#include <linux/string.h>
36	#include <linux/mount.h>
37	#include <linux/rwsem.h>
38	#include <linux/slab.h>
39	#include <linux/ctype.h>
40	#include <linux/init.h>
41	#include <linux/panic_notifier.h>
42	#include <linux/poll.h>
43	#include <linux/nmi.h>
44	#include <linux/fs.h>
45	#include <linux/trace.h>
46	#include <linux/sched/clock.h>
47	#include <linux/sched/rt.h>
48	#include <linux/fsnotify.h>
49	#include <linux/irq_work.h>
50	#include <linux/workqueue.h>
51
52	#include <asm/setup.h> /* COMMAND_LINE_SIZE */
53
54	#include "trace.h"
55	#include "trace_output.h"
56
57	#ifdef CONFIG_FTRACE_STARTUP_TEST
58	/*
59	* We need to change this state when a selftest is running.
60	* A selftest will lurk into the ring-buffer to count the
61	* entries inserted during the selftest although some concurrent
62	* insertions into the ring-buffer such as trace_printk could occurred
63	* at the same time, giving false positive or negative results.
64	*/
65	static bool __read_mostly tracing_selftest_running;
66
67	/*
68	* If boot-time tracing including tracers/events via kernel cmdline
69	* is running, we do not want to run SELFTEST.
70	*/
71	bool __read_mostly tracing_selftest_disabled;
72
73	void __init disable_tracing_selftest(const char *reason)
74	{
75	if (!tracing_selftest_disabled) {
76	tracing_selftest_disabled = true;
77	pr_info("Ftrace startup test is disabled due to %s\n", reason);
78	}
79	}
80	#else
81	#define tracing_selftest_running 0
82	#define tracing_selftest_disabled 0
83	#endif
84
85	/* Pipe tracepoints to printk */
86	static struct trace_iterator *tracepoint_print_iter;
87	int tracepoint_printk;
88	static bool tracepoint_printk_stop_on_boot __initdata;
89	static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
90
91	/* For tracers that don't implement custom flags */
92	static struct tracer_opt dummy_tracer_opt[] = {
93	{ }
94	};
95
96	static int
97	dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
98	{
99	return 0;
100	}
101
102	/*
103	* To prevent the comm cache from being overwritten when no
104	* tracing is active, only save the comm when a trace event
105	* occurred.
106	*/
107	DEFINE_PER_CPU(bool, trace_taskinfo_save);
108
109	/*
110	* Kill all tracing for good (never come back).
111	* It is initialized to 1 but will turn to zero if the initialization
112	* of the tracer is successful. But that is the only place that sets
113	* this back to zero.
114	*/
115	static int tracing_disabled = 1;
116
117	cpumask_var_t __read_mostly tracing_buffer_mask;
118
119	/*
120	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
121	*
122	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
123	* is set, then ftrace_dump is called. This will output the contents
124	* of the ftrace buffers to the console. This is very useful for
125	* capturing traces that lead to crashes and outputing it to a
126	* serial console.
127	*
128	* It is default off, but you can enable it with either specifying
129	* "ftrace_dump_on_oops" in the kernel command line, or setting
130	* /proc/sys/kernel/ftrace_dump_on_oops
131	* Set 1 if you want to dump buffers of all CPUs
132	* Set 2 if you want to dump the buffer of the CPU that triggered oops
133	* Set instance name if you want to dump the specific trace instance
134	* Multiple instance dump is also supported, and instances are seperated
135	* by commas.
136	*/
137	/* Set to string format zero to disable by default */
138	char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0";
139
140	/* When set, tracing will stop when a WARN() is hit /
141	int __disable_trace_on_warning;
142
143	#ifdef CONFIG_TRACE_EVAL_MAP_FILE
144	/* Map of enums to their values, for "eval_map" file */
145	struct trace_eval_map_head {
146	struct module *mod;
147	unsigned long length;
148	};
149
150	union trace_eval_map_item;
151
152	struct trace_eval_map_tail {
153	/*
154	* "end" is first and points to NULL as it must be different
155	* than "mod" or "eval_string"
156	*/
157	union trace_eval_map_item *next;
158	const char end; / points to NULL */
159	};
160
161	static DEFINE_MUTEX(trace_eval_mutex);
162
163	/*
164	* The trace_eval_maps are saved in an array with two extra elements,
165	* one at the beginning, and one at the end. The beginning item contains
166	* the count of the saved maps (head.length), and the module they
167	* belong to if not built in (head.mod). The ending item contains a
168	* pointer to the next array of saved eval_map items.
169	*/
170	union trace_eval_map_item {
171	struct trace_eval_map map;
172	struct trace_eval_map_head head;
173	struct trace_eval_map_tail tail;
174	};
175
176	static union trace_eval_map_item *trace_eval_maps;
177	#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
178
179	int tracing_set_tracer(struct trace_array tr, const char buf);
180	static void ftrace_trace_userstack(struct trace_array *tr,
181	struct trace_buffer *buffer,
182	unsigned int trace_ctx);
183
184	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
185	static char *default_bootup_tracer;
186
187	static bool allocate_snapshot;
188	static bool snapshot_at_boot;
189
190	static char boot_instance_info[COMMAND_LINE_SIZE] __initdata;
191	static int boot_instance_index;
192
193	static char boot_snapshot_info[COMMAND_LINE_SIZE] __initdata;
194	static int boot_snapshot_index;
195
196	static int __init set_cmdline_ftrace(char *str)
197	{
198	strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
199	default_bootup_tracer = bootup_tracer_buf;
200	/* We are using ftrace early, expand it */
201	trace_set_ring_buffer_expanded(NULL);
202	return 1;
203	}
204	__setup("ftrace=", set_cmdline_ftrace);
205
206	int ftrace_dump_on_oops_enabled(void)
207	{
208	if (!strcmp("0", ftrace_dump_on_oops))
209	return 0;
210	else
211	return 1;
212	}
213
214	static int __init set_ftrace_dump_on_oops(char *str)
215	{
216	if (!*str) {
217	strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
218	return 1;
219	}
220
221	if (*str == ',') {
222	strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
223	strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1);
224	return 1;
225	}
226
227	if (*str++ == '=') {
228	strscpy(ftrace_dump_on_oops, str, MAX_TRACER_SIZE);
229	return 1;
230	}
231
232	return 0;
233	}
234	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
235
236	static int __init stop_trace_on_warning(char *str)
237	{
238	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
239	__disable_trace_on_warning = 1;
240	return 1;
241	}
242	__setup("traceoff_on_warning", stop_trace_on_warning);
243
244	static int __init boot_alloc_snapshot(char *str)
245	{
246	char *slot = boot_snapshot_info + boot_snapshot_index;
247	int left = sizeof(boot_snapshot_info) - boot_snapshot_index;
248	int ret;
249
250	if (str[0] == '=') {
251	str++;
252	if (strlen(str) >= left)
253	return -1;
254
255	ret = snprintf(slot, left, "%s\t", str);
256	boot_snapshot_index += ret;
257	} else {
258	allocate_snapshot = true;
259	/* We also need the main ring buffer expanded */
260	trace_set_ring_buffer_expanded(NULL);
261	}
262	return 1;
263	}
264	__setup("alloc_snapshot", boot_alloc_snapshot);
265
266
267	static int __init boot_snapshot(char *str)
268	{
269	snapshot_at_boot = true;
270	boot_alloc_snapshot(str);
271	return 1;
272	}
273	__setup("ftrace_boot_snapshot", boot_snapshot);
274
275
276	static int __init boot_instance(char *str)
277	{
278	char *slot = boot_instance_info + boot_instance_index;
279	int left = sizeof(boot_instance_info) - boot_instance_index;
280	int ret;
281
282	if (strlen(str) >= left)
283	return -1;
284
285	ret = snprintf(slot, left, "%s\t", str);
286	boot_instance_index += ret;
287
288	return 1;
289	}
290	__setup("trace_instance=", boot_instance);
291
292
293	static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
294
295	static int __init set_trace_boot_options(char *str)
296	{
297	strscpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
298	return 1;
299	}
300	__setup("trace_options=", set_trace_boot_options);
301
302	static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
303	static char *trace_boot_clock __initdata;
304
305	static int __init set_trace_boot_clock(char *str)
306	{
307	strscpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
308	trace_boot_clock = trace_boot_clock_buf;
309	return 1;
310	}
311	__setup("trace_clock=", set_trace_boot_clock);
312
313	static int __init set_tracepoint_printk(char *str)
314	{
315	/* Ignore the "tp_printk_stop_on_boot" param */
316	if (*str == '_')
317	return 0;
318
319	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
320	tracepoint_printk = 1;
321	return 1;
322	}
323	__setup("tp_printk", set_tracepoint_printk);
324
325	static int __init set_tracepoint_printk_stop(char *str)
326	{
327	tracepoint_printk_stop_on_boot = true;
328	return 1;
329	}
330	__setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);
331
332	unsigned long long ns2usecs(u64 nsec)
333	{
334	nsec += 500;
335	do_div(nsec, 1000);
336	return nsec;
337	}
338
339	static void
340	trace_process_export(struct trace_export *export,
341	struct ring_buffer_event *event, int flag)
342	{
343	struct trace_entry *entry;
344	unsigned int size = 0;
345
346	if (export->flags & flag) {
347	entry = ring_buffer_event_data(event);
348	size = ring_buffer_event_length(event);
349	export->write(export, entry, size);
350	}
351	}
352
353	static DEFINE_MUTEX(ftrace_export_lock);
354
355	static struct trace_export __rcu *ftrace_exports_list __read_mostly;
356
357	static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
358	static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
359	static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
360
361	static inline void ftrace_exports_enable(struct trace_export *export)
362	{
363	if (export->flags & TRACE_EXPORT_FUNCTION)
364	static_branch_inc(&trace_function_exports_enabled);
365
366	if (export->flags & TRACE_EXPORT_EVENT)
367	static_branch_inc(&trace_event_exports_enabled);
368
369	if (export->flags & TRACE_EXPORT_MARKER)
370	static_branch_inc(&trace_marker_exports_enabled);
371	}
372
373	static inline void ftrace_exports_disable(struct trace_export *export)
374	{
375	if (export->flags & TRACE_EXPORT_FUNCTION)
376	static_branch_dec(&trace_function_exports_enabled);
377
378	if (export->flags & TRACE_EXPORT_EVENT)
379	static_branch_dec(&trace_event_exports_enabled);
380
381	if (export->flags & TRACE_EXPORT_MARKER)
382	static_branch_dec(&trace_marker_exports_enabled);
383	}
384
385	static void ftrace_exports(struct ring_buffer_event *event, int flag)
386	{
387	struct trace_export *export;
388
389	preempt_disable_notrace();
390
391	export = rcu_dereference_raw_check(ftrace_exports_list);
392	while (export) {
393	trace_process_export(export, event, flag);
394	export = rcu_dereference_raw_check(export->next);
395	}
396
397	preempt_enable_notrace();
398	}
399
400	static inline void
401	add_trace_export(struct trace_export *list, struct trace_export export)
402	{
403	rcu_assign_pointer(export->next, *list);
404	/*
405	* We are entering export into the list but another
406	* CPU might be walking that list. We need to make sure
407	* the export->next pointer is valid before another CPU sees
408	* the export pointer included into the list.
409	*/
410	rcu_assign_pointer(*list, export);
411	}
412
413	static inline int
414	rm_trace_export(struct trace_export *list, struct trace_export export)
415	{
416	struct trace_export **p;
417
418	for (p = list; p != NULL; p = &(p)->next)
419	if (*p == export)
420	break;
421
422	if (*p != export)
423	return -1;
424
425	rcu_assign_pointer(p, (p)->next);
426
427	return 0;
428	}
429
430	static inline void
431	add_ftrace_export(struct trace_export *list, struct trace_export export)
432	{
433	ftrace_exports_enable(export);
434
435	add_trace_export(list, export);
436	}
437
438	static inline int
439	rm_ftrace_export(struct trace_export *list, struct trace_export export)
440	{
441	int ret;
442
443	ret = rm_trace_export(list, export);
444	ftrace_exports_disable(export);
445
446	return ret;
447	}
448
449	int register_ftrace_export(struct trace_export *export)
450	{
451	if (WARN_ON_ONCE(!export->write))
452	return -1;
453
454	mutex_lock(&ftrace_export_lock);
455
456	add_ftrace_export(&ftrace_exports_list, export);
457
458	mutex_unlock(&ftrace_export_lock);
459
460	return 0;
461	}
462	EXPORT_SYMBOL_GPL(register_ftrace_export);
463
464	int unregister_ftrace_export(struct trace_export *export)
465	{
466	int ret;
467
468	mutex_lock(&ftrace_export_lock);
469
470	ret = rm_ftrace_export(&ftrace_exports_list, export);
471
472	mutex_unlock(&ftrace_export_lock);
473
474	return ret;
475	}
476	EXPORT_SYMBOL_GPL(unregister_ftrace_export);
477
478	/* trace_flags holds trace_options default values */
479	#define TRACE_DEFAULT_FLAGS \
480	(FUNCTION_DEFAULT_FLAGS \| \
481	TRACE_ITER_PRINT_PARENT \| TRACE_ITER_PRINTK \| \
482	TRACE_ITER_ANNOTATE \| TRACE_ITER_CONTEXT_INFO \| \
483	TRACE_ITER_RECORD_CMD \| TRACE_ITER_OVERWRITE \| \
484	TRACE_ITER_IRQ_INFO \| TRACE_ITER_MARKERS \| \
485	TRACE_ITER_HASH_PTR \| TRACE_ITER_TRACE_PRINTK)
486
487	/* trace_options that are only supported by global_trace */
488	#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK \| \
489	TRACE_ITER_PRINTK_MSGONLY \| TRACE_ITER_RECORD_CMD)
490
491	/* trace_flags that are default zero for instances */
492	#define ZEROED_TRACE_FLAGS \
493	(TRACE_ITER_EVENT_FORK \| TRACE_ITER_FUNC_FORK \| TRACE_ITER_TRACE_PRINTK)
494
495	/*
496	* The global_trace is the descriptor that holds the top-level tracing
497	* buffers for the live tracing.
498	*/
499	static struct trace_array global_trace = {
500	.trace_flags = TRACE_DEFAULT_FLAGS,
501	};
502
503	static struct trace_array *printk_trace = &global_trace;
504
505	static __always_inline bool printk_binsafe(struct trace_array *tr)
506	{
507	/*
508	* The binary format of traceprintk can cause a crash if used
509	* by a buffer from another boot. Force the use of the
510	* non binary version of trace_printk if the trace_printk
511	* buffer is a boot mapped ring buffer.
512	*/
513	return !(tr->flags & TRACE_ARRAY_FL_BOOT);
514	}
515
516	static void update_printk_trace(struct trace_array *tr)
517	{
518	if (printk_trace == tr)
519	return;
520
521	printk_trace->trace_flags &= ~TRACE_ITER_TRACE_PRINTK;
522	printk_trace = tr;
523	tr->trace_flags \|= TRACE_ITER_TRACE_PRINTK;
524	}
525
526	void trace_set_ring_buffer_expanded(struct trace_array *tr)
527	{
528	if (!tr)
529	tr = &global_trace;
530	tr->ring_buffer_expanded = true;
531	}
532
533	LIST_HEAD(ftrace_trace_arrays);
534
535	int trace_array_get(struct trace_array *this_tr)
536	{
537	struct trace_array *tr;
538	int ret = -ENODEV;
539
540	mutex_lock(&trace_types_lock);
541	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
542	if (tr == this_tr) {
543	tr->ref++;
544	ret = 0;
545	break;
546	}
547	}
548	mutex_unlock(&trace_types_lock);
549
550	return ret;
551	}
552
553	static void __trace_array_put(struct trace_array *this_tr)
554	{
555	WARN_ON(!this_tr->ref);
556	this_tr->ref--;
557	}
558
559	/**
560	* trace_array_put - Decrement the reference counter for this trace array.
561	* @this_tr : pointer to the trace array
562	*
563	* NOTE: Use this when we no longer need the trace array returned by
564	* trace_array_get_by_name(). This ensures the trace array can be later
565	* destroyed.
566	*
567	*/
568	void trace_array_put(struct trace_array *this_tr)
569	{
570	if (!this_tr)
571	return;
572
573	mutex_lock(&trace_types_lock);
574	__trace_array_put(this_tr);
575	mutex_unlock(&trace_types_lock);
576	}
577	EXPORT_SYMBOL_GPL(trace_array_put);
578
579	int tracing_check_open_get_tr(struct trace_array *tr)
580	{
581	int ret;
582
583	ret = security_locked_down(LOCKDOWN_TRACEFS);
584	if (ret)
585	return ret;
586
587	if (tracing_disabled)
588	return -ENODEV;
589
590	if (tr && trace_array_get(tr) < 0)
591	return -ENODEV;
592
593	return 0;
594	}
595
596	/**
597	* trace_find_filtered_pid - check if a pid exists in a filtered_pid list
598	* @filtered_pids: The list of pids to check
599	* @search_pid: The PID to find in @filtered_pids
600	*
601	* Returns true if @search_pid is found in @filtered_pids, and false otherwise.
602	*/
603	bool
604	trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
605	{
606	return trace_pid_list_is_set(filtered_pids, search_pid);
607	}
608
609	/**
610	* trace_ignore_this_task - should a task be ignored for tracing
611	* @filtered_pids: The list of pids to check
612	* @filtered_no_pids: The list of pids not to be traced
613	* @task: The task that should be ignored if not filtered
614	*
615	* Checks if @task should be traced or not from @filtered_pids.
616	* Returns true if @task should NOT be traced.
617	* Returns false if @task should be traced.
618	*/
619	bool
620	trace_ignore_this_task(struct trace_pid_list *filtered_pids,
621	struct trace_pid_list *filtered_no_pids,
622	struct task_struct *task)
623	{
624	/*
625	* If filtered_no_pids is not empty, and the task's pid is listed
626	* in filtered_no_pids, then return true.
627	* Otherwise, if filtered_pids is empty, that means we can
628	* trace all tasks. If it has content, then only trace pids
629	* within filtered_pids.
630	*/
631
632	return (filtered_pids &&
633	!trace_find_filtered_pid(filtered_pids, task->pid)) \|\|
634	(filtered_no_pids &&
635	trace_find_filtered_pid(filtered_no_pids, task->pid));
636	}
637
638	/**
639	* trace_filter_add_remove_task - Add or remove a task from a pid_list
640	* @pid_list: The list to modify
641	* @self: The current task for fork or NULL for exit
642	* @task: The task to add or remove
643	*
644	* If adding a task, if @self is defined, the task is only added if @self
645	* is also included in @pid_list. This happens on fork and tasks should
646	* only be added when the parent is listed. If @self is NULL, then the
647	* @task pid will be removed from the list, which would happen on exit
648	* of a task.
649	*/
650	void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
651	struct task_struct *self,
652	struct task_struct *task)
653	{
654	if (!pid_list)
655	return;
656
657	/* For forks, we only add if the forking task is listed */
658	if (self) {
659	if (!trace_find_filtered_pid(pid_list, self->pid))
660	return;
661	}
662
663	/* "self" is set for forks, and NULL for exits */
664	if (self)
665	trace_pid_list_set(pid_list, task->pid);
666	else
667	trace_pid_list_clear(pid_list, task->pid);
668	}
669
670	/**
671	* trace_pid_next - Used for seq_file to get to the next pid of a pid_list
672	* @pid_list: The pid list to show
673	* @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
674	* @pos: The position of the file
675	*
676	* This is used by the seq_file "next" operation to iterate the pids
677	* listed in a trace_pid_list structure.
678	*
679	* Returns the pid+1 as we want to display pid of zero, but NULL would
680	* stop the iteration.
681	*/
682	void trace_pid_next(struct trace_pid_list pid_list, void v, loff_t pos)
683	{
684	long pid = (unsigned long)v;
685	unsigned int next;
686
687	(*pos)++;
688
689	/* pid already is +1 of the actual previous bit */
690	if (trace_pid_list_next(pid_list, pid, &next) < 0)
691	return NULL;
692
693	pid = next;
694
695	/* Return pid + 1 to allow zero to be represented */
696	return (void *)(pid + 1);
697	}
698
699	/**
700	* trace_pid_start - Used for seq_file to start reading pid lists
701	* @pid_list: The pid list to show
702	* @pos: The position of the file
703	*
704	* This is used by seq_file "start" operation to start the iteration
705	* of listing pids.
706	*
707	* Returns the pid+1 as we want to display pid of zero, but NULL would
708	* stop the iteration.
709	*/
710	void trace_pid_start(struct trace_pid_list pid_list, loff_t *pos)
711	{
712	unsigned long pid;
713	unsigned int first;
714	loff_t l = 0;
715
716	if (trace_pid_list_first(pid_list, &first) < 0)
717	return NULL;
718
719	pid = first;
720
721	/* Return pid + 1 so that zero can be the exit value */
722	for (pid++; pid && l < *pos;
723	pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
724	;
725	return (void *)pid;
726	}
727
728	/**
729	* trace_pid_show - show the current pid in seq_file processing
730	* @m: The seq_file structure to write into
731	* @v: A void pointer of the pid (+1) value to display
732	*
733	* Can be directly used by seq_file operations to display the current
734	* pid value.
735	*/
736	int trace_pid_show(struct seq_file m, void v)
737	{
738	unsigned long pid = (unsigned long)v - 1;
739
740	seq_printf(m, "%lu\n", pid);
741	return 0;
742	}
743
744	/* 128 should be much more than enough */
745	#define PID_BUF_SIZE 127
746
747	int trace_pid_write(struct trace_pid_list *filtered_pids,
748	struct trace_pid_list **new_pid_list,
749	const char __user *ubuf, size_t cnt)
750	{
751	struct trace_pid_list *pid_list;
752	struct trace_parser parser;
753	unsigned long val;
754	int nr_pids = 0;
755	ssize_t read = 0;
756	ssize_t ret;
757	loff_t pos;
758	pid_t pid;
759
760	if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
761	return -ENOMEM;
762
763	/*
764	* Always recreate a new array. The write is an all or nothing
765	* operation. Always create a new array when adding new pids by
766	* the user. If the operation fails, then the current list is
767	* not modified.
768	*/
769	pid_list = trace_pid_list_alloc();
770	if (!pid_list) {
771	trace_parser_put(&parser);
772	return -ENOMEM;
773	}
774
775	if (filtered_pids) {
776	/* copy the current bits to the new max */
777	ret = trace_pid_list_first(filtered_pids, &pid);
778	while (!ret) {
779	trace_pid_list_set(pid_list, pid);
780	ret = trace_pid_list_next(filtered_pids, pid + 1, &pid);
781	nr_pids++;
782	}
783	}
784
785	ret = 0;
786	while (cnt > 0) {
787
788	pos = 0;
789
790	ret = trace_get_user(&parser, ubuf, cnt, &pos);
791	if (ret < 0)
792	break;
793
794	read += ret;
795	ubuf += ret;
796	cnt -= ret;
797
798	if (!trace_parser_loaded(&parser))
799	break;
800
801	ret = -EINVAL;
802	if (kstrtoul(parser.buffer, 0, &val))
803	break;
804
805	pid = (pid_t)val;
806
807	if (trace_pid_list_set(pid_list, pid) < 0) {
808	ret = -1;
809	break;
810	}
811	nr_pids++;
812
813	trace_parser_clear(&parser);
814	ret = 0;
815	}
816	trace_parser_put(&parser);
817
818	if (ret < 0) {
819	trace_pid_list_free(pid_list);
820	return ret;
821	}
822
823	if (!nr_pids) {
824	/* Cleared the list of pids */
825	trace_pid_list_free(pid_list);
826	pid_list = NULL;
827	}
828
829	*new_pid_list = pid_list;
830
831	return read;
832	}
833
834	static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
835	{
836	u64 ts;
837
838	/* Early boot up does not have a buffer yet */
839	if (!buf->buffer)
840	return trace_clock_local();
841
842	ts = ring_buffer_time_stamp(buf->buffer);
843	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
844
845	return ts;
846	}
847
848	u64 ftrace_now(int cpu)
849	{
850	return buffer_ftrace_now(&global_trace.array_buffer, cpu);
851	}
852
853	/**
854	* tracing_is_enabled - Show if global_trace has been enabled
855	*
856	* Shows if the global trace has been enabled or not. It uses the
857	* mirror flag "buffer_disabled" to be used in fast paths such as for
858	* the irqsoff tracer. But it may be inaccurate due to races. If you
859	* need to know the accurate state, use tracing_is_on() which is a little
860	* slower, but accurate.
861	*/
862	int tracing_is_enabled(void)
863	{
864	/*
865	* For quick access (irqsoff uses this in fast path), just
866	* return the mirror variable of the state of the ring buffer.
867	* It's a little racy, but we don't really care.
868	*/
869	smp_rmb();
870	return !global_trace.buffer_disabled;
871	}
872
873	/*
874	* trace_buf_size is the size in bytes that is allocated
875	* for a buffer. Note, the number of bytes is always rounded
876	* to page size.
877	*
878	* This number is purposely set to a low number of 16384.
879	* If the dump on oops happens, it will be much appreciated
880	* to not have to wait for all that output. Anyway this can be
881	* boot time and run time configurable.
882	*/
883	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
884
885	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
886
887	/* trace_types holds a link list of available tracers. */
888	static struct tracer *trace_types __read_mostly;
889
890	/*
891	* trace_types_lock is used to protect the trace_types list.
892	*/
893	DEFINE_MUTEX(trace_types_lock);
894
895	/*
896	* serialize the access of the ring buffer
897	*
898	* ring buffer serializes readers, but it is low level protection.
899	* The validity of the events (which returns by ring_buffer_peek() ..etc)
900	* are not protected by ring buffer.
901	*
902	* The content of events may become garbage if we allow other process consumes
903	* these events concurrently:
904	* A) the page of the consumed events may become a normal page
905	* (not reader page) in ring buffer, and this page will be rewritten
906	* by events producer.
907	* B) The page of the consumed events may become a page for splice_read,
908	* and this page will be returned to system.
909	*
910	* These primitives allow multi process access to different cpu ring buffer
911	* concurrently.
912	*
913	* These primitives don't distinguish read-only and read-consume access.
914	* Multi read-only access are also serialized.
915	*/
916
917	#ifdef CONFIG_SMP
918	static DECLARE_RWSEM(all_cpu_access_lock);
919	static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
920
921	static inline void trace_access_lock(int cpu)
922	{
923	if (cpu == RING_BUFFER_ALL_CPUS) {
924	/* gain it for accessing the whole ring buffer. */
925	down_write(&all_cpu_access_lock);
926	} else {
927	/* gain it for accessing a cpu ring buffer. */
928
929	/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
930	down_read(&all_cpu_access_lock);
931
932	/* Secondly block other access to this @cpu ring buffer. */
933	mutex_lock(&per_cpu(cpu_access_lock, cpu));
934	}
935	}
936
937	static inline void trace_access_unlock(int cpu)
938	{
939	if (cpu == RING_BUFFER_ALL_CPUS) {
940	up_write(&all_cpu_access_lock);
941	} else {
942	mutex_unlock(&per_cpu(cpu_access_lock, cpu));
943	up_read(&all_cpu_access_lock);
944	}
945	}
946
947	static inline void trace_access_lock_init(void)
948	{
949	int cpu;
950
951	for_each_possible_cpu(cpu)
952	mutex_init(&per_cpu(cpu_access_lock, cpu));
953	}
954
955	#else
956
957	static DEFINE_MUTEX(access_lock);
958
959	static inline void trace_access_lock(int cpu)
960	{
961	(void)cpu;
962	mutex_lock(&access_lock);
963	}
964
965	static inline void trace_access_unlock(int cpu)
966	{
967	(void)cpu;
968	mutex_unlock(&access_lock);
969	}
970
971	static inline void trace_access_lock_init(void)
972	{
973	}
974
975	#endif
976
977	#ifdef CONFIG_STACKTRACE
978	static void __ftrace_trace_stack(struct trace_array *tr,
979	struct trace_buffer *buffer,
980	unsigned int trace_ctx,
981	int skip, struct pt_regs *regs);
982	static inline void ftrace_trace_stack(struct trace_array *tr,
983	struct trace_buffer *buffer,
984	unsigned int trace_ctx,
985	int skip, struct pt_regs *regs);
986
987	#else
988	static inline void __ftrace_trace_stack(struct trace_array *tr,
989	struct trace_buffer *buffer,
990	unsigned int trace_ctx,
991	int skip, struct pt_regs *regs)
992	{
993	}
994	static inline void ftrace_trace_stack(struct trace_array *tr,
995	struct trace_buffer *buffer,
996	unsigned long trace_ctx,
997	int skip, struct pt_regs *regs)
998	{
999	}
1000
1001	#endif
1002
1003	static __always_inline void
1004	trace_event_setup(struct ring_buffer_event *event,
1005	int type, unsigned int trace_ctx)
1006	{
1007	struct trace_entry *ent = ring_buffer_event_data(event);
1008
1009	tracing_generic_entry_update(ent, type, trace_ctx);
1010	}
1011
1012	static __always_inline struct ring_buffer_event *
1013	__trace_buffer_lock_reserve(struct trace_buffer *buffer,
1014	int type,
1015	unsigned long len,
1016	unsigned int trace_ctx)
1017	{
1018	struct ring_buffer_event *event;
1019
1020	event = ring_buffer_lock_reserve(buffer, len);
1021	if (event != NULL)
1022	trace_event_setup(event, type, trace_ctx);
1023
1024	return event;
1025	}
1026
1027	void tracer_tracing_on(struct trace_array *tr)
1028	{
1029	if (tr->array_buffer.buffer)
1030	ring_buffer_record_on(tr->array_buffer.buffer);
1031	/*
1032	* This flag is looked at when buffers haven't been allocated
1033	* yet, or by some tracers (like irqsoff), that just want to
1034	* know if the ring buffer has been disabled, but it can handle
1035	* races of where it gets disabled but we still do a record.
1036	* As the check is in the fast path of the tracers, it is more
1037	* important to be fast than accurate.
1038	*/
1039	tr->buffer_disabled = 0;
1040	/* Make the flag seen by readers */
1041	smp_wmb();
1042	}
1043
1044	/**
1045	* tracing_on - enable tracing buffers
1046	*
1047	* This function enables tracing buffers that may have been
1048	* disabled with tracing_off.
1049	*/
1050	void tracing_on(void)
1051	{
1052	tracer_tracing_on(&global_trace);
1053	}
1054	EXPORT_SYMBOL_GPL(tracing_on);
1055
1056
1057	static __always_inline void
1058	__buffer_unlock_commit(struct trace_buffer buffer, struct ring_buffer_event event)
1059	{
1060	__this_cpu_write(trace_taskinfo_save, true);
1061
1062	/* If this is the temp buffer, we need to commit fully */
1063	if (this_cpu_read(trace_buffered_event) == event) {
1064	/* Length is in event->array[0] */
1065	ring_buffer_write(buffer, event->array[0], &event->array[1]);
1066	/* Release the temp buffer */
1067	this_cpu_dec(trace_buffered_event_cnt);
1068	/* ring_buffer_unlock_commit() enables preemption */
1069	preempt_enable_notrace();
1070	} else
1071	ring_buffer_unlock_commit(buffer);
1072	}
1073
1074	int __trace_array_puts(struct trace_array *tr, unsigned long ip,
1075	const char *str, int size)
1076	{
1077	struct ring_buffer_event *event;
1078	struct trace_buffer *buffer;
1079	struct print_entry *entry;
1080	unsigned int trace_ctx;
1081	int alloc;
1082
1083	if (!(tr->trace_flags & TRACE_ITER_PRINTK))
1084	return 0;
1085
1086	if (unlikely(tracing_selftest_running && tr == &global_trace))
1087	return 0;
1088
1089	if (unlikely(tracing_disabled))
1090	return 0;
1091
1092	alloc = sizeof(entry) + size + 2; / possible \n added */
1093
1094	trace_ctx = tracing_gen_ctx();
1095	buffer = tr->array_buffer.buffer;
1096	ring_buffer_nest_start(buffer);
1097	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
1098	trace_ctx);
1099	if (!event) {
1100	size = 0;
1101	goto out;
1102	}
1103
1104	entry = ring_buffer_event_data(event);
1105	entry->ip = ip;
1106
1107	memcpy(&entry->buf, str, size);
1108
1109	/* Add a newline if necessary */
1110	if (entry->buf[size - 1] != '\n') {
1111	entry->buf[size] = '\n';
1112	entry->buf[size + 1] = '\0';
1113	} else
1114	entry->buf[size] = '\0';
1115
1116	__buffer_unlock_commit(buffer, event);
1117	ftrace_trace_stack(tr, buffer, trace_ctx, 4, NULL);
1118	out:
1119	ring_buffer_nest_end(buffer);
1120	return size;
1121	}
1122	EXPORT_SYMBOL_GPL(__trace_array_puts);
1123
1124	/**
1125	* __trace_puts - write a constant string into the trace buffer.
1126	* @ip: The address of the caller
1127	* @str: The constant string to write
1128	* @size: The size of the string.
1129	*/
1130	int __trace_puts(unsigned long ip, const char *str, int size)
1131	{
1132	return __trace_array_puts(printk_trace, ip, str, size);
1133	}
1134	EXPORT_SYMBOL_GPL(__trace_puts);
1135
1136	/**
1137	* __trace_bputs - write the pointer to a constant string into trace buffer
1138	* @ip: The address of the caller
1139	* @str: The constant string to write to the buffer to
1140	*/
1141	int __trace_bputs(unsigned long ip, const char *str)
1142	{
1143	struct trace_array *tr = READ_ONCE(printk_trace);
1144	struct ring_buffer_event *event;
1145	struct trace_buffer *buffer;
1146	struct bputs_entry *entry;
1147	unsigned int trace_ctx;
1148	int size = sizeof(struct bputs_entry);
1149	int ret = 0;
1150
1151	if (!printk_binsafe(tr))
1152	return __trace_puts(ip, str, strlen(str));
1153
1154	if (!(tr->trace_flags & TRACE_ITER_PRINTK))
1155	return 0;
1156
1157	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
1158	return 0;
1159
1160	trace_ctx = tracing_gen_ctx();
1161	buffer = tr->array_buffer.buffer;
1162
1163	ring_buffer_nest_start(buffer);
1164	event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
1165	trace_ctx);
1166	if (!event)
1167	goto out;
1168
1169	entry = ring_buffer_event_data(event);
1170	entry->ip = ip;
1171	entry->str = str;
1172
1173	__buffer_unlock_commit(buffer, event);
1174	ftrace_trace_stack(tr, buffer, trace_ctx, 4, NULL);
1175
1176	ret = 1;
1177	out:
1178	ring_buffer_nest_end(buffer);
1179	return ret;
1180	}
1181	EXPORT_SYMBOL_GPL(__trace_bputs);
1182
1183	#ifdef CONFIG_TRACER_SNAPSHOT
1184	static void tracing_snapshot_instance_cond(struct trace_array *tr,
1185	void *cond_data)
1186	{
1187	struct tracer *tracer = tr->current_trace;
1188	unsigned long flags;
1189
1190	if (in_nmi()) {
1191	trace_array_puts(tr, "* SNAPSHOT CALLED FROM NMI CONTEXT *\n");
1192	trace_array_puts(tr, "* snapshot is being ignored *\n");
1193	return;
1194	}
1195
1196	if (!tr->allocated_snapshot) {
1197	trace_array_puts(tr, "* SNAPSHOT NOT ALLOCATED *\n");
1198	trace_array_puts(tr, "* stopping trace here! *\n");
1199	tracer_tracing_off(tr);
1200	return;
1201	}
1202
1203	/* Note, snapshot can not be used when the tracer uses it */
1204	if (tracer->use_max_tr) {
1205	trace_array_puts(tr, "* LATENCY TRACER ACTIVE *\n");
1206	trace_array_puts(tr, "* Can not use snapshot (sorry) *\n");
1207	return;
1208	}
1209
1210	if (tr->mapped) {
1211	trace_array_puts(tr, "* BUFFER MEMORY MAPPED *\n");
1212	trace_array_puts(tr, "* Can not use snapshot (sorry) *\n");
1213	return;
1214	}
1215
1216	local_irq_save(flags);
1217	update_max_tr(tr, current, smp_processor_id(), cond_data);
1218	local_irq_restore(flags);
1219	}
1220
1221	void tracing_snapshot_instance(struct trace_array *tr)
1222	{
1223	tracing_snapshot_instance_cond(tr, NULL);
1224	}
1225
1226	/**
1227	* tracing_snapshot - take a snapshot of the current buffer.
1228	*
1229	* This causes a swap between the snapshot buffer and the current live
1230	* tracing buffer. You can use this to take snapshots of the live
1231	* trace when some condition is triggered, but continue to trace.
1232	*
1233	* Note, make sure to allocate the snapshot with either
1234	* a tracing_snapshot_alloc(), or by doing it manually
1235	* with: echo 1 > /sys/kernel/tracing/snapshot
1236	*
1237	* If the snapshot buffer is not allocated, it will stop tracing.
1238	* Basically making a permanent snapshot.
1239	*/
1240	void tracing_snapshot(void)
1241	{
1242	struct trace_array *tr = &global_trace;
1243
1244	tracing_snapshot_instance(tr);
1245	}
1246	EXPORT_SYMBOL_GPL(tracing_snapshot);
1247
1248	/**
1249	* tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1250	* @tr: The tracing instance to snapshot
1251	* @cond_data: The data to be tested conditionally, and possibly saved
1252	*
1253	* This is the same as tracing_snapshot() except that the snapshot is
1254	* conditional - the snapshot will only happen if the
1255	* cond_snapshot.update() implementation receiving the cond_data
1256	* returns true, which means that the trace array's cond_snapshot
1257	* update() operation used the cond_data to determine whether the
1258	* snapshot should be taken, and if it was, presumably saved it along
1259	* with the snapshot.
1260	*/
1261	void tracing_snapshot_cond(struct trace_array tr, void cond_data)
1262	{
1263	tracing_snapshot_instance_cond(tr, cond_data);
1264	}
1265	EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1266
1267	/**
1268	* tracing_cond_snapshot_data - get the user data associated with a snapshot
1269	* @tr: The tracing instance
1270	*
1271	* When the user enables a conditional snapshot using
1272	* tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1273	* with the snapshot. This accessor is used to retrieve it.
1274	*
1275	* Should not be called from cond_snapshot.update(), since it takes
1276	* the tr->max_lock lock, which the code calling
1277	* cond_snapshot.update() has already done.
1278	*
1279	* Returns the cond_data associated with the trace array's snapshot.
1280	*/
1281	void tracing_cond_snapshot_data(struct trace_array tr)
1282	{
1283	void *cond_data = NULL;
1284
1285	local_irq_disable();
1286	arch_spin_lock(&tr->max_lock);
1287
1288	if (tr->cond_snapshot)
1289	cond_data = tr->cond_snapshot->cond_data;
1290
1291	arch_spin_unlock(&tr->max_lock);
1292	local_irq_enable();
1293
1294	return cond_data;
1295	}
1296	EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1297
1298	static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1299	struct array_buffer *size_buf, int cpu_id);
1300	static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1301
1302	int tracing_alloc_snapshot_instance(struct trace_array *tr)
1303	{
1304	int order;
1305	int ret;
1306
1307	if (!tr->allocated_snapshot) {
1308
1309	/* Make the snapshot buffer have the same order as main buffer */
1310	order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
1311	ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order);
1312	if (ret < 0)
1313	return ret;
1314
1315	/* allocate spare buffer */
1316	ret = resize_buffer_duplicate_size(&tr->max_buffer,
1317	&tr->array_buffer, RING_BUFFER_ALL_CPUS);
1318	if (ret < 0)
1319	return ret;
1320
1321	tr->allocated_snapshot = true;
1322	}
1323
1324	return 0;
1325	}
1326
1327	static void free_snapshot(struct trace_array *tr)
1328	{
1329	/*
1330	* We don't free the ring buffer. instead, resize it because
1331	* The max_tr ring buffer has some state (e.g. ring->clock) and
1332	* we want preserve it.
1333	*/
1334	ring_buffer_subbuf_order_set(tr->max_buffer.buffer, 0);
1335	ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1336	set_buffer_entries(&tr->max_buffer, 1);
1337	tracing_reset_online_cpus(&tr->max_buffer);
1338	tr->allocated_snapshot = false;
1339	}
1340
1341	static int tracing_arm_snapshot_locked(struct trace_array *tr)
1342	{
1343	int ret;
1344
1345	lockdep_assert_held(&trace_types_lock);
1346
1347	spin_lock(&tr->snapshot_trigger_lock);
1348	if (tr->snapshot == UINT_MAX \|\| tr->mapped) {
1349	spin_unlock(&tr->snapshot_trigger_lock);
1350	return -EBUSY;
1351	}
1352
1353	tr->snapshot++;
1354	spin_unlock(&tr->snapshot_trigger_lock);
1355
1356	ret = tracing_alloc_snapshot_instance(tr);
1357	if (ret) {
1358	spin_lock(&tr->snapshot_trigger_lock);
1359	tr->snapshot--;
1360	spin_unlock(&tr->snapshot_trigger_lock);
1361	}
1362
1363	return ret;
1364	}
1365
1366	int tracing_arm_snapshot(struct trace_array *tr)
1367	{
1368	int ret;
1369
1370	mutex_lock(&trace_types_lock);
1371	ret = tracing_arm_snapshot_locked(tr);
1372	mutex_unlock(&trace_types_lock);
1373
1374	return ret;
1375	}
1376
1377	void tracing_disarm_snapshot(struct trace_array *tr)
1378	{
1379	spin_lock(&tr->snapshot_trigger_lock);
1380	if (!WARN_ON(!tr->snapshot))
1381	tr->snapshot--;
1382	spin_unlock(&tr->snapshot_trigger_lock);
1383	}
1384
1385	/**
1386	* tracing_alloc_snapshot - allocate snapshot buffer.
1387	*
1388	* This only allocates the snapshot buffer if it isn't already
1389	* allocated - it doesn't also take a snapshot.
1390	*
1391	* This is meant to be used in cases where the snapshot buffer needs
1392	* to be set up for events that can't sleep but need to be able to
1393	* trigger a snapshot.
1394	*/
1395	int tracing_alloc_snapshot(void)
1396	{
1397	struct trace_array *tr = &global_trace;
1398	int ret;
1399
1400	ret = tracing_alloc_snapshot_instance(tr);
1401	WARN_ON(ret < 0);
1402
1403	return ret;
1404	}
1405	EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1406
1407	/**
1408	* tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1409	*
1410	* This is similar to tracing_snapshot(), but it will allocate the
1411	* snapshot buffer if it isn't already allocated. Use this only
1412	* where it is safe to sleep, as the allocation may sleep.
1413	*
1414	* This causes a swap between the snapshot buffer and the current live
1415	* tracing buffer. You can use this to take snapshots of the live
1416	* trace when some condition is triggered, but continue to trace.
1417	*/
1418	void tracing_snapshot_alloc(void)
1419	{
1420	int ret;
1421
1422	ret = tracing_alloc_snapshot();
1423	if (ret < 0)
1424	return;
1425
1426	tracing_snapshot();
1427	}
1428	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1429
1430	/**
1431	* tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1432	* @tr: The tracing instance
1433	* @cond_data: User data to associate with the snapshot
1434	* @update: Implementation of the cond_snapshot update function
1435	*
1436	* Check whether the conditional snapshot for the given instance has
1437	* already been enabled, or if the current tracer is already using a
1438	* snapshot; if so, return -EBUSY, else create a cond_snapshot and
1439	* save the cond_data and update function inside.
1440	*
1441	* Returns 0 if successful, error otherwise.
1442	*/
1443	int tracing_snapshot_cond_enable(struct trace_array tr, void cond_data,
1444	cond_update_fn_t update)
1445	{
1446	struct cond_snapshot *cond_snapshot;
1447	int ret = 0;
1448
1449	cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1450	if (!cond_snapshot)
1451	return -ENOMEM;
1452
1453	cond_snapshot->cond_data = cond_data;
1454	cond_snapshot->update = update;
1455
1456	mutex_lock(&trace_types_lock);
1457
1458	if (tr->current_trace->use_max_tr) {
1459	ret = -EBUSY;
1460	goto fail_unlock;
1461	}
1462
1463	/*
1464	* The cond_snapshot can only change to NULL without the
1465	* trace_types_lock. We don't care if we race with it going
1466	* to NULL, but we want to make sure that it's not set to
1467	* something other than NULL when we get here, which we can
1468	* do safely with only holding the trace_types_lock and not
1469	* having to take the max_lock.
1470	*/
1471	if (tr->cond_snapshot) {
1472	ret = -EBUSY;
1473	goto fail_unlock;
1474	}
1475
1476	ret = tracing_arm_snapshot_locked(tr);
1477	if (ret)
1478	goto fail_unlock;
1479
1480	local_irq_disable();
1481	arch_spin_lock(&tr->max_lock);
1482	tr->cond_snapshot = cond_snapshot;
1483	arch_spin_unlock(&tr->max_lock);
1484	local_irq_enable();
1485
1486	mutex_unlock(&trace_types_lock);
1487
1488	return ret;
1489
1490	fail_unlock:
1491	mutex_unlock(&trace_types_lock);
1492	kfree(cond_snapshot);
1493	return ret;
1494	}
1495	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1496
1497	/**
1498	* tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1499	* @tr: The tracing instance
1500	*
1501	* Check whether the conditional snapshot for the given instance is
1502	* enabled; if so, free the cond_snapshot associated with it,
1503	* otherwise return -EINVAL.
1504	*
1505	* Returns 0 if successful, error otherwise.
1506	*/
1507	int tracing_snapshot_cond_disable(struct trace_array *tr)
1508	{
1509	int ret = 0;
1510
1511	local_irq_disable();
1512	arch_spin_lock(&tr->max_lock);
1513
1514	if (!tr->cond_snapshot)
1515	ret = -EINVAL;
1516	else {
1517	kfree(tr->cond_snapshot);
1518	tr->cond_snapshot = NULL;
1519	}
1520
1521	arch_spin_unlock(&tr->max_lock);
1522	local_irq_enable();
1523
1524	tracing_disarm_snapshot(tr);
1525
1526	return ret;
1527	}
1528	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1529	#else
1530	void tracing_snapshot(void)
1531	{
1532	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1533	}
1534	EXPORT_SYMBOL_GPL(tracing_snapshot);
1535	void tracing_snapshot_cond(struct trace_array tr, void cond_data)
1536	{
1537	WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1538	}
1539	EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1540	int tracing_alloc_snapshot(void)
1541	{
1542	WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1543	return -ENODEV;
1544	}
1545	EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1546	void tracing_snapshot_alloc(void)
1547	{
1548	/* Give warning */
1549	tracing_snapshot();
1550	}
1551	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1552	void tracing_cond_snapshot_data(struct trace_array tr)
1553	{
1554	return NULL;
1555	}
1556	EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1557	int tracing_snapshot_cond_enable(struct trace_array tr, void cond_data, cond_update_fn_t update)
1558	{
1559	return -ENODEV;
1560	}
1561	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1562	int tracing_snapshot_cond_disable(struct trace_array *tr)
1563	{
1564	return false;
1565	}
1566	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1567	#define free_snapshot(tr) do { } while (0)
1568	#define tracing_arm_snapshot_locked(tr) ({ -EBUSY; })
1569	#endif /* CONFIG_TRACER_SNAPSHOT */
1570
1571	void tracer_tracing_off(struct trace_array *tr)
1572	{
1573	if (tr->array_buffer.buffer)
1574	ring_buffer_record_off(tr->array_buffer.buffer);
1575	/*
1576	* This flag is looked at when buffers haven't been allocated
1577	* yet, or by some tracers (like irqsoff), that just want to
1578	* know if the ring buffer has been disabled, but it can handle
1579	* races of where it gets disabled but we still do a record.
1580	* As the check is in the fast path of the tracers, it is more
1581	* important to be fast than accurate.
1582	*/
1583	tr->buffer_disabled = 1;
1584	/* Make the flag seen by readers */
1585	smp_wmb();
1586	}
1587
1588	/**
1589	* tracing_off - turn off tracing buffers
1590	*
1591	* This function stops the tracing buffers from recording data.
1592	* It does not disable any overhead the tracers themselves may
1593	* be causing. This function simply causes all recording to
1594	* the ring buffers to fail.
1595	*/
1596	void tracing_off(void)
1597	{
1598	tracer_tracing_off(&global_trace);
1599	}
1600	EXPORT_SYMBOL_GPL(tracing_off);
1601
1602	void disable_trace_on_warning(void)
1603	{
1604	if (__disable_trace_on_warning) {
1605	trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
1606	"Disabling tracing due to warning\n");
1607	tracing_off();
1608	}
1609	}
1610
1611	/**
1612	* tracer_tracing_is_on - show real state of ring buffer enabled
1613	* @tr : the trace array to know if ring buffer is enabled
1614	*
1615	* Shows real state of the ring buffer if it is enabled or not.
1616	*/
1617	bool tracer_tracing_is_on(struct trace_array *tr)
1618	{
1619	if (tr->array_buffer.buffer)
1620	return ring_buffer_record_is_set_on(tr->array_buffer.buffer);
1621	return !tr->buffer_disabled;
1622	}
1623
1624	/**
1625	* tracing_is_on - show state of ring buffers enabled
1626	*/
1627	int tracing_is_on(void)
1628	{
1629	return tracer_tracing_is_on(&global_trace);
1630	}
1631	EXPORT_SYMBOL_GPL(tracing_is_on);
1632
1633	static int __init set_buf_size(char *str)
1634	{
1635	unsigned long buf_size;
1636
1637	if (!str)
1638	return 0;
1639	buf_size = memparse(str, &str);
1640	/*
1641	* nr_entries can not be zero and the startup
1642	* tests require some buffer space. Therefore
1643	* ensure we have at least 4096 bytes of buffer.
1644	*/
1645	trace_buf_size = max(4096UL, buf_size);
1646	return 1;
1647	}
1648	__setup("trace_buf_size=", set_buf_size);
1649
1650	static int __init set_tracing_thresh(char *str)
1651	{
1652	unsigned long threshold;
1653	int ret;
1654
1655	if (!str)
1656	return 0;
1657	ret = kstrtoul(str, 0, &threshold);
1658	if (ret < 0)
1659	return 0;
1660	tracing_thresh = threshold * 1000;
1661	return 1;
1662	}
1663	__setup("tracing_thresh=", set_tracing_thresh);
1664
1665	unsigned long nsecs_to_usecs(unsigned long nsecs)
1666	{
1667	return nsecs / 1000;
1668	}
1669
1670	/*
1671	* TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1672	* It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1673	* matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1674	* of strings in the order that the evals (enum) were defined.
1675	*/
1676	#undef C
1677	#define C(a, b) b
1678
1679	/* These must match the bit positions in trace_iterator_flags */
1680	static const char *trace_options[] = {
1681	TRACE_FLAGS
1682	NULL
1683	};
1684
1685	static struct {
1686	u64 (*func)(void);
1687	const char *name;
1688	int in_ns; /* is this clock in nanoseconds? */
1689	} trace_clocks[] = {
1690	{ trace_clock_local, "local", 1 },
1691	{ trace_clock_global, "global", 1 },
1692	{ trace_clock_counter, "counter", 0 },
1693	{ trace_clock_jiffies, "uptime", 0 },
1694	{ trace_clock, "perf", 1 },
1695	{ ktime_get_mono_fast_ns, "mono", 1 },
1696	{ ktime_get_raw_fast_ns, "mono_raw", 1 },
1697	{ ktime_get_boot_fast_ns, "boot", 1 },
1698	{ ktime_get_tai_fast_ns, "tai", 1 },
1699	ARCH_TRACE_CLOCKS
1700	};
1701
1702	bool trace_clock_in_ns(struct trace_array *tr)
1703	{
1704	if (trace_clocks[tr->clock_id].in_ns)
1705	return true;
1706
1707	return false;
1708	}
1709
1710	/*
1711	* trace_parser_get_init - gets the buffer for trace parser
1712	*/
1713	int trace_parser_get_init(struct trace_parser *parser, int size)
1714	{
1715	memset(parser, 0, sizeof(*parser));
1716
1717	parser->buffer = kmalloc(size, GFP_KERNEL);
1718	if (!parser->buffer)
1719	return 1;
1720
1721	parser->size = size;
1722	return 0;
1723	}
1724
1725	/*
1726	* trace_parser_put - frees the buffer for trace parser
1727	*/
1728	void trace_parser_put(struct trace_parser *parser)
1729	{
1730	kfree(parser->buffer);
1731	parser->buffer = NULL;
1732	}
1733
1734	/*
1735	* trace_get_user - reads the user input string separated by space
1736	* (matched by isspace(ch))
1737	*
1738	* For each string found the 'struct trace_parser' is updated,
1739	* and the function returns.
1740	*
1741	* Returns number of bytes read.
1742	*
1743	* See kernel/trace/trace.h for 'struct trace_parser' details.
1744	*/
1745	int trace_get_user(struct trace_parser parser, const char __user ubuf,
1746	size_t cnt, loff_t *ppos)
1747	{
1748	char ch;
1749	size_t read = 0;
1750	ssize_t ret;
1751
1752	if (!*ppos)
1753	trace_parser_clear(parser);
1754
1755	ret = get_user(ch, ubuf++);
1756	if (ret)
1757	goto out;
1758
1759	read++;
1760	cnt--;
1761
1762	/*
1763	* The parser is not finished with the last write,
1764	* continue reading the user input without skipping spaces.
1765	*/
1766	if (!parser->cont) {
1767	/* skip white space */
1768	while (cnt && isspace(ch)) {
1769	ret = get_user(ch, ubuf++);
1770	if (ret)
1771	goto out;
1772	read++;
1773	cnt--;
1774	}
1775
1776	parser->idx = 0;
1777
1778	/* only spaces were written */
1779	if (isspace(ch) \|\| !ch) {
1780	*ppos += read;
1781	ret = read;
1782	goto out;
1783	}
1784	}
1785
1786	/* read the non-space input */
1787	while (cnt && !isspace(ch) && ch) {
1788	if (parser->idx < parser->size - 1)
1789	parser->buffer[parser->idx++] = ch;
1790	else {
1791	ret = -EINVAL;
1792	goto out;
1793	}
1794	ret = get_user(ch, ubuf++);
1795	if (ret)
1796	goto out;
1797	read++;
1798	cnt--;
1799	}
1800
1801	/* We either got finished input or we have to wait for another call. */
1802	if (isspace(ch) \|\| !ch) {
1803	parser->buffer[parser->idx] = 0;
1804	parser->cont = false;
1805	} else if (parser->idx < parser->size - 1) {
1806	parser->cont = true;
1807	parser->buffer[parser->idx++] = ch;
1808	/* Make sure the parsed string always terminates with '\0'. */
1809	parser->buffer[parser->idx] = 0;
1810	} else {
1811	ret = -EINVAL;
1812	goto out;
1813	}
1814
1815	*ppos += read;
1816	ret = read;
1817
1818	out:
1819	return ret;
1820	}
1821
1822	/* TODO add a seq_buf_to_buffer() */
1823	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
1824	{
1825	int len;
1826
1827	if (trace_seq_used(s) <= s->readpos)
1828	return -EBUSY;
1829
1830	len = trace_seq_used(s) - s->readpos;
1831	if (cnt > len)
1832	cnt = len;
1833	memcpy(buf, s->buffer + s->readpos, cnt);
1834
1835	s->readpos += cnt;
1836	return cnt;
1837	}
1838
1839	unsigned long __read_mostly tracing_thresh;
1840
1841	#ifdef CONFIG_TRACER_MAX_TRACE
1842	static const struct file_operations tracing_max_lat_fops;
1843
1844	#ifdef LATENCY_FS_NOTIFY
1845
1846	static struct workqueue_struct *fsnotify_wq;
1847
1848	static void latency_fsnotify_workfn(struct work_struct *work)
1849	{
1850	struct trace_array *tr = container_of(work, struct trace_array,
1851	fsnotify_work);
1852	fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);
1853	}
1854
1855	static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1856	{
1857	struct trace_array *tr = container_of(iwork, struct trace_array,
1858	fsnotify_irqwork);
1859	queue_work(fsnotify_wq, &tr->fsnotify_work);
1860	}
1861
1862	static void trace_create_maxlat_file(struct trace_array *tr,
1863	struct dentry *d_tracer)
1864	{
1865	INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1866	init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1867	tr->d_max_latency = trace_create_file("tracing_max_latency",
1868	TRACE_MODE_WRITE,
1869	d_tracer, tr,
1870	&tracing_max_lat_fops);
1871	}
1872
1873	__init static int latency_fsnotify_init(void)
1874	{
1875	fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1876	WQ_UNBOUND \| WQ_HIGHPRI, 0);
1877	if (!fsnotify_wq) {
1878	pr_err("Unable to allocate tr_max_lat_wq\n");
1879	return -ENOMEM;
1880	}
1881	return 0;
1882	}
1883
1884	late_initcall_sync(latency_fsnotify_init);
1885
1886	void latency_fsnotify(struct trace_array *tr)
1887	{
1888	if (!fsnotify_wq)
1889	return;
1890	/*
1891	* We cannot call queue_work(&tr->fsnotify_work) from here because it's
1892	* possible that we are called from __schedule() or do_idle(), which
1893	* could cause a deadlock.
1894	*/
1895	irq_work_queue(&tr->fsnotify_irqwork);
1896	}
1897
1898	#else /* !LATENCY_FS_NOTIFY */
1899
1900	#define trace_create_maxlat_file(tr, d_tracer) \
1901	trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \
1902	d_tracer, tr, &tracing_max_lat_fops)
1903
1904	#endif
1905
1906	/*
1907	* Copy the new maximum trace into the separate maximum-trace
1908	* structure. (this way the maximum trace is permanently saved,
1909	* for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1910	*/
1911	static void
1912	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
1913	{
1914	struct array_buffer *trace_buf = &tr->array_buffer;
1915	struct array_buffer *max_buf = &tr->max_buffer;
1916	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1917	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1918
1919	max_buf->cpu = cpu;
1920	max_buf->time_start = data->preempt_timestamp;
1921
1922	max_data->saved_latency = tr->max_latency;
1923	max_data->critical_start = data->critical_start;
1924	max_data->critical_end = data->critical_end;
1925
1926	strscpy(max_data->comm, tsk->comm);
1927	max_data->pid = tsk->pid;
1928	/*
1929	* If tsk == current, then use current_uid(), as that does not use
1930	* RCU. The irq tracer can be called out of RCU scope.
1931	*/
1932	if (tsk == current)
1933	max_data->uid = current_uid();
1934	else
1935	max_data->uid = task_uid(tsk);
1936
1937	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1938	max_data->policy = tsk->policy;
1939	max_data->rt_priority = tsk->rt_priority;
1940
1941	/* record this tasks comm */
1942	tracing_record_cmdline(tsk);
1943	latency_fsnotify(tr);
1944	}
1945
1946	/**
1947	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
1948	* @tr: tracer
1949	* @tsk: the task with the latency
1950	* @cpu: The cpu that initiated the trace.
1951	* @cond_data: User data associated with a conditional snapshot
1952	*
1953	* Flip the buffers between the @tr and the max_tr and record information
1954	* about which task was the cause of this latency.
1955	*/
1956	void
1957	update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu,
1958	void *cond_data)
1959	{
1960	if (tr->stop_count)
1961	return;
1962
1963	WARN_ON_ONCE(!irqs_disabled());
1964
1965	if (!tr->allocated_snapshot) {
1966	/* Only the nop tracer should hit this when disabling */
1967	WARN_ON_ONCE(tr->current_trace != &nop_trace);
1968	return;
1969	}
1970
1971	arch_spin_lock(&tr->max_lock);
1972
1973	/* Inherit the recordable setting from array_buffer */
1974	if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1975	ring_buffer_record_on(tr->max_buffer.buffer);
1976	else
1977	ring_buffer_record_off(tr->max_buffer.buffer);
1978
1979	#ifdef CONFIG_TRACER_SNAPSHOT
1980	if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) {
1981	arch_spin_unlock(&tr->max_lock);
1982	return;
1983	}
1984	#endif
1985	swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1986
1987	__update_max_tr(tr, tsk, cpu);
1988
1989	arch_spin_unlock(&tr->max_lock);
1990
1991	/* Any waiters on the old snapshot buffer need to wake up */
1992	ring_buffer_wake_waiters(tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS);
1993	}
1994
1995	/**
1996	* update_max_tr_single - only copy one trace over, and reset the rest
1997	* @tr: tracer
1998	* @tsk: task with the latency
1999	* @cpu: the cpu of the buffer to copy.
2000	*
2001	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
2002	*/
2003	void
2004	update_max_tr_single(struct trace_array tr, struct task_struct tsk, int cpu)
2005	{
2006	int ret;
2007
2008	if (tr->stop_count)
2009	return;
2010
2011	WARN_ON_ONCE(!irqs_disabled());
2012	if (!tr->allocated_snapshot) {
2013	/* Only the nop tracer should hit this when disabling */
2014	WARN_ON_ONCE(tr->current_trace != &nop_trace);
2015	return;
2016	}
2017
2018	arch_spin_lock(&tr->max_lock);
2019
2020	ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
2021
2022	if (ret == -EBUSY) {
2023	/*
2024	* We failed to swap the buffer due to a commit taking
2025	* place on this CPU. We fail to record, but we reset
2026	* the max trace buffer (no one writes directly to it)
2027	* and flag that it failed.
2028	* Another reason is resize is in progress.
2029	*/
2030	trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
2031	"Failed to swap buffers due to commit or resize in progress\n");
2032	}
2033
2034	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
2035
2036	__update_max_tr(tr, tsk, cpu);
2037	arch_spin_unlock(&tr->max_lock);
2038	}
2039
2040	#endif /* CONFIG_TRACER_MAX_TRACE */
2041
2042	struct pipe_wait {
2043	struct trace_iterator *iter;
2044	int wait_index;
2045	};
2046
2047	static bool wait_pipe_cond(void *data)
2048	{
2049	struct pipe_wait *pwait = data;
2050	struct trace_iterator *iter = pwait->iter;
2051
2052	if (atomic_read_acquire(&iter->wait_index) != pwait->wait_index)
2053	return true;
2054
2055	return iter->closed;
2056	}
2057
2058	static int wait_on_pipe(struct trace_iterator *iter, int full)
2059	{
2060	struct pipe_wait pwait;
2061	int ret;
2062
2063	/* Iterators are static, they should be filled or empty */
2064	if (trace_buffer_iter(iter, iter->cpu_file))
2065	return 0;
2066
2067	pwait.wait_index = atomic_read_acquire(&iter->wait_index);
2068	pwait.iter = iter;
2069
2070	ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full,
2071	wait_pipe_cond, &pwait);
2072
2073	#ifdef CONFIG_TRACER_MAX_TRACE
2074	/*
2075	* Make sure this is still the snapshot buffer, as if a snapshot were
2076	* to happen, this would now be the main buffer.
2077	*/
2078	if (iter->snapshot)
2079	iter->array_buffer = &iter->tr->max_buffer;
2080	#endif
2081	return ret;
2082	}
2083
2084	#ifdef CONFIG_FTRACE_STARTUP_TEST
2085	static bool selftests_can_run;
2086
2087	struct trace_selftests {
2088	struct list_head list;
2089	struct tracer *type;
2090	};
2091
2092	static LIST_HEAD(postponed_selftests);
2093
2094	static int save_selftest(struct tracer *type)
2095	{
2096	struct trace_selftests *selftest;
2097
2098	selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
2099	if (!selftest)
2100	return -ENOMEM;
2101
2102	selftest->type = type;
2103	list_add(&selftest->list, &postponed_selftests);
2104	return 0;
2105	}
2106
2107	static int run_tracer_selftest(struct tracer *type)
2108	{
2109	struct trace_array *tr = &global_trace;
2110	struct tracer *saved_tracer = tr->current_trace;
2111	int ret;
2112
2113	if (!type->selftest \|\| tracing_selftest_disabled)
2114	return 0;
2115
2116	/*
2117	* If a tracer registers early in boot up (before scheduling is
2118	* initialized and such), then do not run its selftests yet.
2119	* Instead, run it a little later in the boot process.
2120	*/
2121	if (!selftests_can_run)
2122	return save_selftest(type);
2123
2124	if (!tracing_is_on()) {
2125	pr_warn("Selftest for tracer %s skipped due to tracing disabled\n",
2126	type->name);
2127	return 0;
2128	}
2129
2130	/*
2131	* Run a selftest on this tracer.
2132	* Here we reset the trace buffer, and set the current
2133	* tracer to be this tracer. The tracer can then run some
2134	* internal tracing to verify that everything is in order.
2135	* If we fail, we do not register this tracer.
2136	*/
2137	tracing_reset_online_cpus(&tr->array_buffer);
2138
2139	tr->current_trace = type;
2140
2141	#ifdef CONFIG_TRACER_MAX_TRACE
2142	if (type->use_max_tr) {
2143	/* If we expanded the buffers, make sure the max is expanded too */
2144	if (tr->ring_buffer_expanded)
2145	ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
2146	RING_BUFFER_ALL_CPUS);
2147	tr->allocated_snapshot = true;
2148	}
2149	#endif
2150
2151	/* the test is responsible for initializing and enabling */
2152	pr_info("Testing tracer %s: ", type->name);
2153	ret = type->selftest(type, tr);
2154	/* the test is responsible for resetting too */
2155	tr->current_trace = saved_tracer;
2156	if (ret) {
2157	printk(KERN_CONT "FAILED!\n");
2158	/* Add the warning after printing 'FAILED' */
2159	WARN_ON(1);
2160	return -1;
2161	}
2162	/* Only reset on passing, to avoid touching corrupted buffers */
2163	tracing_reset_online_cpus(&tr->array_buffer);
2164
2165	#ifdef CONFIG_TRACER_MAX_TRACE
2166	if (type->use_max_tr) {
2167	tr->allocated_snapshot = false;
2168
2169	/* Shrink the max buffer again */
2170	if (tr->ring_buffer_expanded)
2171	ring_buffer_resize(tr->max_buffer.buffer, 1,
2172	RING_BUFFER_ALL_CPUS);
2173	}
2174	#endif
2175
2176	printk(KERN_CONT "PASSED\n");
2177	return 0;
2178	}
2179
2180	static int do_run_tracer_selftest(struct tracer *type)
2181	{
2182	int ret;
2183
2184	/*
2185	* Tests can take a long time, especially if they are run one after the
2186	* other, as does happen during bootup when all the tracers are
2187	* registered. This could cause the soft lockup watchdog to trigger.
2188	*/
2189	cond_resched();
2190
2191	tracing_selftest_running = true;
2192	ret = run_tracer_selftest(type);
2193	tracing_selftest_running = false;
2194
2195	return ret;
2196	}
2197
2198	static __init int init_trace_selftests(void)
2199	{
2200	struct trace_selftests p, n;
2201	struct tracer t, *last;
2202	int ret;
2203
2204	selftests_can_run = true;
2205
2206	mutex_lock(&trace_types_lock);
2207
2208	if (list_empty(&postponed_selftests))
2209	goto out;
2210
2211	pr_info("Running postponed tracer tests:\n");
2212
2213	tracing_selftest_running = true;
2214	list_for_each_entry_safe(p, n, &postponed_selftests, list) {
2215	/* This loop can take minutes when sanitizers are enabled, so
2216	* lets make sure we allow RCU processing.
2217	*/
2218	cond_resched();
2219	ret = run_tracer_selftest(p->type);
2220	/* If the test fails, then warn and remove from available_tracers */
2221	if (ret < 0) {
2222	WARN(1, "tracer: %s failed selftest, disabling\n",
2223	p->type->name);
2224	last = &trace_types;
2225	for (t = trace_types; t; t = t->next) {
2226	if (t == p->type) {
2227	*last = t->next;
2228	break;
2229	}
2230	last = &t->next;
2231	}
2232	}
2233	list_del(&p->list);
2234	kfree(p);
2235	}
2236	tracing_selftest_running = false;
2237
2238	out:
2239	mutex_unlock(&trace_types_lock);
2240
2241	return 0;
2242	}
2243	core_initcall(init_trace_selftests);
2244	#else
2245	static inline int do_run_tracer_selftest(struct tracer *type)
2246	{
2247	return 0;
2248	}
2249	#endif /* CONFIG_FTRACE_STARTUP_TEST */
2250
2251	static void add_tracer_options(struct trace_array tr, struct tracer t);
2252
2253	static void __init apply_trace_boot_options(void);
2254
2255	/**
2256	* register_tracer - register a tracer with the ftrace system.
2257	* @type: the plugin for the tracer
2258	*
2259	* Register a new plugin tracer.
2260	*/
2261	int __init register_tracer(struct tracer *type)
2262	{
2263	struct tracer *t;
2264	int ret = 0;
2265
2266	if (!type->name) {
2267	pr_info("Tracer must have a name\n");
2268	return -1;
2269	}
2270
2271	if (strlen(type->name) >= MAX_TRACER_SIZE) {
2272	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
2273	return -1;
2274	}
2275
2276	if (security_locked_down(LOCKDOWN_TRACEFS)) {
2277	pr_warn("Can not register tracer %s due to lockdown\n",
2278	type->name);
2279	return -EPERM;
2280	}
2281
2282	mutex_lock(&trace_types_lock);
2283
2284	for (t = trace_types; t; t = t->next) {
2285	if (strcmp(type->name, t->name) == 0) {
2286	/* already found */
2287	pr_info("Tracer %s already registered\n",
2288	type->name);
2289	ret = -1;
2290	goto out;
2291	}
2292	}
2293
2294	if (!type->set_flag)
2295	type->set_flag = &dummy_set_flag;
2296	if (!type->flags) {
2297	/allocate a dummy tracer_flags/
2298	type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
2299	if (!type->flags) {
2300	ret = -ENOMEM;
2301	goto out;
2302	}
2303	type->flags->val = 0;
2304	type->flags->opts = dummy_tracer_opt;
2305	} else
2306	if (!type->flags->opts)
2307	type->flags->opts = dummy_tracer_opt;
2308
2309	/* store the tracer for __set_tracer_option */
2310	type->flags->trace = type;
2311
2312	ret = do_run_tracer_selftest(type);
2313	if (ret < 0)
2314	goto out;
2315
2316	type->next = trace_types;
2317	trace_types = type;
2318	add_tracer_options(&global_trace, type);
2319
2320	out:
2321	mutex_unlock(&trace_types_lock);
2322
2323	if (ret \|\| !default_bootup_tracer)
2324	goto out_unlock;
2325
2326	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
2327	goto out_unlock;
2328
2329	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
2330	/* Do we want this tracer to start on bootup? */
2331	tracing_set_tracer(&global_trace, type->name);
2332	default_bootup_tracer = NULL;
2333
2334	apply_trace_boot_options();
2335
2336	/* disable other selftests, since this will break it. */
2337	disable_tracing_selftest("running a tracer");
2338
2339	out_unlock:
2340	return ret;
2341	}
2342
2343	static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
2344	{
2345	struct trace_buffer *buffer = buf->buffer;
2346
2347	if (!buffer)
2348	return;
2349
2350	ring_buffer_record_disable(buffer);
2351
2352	/* Make sure all commits have finished */
2353	synchronize_rcu();
2354	ring_buffer_reset_cpu(buffer, cpu);
2355
2356	ring_buffer_record_enable(buffer);
2357	}
2358
2359	void tracing_reset_online_cpus(struct array_buffer *buf)
2360	{
2361	struct trace_buffer *buffer = buf->buffer;
2362
2363	if (!buffer)
2364	return;
2365
2366	ring_buffer_record_disable(buffer);
2367
2368	/* Make sure all commits have finished */
2369	synchronize_rcu();
2370
2371	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2372
2373	ring_buffer_reset_online_cpus(buffer);
2374
2375	ring_buffer_record_enable(buffer);
2376	}
2377
2378	static void tracing_reset_all_cpus(struct array_buffer *buf)
2379	{
2380	struct trace_buffer *buffer = buf->buffer;
2381
2382	if (!buffer)
2383	return;
2384
2385	ring_buffer_record_disable(buffer);
2386
2387	/* Make sure all commits have finished */
2388	synchronize_rcu();
2389
2390	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2391
2392	ring_buffer_reset(buffer);
2393
2394	ring_buffer_record_enable(buffer);
2395	}
2396
2397	/* Must have trace_types_lock held */
2398	void tracing_reset_all_online_cpus_unlocked(void)
2399	{
2400	struct trace_array *tr;
2401
2402	lockdep_assert_held(&trace_types_lock);
2403
2404	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2405	if (!tr->clear_trace)
2406	continue;
2407	tr->clear_trace = false;
2408	tracing_reset_online_cpus(&tr->array_buffer);
2409	#ifdef CONFIG_TRACER_MAX_TRACE
2410	tracing_reset_online_cpus(&tr->max_buffer);
2411	#endif
2412	}
2413	}
2414
2415	void tracing_reset_all_online_cpus(void)
2416	{
2417	mutex_lock(&trace_types_lock);
2418	tracing_reset_all_online_cpus_unlocked();
2419	mutex_unlock(&trace_types_lock);
2420	}
2421
2422	int is_tracing_stopped(void)
2423	{
2424	return global_trace.stop_count;
2425	}
2426
2427	static void tracing_start_tr(struct trace_array *tr)
2428	{
2429	struct trace_buffer *buffer;
2430	unsigned long flags;
2431
2432	if (tracing_disabled)
2433	return;
2434
2435	raw_spin_lock_irqsave(&tr->start_lock, flags);
2436	if (--tr->stop_count) {
2437	if (WARN_ON_ONCE(tr->stop_count < 0)) {
2438	/* Someone screwed up their debugging */
2439	tr->stop_count = 0;
2440	}
2441	goto out;
2442	}
2443
2444	/* Prevent the buffers from switching */
2445	arch_spin_lock(&tr->max_lock);
2446
2447	buffer = tr->array_buffer.buffer;
2448	if (buffer)
2449	ring_buffer_record_enable(buffer);
2450
2451	#ifdef CONFIG_TRACER_MAX_TRACE
2452	buffer = tr->max_buffer.buffer;
2453	if (buffer)
2454	ring_buffer_record_enable(buffer);
2455	#endif
2456
2457	arch_spin_unlock(&tr->max_lock);
2458
2459	out:
2460	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2461	}
2462
2463	/**
2464	* tracing_start - quick start of the tracer
2465	*
2466	* If tracing is enabled but was stopped by tracing_stop,
2467	* this will start the tracer back up.
2468	*/
2469	void tracing_start(void)
2470
2471	{
2472	return tracing_start_tr(&global_trace);
2473	}
2474
2475	static void tracing_stop_tr(struct trace_array *tr)
2476	{
2477	struct trace_buffer *buffer;
2478	unsigned long flags;
2479
2480	raw_spin_lock_irqsave(&tr->start_lock, flags);
2481	if (tr->stop_count++)
2482	goto out;
2483
2484	/* Prevent the buffers from switching */
2485	arch_spin_lock(&tr->max_lock);
2486
2487	buffer = tr->array_buffer.buffer;
2488	if (buffer)
2489	ring_buffer_record_disable(buffer);
2490
2491	#ifdef CONFIG_TRACER_MAX_TRACE
2492	buffer = tr->max_buffer.buffer;
2493	if (buffer)
2494	ring_buffer_record_disable(buffer);
2495	#endif
2496
2497	arch_spin_unlock(&tr->max_lock);
2498
2499	out:
2500	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2501	}
2502
2503	/**
2504	* tracing_stop - quick stop of the tracer
2505	*
2506	* Light weight way to stop tracing. Use in conjunction with
2507	* tracing_start.
2508	*/
2509	void tracing_stop(void)
2510	{
2511	return tracing_stop_tr(&global_trace);
2512	}
2513
2514	/*
2515	* Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2516	* overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2517	* simplifies those functions and keeps them in sync.
2518	*/
2519	enum print_line_t trace_handle_return(struct trace_seq *s)
2520	{
2521	return trace_seq_has_overflowed(s) ?
2522	TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2523	}
2524	EXPORT_SYMBOL_GPL(trace_handle_return);
2525
2526	static unsigned short migration_disable_value(void)
2527	{
2528	#if defined(CONFIG_SMP)
2529	return current->migration_disabled;
2530	#else
2531	return 0;
2532	#endif
2533	}
2534
2535	unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
2536	{
2537	unsigned int trace_flags = irqs_status;
2538	unsigned int pc;
2539
2540	pc = preempt_count();
2541
2542	if (pc & NMI_MASK)
2543	trace_flags \|= TRACE_FLAG_NMI;
2544	if (pc & HARDIRQ_MASK)
2545	trace_flags \|= TRACE_FLAG_HARDIRQ;
2546	if (in_serving_softirq())
2547	trace_flags \|= TRACE_FLAG_SOFTIRQ;
2548	if (softirq_count() >> (SOFTIRQ_SHIFT + 1))
2549	trace_flags \|= TRACE_FLAG_BH_OFF;
2550
2551	if (tif_need_resched())
2552	trace_flags \|= TRACE_FLAG_NEED_RESCHED;
2553	if (test_preempt_need_resched())
2554	trace_flags \|= TRACE_FLAG_PREEMPT_RESCHED;
2555	if (IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY) && tif_test_bit(TIF_NEED_RESCHED_LAZY))
2556	trace_flags \|= TRACE_FLAG_NEED_RESCHED_LAZY;
2557	return (trace_flags << 16) \| (min_t(unsigned int, pc & 0xff, 0xf)) \|
2558	(min_t(unsigned int, migration_disable_value(), 0xf)) << 4;
2559	}
2560
2561	struct ring_buffer_event *
2562	trace_buffer_lock_reserve(struct trace_buffer *buffer,
2563	int type,
2564	unsigned long len,
2565	unsigned int trace_ctx)
2566	{
2567	return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
2568	}
2569
2570	DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2571	DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2572	static int trace_buffered_event_ref;
2573
2574	/**
2575	* trace_buffered_event_enable - enable buffering events
2576	*
2577	* When events are being filtered, it is quicker to use a temporary
2578	* buffer to write the event data into if there's a likely chance
2579	* that it will not be committed. The discard of the ring buffer
2580	* is not as fast as committing, and is much slower than copying
2581	* a commit.
2582	*
2583	* When an event is to be filtered, allocate per cpu buffers to
2584	* write the event data into, and if the event is filtered and discarded
2585	* it is simply dropped, otherwise, the entire data is to be committed
2586	* in one shot.
2587	*/
2588	void trace_buffered_event_enable(void)
2589	{
2590	struct ring_buffer_event *event;
2591	struct page *page;
2592	int cpu;
2593
2594	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2595
2596	if (trace_buffered_event_ref++)
2597	return;
2598
2599	for_each_tracing_cpu(cpu) {
2600	page = alloc_pages_node(cpu_to_node(cpu),
2601	GFP_KERNEL \| __GFP_NORETRY, 0);
2602	/* This is just an optimization and can handle failures */
2603	if (!page) {
2604	pr_err("Failed to allocate event buffer\n");
2605	break;
2606	}
2607
2608	event = page_address(page);
2609	memset(event, 0, sizeof(*event));
2610
2611	per_cpu(trace_buffered_event, cpu) = event;
2612
2613	preempt_disable();
2614	if (cpu == smp_processor_id() &&
2615	__this_cpu_read(trace_buffered_event) !=
2616	per_cpu(trace_buffered_event, cpu))
2617	WARN_ON_ONCE(1);
2618	preempt_enable();
2619	}
2620	}
2621
2622	static void enable_trace_buffered_event(void *data)
2623	{
2624	/* Probably not needed, but do it anyway */
2625	smp_rmb();
2626	this_cpu_dec(trace_buffered_event_cnt);
2627	}
2628
2629	static void disable_trace_buffered_event(void *data)
2630	{
2631	this_cpu_inc(trace_buffered_event_cnt);
2632	}
2633
2634	/**
2635	* trace_buffered_event_disable - disable buffering events
2636	*
2637	* When a filter is removed, it is faster to not use the buffered
2638	* events, and to commit directly into the ring buffer. Free up
2639	* the temp buffers when there are no more users. This requires
2640	* special synchronization with current events.
2641	*/
2642	void trace_buffered_event_disable(void)
2643	{
2644	int cpu;
2645
2646	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2647
2648	if (WARN_ON_ONCE(!trace_buffered_event_ref))
2649	return;
2650
2651	if (--trace_buffered_event_ref)
2652	return;
2653
2654	/* For each CPU, set the buffer as used. */
2655	on_each_cpu_mask(tracing_buffer_mask, disable_trace_buffered_event,
2656	NULL, true);
2657
2658	/* Wait for all current users to finish */
2659	synchronize_rcu();
2660
2661	for_each_tracing_cpu(cpu) {
2662	free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2663	per_cpu(trace_buffered_event, cpu) = NULL;
2664	}
2665
2666	/*
2667	* Wait for all CPUs that potentially started checking if they can use
2668	* their event buffer only after the previous synchronize_rcu() call and
2669	* they still read a valid pointer from trace_buffered_event. It must be
2670	* ensured they don't see cleared trace_buffered_event_cnt else they
2671	* could wrongly decide to use the pointed-to buffer which is now freed.
2672	*/
2673	synchronize_rcu();
2674
2675	/* For each CPU, relinquish the buffer */
2676	on_each_cpu_mask(tracing_buffer_mask, enable_trace_buffered_event, NULL,
2677	true);
2678	}
2679
2680	static struct trace_buffer *temp_buffer;
2681
2682	struct ring_buffer_event *
2683	trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2684	struct trace_event_file *trace_file,
2685	int type, unsigned long len,
2686	unsigned int trace_ctx)
2687	{
2688	struct ring_buffer_event *entry;
2689	struct trace_array *tr = trace_file->tr;
2690	int val;
2691
2692	*current_rb = tr->array_buffer.buffer;
2693
2694	if (!tr->no_filter_buffering_ref &&
2695	(trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED \| EVENT_FILE_FL_FILTERED))) {
2696	preempt_disable_notrace();
2697	/*
2698	* Filtering is on, so try to use the per cpu buffer first.
2699	* This buffer will simulate a ring_buffer_event,
2700	* where the type_len is zero and the array[0] will
2701	* hold the full length.
2702	* (see include/linux/ring-buffer.h for details on
2703	* how the ring_buffer_event is structured).
2704	*
2705	* Using a temp buffer during filtering and copying it
2706	* on a matched filter is quicker than writing directly
2707	* into the ring buffer and then discarding it when
2708	* it doesn't match. That is because the discard
2709	* requires several atomic operations to get right.
2710	* Copying on match and doing nothing on a failed match
2711	* is still quicker than no copy on match, but having
2712	* to discard out of the ring buffer on a failed match.
2713	*/
2714	if ((entry = __this_cpu_read(trace_buffered_event))) {
2715	int max_len = PAGE_SIZE - struct_size(entry, array, 1);
2716
2717	val = this_cpu_inc_return(trace_buffered_event_cnt);
2718
2719	/*
2720	* Preemption is disabled, but interrupts and NMIs
2721	* can still come in now. If that happens after
2722	* the above increment, then it will have to go
2723	* back to the old method of allocating the event
2724	* on the ring buffer, and if the filter fails, it
2725	* will have to call ring_buffer_discard_commit()
2726	* to remove it.
2727	*
2728	* Need to also check the unlikely case that the
2729	* length is bigger than the temp buffer size.
2730	* If that happens, then the reserve is pretty much
2731	* guaranteed to fail, as the ring buffer currently
2732	* only allows events less than a page. But that may
2733	* change in the future, so let the ring buffer reserve
2734	* handle the failure in that case.
2735	*/
2736	if (val == 1 && likely(len <= max_len)) {
2737	trace_event_setup(entry, type, trace_ctx);
2738	entry->array[0] = len;
2739	/* Return with preemption disabled */
2740	return entry;
2741	}
2742	this_cpu_dec(trace_buffered_event_cnt);
2743	}
2744	/* __trace_buffer_lock_reserve() disables preemption */
2745	preempt_enable_notrace();
2746	}
2747
2748	entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2749	trace_ctx);
2750	/*
2751	* If tracing is off, but we have triggers enabled
2752	* we still need to look at the event data. Use the temp_buffer
2753	* to store the trace event for the trigger to use. It's recursive
2754	* safe and will not be recorded anywhere.
2755	*/
2756	if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2757	*current_rb = temp_buffer;
2758	entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2759	trace_ctx);
2760	}
2761	return entry;
2762	}
2763	EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2764
2765	static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock);
2766	static DEFINE_MUTEX(tracepoint_printk_mutex);
2767
2768	static void output_printk(struct trace_event_buffer *fbuffer)
2769	{
2770	struct trace_event_call *event_call;
2771	struct trace_event_file *file;
2772	struct trace_event *event;
2773	unsigned long flags;
2774	struct trace_iterator *iter = tracepoint_print_iter;
2775
2776	/* We should never get here if iter is NULL */
2777	if (WARN_ON_ONCE(!iter))
2778	return;
2779
2780	event_call = fbuffer->trace_file->event_call;
2781	if (!event_call \|\| !event_call->event.funcs \|\|
2782	!event_call->event.funcs->trace)
2783	return;
2784
2785	file = fbuffer->trace_file;
2786	if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) \|\|
2787	(unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2788	!filter_match_preds(file->filter, fbuffer->entry)))
2789	return;
2790
2791	event = &fbuffer->trace_file->event_call->event;
2792
2793	raw_spin_lock_irqsave(&tracepoint_iter_lock, flags);
2794	trace_seq_init(&iter->seq);
2795	iter->ent = fbuffer->entry;
2796	event_call->event.funcs->trace(iter, 0, event);
2797	trace_seq_putc(&iter->seq, 0);
2798	printk("%s", iter->seq.buffer);
2799
2800	raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2801	}
2802
2803	int tracepoint_printk_sysctl(const struct ctl_table *table, int write,
2804	void buffer, size_t lenp,
2805	loff_t *ppos)
2806	{
2807	int save_tracepoint_printk;
2808	int ret;
2809
2810	mutex_lock(&tracepoint_printk_mutex);
2811	save_tracepoint_printk = tracepoint_printk;
2812
2813	ret = proc_dointvec(table, write, buffer, lenp, ppos);
2814
2815	/*
2816	* This will force exiting early, as tracepoint_printk
2817	* is always zero when tracepoint_printk_iter is not allocated
2818	*/
2819	if (!tracepoint_print_iter)
2820	tracepoint_printk = 0;
2821
2822	if (save_tracepoint_printk == tracepoint_printk)
2823	goto out;
2824
2825	if (tracepoint_printk)
2826	static_key_enable(&tracepoint_printk_key.key);
2827	else
2828	static_key_disable(&tracepoint_printk_key.key);
2829
2830	out:
2831	mutex_unlock(&tracepoint_printk_mutex);
2832
2833	return ret;
2834	}
2835
2836	void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2837	{
2838	enum event_trigger_type tt = ETT_NONE;
2839	struct trace_event_file *file = fbuffer->trace_file;
2840
2841	if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event,
2842	fbuffer->entry, &tt))
2843	goto discard;
2844
2845	if (static_key_false(&tracepoint_printk_key.key))
2846	output_printk(fbuffer);
2847
2848	if (static_branch_unlikely(&trace_event_exports_enabled))
2849	ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
2850
2851	trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer,
2852	fbuffer->event, fbuffer->trace_ctx, fbuffer->regs);
2853
2854	discard:
2855	if (tt)
2856	event_triggers_post_call(file, tt);
2857
2858	}
2859	EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2860
2861	/*
2862	* Skip 3:
2863	*
2864	* trace_buffer_unlock_commit_regs()
2865	* trace_event_buffer_commit()
2866	* trace_event_raw_event_xxx()
2867	*/
2868	# define STACK_SKIP 3
2869
2870	void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2871	struct trace_buffer *buffer,
2872	struct ring_buffer_event *event,
2873	unsigned int trace_ctx,
2874	struct pt_regs *regs)
2875	{
2876	__buffer_unlock_commit(buffer, event);
2877
2878	/*
2879	* If regs is not set, then skip the necessary functions.
2880	* Note, we can still get here via blktrace, wakeup tracer
2881	* and mmiotrace, but that's ok if they lose a function or
2882	* two. They are not that meaningful.
2883	*/
2884	ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs);
2885	ftrace_trace_userstack(tr, buffer, trace_ctx);
2886	}
2887
2888	/*
2889	* Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2890	*/
2891	void
2892	trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2893	struct ring_buffer_event *event)
2894	{
2895	__buffer_unlock_commit(buffer, event);
2896	}
2897
2898	void
2899	trace_function(struct trace_array *tr, unsigned long ip, unsigned long
2900	parent_ip, unsigned int trace_ctx)
2901	{
2902	struct trace_buffer *buffer = tr->array_buffer.buffer;
2903	struct ring_buffer_event *event;
2904	struct ftrace_entry *entry;
2905
2906	event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2907	trace_ctx);
2908	if (!event)
2909	return;
2910	entry = ring_buffer_event_data(event);
2911	entry->ip = ip;
2912	entry->parent_ip = parent_ip;
2913
2914	if (static_branch_unlikely(&trace_function_exports_enabled))
2915	ftrace_exports(event, TRACE_EXPORT_FUNCTION);
2916	__buffer_unlock_commit(buffer, event);
2917	}
2918
2919	#ifdef CONFIG_STACKTRACE
2920
2921	/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2922	#define FTRACE_KSTACK_NESTING 4
2923
2924	#define FTRACE_KSTACK_ENTRIES (SZ_4K / FTRACE_KSTACK_NESTING)
2925
2926	struct ftrace_stack {
2927	unsigned long calls[FTRACE_KSTACK_ENTRIES];
2928	};
2929
2930
2931	struct ftrace_stacks {
2932	struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2933	};
2934
2935	static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2936	static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2937
2938	static void __ftrace_trace_stack(struct trace_array *tr,
2939	struct trace_buffer *buffer,
2940	unsigned int trace_ctx,
2941	int skip, struct pt_regs *regs)
2942	{
2943	struct ring_buffer_event *event;
2944	unsigned int size, nr_entries;
2945	struct ftrace_stack *fstack;
2946	struct stack_entry *entry;
2947	int stackidx;
2948
2949	/*
2950	* Add one, for this function and the call to save_stack_trace()
2951	* If regs is set, then these functions will not be in the way.
2952	*/
2953	#ifndef CONFIG_UNWINDER_ORC
2954	if (!regs)
2955	skip++;
2956	#endif
2957
2958	preempt_disable_notrace();
2959
2960	stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2961
2962	/* This should never happen. If it does, yell once and skip */
2963	if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
2964	goto out;
2965
2966	/*
2967	* The above __this_cpu_inc_return() is 'atomic' cpu local. An
2968	* interrupt will either see the value pre increment or post
2969	* increment. If the interrupt happens pre increment it will have
2970	* restored the counter when it returns. We just need a barrier to
2971	* keep gcc from moving things around.
2972	*/
2973	barrier();
2974
2975	fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2976	size = ARRAY_SIZE(fstack->calls);
2977
2978	if (regs) {
2979	nr_entries = stack_trace_save_regs(regs, fstack->calls,
2980	size, skip);
2981	} else {
2982	nr_entries = stack_trace_save(fstack->calls, size, skip);
2983	}
2984
2985	#ifdef CONFIG_DYNAMIC_FTRACE
2986	/* Mark entry of stack trace as trampoline code */
2987	if (tr->ops && tr->ops->trampoline) {
2988	unsigned long tramp_start = tr->ops->trampoline;
2989	unsigned long tramp_end = tramp_start + tr->ops->trampoline_size;
2990	unsigned long *calls = fstack->calls;
2991
2992	for (int i = 0; i < nr_entries; i++) {
2993	if (calls[i] >= tramp_start && calls[i] < tramp_end)
2994	calls[i] = FTRACE_TRAMPOLINE_MARKER;
2995	}
2996	}
2997	#endif
2998
2999	event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
3000	struct_size(entry, caller, nr_entries),
3001	trace_ctx);
3002	if (!event)
3003	goto out;
3004	entry = ring_buffer_event_data(event);
3005
3006	entry->size = nr_entries;
3007	memcpy(&entry->caller, fstack->calls,
3008	flex_array_size(entry, caller, nr_entries));
3009
3010	__buffer_unlock_commit(buffer, event);
3011
3012	out:
3013	/* Again, don't let gcc optimize things here */
3014	barrier();
3015	__this_cpu_dec(ftrace_stack_reserve);
3016	preempt_enable_notrace();
3017
3018	}
3019
3020	static inline void ftrace_trace_stack(struct trace_array *tr,
3021	struct trace_buffer *buffer,
3022	unsigned int trace_ctx,
3023	int skip, struct pt_regs *regs)
3024	{
3025	if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
3026	return;
3027
3028	__ftrace_trace_stack(tr, buffer, trace_ctx, skip, regs);
3029	}
3030
3031	void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
3032	int skip)
3033	{
3034	struct trace_buffer *buffer = tr->array_buffer.buffer;
3035
3036	if (rcu_is_watching()) {
3037	__ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL);
3038	return;
3039	}
3040
3041	if (WARN_ON_ONCE(IS_ENABLED(CONFIG_GENERIC_ENTRY)))
3042	return;
3043
3044	/*
3045	* When an NMI triggers, RCU is enabled via ct_nmi_enter(),
3046	* but if the above rcu_is_watching() failed, then the NMI
3047	* triggered someplace critical, and ct_irq_enter() should
3048	* not be called from NMI.
3049	*/
3050	if (unlikely(in_nmi()))
3051	return;
3052
3053	ct_irq_enter_irqson();
3054	__ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL);
3055	ct_irq_exit_irqson();
3056	}
3057
3058	/**
3059	* trace_dump_stack - record a stack back trace in the trace buffer
3060	* @skip: Number of functions to skip (helper handlers)
3061	*/
3062	void trace_dump_stack(int skip)
3063	{
3064	if (tracing_disabled \|\| tracing_selftest_running)
3065	return;
3066
3067	#ifndef CONFIG_UNWINDER_ORC
3068	/* Skip 1 to skip this function. */
3069	skip++;
3070	#endif
3071	__ftrace_trace_stack(printk_trace, printk_trace->array_buffer.buffer,
3072	tracing_gen_ctx(), skip, NULL);
3073	}
3074	EXPORT_SYMBOL_GPL(trace_dump_stack);
3075
3076	#ifdef CONFIG_USER_STACKTRACE_SUPPORT
3077	static DEFINE_PER_CPU(int, user_stack_count);
3078
3079	static void
3080	ftrace_trace_userstack(struct trace_array *tr,
3081	struct trace_buffer *buffer, unsigned int trace_ctx)
3082	{
3083	struct ring_buffer_event *event;
3084	struct userstack_entry *entry;
3085
3086	if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
3087	return;
3088
3089	/*
3090	* NMIs can not handle page faults, even with fix ups.
3091	* The save user stack can (and often does) fault.
3092	*/
3093	if (unlikely(in_nmi()))
3094	return;
3095
3096	/*
3097	* prevent recursion, since the user stack tracing may
3098	* trigger other kernel events.
3099	*/
3100	preempt_disable();
3101	if (__this_cpu_read(user_stack_count))
3102	goto out;
3103
3104	__this_cpu_inc(user_stack_count);
3105
3106	event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3107	sizeof(*entry), trace_ctx);
3108	if (!event)
3109	goto out_drop_count;
3110	entry = ring_buffer_event_data(event);
3111
3112	entry->tgid = current->tgid;
3113	memset(&entry->caller, 0, sizeof(entry->caller));
3114
3115	stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3116	__buffer_unlock_commit(buffer, event);
3117
3118	out_drop_count:
3119	__this_cpu_dec(user_stack_count);
3120	out:
3121	preempt_enable();
3122	}
3123	#else /* CONFIG_USER_STACKTRACE_SUPPORT */
3124	static void ftrace_trace_userstack(struct trace_array *tr,
3125	struct trace_buffer *buffer,
3126	unsigned int trace_ctx)
3127	{
3128	}
3129	#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3130
3131	#endif /* CONFIG_STACKTRACE */
3132
3133	static inline void
3134	func_repeats_set_delta_ts(struct func_repeats_entry *entry,
3135	unsigned long long delta)
3136	{
3137	entry->bottom_delta_ts = delta & U32_MAX;
3138	entry->top_delta_ts = (delta >> 32);
3139	}
3140
3141	void trace_last_func_repeats(struct trace_array *tr,
3142	struct trace_func_repeats *last_info,
3143	unsigned int trace_ctx)
3144	{
3145	struct trace_buffer *buffer = tr->array_buffer.buffer;
3146	struct func_repeats_entry *entry;
3147	struct ring_buffer_event *event;
3148	u64 delta;
3149
3150	event = __trace_buffer_lock_reserve(buffer, TRACE_FUNC_REPEATS,
3151	sizeof(*entry), trace_ctx);
3152	if (!event)
3153	return;
3154
3155	delta = ring_buffer_event_time_stamp(buffer, event) -
3156	last_info->ts_last_call;
3157
3158	entry = ring_buffer_event_data(event);
3159	entry->ip = last_info->ip;
3160	entry->parent_ip = last_info->parent_ip;
3161	entry->count = last_info->count;
3162	func_repeats_set_delta_ts(entry, delta);
3163
3164	__buffer_unlock_commit(buffer, event);
3165	}
3166
3167	/* created for use with alloc_percpu */
3168	struct trace_buffer_struct {
3169	int nesting;
3170	char buffer[4][TRACE_BUF_SIZE];
3171	};
3172
3173	static struct trace_buffer_struct __percpu *trace_percpu_buffer;
3174
3175	/*
3176	* This allows for lockless recording. If we're nested too deeply, then
3177	* this returns NULL.
3178	*/
3179	static char *get_trace_buf(void)
3180	{
3181	struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3182
3183	if (!trace_percpu_buffer \|\| buffer->nesting >= 4)
3184	return NULL;
3185
3186	buffer->nesting++;
3187
3188	/* Interrupts must see nesting incremented before we use the buffer */
3189	barrier();
3190	return &buffer->buffer[buffer->nesting - 1][0];
3191	}
3192
3193	static void put_trace_buf(void)
3194	{
3195	/* Don't let the decrement of nesting leak before this */
3196	barrier();
3197	this_cpu_dec(trace_percpu_buffer->nesting);
3198	}
3199
3200	static int alloc_percpu_trace_buffer(void)
3201	{
3202	struct trace_buffer_struct __percpu *buffers;
3203
3204	if (trace_percpu_buffer)
3205	return 0;
3206
3207	buffers = alloc_percpu(struct trace_buffer_struct);
3208	if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3209	return -ENOMEM;
3210
3211	trace_percpu_buffer = buffers;
3212	return 0;
3213	}
3214
3215	static int buffers_allocated;
3216
3217	void trace_printk_init_buffers(void)
3218	{
3219	if (buffers_allocated)
3220	return;
3221
3222	if (alloc_percpu_trace_buffer())
3223	return;
3224
3225	/* trace_printk() is for debug use only. Don't use it in production. */
3226
3227	pr_warn("\n");
3228	pr_warn("**********************************************************\n");
3229	pr_warn(" NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE \n");
3230	pr_warn(" \n");
3231	pr_warn(" trace_printk() being used. Allocating extra memory. \n");
3232	pr_warn(" \n");
3233	pr_warn(" This means that this is a DEBUG kernel and it is \n");
3234	pr_warn(" unsafe for production use. \n");
3235	pr_warn(" \n");
3236	pr_warn(" If you see this message and you are not debugging \n");
3237	pr_warn(" the kernel, report this immediately to your vendor! \n");
3238	pr_warn(" \n");
3239	pr_warn(" NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE \n");
3240	pr_warn("**********************************************************\n");
3241
3242	/* Expand the buffers to set size */
3243	tracing_update_buffers(&global_trace);
3244
3245	buffers_allocated = 1;
3246
3247	/*
3248	* trace_printk_init_buffers() can be called by modules.
3249	* If that happens, then we need to start cmdline recording
3250	* directly here. If the global_trace.buffer is already
3251	* allocated here, then this was called by module code.
3252	*/
3253	if (global_trace.array_buffer.buffer)
3254	tracing_start_cmdline_record();
3255	}
3256	EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3257
3258	void trace_printk_start_comm(void)
3259	{
3260	/* Start tracing comms if trace printk is set */
3261	if (!buffers_allocated)
3262	return;
3263	tracing_start_cmdline_record();
3264	}
3265
3266	static void trace_printk_start_stop_comm(int enabled)
3267	{
3268	if (!buffers_allocated)
3269	return;
3270
3271	if (enabled)
3272	tracing_start_cmdline_record();
3273	else
3274	tracing_stop_cmdline_record();
3275	}
3276
3277	/**
3278	* trace_vbprintk - write binary msg to tracing buffer
3279	* @ip: The address of the caller
3280	* @fmt: The string format to write to the buffer
3281	* @args: Arguments for @fmt
3282	*/
3283	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3284	{
3285	struct ring_buffer_event *event;
3286	struct trace_buffer *buffer;
3287	struct trace_array *tr = READ_ONCE(printk_trace);
3288	struct bprint_entry *entry;
3289	unsigned int trace_ctx;
3290	char *tbuffer;
3291	int len = 0, size;
3292
3293	if (!printk_binsafe(tr))
3294	return trace_vprintk(ip, fmt, args);
3295
3296	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
3297	return 0;
3298
3299	/* Don't pollute graph traces with trace_vprintk internals */
3300	pause_graph_tracing();
3301
3302	trace_ctx = tracing_gen_ctx();
3303	preempt_disable_notrace();
3304
3305	tbuffer = get_trace_buf();
3306	if (!tbuffer) {
3307	len = 0;
3308	goto out_nobuffer;
3309	}
3310
3311	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3312
3313	if (len > TRACE_BUF_SIZE/sizeof(int) \|\| len < 0)
3314	goto out_put;
3315
3316	size = sizeof(entry) + sizeof(u32) len;
3317	buffer = tr->array_buffer.buffer;
3318	ring_buffer_nest_start(buffer);
3319	event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3320	trace_ctx);
3321	if (!event)
3322	goto out;
3323	entry = ring_buffer_event_data(event);
3324	entry->ip = ip;
3325	entry->fmt = fmt;
3326
3327	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3328	__buffer_unlock_commit(buffer, event);
3329	ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL);
3330
3331	out:
3332	ring_buffer_nest_end(buffer);
3333	out_put:
3334	put_trace_buf();
3335
3336	out_nobuffer:
3337	preempt_enable_notrace();
3338	unpause_graph_tracing();
3339
3340	return len;
3341	}
3342	EXPORT_SYMBOL_GPL(trace_vbprintk);
3343
3344	__printf(3, 0)
3345	static int
3346	__trace_array_vprintk(struct trace_buffer *buffer,
3347	unsigned long ip, const char *fmt, va_list args)
3348	{
3349	struct ring_buffer_event *event;
3350	int len = 0, size;
3351	struct print_entry *entry;
3352	unsigned int trace_ctx;
3353	char *tbuffer;
3354
3355	if (tracing_disabled)
3356	return 0;
3357
3358	/* Don't pollute graph traces with trace_vprintk internals */
3359	pause_graph_tracing();
3360
3361	trace_ctx = tracing_gen_ctx();
3362	preempt_disable_notrace();
3363
3364
3365	tbuffer = get_trace_buf();
3366	if (!tbuffer) {
3367	len = 0;
3368	goto out_nobuffer;
3369	}
3370
3371	len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3372
3373	size = sizeof(*entry) + len + 1;
3374	ring_buffer_nest_start(buffer);
3375	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3376	trace_ctx);
3377	if (!event)
3378	goto out;
3379	entry = ring_buffer_event_data(event);
3380	entry->ip = ip;
3381
3382	memcpy(&entry->buf, tbuffer, len + 1);
3383	__buffer_unlock_commit(buffer, event);
3384	ftrace_trace_stack(printk_trace, buffer, trace_ctx, 6, NULL);
3385
3386	out:
3387	ring_buffer_nest_end(buffer);
3388	put_trace_buf();
3389
3390	out_nobuffer:
3391	preempt_enable_notrace();
3392	unpause_graph_tracing();
3393
3394	return len;
3395	}
3396
3397	__printf(3, 0)
3398	int trace_array_vprintk(struct trace_array *tr,
3399	unsigned long ip, const char *fmt, va_list args)
3400	{
3401	if (tracing_selftest_running && tr == &global_trace)
3402	return 0;
3403
3404	return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3405	}
3406
3407	/**
3408	* trace_array_printk - Print a message to a specific instance
3409	* @tr: The instance trace_array descriptor
3410	* @ip: The instruction pointer that this is called from.
3411	* @fmt: The format to print (printf format)
3412	*
3413	* If a subsystem sets up its own instance, they have the right to
3414	* printk strings into their tracing instance buffer using this
3415	* function. Note, this function will not write into the top level
3416	* buffer (use trace_printk() for that), as writing into the top level
3417	* buffer should only have events that can be individually disabled.
3418	* trace_printk() is only used for debugging a kernel, and should not
3419	* be ever incorporated in normal use.
3420	*
3421	* trace_array_printk() can be used, as it will not add noise to the
3422	* top level tracing buffer.
3423	*
3424	* Note, trace_array_init_printk() must be called on @tr before this
3425	* can be used.
3426	*/
3427	__printf(3, 0)
3428	int trace_array_printk(struct trace_array *tr,
3429	unsigned long ip, const char *fmt, ...)
3430	{
3431	int ret;
3432	va_list ap;
3433
3434	if (!tr)
3435	return -ENOENT;
3436
3437	/* This is only allowed for created instances */
3438	if (tr == &global_trace)
3439	return 0;
3440
3441	if (!(tr->trace_flags & TRACE_ITER_PRINTK))
3442	return 0;
3443
3444	va_start(ap, fmt);
3445	ret = trace_array_vprintk(tr, ip, fmt, ap);
3446	va_end(ap);
3447	return ret;
3448	}
3449	EXPORT_SYMBOL_GPL(trace_array_printk);
3450
3451	/**
3452	* trace_array_init_printk - Initialize buffers for trace_array_printk()
3453	* @tr: The trace array to initialize the buffers for
3454	*
3455	* As trace_array_printk() only writes into instances, they are OK to
3456	* have in the kernel (unlike trace_printk()). This needs to be called
3457	* before trace_array_printk() can be used on a trace_array.
3458	*/
3459	int trace_array_init_printk(struct trace_array *tr)
3460	{
3461	if (!tr)
3462	return -ENOENT;
3463
3464	/* This is only allowed for created instances */
3465	if (tr == &global_trace)
3466	return -EINVAL;
3467
3468	return alloc_percpu_trace_buffer();
3469	}
3470	EXPORT_SYMBOL_GPL(trace_array_init_printk);
3471
3472	__printf(3, 4)
3473	int trace_array_printk_buf(struct trace_buffer *buffer,
3474	unsigned long ip, const char *fmt, ...)
3475	{
3476	int ret;
3477	va_list ap;
3478
3479	if (!(printk_trace->trace_flags & TRACE_ITER_PRINTK))
3480	return 0;
3481
3482	va_start(ap, fmt);
3483	ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3484	va_end(ap);
3485	return ret;
3486	}
3487
3488	__printf(2, 0)
3489	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3490	{
3491	return trace_array_vprintk(printk_trace, ip, fmt, args);
3492	}
3493	EXPORT_SYMBOL_GPL(trace_vprintk);
3494
3495	static void trace_iterator_increment(struct trace_iterator *iter)
3496	{
3497	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3498
3499	iter->idx++;
3500	if (buf_iter)
3501	ring_buffer_iter_advance(buf_iter);
3502	}
3503
3504	static struct trace_entry *
3505	peek_next_entry(struct trace_iterator iter, int cpu, u64 ts,
3506	unsigned long *lost_events)
3507	{
3508	struct ring_buffer_event *event;
3509	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3510
3511	if (buf_iter) {
3512	event = ring_buffer_iter_peek(buf_iter, ts);
3513	if (lost_events)
3514	*lost_events = ring_buffer_iter_dropped(buf_iter) ?
3515	(unsigned long)-1 : 0;
3516	} else {
3517	event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3518	lost_events);
3519	}
3520
3521	if (event) {
3522	iter->ent_size = ring_buffer_event_length(event);
3523	return ring_buffer_event_data(event);
3524	}
3525	iter->ent_size = 0;
3526	return NULL;
3527	}
3528
3529	static struct trace_entry *
3530	__find_next_entry(struct trace_iterator iter, int ent_cpu,
3531	unsigned long missing_events, u64 ent_ts)
3532	{
3533	struct trace_buffer *buffer = iter->array_buffer->buffer;
3534	struct trace_entry ent, next = NULL;
3535	unsigned long lost_events = 0, next_lost = 0;
3536	int cpu_file = iter->cpu_file;
3537	u64 next_ts = 0, ts;
3538	int next_cpu = -1;
3539	int next_size = 0;
3540	int cpu;
3541
3542	/*
3543	* If we are in a per_cpu trace file, don't bother by iterating over
3544	* all cpu and peek directly.
3545	*/
3546	if (cpu_file > RING_BUFFER_ALL_CPUS) {
3547	if (ring_buffer_empty_cpu(buffer, cpu_file))
3548	return NULL;
3549	ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3550	if (ent_cpu)
3551	*ent_cpu = cpu_file;
3552
3553	return ent;
3554	}
3555
3556	for_each_tracing_cpu(cpu) {
3557
3558	if (ring_buffer_empty_cpu(buffer, cpu))
3559	continue;
3560
3561	ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3562
3563	/*
3564	* Pick the entry with the smallest timestamp:
3565	*/
3566	if (ent && (!next \|\| ts < next_ts)) {
3567	next = ent;
3568	next_cpu = cpu;
3569	next_ts = ts;
3570	next_lost = lost_events;
3571	next_size = iter->ent_size;
3572	}
3573	}
3574
3575	iter->ent_size = next_size;
3576
3577	if (ent_cpu)
3578	*ent_cpu = next_cpu;
3579
3580	if (ent_ts)
3581	*ent_ts = next_ts;
3582
3583	if (missing_events)
3584	*missing_events = next_lost;
3585
3586	return next;
3587	}
3588
3589	#define STATIC_FMT_BUF_SIZE 128
3590	static char static_fmt_buf[STATIC_FMT_BUF_SIZE];
3591
3592	char trace_iter_expand_format(struct trace_iterator iter)
3593	{
3594	char *tmp;
3595
3596	/*
3597	* iter->tr is NULL when used with tp_printk, which makes
3598	* this get called where it is not safe to call krealloc().
3599	*/
3600	if (!iter->tr \|\| iter->fmt == static_fmt_buf)
3601	return NULL;
3602
3603	tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE,
3604	GFP_KERNEL);
3605	if (tmp) {
3606	iter->fmt_size += STATIC_FMT_BUF_SIZE;
3607	iter->fmt = tmp;
3608	}
3609
3610	return tmp;
3611	}
3612
3613	/* Returns true if the string is safe to dereference from an event */
3614	static bool trace_safe_str(struct trace_iterator iter, const char str)
3615	{
3616	unsigned long addr = (unsigned long)str;
3617	struct trace_event *trace_event;
3618	struct trace_event_call *event;
3619
3620	/* OK if part of the event data */
3621	if ((addr >= (unsigned long)iter->ent) &&
3622	(addr < (unsigned long)iter->ent + iter->ent_size))
3623	return true;
3624
3625	/* OK if part of the temp seq buffer */
3626	if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&
3627	(addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE))
3628	return true;
3629
3630	/* Core rodata can not be freed */
3631	if (is_kernel_rodata(addr))
3632	return true;
3633
3634	if (trace_is_tracepoint_string(str))
3635	return true;
3636
3637	/*
3638	* Now this could be a module event, referencing core module
3639	* data, which is OK.
3640	*/
3641	if (!iter->ent)
3642	return false;
3643
3644	trace_event = ftrace_find_event(iter->ent->type);
3645	if (!trace_event)
3646	return false;
3647
3648	event = container_of(trace_event, struct trace_event_call, event);
3649	if ((event->flags & TRACE_EVENT_FL_DYNAMIC) \|\| !event->module)
3650	return false;
3651
3652	/* Would rather have rodata, but this will suffice */
3653	if (within_module_core(addr, event->module))
3654	return true;
3655
3656	return false;
3657	}
3658
3659	/**
3660	* ignore_event - Check dereferenced fields while writing to the seq buffer
3661	* @iter: The iterator that holds the seq buffer and the event being printed
3662	*
3663	* At boot up, test_event_printk() will flag any event that dereferences
3664	* a string with "%s" that does exist in the ring buffer. It may still
3665	* be valid, as the string may point to a static string in the kernel
3666	* rodata that never gets freed. But if the string pointer is pointing
3667	* to something that was allocated, there's a chance that it can be freed
3668	* by the time the user reads the trace. This would cause a bad memory
3669	* access by the kernel and possibly crash the system.
3670	*
3671	* This function will check if the event has any fields flagged as needing
3672	* to be checked at runtime and perform those checks.
3673	*
3674	* If it is found that a field is unsafe, it will write into the @iter->seq
3675	* a message stating what was found to be unsafe.
3676	*
3677	* @return: true if the event is unsafe and should be ignored,
3678	* false otherwise.
3679	*/
3680	bool ignore_event(struct trace_iterator *iter)
3681	{
3682	struct ftrace_event_field *field;
3683	struct trace_event *trace_event;
3684	struct trace_event_call *event;
3685	struct list_head *head;
3686	struct trace_seq *seq;
3687	const void *ptr;
3688
3689	trace_event = ftrace_find_event(iter->ent->type);
3690
3691	seq = &iter->seq;
3692
3693	if (!trace_event) {
3694	trace_seq_printf(seq, "EVENT ID %d NOT FOUND?\n", iter->ent->type);
3695	return true;
3696	}
3697
3698	event = container_of(trace_event, struct trace_event_call, event);
3699	if (!(event->flags & TRACE_EVENT_FL_TEST_STR))
3700	return false;
3701
3702	head = trace_get_fields(event);
3703	if (!head) {
3704	trace_seq_printf(seq, "FIELDS FOR EVENT '%s' NOT FOUND?\n",
3705	trace_event_name(event));
3706	return true;
3707	}
3708
3709	/* Offsets are from the iter->ent that points to the raw event */
3710	ptr = iter->ent;
3711
3712	list_for_each_entry(field, head, link) {
3713	const char *str;
3714	bool good;
3715
3716	if (!field->needs_test)
3717	continue;
3718
3719	str = (const char *)(ptr + field->offset);
3720
3721	good = trace_safe_str(iter, str);
3722
3723	/*
3724	* If you hit this warning, it is likely that the
3725	* trace event in question used %s on a string that
3726	* was saved at the time of the event, but may not be
3727	* around when the trace is read. Use __string(),
3728	* __assign_str() and __get_str() helpers in the TRACE_EVENT()
3729	* instead. See samples/trace_events/trace-events-sample.h
3730	* for reference.
3731	*/
3732	if (WARN_ONCE(!good, "event '%s' has unsafe pointer field '%s'",
3733	trace_event_name(event), field->name)) {
3734	trace_seq_printf(seq, "EVENT %s: HAS UNSAFE POINTER FIELD '%s'\n",
3735	trace_event_name(event), field->name);
3736	return true;
3737	}
3738	}
3739	return false;
3740	}
3741
3742	const char trace_event_format(struct trace_iterator iter, const char *fmt)
3743	{
3744	const char p, new_fmt;
3745	char *q;
3746
3747	if (WARN_ON_ONCE(!fmt))
3748	return fmt;
3749
3750	if (!iter->tr \|\| iter->tr->trace_flags & TRACE_ITER_HASH_PTR)
3751	return fmt;
3752
3753	p = fmt;
3754	new_fmt = q = iter->fmt;
3755	while (*p) {
3756	if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {
3757	if (!trace_iter_expand_format(iter))
3758	return fmt;
3759
3760	q += iter->fmt - new_fmt;
3761	new_fmt = iter->fmt;
3762	}
3763
3764	q++ = p++;
3765
3766	/* Replace %p with %px */
3767	if (p[-1] == '%') {
3768	if (p[0] == '%') {
3769	q++ = p++;
3770	} else if (p[0] == 'p' && !isalnum(p[1])) {
3771	q++ = p++;
3772	*q++ = 'x';
3773	}
3774	}
3775	}
3776	*q = '\0';
3777
3778	return new_fmt;
3779	}
3780
3781	#define STATIC_TEMP_BUF_SIZE 128
3782	static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
3783
3784	/* Find the next real entry, without updating the iterator itself */
3785	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
3786	int ent_cpu, u64 ent_ts)
3787	{
3788	/* __find_next_entry will reset ent_size */
3789	int ent_size = iter->ent_size;
3790	struct trace_entry *entry;
3791
3792	/*
3793	* If called from ftrace_dump(), then the iter->temp buffer
3794	* will be the static_temp_buf and not created from kmalloc.
3795	* If the entry size is greater than the buffer, we can
3796	* not save it. Just return NULL in that case. This is only
3797	* used to add markers when two consecutive events' time
3798	* stamps have a large delta. See trace_print_lat_context()
3799	*/
3800	if (iter->temp == static_temp_buf &&
3801	STATIC_TEMP_BUF_SIZE < ent_size)
3802	return NULL;
3803
3804	/*
3805	* The __find_next_entry() may call peek_next_entry(), which may
3806	* call ring_buffer_peek() that may make the contents of iter->ent
3807	* undefined. Need to copy iter->ent now.
3808	*/
3809	if (iter->ent && iter->ent != iter->temp) {
3810	if ((!iter->temp \|\| iter->temp_size < iter->ent_size) &&
3811	!WARN_ON_ONCE(iter->temp == static_temp_buf)) {
3812	void *temp;
3813	temp = kmalloc(iter->ent_size, GFP_KERNEL);
3814	if (!temp)
3815	return NULL;
3816	kfree(iter->temp);
3817	iter->temp = temp;
3818	iter->temp_size = iter->ent_size;
3819	}
3820	memcpy(iter->temp, iter->ent, iter->ent_size);
3821	iter->ent = iter->temp;
3822	}
3823	entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3824	/* Put back the original ent_size */
3825	iter->ent_size = ent_size;
3826
3827	return entry;
3828	}
3829
3830	/* Find the next real entry, and increment the iterator to the next entry */
3831	void trace_find_next_entry_inc(struct trace_iterator iter)
3832	{
3833	iter->ent = __find_next_entry(iter, &iter->cpu,
3834	&iter->lost_events, &iter->ts);
3835
3836	if (iter->ent)
3837	trace_iterator_increment(iter);
3838
3839	return iter->ent ? iter : NULL;
3840	}
3841
3842	static void trace_consume(struct trace_iterator *iter)
3843	{
3844	ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
3845	&iter->lost_events);
3846	}
3847
3848	static void s_next(struct seq_file m, void v, loff_t pos)
3849	{
3850	struct trace_iterator *iter = m->private;
3851	int i = (int)*pos;
3852	void *ent;
3853
3854	WARN_ON_ONCE(iter->leftover);
3855
3856	(*pos)++;
3857
3858	/* can't go backwards */
3859	if (iter->idx > i)
3860	return NULL;
3861
3862	if (iter->idx < 0)
3863	ent = trace_find_next_entry_inc(iter);
3864	else
3865	ent = iter;
3866
3867	while (ent && iter->idx < i)
3868	ent = trace_find_next_entry_inc(iter);
3869
3870	iter->pos = *pos;
3871
3872	return ent;
3873	}
3874
3875	void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3876	{
3877	struct ring_buffer_iter *buf_iter;
3878	unsigned long entries = 0;
3879	u64 ts;
3880
3881	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
3882
3883	buf_iter = trace_buffer_iter(iter, cpu);
3884	if (!buf_iter)
3885	return;
3886
3887	ring_buffer_iter_reset(buf_iter);
3888
3889	/*
3890	* We could have the case with the max latency tracers
3891	* that a reset never took place on a cpu. This is evident
3892	* by the timestamp being before the start of the buffer.
3893	*/
3894	while (ring_buffer_iter_peek(buf_iter, &ts)) {
3895	if (ts >= iter->array_buffer->time_start)
3896	break;
3897	entries++;
3898	ring_buffer_iter_advance(buf_iter);
3899	/* This could be a big loop */
3900	cond_resched();
3901	}
3902
3903	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
3904	}
3905
3906	/*
3907	* The current tracer is copied to avoid a global locking
3908	* all around.
3909	*/
3910	static void s_start(struct seq_file m, loff_t *pos)
3911	{
3912	struct trace_iterator *iter = m->private;
3913	struct trace_array *tr = iter->tr;
3914	int cpu_file = iter->cpu_file;
3915	void *p = NULL;
3916	loff_t l = 0;
3917	int cpu;
3918
3919	mutex_lock(&trace_types_lock);
3920	if (unlikely(tr->current_trace != iter->trace)) {
3921	/* Close iter->trace before switching to the new current tracer */
3922	if (iter->trace->close)
3923	iter->trace->close(iter);
3924	iter->trace = tr->current_trace;
3925	/* Reopen the new current tracer */
3926	if (iter->trace->open)
3927	iter->trace->open(iter);
3928	}
3929	mutex_unlock(&trace_types_lock);
3930
3931	#ifdef CONFIG_TRACER_MAX_TRACE
3932	if (iter->snapshot && iter->trace->use_max_tr)
3933	return ERR_PTR(-EBUSY);
3934	#endif
3935
3936	if (*pos != iter->pos) {
3937	iter->ent = NULL;
3938	iter->cpu = 0;
3939	iter->idx = -1;
3940
3941	if (cpu_file == RING_BUFFER_ALL_CPUS) {
3942	for_each_tracing_cpu(cpu)
3943	tracing_iter_reset(iter, cpu);
3944	} else
3945	tracing_iter_reset(iter, cpu_file);
3946
3947	iter->leftover = 0;
3948	for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3949	;
3950
3951	} else {
3952	/*
3953	* If we overflowed the seq_file before, then we want
3954	* to just reuse the trace_seq buffer again.
3955	*/
3956	if (iter->leftover)
3957	p = iter;
3958	else {
3959	l = *pos - 1;
3960	p = s_next(m, p, &l);
3961	}
3962	}
3963
3964	trace_event_read_lock();
3965	trace_access_lock(cpu_file);
3966	return p;
3967	}
3968
3969	static void s_stop(struct seq_file m, void p)
3970	{
3971	struct trace_iterator *iter = m->private;
3972
3973	#ifdef CONFIG_TRACER_MAX_TRACE
3974	if (iter->snapshot && iter->trace->use_max_tr)
3975	return;
3976	#endif
3977
3978	trace_access_unlock(iter->cpu_file);
3979	trace_event_read_unlock();
3980	}
3981
3982	static void
3983	get_total_entries_cpu(struct array_buffer buf, unsigned long total,
3984	unsigned long *entries, int cpu)
3985	{
3986	unsigned long count;
3987
3988	count = ring_buffer_entries_cpu(buf->buffer, cpu);
3989	/*
3990	* If this buffer has skipped entries, then we hold all
3991	* entries for the trace and we need to ignore the
3992	* ones before the time stamp.
3993	*/
3994	if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3995	count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3996	/* total is the same as the entries */
3997	*total = count;
3998	} else
3999	*total = count +
4000	ring_buffer_overrun_cpu(buf->buffer, cpu);
4001	*entries = count;
4002	}
4003
4004	static void
4005	get_total_entries(struct array_buffer *buf,
4006	unsigned long total, unsigned long entries)
4007	{
4008	unsigned long t, e;
4009	int cpu;
4010
4011	*total = 0;
4012	*entries = 0;
4013
4014	for_each_tracing_cpu(cpu) {
4015	get_total_entries_cpu(buf, &t, &e, cpu);
4016	*total += t;
4017	*entries += e;
4018	}
4019	}
4020
4021	unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
4022	{
4023	unsigned long total, entries;
4024
4025	if (!tr)
4026	tr = &global_trace;
4027
4028	get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
4029
4030	return entries;
4031	}
4032
4033	unsigned long trace_total_entries(struct trace_array *tr)
4034	{
4035	unsigned long total, entries;
4036
4037	if (!tr)
4038	tr = &global_trace;
4039
4040	get_total_entries(&tr->array_buffer, &total, &entries);
4041
4042	return entries;
4043	}
4044
4045	static void print_lat_help_header(struct seq_file *m)
4046	{
4047	seq_puts(m, "# _------=> CPU# \n"
4048	"# / _-----=> irqs-off/BH-disabled\n"
4049	"# \| / _----=> need-resched \n"
4050	"# \|\| / _---=> hardirq/softirq \n"
4051	"# \|\|\| / _--=> preempt-depth \n"
4052	"# \|\|\|\| / _-=> migrate-disable \n"
4053	"# \|\|\|\|\| / delay \n"
4054	"# cmd pid \|\|\|\|\|\| time \| caller \n"
4055	"# \\ / \|\|\|\|\|\| \\ \| / \n");
4056	}
4057
4058	static void print_event_info(struct array_buffer buf, struct seq_file m)
4059	{
4060	unsigned long total;
4061	unsigned long entries;
4062
4063	get_total_entries(buf, &total, &entries);
4064	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
4065	entries, total, num_online_cpus());
4066	seq_puts(m, "#\n");
4067	}
4068
4069	static void print_func_help_header(struct array_buffer buf, struct seq_file m,
4070	unsigned int flags)
4071	{
4072	bool tgid = flags & TRACE_ITER_RECORD_TGID;
4073
4074	print_event_info(buf, m);
4075
4076	seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
4077	seq_printf(m, "# \| \| %s \| \| \|\n", tgid ? " \| " : "");
4078	}
4079
4080	static void print_func_help_header_irq(struct array_buffer buf, struct seq_file m,
4081	unsigned int flags)
4082	{
4083	bool tgid = flags & TRACE_ITER_RECORD_TGID;
4084	static const char space[] = " ";
4085	int prec = tgid ? 12 : 2;
4086
4087	print_event_info(buf, m);
4088
4089	seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space);
4090	seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
4091	seq_printf(m, "# %.*s\| / _---=> hardirq/softirq\n", prec, space);
4092	seq_printf(m, "# %.*s\|\| / _--=> preempt-depth\n", prec, space);
4093	seq_printf(m, "# %.*s\|\|\| / _-=> migrate-disable\n", prec, space);
4094	seq_printf(m, "# %.*s\|\|\|\| / delay\n", prec, space);
4095	seq_printf(m, "# TASK-PID %.*s CPU# \|\|\|\|\| TIMESTAMP FUNCTION\n", prec, " TGID ");
4096	seq_printf(m, "# \| \| %.*s \| \|\|\|\|\| \| \|\n", prec, " \| ");
4097	}
4098
4099	void
4100	print_trace_header(struct seq_file m, struct trace_iterator iter)
4101	{
4102	unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
4103	struct array_buffer *buf = iter->array_buffer;
4104	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
4105	struct tracer *type = iter->trace;
4106	unsigned long entries;
4107	unsigned long total;
4108	const char *name = type->name;
4109
4110	get_total_entries(buf, &total, &entries);
4111
4112	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
4113	name, init_utsname()->release);
4114	seq_puts(m, "# -----------------------------------"
4115	"---------------------------------\n");
4116	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d \|"
4117	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
4118	nsecs_to_usecs(data->saved_latency),
4119	entries,
4120	total,
4121	buf->cpu,
4122	preempt_model_none() ? "server" :
4123	preempt_model_voluntary() ? "desktop" :
4124	preempt_model_full() ? "preempt" :
4125	preempt_model_lazy() ? "lazy" :
4126	preempt_model_rt() ? "preempt_rt" :
4127	"unknown",
4128	/* These are reserved for later use */
4129	0, 0, 0, 0);
4130	#ifdef CONFIG_SMP
4131	seq_printf(m, " #P:%d)\n", num_online_cpus());
4132	#else
4133	seq_puts(m, ")\n");
4134	#endif
4135	seq_puts(m, "# -----------------\n");
4136	seq_printf(m, "# \| task: %.16s-%d "
4137	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
4138	data->comm, data->pid,
4139	from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
4140	data->policy, data->rt_priority);
4141	seq_puts(m, "# -----------------\n");
4142
4143	if (data->critical_start) {
4144	seq_puts(m, "# => started at: ");
4145	seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
4146	trace_print_seq(m, &iter->seq);
4147	seq_puts(m, "\n# => ended at: ");
4148	seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
4149	trace_print_seq(m, &iter->seq);
4150	seq_puts(m, "\n#\n");
4151	}
4152
4153	seq_puts(m, "#\n");
4154	}
4155
4156	static void test_cpu_buff_start(struct trace_iterator *iter)
4157	{
4158	struct trace_seq *s = &iter->seq;
4159	struct trace_array *tr = iter->tr;
4160
4161	if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
4162	return;
4163
4164	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
4165	return;
4166
4167	if (cpumask_available(iter->started) &&
4168	cpumask_test_cpu(iter->cpu, iter->started))
4169	return;
4170
4171	if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
4172	return;
4173
4174	if (cpumask_available(iter->started))
4175	cpumask_set_cpu(iter->cpu, iter->started);
4176
4177	/* Don't print started cpu buffer for the first entry of the trace */
4178	if (iter->idx > 1)
4179	trace_seq_printf(s, "##### CPU %u buffer started ####\n",
4180	iter->cpu);
4181	}
4182
4183	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
4184	{
4185	struct trace_array *tr = iter->tr;
4186	struct trace_seq *s = &iter->seq;
4187	unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4188	struct trace_entry *entry;
4189	struct trace_event *event;
4190
4191	entry = iter->ent;
4192
4193	test_cpu_buff_start(iter);
4194
4195	event = ftrace_find_event(entry->type);
4196
4197	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4198	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4199	trace_print_lat_context(iter);
4200	else
4201	trace_print_context(iter);
4202	}
4203
4204	if (trace_seq_has_overflowed(s))
4205	return TRACE_TYPE_PARTIAL_LINE;
4206
4207	if (event) {
4208	if (tr->trace_flags & TRACE_ITER_FIELDS)
4209	return print_event_fields(iter, event);
4210	/*
4211	* For TRACE_EVENT() events, the print_fmt is not
4212	* safe to use if the array has delta offsets
4213	* Force printing via the fields.
4214	*/
4215	if ((tr->text_delta \|\| tr->data_delta) &&
4216	event->type > __TRACE_LAST_TYPE)
4217	return print_event_fields(iter, event);
4218
4219	return event->funcs->trace(iter, sym_flags, event);
4220	}
4221
4222	trace_seq_printf(s, "Unknown type %d\n", entry->type);
4223
4224	return trace_handle_return(s);
4225	}
4226
4227	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
4228	{
4229	struct trace_array *tr = iter->tr;
4230	struct trace_seq *s = &iter->seq;
4231	struct trace_entry *entry;
4232	struct trace_event *event;
4233
4234	entry = iter->ent;
4235
4236	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
4237	trace_seq_printf(s, "%d %d %llu ",
4238	entry->pid, iter->cpu, iter->ts);
4239
4240	if (trace_seq_has_overflowed(s))
4241	return TRACE_TYPE_PARTIAL_LINE;
4242
4243	event = ftrace_find_event(entry->type);
4244	if (event)
4245	return event->funcs->raw(iter, 0, event);
4246
4247	trace_seq_printf(s, "%d ?\n", entry->type);
4248
4249	return trace_handle_return(s);
4250	}
4251
4252	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
4253	{
4254	struct trace_array *tr = iter->tr;
4255	struct trace_seq *s = &iter->seq;
4256	unsigned char newline = '\n';
4257	struct trace_entry *entry;
4258	struct trace_event *event;
4259
4260	entry = iter->ent;
4261
4262	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4263	SEQ_PUT_HEX_FIELD(s, entry->pid);
4264	SEQ_PUT_HEX_FIELD(s, iter->cpu);
4265	SEQ_PUT_HEX_FIELD(s, iter->ts);
4266	if (trace_seq_has_overflowed(s))
4267	return TRACE_TYPE_PARTIAL_LINE;
4268	}
4269
4270	event = ftrace_find_event(entry->type);
4271	if (event) {
4272	enum print_line_t ret = event->funcs->hex(iter, 0, event);
4273	if (ret != TRACE_TYPE_HANDLED)
4274	return ret;
4275	}
4276
4277	SEQ_PUT_FIELD(s, newline);
4278
4279	return trace_handle_return(s);
4280	}
4281
4282	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
4283	{
4284	struct trace_array *tr = iter->tr;
4285	struct trace_seq *s = &iter->seq;
4286	struct trace_entry *entry;
4287	struct trace_event *event;
4288
4289	entry = iter->ent;
4290
4291	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4292	SEQ_PUT_FIELD(s, entry->pid);
4293	SEQ_PUT_FIELD(s, iter->cpu);
4294	SEQ_PUT_FIELD(s, iter->ts);
4295	if (trace_seq_has_overflowed(s))
4296	return TRACE_TYPE_PARTIAL_LINE;
4297	}
4298
4299	event = ftrace_find_event(entry->type);
4300	return event ? event->funcs->binary(iter, 0, event) :
4301	TRACE_TYPE_HANDLED;
4302	}
4303
4304	int trace_empty(struct trace_iterator *iter)
4305	{
4306	struct ring_buffer_iter *buf_iter;
4307	int cpu;
4308
4309	/* If we are looking at one CPU buffer, only check that one */
4310	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4311	cpu = iter->cpu_file;
4312	buf_iter = trace_buffer_iter(iter, cpu);
4313	if (buf_iter) {
4314	if (!ring_buffer_iter_empty(buf_iter))
4315	return 0;
4316	} else {
4317	if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4318	return 0;
4319	}
4320	return 1;
4321	}
4322
4323	for_each_tracing_cpu(cpu) {
4324	buf_iter = trace_buffer_iter(iter, cpu);
4325	if (buf_iter) {
4326	if (!ring_buffer_iter_empty(buf_iter))
4327	return 0;
4328	} else {
4329	if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4330	return 0;
4331	}
4332	}
4333
4334	return 1;
4335	}
4336
4337	/* Called with trace_event_read_lock() held. */
4338	enum print_line_t print_trace_line(struct trace_iterator *iter)
4339	{
4340	struct trace_array *tr = iter->tr;
4341	unsigned long trace_flags = tr->trace_flags;
4342	enum print_line_t ret;
4343
4344	if (iter->lost_events) {
4345	if (iter->lost_events == (unsigned long)-1)
4346	trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
4347	iter->cpu);
4348	else
4349	trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
4350	iter->cpu, iter->lost_events);
4351	if (trace_seq_has_overflowed(&iter->seq))
4352	return TRACE_TYPE_PARTIAL_LINE;
4353	}
4354
4355	if (iter->trace && iter->trace->print_line) {
4356	ret = iter->trace->print_line(iter);
4357	if (ret != TRACE_TYPE_UNHANDLED)
4358	return ret;
4359	}
4360
4361	if (iter->ent->type == TRACE_BPUTS &&
4362	trace_flags & TRACE_ITER_PRINTK &&
4363	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4364	return trace_print_bputs_msg_only(iter);
4365
4366	if (iter->ent->type == TRACE_BPRINT &&
4367	trace_flags & TRACE_ITER_PRINTK &&
4368	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4369	return trace_print_bprintk_msg_only(iter);
4370
4371	if (iter->ent->type == TRACE_PRINT &&
4372	trace_flags & TRACE_ITER_PRINTK &&
4373	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4374	return trace_print_printk_msg_only(iter);
4375
4376	if (trace_flags & TRACE_ITER_BIN)
4377	return print_bin_fmt(iter);
4378
4379	if (trace_flags & TRACE_ITER_HEX)
4380	return print_hex_fmt(iter);
4381
4382	if (trace_flags & TRACE_ITER_RAW)
4383	return print_raw_fmt(iter);
4384
4385	return print_trace_fmt(iter);
4386	}
4387
4388	void trace_latency_header(struct seq_file *m)
4389	{
4390	struct trace_iterator *iter = m->private;
4391	struct trace_array *tr = iter->tr;
4392
4393	/* print nothing if the buffers are empty */
4394	if (trace_empty(iter))
4395	return;
4396
4397	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4398	print_trace_header(m, iter);
4399
4400	if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4401	print_lat_help_header(m);
4402	}
4403
4404	void trace_default_header(struct seq_file *m)
4405	{
4406	struct trace_iterator *iter = m->private;
4407	struct trace_array *tr = iter->tr;
4408	unsigned long trace_flags = tr->trace_flags;
4409
4410	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4411	return;
4412
4413	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4414	/* print nothing if the buffers are empty */
4415	if (trace_empty(iter))
4416	return;
4417	print_trace_header(m, iter);
4418	if (!(trace_flags & TRACE_ITER_VERBOSE))
4419	print_lat_help_header(m);
4420	} else {
4421	if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4422	if (trace_flags & TRACE_ITER_IRQ_INFO)
4423	print_func_help_header_irq(iter->array_buffer,
4424	m, trace_flags);
4425	else
4426	print_func_help_header(iter->array_buffer, m,
4427	trace_flags);
4428	}
4429	}
4430	}
4431
4432	static void test_ftrace_alive(struct seq_file *m)
4433	{
4434	if (!ftrace_is_dead())
4435	return;
4436	seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4437	"# MAY BE MISSING FUNCTION EVENTS\n");
4438	}
4439
4440	#ifdef CONFIG_TRACER_MAX_TRACE
4441	static void show_snapshot_main_help(struct seq_file *m)
4442	{
4443	seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4444	"# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4445	"# Takes a snapshot of the main buffer.\n"
4446	"# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4447	"# (Doesn't have to be '2' works with any number that\n"
4448	"# is not a '0' or '1')\n");
4449	}
4450
4451	static void show_snapshot_percpu_help(struct seq_file *m)
4452	{
4453	seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4454	#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4455	seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4456	"# Takes a snapshot of the main buffer for this cpu.\n");
4457	#else
4458	seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4459	"# Must use main snapshot file to allocate.\n");
4460	#endif
4461	seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4462	"# (Doesn't have to be '2' works with any number that\n"
4463	"# is not a '0' or '1')\n");
4464	}
4465
4466	static void print_snapshot_help(struct seq_file m, struct trace_iterator iter)
4467	{
4468	if (iter->tr->allocated_snapshot)
4469	seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4470	else
4471	seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4472
4473	seq_puts(m, "# Snapshot commands:\n");
4474	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4475	show_snapshot_main_help(m);
4476	else
4477	show_snapshot_percpu_help(m);
4478	}
4479	#else
4480	/* Should never be called */
4481	static inline void print_snapshot_help(struct seq_file m, struct trace_iterator iter) { }
4482	#endif
4483
4484	static int s_show(struct seq_file m, void v)
4485	{
4486	struct trace_iterator *iter = v;
4487	int ret;
4488
4489	if (iter->ent == NULL) {
4490	if (iter->tr) {
4491	seq_printf(m, "# tracer: %s\n", iter->trace->name);
4492	seq_puts(m, "#\n");
4493	test_ftrace_alive(m);
4494	}
4495	if (iter->snapshot && trace_empty(iter))
4496	print_snapshot_help(m, iter);
4497	else if (iter->trace && iter->trace->print_header)
4498	iter->trace->print_header(m);
4499	else
4500	trace_default_header(m);
4501
4502	} else if (iter->leftover) {
4503	/*
4504	* If we filled the seq_file buffer earlier, we
4505	* want to just show it now.
4506	*/
4507	ret = trace_print_seq(m, &iter->seq);
4508
4509	/* ret should this time be zero, but you never know */
4510	iter->leftover = ret;
4511
4512	} else {
4513	ret = print_trace_line(iter);
4514	if (ret == TRACE_TYPE_PARTIAL_LINE) {
4515	iter->seq.full = 0;
4516	trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
4517	}
4518	ret = trace_print_seq(m, &iter->seq);
4519	/*
4520	* If we overflow the seq_file buffer, then it will
4521	* ask us for this data again at start up.
4522	* Use that instead.
4523	* ret is 0 if seq_file write succeeded.
4524	* -1 otherwise.
4525	*/
4526	iter->leftover = ret;
4527	}
4528
4529	return 0;
4530	}
4531
4532	/*
4533	* Should be used after trace_array_get(), trace_types_lock
4534	* ensures that i_cdev was already initialized.
4535	*/
4536	static inline int tracing_get_cpu(struct inode *inode)
4537	{
4538	if (inode->i_cdev) /* See trace_create_cpu_file() */
4539	return (long)inode->i_cdev - 1;
4540	return RING_BUFFER_ALL_CPUS;
4541	}
4542
4543	static const struct seq_operations tracer_seq_ops = {
4544	.start = s_start,
4545	.next = s_next,
4546	.stop = s_stop,
4547	.show = s_show,
4548	};
4549
4550	/*
4551	* Note, as iter itself can be allocated and freed in different
4552	* ways, this function is only used to free its content, and not
4553	* the iterator itself. The only requirement to all the allocations
4554	* is that it must zero all fields (kzalloc), as freeing works with
4555	* ethier allocated content or NULL.
4556	*/
4557	static void free_trace_iter_content(struct trace_iterator *iter)
4558	{
4559	/* The fmt is either NULL, allocated or points to static_fmt_buf */
4560	if (iter->fmt != static_fmt_buf)
4561	kfree(iter->fmt);
4562
4563	kfree(iter->temp);
4564	kfree(iter->buffer_iter);
4565	mutex_destroy(&iter->mutex);
4566	free_cpumask_var(iter->started);
4567	}
4568
4569	static struct trace_iterator *
4570	__tracing_open(struct inode inode, struct file file, bool snapshot)
4571	{
4572	struct trace_array *tr = inode->i_private;
4573	struct trace_iterator *iter;
4574	int cpu;
4575
4576	if (tracing_disabled)
4577	return ERR_PTR(-ENODEV);
4578
4579	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4580	if (!iter)
4581	return ERR_PTR(-ENOMEM);
4582
4583	iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4584	GFP_KERNEL);
4585	if (!iter->buffer_iter)
4586	goto release;
4587
4588	/*
4589	* trace_find_next_entry() may need to save off iter->ent.
4590	* It will place it into the iter->temp buffer. As most
4591	* events are less than 128, allocate a buffer of that size.
4592	* If one is greater, then trace_find_next_entry() will
4593	* allocate a new buffer to adjust for the bigger iter->ent.
4594	* It's not critical if it fails to get allocated here.
4595	*/
4596	iter->temp = kmalloc(128, GFP_KERNEL);
4597	if (iter->temp)
4598	iter->temp_size = 128;
4599
4600	/*
4601	* trace_event_printf() may need to modify given format
4602	* string to replace %p with %px so that it shows real address
4603	* instead of hash value. However, that is only for the event
4604	* tracing, other tracer may not need. Defer the allocation
4605	* until it is needed.
4606	*/
4607	iter->fmt = NULL;
4608	iter->fmt_size = 0;
4609
4610	mutex_lock(&trace_types_lock);
4611	iter->trace = tr->current_trace;
4612
4613	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4614	goto fail;
4615
4616	iter->tr = tr;
4617
4618	#ifdef CONFIG_TRACER_MAX_TRACE
4619	/* Currently only the top directory has a snapshot */
4620	if (tr->current_trace->print_max \|\| snapshot)
4621	iter->array_buffer = &tr->max_buffer;
4622	else
4623	#endif
4624	iter->array_buffer = &tr->array_buffer;
4625	iter->snapshot = snapshot;
4626	iter->pos = -1;
4627	iter->cpu_file = tracing_get_cpu(inode);
4628	mutex_init(&iter->mutex);
4629
4630	/* Notify the tracer early; before we stop tracing. */
4631	if (iter->trace->open)
4632	iter->trace->open(iter);
4633
4634	/* Annotate start of buffers if we had overruns */
4635	if (ring_buffer_overruns(iter->array_buffer->buffer))
4636	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
4637
4638	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
4639	if (trace_clocks[tr->clock_id].in_ns)
4640	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
4641
4642	/*
4643	* If pause-on-trace is enabled, then stop the trace while
4644	* dumping, unless this is the "snapshot" file
4645	*/
4646	if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4647	tracing_stop_tr(tr);
4648
4649	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4650	for_each_tracing_cpu(cpu) {
4651	iter->buffer_iter[cpu] =
4652	ring_buffer_read_prepare(iter->array_buffer->buffer,
4653	cpu, GFP_KERNEL);
4654	}
4655	ring_buffer_read_prepare_sync();
4656	for_each_tracing_cpu(cpu) {
4657	ring_buffer_read_start(iter->buffer_iter[cpu]);
4658	tracing_iter_reset(iter, cpu);
4659	}
4660	} else {
4661	cpu = iter->cpu_file;
4662	iter->buffer_iter[cpu] =
4663	ring_buffer_read_prepare(iter->array_buffer->buffer,
4664	cpu, GFP_KERNEL);
4665	ring_buffer_read_prepare_sync();
4666	ring_buffer_read_start(iter->buffer_iter[cpu]);
4667	tracing_iter_reset(iter, cpu);
4668	}
4669
4670	mutex_unlock(&trace_types_lock);
4671
4672	return iter;
4673
4674	fail:
4675	mutex_unlock(&trace_types_lock);
4676	free_trace_iter_content(iter);
4677	release:
4678	seq_release_private(inode, file);
4679	return ERR_PTR(-ENOMEM);
4680	}
4681
4682	int tracing_open_generic(struct inode inode, struct file filp)
4683	{
4684	int ret;
4685
4686	ret = tracing_check_open_get_tr(NULL);
4687	if (ret)
4688	return ret;
4689
4690	filp->private_data = inode->i_private;
4691	return 0;
4692	}
4693
4694	bool tracing_is_disabled(void)
4695	{
4696	return (tracing_disabled) ? true: false;
4697	}
4698
4699	/*
4700	* Open and update trace_array ref count.
4701	* Must have the current trace_array passed to it.
4702	*/
4703	int tracing_open_generic_tr(struct inode inode, struct file filp)
4704	{
4705	struct trace_array *tr = inode->i_private;
4706	int ret;
4707
4708	ret = tracing_check_open_get_tr(tr);
4709	if (ret)
4710	return ret;
4711
4712	filp->private_data = inode->i_private;
4713
4714	return 0;
4715	}
4716
4717	/*
4718	* The private pointer of the inode is the trace_event_file.
4719	* Update the tr ref count associated to it.
4720	*/
4721	int tracing_open_file_tr(struct inode inode, struct file filp)
4722	{
4723	struct trace_event_file *file = inode->i_private;
4724	int ret;
4725
4726	ret = tracing_check_open_get_tr(file->tr);
4727	if (ret)
4728	return ret;
4729
4730	mutex_lock(&event_mutex);
4731
4732	/* Fail if the file is marked for removal */
4733	if (file->flags & EVENT_FILE_FL_FREED) {
4734	trace_array_put(file->tr);
4735	ret = -ENODEV;
4736	} else {
4737	event_file_get(file);
4738	}
4739
4740	mutex_unlock(&event_mutex);
4741	if (ret)
4742	return ret;
4743
4744	filp->private_data = inode->i_private;
4745
4746	return 0;
4747	}
4748
4749	int tracing_release_file_tr(struct inode inode, struct file filp)
4750	{
4751	struct trace_event_file *file = inode->i_private;
4752
4753	trace_array_put(file->tr);
4754	event_file_put(file);
4755
4756	return 0;
4757	}
4758
4759	int tracing_single_release_file_tr(struct inode inode, struct file filp)
4760	{
4761	tracing_release_file_tr(inode, filp);
4762	return single_release(inode, filp);
4763	}
4764
4765	static int tracing_mark_open(struct inode inode, struct file filp)
4766	{
4767	stream_open(inode, filp);
4768	return tracing_open_generic_tr(inode, filp);
4769	}
4770
4771	static int tracing_release(struct inode inode, struct file file)
4772	{
4773	struct trace_array *tr = inode->i_private;
4774	struct seq_file *m = file->private_data;
4775	struct trace_iterator *iter;
4776	int cpu;
4777
4778	if (!(file->f_mode & FMODE_READ)) {
4779	trace_array_put(tr);
4780	return 0;
4781	}
4782
4783	/* Writes do not use seq_file */
4784	iter = m->private;
4785	mutex_lock(&trace_types_lock);
4786
4787	for_each_tracing_cpu(cpu) {
4788	if (iter->buffer_iter[cpu])
4789	ring_buffer_read_finish(iter->buffer_iter[cpu]);
4790	}
4791
4792	if (iter->trace && iter->trace->close)
4793	iter->trace->close(iter);
4794
4795	if (!iter->snapshot && tr->stop_count)
4796	/* reenable tracing if it was previously enabled */
4797	tracing_start_tr(tr);
4798
4799	__trace_array_put(tr);
4800
4801	mutex_unlock(&trace_types_lock);
4802
4803	free_trace_iter_content(iter);
4804	seq_release_private(inode, file);
4805
4806	return 0;
4807	}
4808
4809	int tracing_release_generic_tr(struct inode inode, struct file file)
4810	{
4811	struct trace_array *tr = inode->i_private;
4812
4813	trace_array_put(tr);
4814	return 0;
4815	}
4816
4817	static int tracing_single_release_tr(struct inode inode, struct file file)
4818	{
4819	struct trace_array *tr = inode->i_private;
4820
4821	trace_array_put(tr);
4822
4823	return single_release(inode, file);
4824	}
4825
4826	static int tracing_open(struct inode inode, struct file file)
4827	{
4828	struct trace_array *tr = inode->i_private;
4829	struct trace_iterator *iter;
4830	int ret;
4831
4832	ret = tracing_check_open_get_tr(tr);
4833	if (ret)
4834	return ret;
4835
4836	/* If this file was open for write, then erase contents */
4837	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4838	int cpu = tracing_get_cpu(inode);
4839	struct array_buffer *trace_buf = &tr->array_buffer;
4840
4841	#ifdef CONFIG_TRACER_MAX_TRACE
4842	if (tr->current_trace->print_max)
4843	trace_buf = &tr->max_buffer;
4844	#endif
4845
4846	if (cpu == RING_BUFFER_ALL_CPUS)
4847	tracing_reset_online_cpus(trace_buf);
4848	else
4849	tracing_reset_cpu(trace_buf, cpu);
4850	}
4851
4852	if (file->f_mode & FMODE_READ) {
4853	iter = __tracing_open(inode, file, false);
4854	if (IS_ERR(iter))
4855	ret = PTR_ERR(iter);
4856	else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4857	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
4858	}
4859
4860	if (ret < 0)
4861	trace_array_put(tr);
4862
4863	return ret;
4864	}
4865
4866	/*
4867	* Some tracers are not suitable for instance buffers.
4868	* A tracer is always available for the global array (toplevel)
4869	* or if it explicitly states that it is.
4870	*/
4871	static bool
4872	trace_ok_for_array(struct tracer t, struct trace_array tr)
4873	{
4874	#ifdef CONFIG_TRACER_SNAPSHOT
4875	/* arrays with mapped buffer range do not have snapshots */
4876	if (tr->range_addr_start && t->use_max_tr)
4877	return false;
4878	#endif
4879	return (tr->flags & TRACE_ARRAY_FL_GLOBAL) \|\| t->allow_instances;
4880	}
4881
4882	/* Find the next tracer that this trace array may use */
4883	static struct tracer *
4884	get_tracer_for_array(struct trace_array tr, struct tracer t)
4885	{
4886	while (t && !trace_ok_for_array(t, tr))
4887	t = t->next;
4888
4889	return t;
4890	}
4891
4892	static void *
4893	t_next(struct seq_file m, void v, loff_t *pos)
4894	{
4895	struct trace_array *tr = m->private;
4896	struct tracer *t = v;
4897
4898	(*pos)++;
4899
4900	if (t)
4901	t = get_tracer_for_array(tr, t->next);
4902
4903	return t;
4904	}
4905
4906	static void t_start(struct seq_file m, loff_t *pos)
4907	{
4908	struct trace_array *tr = m->private;
4909	struct tracer *t;
4910	loff_t l = 0;
4911
4912	mutex_lock(&trace_types_lock);
4913
4914	t = get_tracer_for_array(tr, trace_types);
4915	for (; t && l < *pos; t = t_next(m, t, &l))
4916	;
4917
4918	return t;
4919	}
4920
4921	static void t_stop(struct seq_file m, void p)
4922	{
4923	mutex_unlock(&trace_types_lock);
4924	}
4925
4926	static int t_show(struct seq_file m, void v)
4927	{
4928	struct tracer *t = v;
4929
4930	if (!t)
4931	return 0;
4932
4933	seq_puts(m, t->name);
4934	if (t->next)
4935	seq_putc(m, ' ');
4936	else
4937	seq_putc(m, '\n');
4938
4939	return 0;
4940	}
4941
4942	static const struct seq_operations show_traces_seq_ops = {
4943	.start = t_start,
4944	.next = t_next,
4945	.stop = t_stop,
4946	.show = t_show,
4947	};
4948
4949	static int show_traces_open(struct inode inode, struct file file)
4950	{
4951	struct trace_array *tr = inode->i_private;
4952	struct seq_file *m;
4953	int ret;
4954
4955	ret = tracing_check_open_get_tr(tr);
4956	if (ret)
4957	return ret;
4958
4959	ret = seq_open(file, &show_traces_seq_ops);
4960	if (ret) {
4961	trace_array_put(tr);
4962	return ret;
4963	}
4964
4965	m = file->private_data;
4966	m->private = tr;
4967
4968	return 0;
4969	}
4970
4971	static int tracing_seq_release(struct inode inode, struct file file)
4972	{
4973	struct trace_array *tr = inode->i_private;
4974
4975	trace_array_put(tr);
4976	return seq_release(inode, file);
4977	}
4978
4979	static ssize_t
4980	tracing_write_stub(struct file filp, const char __user ubuf,
4981	size_t count, loff_t *ppos)
4982	{
4983	return count;
4984	}
4985
4986	loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4987	{
4988	int ret;
4989
4990	if (file->f_mode & FMODE_READ)
4991	ret = seq_lseek(file, offset, whence);
4992	else
4993	file->f_pos = ret = 0;
4994
4995	return ret;
4996	}
4997
4998	static const struct file_operations tracing_fops = {
4999	.open = tracing_open,
5000	.read = seq_read,
5001	.read_iter = seq_read_iter,
5002	.splice_read = copy_splice_read,
5003	.write = tracing_write_stub,
5004	.llseek = tracing_lseek,
5005	.release = tracing_release,
5006	};
5007
5008	static const struct file_operations show_traces_fops = {
5009	.open = show_traces_open,
5010	.read = seq_read,
5011	.llseek = seq_lseek,
5012	.release = tracing_seq_release,
5013	};
5014
5015	static ssize_t
5016	tracing_cpumask_read(struct file filp, char __user ubuf,
5017	size_t count, loff_t *ppos)
5018	{
5019	struct trace_array *tr = file_inode(filp)->i_private;
5020	char *mask_str;
5021	int len;
5022
5023	len = snprintf(NULL, 0, "%*pb\n",
5024	cpumask_pr_args(tr->tracing_cpumask)) + 1;
5025	mask_str = kmalloc(len, GFP_KERNEL);
5026	if (!mask_str)
5027	return -ENOMEM;
5028
5029	len = snprintf(mask_str, len, "%*pb\n",
5030	cpumask_pr_args(tr->tracing_cpumask));
5031	if (len >= count) {
5032	count = -EINVAL;
5033	goto out_err;
5034	}
5035	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
5036
5037	out_err:
5038	kfree(mask_str);
5039
5040	return count;
5041	}
5042
5043	int tracing_set_cpumask(struct trace_array *tr,
5044	cpumask_var_t tracing_cpumask_new)
5045	{
5046	int cpu;
5047
5048	if (!tr)
5049	return -EINVAL;
5050
5051	local_irq_disable();
5052	arch_spin_lock(&tr->max_lock);
5053	for_each_tracing_cpu(cpu) {
5054	/*
5055	* Increase/decrease the disabled counter if we are
5056	* about to flip a bit in the cpumask:
5057	*/
5058	if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5059	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5060	atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5061	ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
5062	#ifdef CONFIG_TRACER_MAX_TRACE
5063	ring_buffer_record_disable_cpu(tr->max_buffer.buffer, cpu);
5064	#endif
5065	}
5066	if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5067	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5068	atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5069	ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
5070	#ifdef CONFIG_TRACER_MAX_TRACE
5071	ring_buffer_record_enable_cpu(tr->max_buffer.buffer, cpu);
5072	#endif
5073	}
5074	}
5075	arch_spin_unlock(&tr->max_lock);
5076	local_irq_enable();
5077
5078	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
5079
5080	return 0;
5081	}
5082
5083	static ssize_t
5084	tracing_cpumask_write(struct file filp, const char __user ubuf,
5085	size_t count, loff_t *ppos)
5086	{
5087	struct trace_array *tr = file_inode(filp)->i_private;
5088	cpumask_var_t tracing_cpumask_new;
5089	int err;
5090
5091	if (count == 0 \|\| count > KMALLOC_MAX_SIZE)
5092	return -EINVAL;
5093
5094	if (!zalloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
5095	return -ENOMEM;
5096
5097	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
5098	if (err)
5099	goto err_free;
5100
5101	err = tracing_set_cpumask(tr, tracing_cpumask_new);
5102	if (err)
5103	goto err_free;
5104
5105	free_cpumask_var(tracing_cpumask_new);
5106
5107	return count;
5108
5109	err_free:
5110	free_cpumask_var(tracing_cpumask_new);
5111
5112	return err;
5113	}
5114
5115	static const struct file_operations tracing_cpumask_fops = {
5116	.open = tracing_open_generic_tr,
5117	.read = tracing_cpumask_read,
5118	.write = tracing_cpumask_write,
5119	.release = tracing_release_generic_tr,
5120	.llseek = generic_file_llseek,
5121	};
5122
5123	static int tracing_trace_options_show(struct seq_file m, void v)
5124	{
5125	struct tracer_opt *trace_opts;
5126	struct trace_array *tr = m->private;
5127	u32 tracer_flags;
5128	int i;
5129
5130	mutex_lock(&trace_types_lock);
5131	tracer_flags = tr->current_trace->flags->val;
5132	trace_opts = tr->current_trace->flags->opts;
5133
5134	for (i = 0; trace_options[i]; i++) {
5135	if (tr->trace_flags & (1 << i))
5136	seq_printf(m, "%s\n", trace_options[i]);
5137	else
5138	seq_printf(m, "no%s\n", trace_options[i]);
5139	}
5140
5141	for (i = 0; trace_opts[i].name; i++) {
5142	if (tracer_flags & trace_opts[i].bit)
5143	seq_printf(m, "%s\n", trace_opts[i].name);
5144	else
5145	seq_printf(m, "no%s\n", trace_opts[i].name);
5146	}
5147	mutex_unlock(&trace_types_lock);
5148
5149	return 0;
5150	}
5151
5152	static int __set_tracer_option(struct trace_array *tr,
5153	struct tracer_flags *tracer_flags,
5154	struct tracer_opt *opts, int neg)
5155	{
5156	struct tracer *trace = tracer_flags->trace;
5157	int ret;
5158
5159	ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
5160	if (ret)
5161	return ret;
5162
5163	if (neg)
5164	tracer_flags->val &= ~opts->bit;
5165	else
5166	tracer_flags->val \|= opts->bit;
5167	return 0;
5168	}
5169
5170	/* Try to assign a tracer specific option */
5171	static int set_tracer_option(struct trace_array tr, char cmp, int neg)
5172	{
5173	struct tracer *trace = tr->current_trace;
5174	struct tracer_flags *tracer_flags = trace->flags;
5175	struct tracer_opt *opts = NULL;
5176	int i;
5177
5178	for (i = 0; tracer_flags->opts[i].name; i++) {
5179	opts = &tracer_flags->opts[i];
5180
5181	if (strcmp(cmp, opts->name) == 0)
5182	return __set_tracer_option(tr, trace->flags, opts, neg);
5183	}
5184
5185	return -EINVAL;
5186	}
5187
5188	/* Some tracers require overwrite to stay enabled */
5189	int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
5190	{
5191	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
5192	return -1;
5193
5194	return 0;
5195	}
5196
5197	int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
5198	{
5199	if ((mask == TRACE_ITER_RECORD_TGID) \|\|
5200	(mask == TRACE_ITER_RECORD_CMD) \|\|
5201	(mask == TRACE_ITER_TRACE_PRINTK))
5202	lockdep_assert_held(&event_mutex);
5203
5204	/* do nothing if flag is already set */
5205	if (!!(tr->trace_flags & mask) == !!enabled)
5206	return 0;
5207
5208	/* Give the tracer a chance to approve the change */
5209	if (tr->current_trace->flag_changed)
5210	if (tr->current_trace->flag_changed(tr, mask, !!enabled))
5211	return -EINVAL;
5212
5213	if (mask == TRACE_ITER_TRACE_PRINTK) {
5214	if (enabled) {
5215	update_printk_trace(tr);
5216	} else {
5217	/*
5218	* The global_trace cannot clear this.
5219	* It's flag only gets cleared if another instance sets it.
5220	*/
5221	if (printk_trace == &global_trace)
5222	return -EINVAL;
5223	/*
5224	* An instance must always have it set.
5225	* by default, that's the global_trace instane.
5226	*/
5227	if (printk_trace == tr)
5228	update_printk_trace(&global_trace);
5229	}
5230	}
5231
5232	if (enabled)
5233	tr->trace_flags \|= mask;
5234	else
5235	tr->trace_flags &= ~mask;
5236
5237	if (mask == TRACE_ITER_RECORD_CMD)
5238	trace_event_enable_cmd_record(enabled);
5239
5240	if (mask == TRACE_ITER_RECORD_TGID) {
5241
5242	if (trace_alloc_tgid_map() < 0) {
5243	tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
5244	return -ENOMEM;
5245	}
5246
5247	trace_event_enable_tgid_record(enabled);
5248	}
5249
5250	if (mask == TRACE_ITER_EVENT_FORK)
5251	trace_event_follow_fork(tr, enabled);
5252
5253	if (mask == TRACE_ITER_FUNC_FORK)
5254	ftrace_pid_follow_fork(tr, enabled);
5255
5256	if (mask == TRACE_ITER_OVERWRITE) {
5257	ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
5258	#ifdef CONFIG_TRACER_MAX_TRACE
5259	ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
5260	#endif
5261	}
5262
5263	if (mask == TRACE_ITER_PRINTK) {
5264	trace_printk_start_stop_comm(enabled);
5265	trace_printk_control(enabled);
5266	}
5267
5268	return 0;
5269	}
5270
5271	int trace_set_options(struct trace_array tr, char option)
5272	{
5273	char *cmp;
5274	int neg = 0;
5275	int ret;
5276	size_t orig_len = strlen(option);
5277	int len;
5278
5279	cmp = strstrip(option);
5280
5281	len = str_has_prefix(cmp, "no");
5282	if (len)
5283	neg = 1;
5284
5285	cmp += len;
5286
5287	mutex_lock(&event_mutex);
5288	mutex_lock(&trace_types_lock);
5289
5290	ret = match_string(trace_options, -1, cmp);
5291	/* If no option could be set, test the specific tracer options */
5292	if (ret < 0)
5293	ret = set_tracer_option(tr, cmp, neg);
5294	else
5295	ret = set_tracer_flag(tr, 1 << ret, !neg);
5296
5297	mutex_unlock(&trace_types_lock);
5298	mutex_unlock(&event_mutex);
5299
5300	/*
5301	* If the first trailing whitespace is replaced with '\0' by strstrip,
5302	* turn it back into a space.
5303	*/
5304	if (orig_len > strlen(option))
5305	option[strlen(option)] = ' ';
5306
5307	return ret;
5308	}
5309
5310	static void __init apply_trace_boot_options(void)
5311	{
5312	char *buf = trace_boot_options_buf;
5313	char *option;
5314
5315	while (true) {
5316	option = strsep(&buf, ",");
5317
5318	if (!option)
5319	break;
5320
5321	if (*option)
5322	trace_set_options(&global_trace, option);
5323
5324	/* Put back the comma to allow this to be called again */
5325	if (buf)
5326	*(buf - 1) = ',';
5327	}
5328	}
5329
5330	static ssize_t
5331	tracing_trace_options_write(struct file filp, const char __user ubuf,
5332	size_t cnt, loff_t *ppos)
5333	{
5334	struct seq_file *m = filp->private_data;
5335	struct trace_array *tr = m->private;
5336	char buf[64];
5337	int ret;
5338
5339	if (cnt >= sizeof(buf))
5340	return -EINVAL;
5341
5342	if (copy_from_user(buf, ubuf, cnt))
5343	return -EFAULT;
5344
5345	buf[cnt] = 0;
5346
5347	ret = trace_set_options(tr, buf);
5348	if (ret < 0)
5349	return ret;
5350
5351	*ppos += cnt;
5352
5353	return cnt;
5354	}
5355
5356	static int tracing_trace_options_open(struct inode inode, struct file file)
5357	{
5358	struct trace_array *tr = inode->i_private;
5359	int ret;
5360
5361	ret = tracing_check_open_get_tr(tr);
5362	if (ret)
5363	return ret;
5364
5365	ret = single_open(file, tracing_trace_options_show, inode->i_private);
5366	if (ret < 0)
5367	trace_array_put(tr);
5368
5369	return ret;
5370	}
5371
5372	static const struct file_operations tracing_iter_fops = {
5373	.open = tracing_trace_options_open,
5374	.read = seq_read,
5375	.llseek = seq_lseek,
5376	.release = tracing_single_release_tr,
5377	.write = tracing_trace_options_write,
5378	};
5379
5380	static const char readme_msg[] =
5381	"tracing mini-HOWTO:\n\n"
5382	"By default tracefs removes all OTH file permission bits.\n"
5383	"When mounting tracefs an optional group id can be specified\n"
5384	"which adds the group to every directory and file in tracefs:\n\n"
5385	"\t e.g. mount -t tracefs [-o [gid=<gid>]] nodev /sys/kernel/tracing\n\n"
5386	"# echo 0 > tracing_on : quick way to disable tracing\n"
5387	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5388	" Important files:\n"
5389	" trace\t\t\t- The static contents of the buffer\n"
5390	"\t\t\t To clear the buffer write into this file: echo > trace\n"
5391	" trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5392	" current_tracer\t- function and latency tracers\n"
5393	" available_tracers\t- list of configured tracers for current_tracer\n"
5394	" error_log\t- error log for failed commands (that support it)\n"
5395	" buffer_size_kb\t- view and modify size of per cpu buffer\n"
5396	" buffer_total_size_kb - view total size of all cpu buffers\n\n"
5397	" trace_clock\t\t- change the clock used to order events\n"
5398	" local: Per cpu clock but may not be synced across CPUs\n"
5399	" global: Synced across CPUs but slows tracing down.\n"
5400	" counter: Not a clock, but just an increment\n"
5401	" uptime: Jiffy counter from time of boot\n"
5402	" perf: Same clock that perf events use\n"
5403	#ifdef CONFIG_X86_64
5404	" x86-tsc: TSC cycle counter\n"
5405	#endif
5406	"\n timestamp_mode\t- view the mode used to timestamp events\n"
5407	" delta: Delta difference against a buffer-wide timestamp\n"
5408	" absolute: Absolute (standalone) timestamp\n"
5409	"\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5410	"\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5411	" tracing_cpumask\t- Limit which CPUs to trace\n"
5412	" instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5413	"\t\t\t Remove sub-buffer with rmdir\n"
5414	" trace_options\t\t- Set format or modify how tracing happens\n"
5415	"\t\t\t Disable an option by prefixing 'no' to the\n"
5416	"\t\t\t option name\n"
5417	" saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5418	#ifdef CONFIG_DYNAMIC_FTRACE
5419	"\n available_filter_functions - list of functions that can be filtered on\n"
5420	" set_ftrace_filter\t- echo function name in here to only trace these\n"
5421	"\t\t\t functions\n"
5422	"\t accepts: func_full_name or glob-matching-pattern\n"
5423	"\t modules: Can select a group via module\n"
5424	"\t Format: :mod:<module-name>\n"
5425	"\t example: echo :mod:ext3 > set_ftrace_filter\n"
5426	"\t triggers: a command to perform when function is hit\n"
5427	"\t Format: <function>:<trigger>[:count]\n"
5428	"\t trigger: traceon, traceoff\n"
5429	"\t\t enable_event:<system>:<event>\n"
5430	"\t\t disable_event:<system>:<event>\n"
5431	#ifdef CONFIG_STACKTRACE
5432	"\t\t stacktrace\n"
5433	#endif
5434	#ifdef CONFIG_TRACER_SNAPSHOT
5435	"\t\t snapshot\n"
5436	#endif
5437	"\t\t dump\n"
5438	"\t\t cpudump\n"
5439	"\t example: echo do_fault:traceoff > set_ftrace_filter\n"
5440	"\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
5441	"\t The first one will disable tracing every time do_fault is hit\n"
5442	"\t The second will disable tracing at most 3 times when do_trap is hit\n"
5443	"\t The first time do trap is hit and it disables tracing, the\n"
5444	"\t counter will decrement to 2. If tracing is already disabled,\n"
5445	"\t the counter will not decrement. It only decrements when the\n"
5446	"\t trigger did work\n"
5447	"\t To remove trigger without count:\n"
5448	"\t echo '!<function>:<trigger> > set_ftrace_filter\n"
5449	"\t To remove trigger with a count:\n"
5450	"\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5451	" set_ftrace_notrace\t- echo function name in here to never trace.\n"
5452	"\t accepts: func_full_name, func_end, func_begin, func_middle\n"
5453	"\t modules: Can select a group via module command :mod:\n"
5454	"\t Does not accept triggers\n"
5455	#endif /* CONFIG_DYNAMIC_FTRACE */
5456	#ifdef CONFIG_FUNCTION_TRACER
5457	" set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5458	"\t\t (function)\n"
5459	" set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5460	"\t\t (function)\n"
5461	#endif
5462	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5463	" set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5464	" set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5465	" max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5466	#endif
5467	#ifdef CONFIG_TRACER_SNAPSHOT
5468	"\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5469	"\t\t\t snapshot buffer. Read the contents for more\n"
5470	"\t\t\t information\n"
5471	#endif
5472	#ifdef CONFIG_STACK_TRACER
5473	" stack_trace\t\t- Shows the max stack trace when active\n"
5474	" stack_max_size\t- Shows current max stack size that was traced\n"
5475	"\t\t\t Write into this file to reset the max size (trigger a\n"
5476	"\t\t\t new trace)\n"
5477	#ifdef CONFIG_DYNAMIC_FTRACE
5478	" stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5479	"\t\t\t traces\n"
5480	#endif
5481	#endif /* CONFIG_STACK_TRACER */
5482	#ifdef CONFIG_DYNAMIC_EVENTS
5483	" dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5484	"\t\t\t Write into this file to define/undefine new trace events.\n"
5485	#endif
5486	#ifdef CONFIG_KPROBE_EVENTS
5487	" kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5488	"\t\t\t Write into this file to define/undefine new trace events.\n"
5489	#endif
5490	#ifdef CONFIG_UPROBE_EVENTS
5491	" uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5492	"\t\t\t Write into this file to define/undefine new trace events.\n"
5493	#endif
5494	#if defined(CONFIG_KPROBE_EVENTS) \|\| defined(CONFIG_UPROBE_EVENTS) \|\| \
5495	defined(CONFIG_FPROBE_EVENTS)
5496	"\t accepts: event-definitions (one definition per line)\n"
5497	#if defined(CONFIG_KPROBE_EVENTS) \|\| defined(CONFIG_UPROBE_EVENTS)
5498	"\t Format: p[:[<group>/][<event>]] <place> [<args>]\n"
5499	"\t r[maxactive][:[<group>/][<event>]] <place> [<args>]\n"
5500	#endif
5501	#ifdef CONFIG_FPROBE_EVENTS
5502	"\t f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n"
5503	"\t t[:[<group>/][<event>]] <tracepoint> [<args>]\n"
5504	#endif
5505	#ifdef CONFIG_HIST_TRIGGERS
5506	"\t s:[synthetic/]<event> <field> [<field>]\n"
5507	#endif
5508	"\t e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n"
5509	"\t -:[<group>/][<event>]\n"
5510	#ifdef CONFIG_KPROBE_EVENTS
5511	"\t place: [<module>:]<symbol>[+<offset>]\|<memaddr>\n"
5512	"place (kretprobe): [<module>:]<symbol>[+<offset>]%return\|<memaddr>\n"
5513	#endif
5514	#ifdef CONFIG_UPROBE_EVENTS
5515	" place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
5516	#endif
5517	"\t args: <name>=fetcharg[:type]\n"
5518	"\t fetcharg: (%<register>\|$<efield>), @<address>, @<symbol>[+\|-<offset>],\n"
5519	#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5520	"\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5521	#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
5522	"\t <argname>[->field[->field\|.field...]],\n"
5523	#endif
5524	#else
5525	"\t $stack<index>, $stack, $retval, $comm,\n"
5526	#endif
5527	"\t +\|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5528	"\t kernel return probes support: $retval, $arg<N>, $comm\n"
5529	"\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n"
5530	"\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5531	"\t symstr, %pd/%pD, <type>\\[<array-size>\\]\n"
5532	#ifdef CONFIG_HIST_TRIGGERS
5533	"\t field: <stype> <name>;\n"
5534	"\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5535	"\t [unsigned] char/int/long\n"
5536	#endif
5537	"\t efield: For event probes ('e' types), the field is on of the fields\n"
5538	"\t of the <attached-group>/<attached-event>.\n"
5539	#endif
5540	" events/\t\t- Directory containing all trace event subsystems:\n"
5541	" enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5542	" events/<system>/\t- Directory containing all trace events for <system>:\n"
5543	" enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5544	"\t\t\t events\n"
5545	" filter\t\t- If set, only events passing filter are traced\n"
5546	" events/<system>/<event>/\t- Directory containing control files for\n"
5547	"\t\t\t <event>:\n"
5548	" enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5549	" filter\t\t- If set, only events passing filter are traced\n"
5550	" trigger\t\t- If set, a command to perform when event is hit\n"
5551	"\t Format: <trigger>[:count][if <filter>]\n"
5552	"\t trigger: traceon, traceoff\n"
5553	"\t enable_event:<system>:<event>\n"
5554	"\t disable_event:<system>:<event>\n"
5555	#ifdef CONFIG_HIST_TRIGGERS
5556	"\t enable_hist:<system>:<event>\n"
5557	"\t disable_hist:<system>:<event>\n"
5558	#endif
5559	#ifdef CONFIG_STACKTRACE
5560	"\t\t stacktrace\n"
5561	#endif
5562	#ifdef CONFIG_TRACER_SNAPSHOT
5563	"\t\t snapshot\n"
5564	#endif
5565	#ifdef CONFIG_HIST_TRIGGERS
5566	"\t\t hist (see below)\n"
5567	#endif
5568	"\t example: echo traceoff > events/block/block_unplug/trigger\n"
5569	"\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5570	"\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5571	"\t events/block/block_unplug/trigger\n"
5572	"\t The first disables tracing every time block_unplug is hit.\n"
5573	"\t The second disables tracing the first 3 times block_unplug is hit.\n"
5574	"\t The third enables the kmalloc event the first 3 times block_unplug\n"
5575	"\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5576	"\t Like function triggers, the counter is only decremented if it\n"
5577	"\t enabled or disabled tracing.\n"
5578	"\t To remove a trigger without a count:\n"
5579	"\t echo '!<trigger> > <system>/<event>/trigger\n"
5580	"\t To remove a trigger with a count:\n"
5581	"\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5582	"\t Filters can be ignored when removing a trigger.\n"
5583	#ifdef CONFIG_HIST_TRIGGERS
5584	" hist trigger\t- If set, event hits are aggregated into a hash table\n"
5585	"\t Format: hist:keys=<field1[,field2,...]>\n"
5586	"\t [:<var1>=<field\|var_ref\|numeric_literal>[,<var2>=...]]\n"
5587	"\t [:values=<field1[,field2,...]>]\n"
5588	"\t [:sort=<field1[,field2,...]>]\n"
5589	"\t [:size=#entries]\n"
5590	"\t [:pause][:continue][:clear]\n"
5591	"\t [:name=histname1]\n"
5592	"\t [:nohitcount]\n"
5593	"\t [:<handler>.<action>]\n"
5594	"\t [if <filter>]\n\n"
5595	"\t Note, special fields can be used as well:\n"
5596	"\t common_timestamp - to record current timestamp\n"
5597	"\t common_cpu - to record the CPU the event happened on\n"
5598	"\n"
5599	"\t A hist trigger variable can be:\n"
5600	"\t - a reference to a field e.g. x=current_timestamp,\n"
5601	"\t - a reference to another variable e.g. y=$x,\n"
5602	"\t - a numeric literal: e.g. ms_per_sec=1000,\n"
5603	"\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
5604	"\n"
5605	"\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
5606	"\t multiplication(*) and division(/) operators. An operand can be either a\n"
5607	"\t variable reference, field or numeric literal.\n"
5608	"\n"
5609	"\t When a matching event is hit, an entry is added to a hash\n"
5610	"\t table using the key(s) and value(s) named, and the value of a\n"
5611	"\t sum called 'hitcount' is incremented. Keys and values\n"
5612	"\t correspond to fields in the event's format description. Keys\n"
5613	"\t can be any field, or the special string 'common_stacktrace'.\n"
5614	"\t Compound keys consisting of up to two fields can be specified\n"
5615	"\t by the 'keys' keyword. Values must correspond to numeric\n"
5616	"\t fields. Sort keys consisting of up to two fields can be\n"
5617	"\t specified using the 'sort' keyword. The sort direction can\n"
5618	"\t be modified by appending '.descending' or '.ascending' to a\n"
5619	"\t sort field. The 'size' parameter can be used to specify more\n"
5620	"\t or fewer than the default 2048 entries for the hashtable size.\n"
5621	"\t If a hist trigger is given a name using the 'name' parameter,\n"
5622	"\t its histogram data will be shared with other triggers of the\n"
5623	"\t same name, and trigger hits will update this common data.\n\n"
5624	"\t Reading the 'hist' file for the event will dump the hash\n"
5625	"\t table in its entirety to stdout. If there are multiple hist\n"
5626	"\t triggers attached to an event, there will be a table for each\n"
5627	"\t trigger in the output. The table displayed for a named\n"
5628	"\t trigger will be the same as any other instance having the\n"
5629	"\t same name. The default format used to display a given field\n"
5630	"\t can be modified by appending any of the following modifiers\n"
5631	"\t to the field name, as applicable:\n\n"
5632	"\t .hex display a number as a hex value\n"
5633	"\t .sym display an address as a symbol\n"
5634	"\t .sym-offset display an address as a symbol and offset\n"
5635	"\t .execname display a common_pid as a program name\n"
5636	"\t .syscall display a syscall id as a syscall name\n"
5637	"\t .log2 display log2 value rather than raw number\n"
5638	"\t .buckets=size display values in groups of size rather than raw number\n"
5639	"\t .usecs display a common_timestamp in microseconds\n"
5640	"\t .percent display a number of percentage value\n"
5641	"\t .graph display a bar-graph of a value\n\n"
5642	"\t The 'pause' parameter can be used to pause an existing hist\n"
5643	"\t trigger or to start a hist trigger but not log any events\n"
5644	"\t until told to do so. 'continue' can be used to start or\n"
5645	"\t restart a paused hist trigger.\n\n"
5646	"\t The 'clear' parameter will clear the contents of a running\n"
5647	"\t hist trigger and leave its current paused/active state\n"
5648	"\t unchanged.\n\n"
5649	"\t The 'nohitcount' (or NOHC) parameter will suppress display of\n"
5650	"\t raw hitcount in the histogram.\n\n"
5651	"\t The enable_hist and disable_hist triggers can be used to\n"
5652	"\t have one event conditionally start and stop another event's\n"
5653	"\t already-attached hist trigger. The syntax is analogous to\n"
5654	"\t the enable_event and disable_event triggers.\n\n"
5655	"\t Hist trigger handlers and actions are executed whenever a\n"
5656	"\t a histogram entry is added or updated. They take the form:\n\n"
5657	"\t <handler>.<action>\n\n"
5658	"\t The available handlers are:\n\n"
5659	"\t onmatch(matching.event) - invoke on addition or update\n"
5660	"\t onmax(var) - invoke if var exceeds current max\n"
5661	"\t onchange(var) - invoke action if var changes\n\n"
5662	"\t The available actions are:\n\n"
5663	"\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5664	"\t save(field,...) - save current event fields\n"
5665	#ifdef CONFIG_TRACER_SNAPSHOT
5666	"\t snapshot() - snapshot the trace buffer\n\n"
5667	#endif
5668	#ifdef CONFIG_SYNTH_EVENTS
5669	" events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5670	"\t Write into this file to define/undefine new synthetic events.\n"
5671	"\t example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n"
5672	#endif
5673	#endif
5674	;
5675
5676	static ssize_t
5677	tracing_readme_read(struct file filp, char __user ubuf,
5678	size_t cnt, loff_t *ppos)
5679	{
5680	return simple_read_from_buffer(ubuf, cnt, ppos,
5681	readme_msg, strlen(readme_msg));
5682	}
5683
5684	static const struct file_operations tracing_readme_fops = {
5685	.open = tracing_open_generic,
5686	.read = tracing_readme_read,
5687	.llseek = generic_file_llseek,
5688	};
5689
5690	#ifdef CONFIG_TRACE_EVAL_MAP_FILE
5691	static union trace_eval_map_item *
5692	update_eval_map(union trace_eval_map_item *ptr)
5693	{
5694	if (!ptr->map.eval_string) {
5695	if (ptr->tail.next) {
5696	ptr = ptr->tail.next;
5697	/* Set ptr to the next real item (skip head) */
5698	ptr++;
5699	} else
5700	return NULL;
5701	}
5702	return ptr;
5703	}
5704
5705	static void eval_map_next(struct seq_file m, void v, loff_t pos)
5706	{
5707	union trace_eval_map_item *ptr = v;
5708
5709	/*
5710	* Paranoid! If ptr points to end, we don't want to increment past it.
5711	* This really should never happen.
5712	*/
5713	(*pos)++;
5714	ptr = update_eval_map(ptr);
5715	if (WARN_ON_ONCE(!ptr))
5716	return NULL;
5717
5718	ptr++;
5719	ptr = update_eval_map(ptr);
5720
5721	return ptr;
5722	}
5723
5724	static void eval_map_start(struct seq_file m, loff_t *pos)
5725	{
5726	union trace_eval_map_item *v;
5727	loff_t l = 0;
5728
5729	mutex_lock(&trace_eval_mutex);
5730
5731	v = trace_eval_maps;
5732	if (v)
5733	v++;
5734
5735	while (v && l < *pos) {
5736	v = eval_map_next(m, v, &l);
5737	}
5738
5739	return v;
5740	}
5741
5742	static void eval_map_stop(struct seq_file m, void v)
5743	{
5744	mutex_unlock(&trace_eval_mutex);
5745	}
5746
5747	static int eval_map_show(struct seq_file m, void v)
5748	{
5749	union trace_eval_map_item *ptr = v;
5750
5751	seq_printf(m, "%s %ld (%s)\n",
5752	ptr->map.eval_string, ptr->map.eval_value,
5753	ptr->map.system);
5754
5755	return 0;
5756	}
5757
5758	static const struct seq_operations tracing_eval_map_seq_ops = {
5759	.start = eval_map_start,
5760	.next = eval_map_next,
5761	.stop = eval_map_stop,
5762	.show = eval_map_show,
5763	};
5764
5765	static int tracing_eval_map_open(struct inode inode, struct file filp)
5766	{
5767	int ret;
5768
5769	ret = tracing_check_open_get_tr(NULL);
5770	if (ret)
5771	return ret;
5772
5773	return seq_open(filp, &tracing_eval_map_seq_ops);
5774	}
5775
5776	static const struct file_operations tracing_eval_map_fops = {
5777	.open = tracing_eval_map_open,
5778	.read = seq_read,
5779	.llseek = seq_lseek,
5780	.release = seq_release,
5781	};
5782
5783	static inline union trace_eval_map_item *
5784	trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5785	{
5786	/* Return tail of array given the head */
5787	return ptr + ptr->head.length + 1;
5788	}
5789
5790	static void
5791	trace_insert_eval_map_file(struct module mod, struct trace_eval_map *start,
5792	int len)
5793	{
5794	struct trace_eval_map **stop;
5795	struct trace_eval_map **map;
5796	union trace_eval_map_item *map_array;
5797	union trace_eval_map_item *ptr;
5798
5799	stop = start + len;
5800
5801	/*
5802	* The trace_eval_maps contains the map plus a head and tail item,
5803	* where the head holds the module and length of array, and the
5804	* tail holds a pointer to the next list.
5805	*/
5806	map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
5807	if (!map_array) {
5808	pr_warn("Unable to allocate trace eval mapping\n");
5809	return;
5810	}
5811
5812	mutex_lock(&trace_eval_mutex);
5813
5814	if (!trace_eval_maps)
5815	trace_eval_maps = map_array;
5816	else {
5817	ptr = trace_eval_maps;
5818	for (;;) {
5819	ptr = trace_eval_jmp_to_tail(ptr);
5820	if (!ptr->tail.next)
5821	break;
5822	ptr = ptr->tail.next;
5823
5824	}
5825	ptr->tail.next = map_array;
5826	}
5827	map_array->head.mod = mod;
5828	map_array->head.length = len;
5829	map_array++;
5830
5831	for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5832	map_array->map = **map;
5833	map_array++;
5834	}
5835	memset(map_array, 0, sizeof(*map_array));
5836
5837	mutex_unlock(&trace_eval_mutex);
5838	}
5839
5840	static void trace_create_eval_file(struct dentry *d_tracer)
5841	{
5842	trace_create_file("eval_map", TRACE_MODE_READ, d_tracer,
5843	NULL, &tracing_eval_map_fops);
5844	}
5845
5846	#else /* CONFIG_TRACE_EVAL_MAP_FILE */
5847	static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5848	static inline void trace_insert_eval_map_file(struct module *mod,
5849	struct trace_eval_map **start, int len) { }
5850	#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5851
5852	static void trace_insert_eval_map(struct module *mod,
5853	struct trace_eval_map **start, int len)
5854	{
5855	struct trace_eval_map **map;
5856
5857	if (len <= 0)
5858	return;
5859
5860	map = start;
5861
5862	trace_event_eval_update(map, len);
5863
5864	trace_insert_eval_map_file(mod, start, len);
5865	}
5866
5867	static ssize_t
5868	tracing_set_trace_read(struct file filp, char __user ubuf,
5869	size_t cnt, loff_t *ppos)
5870	{
5871	struct trace_array *tr = filp->private_data;
5872	char buf[MAX_TRACER_SIZE+2];
5873	int r;
5874
5875	mutex_lock(&trace_types_lock);
5876	r = sprintf(buf, "%s\n", tr->current_trace->name);
5877	mutex_unlock(&trace_types_lock);
5878
5879	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5880	}
5881
5882	int tracer_init(struct tracer t, struct trace_array tr)
5883	{
5884	tracing_reset_online_cpus(&tr->array_buffer);
5885	return t->init(tr);
5886	}
5887
5888	static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
5889	{
5890	int cpu;
5891
5892	for_each_tracing_cpu(cpu)
5893	per_cpu_ptr(buf->data, cpu)->entries = val;
5894	}
5895
5896	static void update_buffer_entries(struct array_buffer *buf, int cpu)
5897	{
5898	if (cpu == RING_BUFFER_ALL_CPUS) {
5899	set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0));
5900	} else {
5901	per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu);
5902	}
5903	}
5904
5905	#ifdef CONFIG_TRACER_MAX_TRACE
5906	/* resize @tr's buffer to the size of @size_tr's entries */
5907	static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
5908	struct array_buffer *size_buf, int cpu_id)
5909	{
5910	int cpu, ret = 0;
5911
5912	if (cpu_id == RING_BUFFER_ALL_CPUS) {
5913	for_each_tracing_cpu(cpu) {
5914	ret = ring_buffer_resize(trace_buf->buffer,
5915	per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5916	if (ret < 0)
5917	break;
5918	per_cpu_ptr(trace_buf->data, cpu)->entries =
5919	per_cpu_ptr(size_buf->data, cpu)->entries;
5920	}
5921	} else {
5922	ret = ring_buffer_resize(trace_buf->buffer,
5923	per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5924	if (ret == 0)
5925	per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5926	per_cpu_ptr(size_buf->data, cpu_id)->entries;
5927	}
5928
5929	return ret;
5930	}
5931	#endif /* CONFIG_TRACER_MAX_TRACE */
5932
5933	static int __tracing_resize_ring_buffer(struct trace_array *tr,
5934	unsigned long size, int cpu)
5935	{
5936	int ret;
5937
5938	/*
5939	* If kernel or user changes the size of the ring buffer
5940	* we use the size that was given, and we can forget about
5941	* expanding it later.
5942	*/
5943	trace_set_ring_buffer_expanded(tr);
5944
5945	/* May be called before buffers are initialized */
5946	if (!tr->array_buffer.buffer)
5947	return 0;
5948
5949	/* Do not allow tracing while resizing ring buffer */
5950	tracing_stop_tr(tr);
5951
5952	ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
5953	if (ret < 0)
5954	goto out_start;
5955
5956	#ifdef CONFIG_TRACER_MAX_TRACE
5957	if (!tr->allocated_snapshot)
5958	goto out;
5959
5960	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5961	if (ret < 0) {
5962	int r = resize_buffer_duplicate_size(&tr->array_buffer,
5963	&tr->array_buffer, cpu);
5964	if (r < 0) {
5965	/*
5966	* AARGH! We are left with different
5967	* size max buffer!!!!
5968	* The max buffer is our "snapshot" buffer.
5969	* When a tracer needs a snapshot (one of the
5970	* latency tracers), it swaps the max buffer
5971	* with the saved snap shot. We succeeded to
5972	* update the size of the main buffer, but failed to
5973	* update the size of the max buffer. But when we tried
5974	* to reset the main buffer to the original size, we
5975	* failed there too. This is very unlikely to
5976	* happen, but if it does, warn and kill all
5977	* tracing.
5978	*/
5979	WARN_ON(1);
5980	tracing_disabled = 1;
5981	}
5982	goto out_start;
5983	}
5984
5985	update_buffer_entries(&tr->max_buffer, cpu);
5986
5987	out:
5988	#endif /* CONFIG_TRACER_MAX_TRACE */
5989
5990	update_buffer_entries(&tr->array_buffer, cpu);
5991	out_start:
5992	tracing_start_tr(tr);
5993	return ret;
5994	}
5995
5996	ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5997	unsigned long size, int cpu_id)
5998	{
5999	int ret;
6000
6001	mutex_lock(&trace_types_lock);
6002
6003	if (cpu_id != RING_BUFFER_ALL_CPUS) {
6004	/* make sure, this cpu is enabled in the mask */
6005	if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
6006	ret = -EINVAL;
6007	goto out;
6008	}
6009	}
6010
6011	ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
6012	if (ret < 0)
6013	ret = -ENOMEM;
6014
6015	out:
6016	mutex_unlock(&trace_types_lock);
6017
6018	return ret;
6019	}
6020
6021	static void update_last_data(struct trace_array *tr)
6022	{
6023	if (!tr->text_delta && !tr->data_delta)
6024	return;
6025
6026	/*
6027	* Need to clear all CPU buffers as there cannot be events
6028	* from the previous boot mixed with events with this boot
6029	* as that will cause a confusing trace. Need to clear all
6030	* CPU buffers, even for those that may currently be offline.
6031	*/
6032	tracing_reset_all_cpus(&tr->array_buffer);
6033
6034	/* Using current data now */
6035	tr->text_delta = 0;
6036	tr->data_delta = 0;
6037	}
6038
6039	/**
6040	* tracing_update_buffers - used by tracing facility to expand ring buffers
6041	* @tr: The tracing instance
6042	*
6043	* To save on memory when the tracing is never used on a system with it
6044	* configured in. The ring buffers are set to a minimum size. But once
6045	* a user starts to use the tracing facility, then they need to grow
6046	* to their default size.
6047	*
6048	* This function is to be called when a tracer is about to be used.
6049	*/
6050	int tracing_update_buffers(struct trace_array *tr)
6051	{
6052	int ret = 0;
6053
6054	mutex_lock(&trace_types_lock);
6055
6056	update_last_data(tr);
6057
6058	if (!tr->ring_buffer_expanded)
6059	ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
6060	RING_BUFFER_ALL_CPUS);
6061	mutex_unlock(&trace_types_lock);
6062
6063	return ret;
6064	}
6065
6066	struct trace_option_dentry;
6067
6068	static void
6069	create_trace_option_files(struct trace_array tr, struct tracer tracer);
6070
6071	/*
6072	* Used to clear out the tracer before deletion of an instance.
6073	* Must have trace_types_lock held.
6074	*/
6075	static void tracing_set_nop(struct trace_array *tr)
6076	{
6077	if (tr->current_trace == &nop_trace)
6078	return;
6079
6080	tr->current_trace->enabled--;
6081
6082	if (tr->current_trace->reset)
6083	tr->current_trace->reset(tr);
6084
6085	tr->current_trace = &nop_trace;
6086	}
6087
6088	static bool tracer_options_updated;
6089
6090	static void add_tracer_options(struct trace_array tr, struct tracer t)
6091	{
6092	/* Only enable if the directory has been created already. */
6093	if (!tr->dir)
6094	return;
6095
6096	/* Only create trace option files after update_tracer_options finish */
6097	if (!tracer_options_updated)
6098	return;
6099
6100	create_trace_option_files(tr, t);
6101	}
6102
6103	int tracing_set_tracer(struct trace_array tr, const char buf)
6104	{
6105	struct tracer *t;
6106	#ifdef CONFIG_TRACER_MAX_TRACE
6107	bool had_max_tr;
6108	#endif
6109	int ret = 0;
6110
6111	mutex_lock(&trace_types_lock);
6112
6113	update_last_data(tr);
6114
6115	if (!tr->ring_buffer_expanded) {
6116	ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
6117	RING_BUFFER_ALL_CPUS);
6118	if (ret < 0)
6119	goto out;
6120	ret = 0;
6121	}
6122
6123	for (t = trace_types; t; t = t->next) {
6124	if (strcmp(t->name, buf) == 0)
6125	break;
6126	}
6127	if (!t) {
6128	ret = -EINVAL;
6129	goto out;
6130	}
6131	if (t == tr->current_trace)
6132	goto out;
6133
6134	#ifdef CONFIG_TRACER_SNAPSHOT
6135	if (t->use_max_tr) {
6136	local_irq_disable();
6137	arch_spin_lock(&tr->max_lock);
6138	if (tr->cond_snapshot)
6139	ret = -EBUSY;
6140	arch_spin_unlock(&tr->max_lock);
6141	local_irq_enable();
6142	if (ret)
6143	goto out;
6144	}
6145	#endif
6146	/* Some tracers won't work on kernel command line */
6147	if (system_state < SYSTEM_RUNNING && t->noboot) {
6148	pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
6149	t->name);
6150	goto out;
6151	}
6152
6153	/* Some tracers are only allowed for the top level buffer */
6154	if (!trace_ok_for_array(t, tr)) {
6155	ret = -EINVAL;
6156	goto out;
6157	}
6158
6159	/* If trace pipe files are being read, we can't change the tracer */
6160	if (tr->trace_ref) {
6161	ret = -EBUSY;
6162	goto out;
6163	}
6164
6165	trace_branch_disable();
6166
6167	tr->current_trace->enabled--;
6168
6169	if (tr->current_trace->reset)
6170	tr->current_trace->reset(tr);
6171
6172	#ifdef CONFIG_TRACER_MAX_TRACE
6173	had_max_tr = tr->current_trace->use_max_tr;
6174
6175	/* Current trace needs to be nop_trace before synchronize_rcu */
6176	tr->current_trace = &nop_trace;
6177
6178	if (had_max_tr && !t->use_max_tr) {
6179	/*
6180	* We need to make sure that the update_max_tr sees that
6181	* current_trace changed to nop_trace to keep it from
6182	* swapping the buffers after we resize it.
6183	* The update_max_tr is called from interrupts disabled
6184	* so a synchronized_sched() is sufficient.
6185	*/
6186	synchronize_rcu();
6187	free_snapshot(tr);
6188	tracing_disarm_snapshot(tr);
6189	}
6190
6191	if (!had_max_tr && t->use_max_tr) {
6192	ret = tracing_arm_snapshot_locked(tr);
6193	if (ret)
6194	goto out;
6195	}
6196	#else
6197	tr->current_trace = &nop_trace;
6198	#endif
6199
6200	if (t->init) {
6201	ret = tracer_init(t, tr);
6202	if (ret) {
6203	#ifdef CONFIG_TRACER_MAX_TRACE
6204	if (t->use_max_tr)
6205	tracing_disarm_snapshot(tr);
6206	#endif
6207	goto out;
6208	}
6209	}
6210
6211	tr->current_trace = t;
6212	tr->current_trace->enabled++;
6213	trace_branch_enable(tr);
6214	out:
6215	mutex_unlock(&trace_types_lock);
6216
6217	return ret;
6218	}
6219
6220	static ssize_t
6221	tracing_set_trace_write(struct file filp, const char __user ubuf,
6222	size_t cnt, loff_t *ppos)
6223	{
6224	struct trace_array *tr = filp->private_data;
6225	char buf[MAX_TRACER_SIZE+1];
6226	char *name;
6227	size_t ret;
6228	int err;
6229
6230	ret = cnt;
6231
6232	if (cnt > MAX_TRACER_SIZE)
6233	cnt = MAX_TRACER_SIZE;
6234
6235	if (copy_from_user(buf, ubuf, cnt))
6236	return -EFAULT;
6237
6238	buf[cnt] = 0;
6239
6240	name = strim(buf);
6241
6242	err = tracing_set_tracer(tr, name);
6243	if (err)
6244	return err;
6245
6246	*ppos += ret;
6247
6248	return ret;
6249	}
6250
6251	static ssize_t
6252	tracing_nsecs_read(unsigned long ptr, char __user ubuf,
6253	size_t cnt, loff_t *ppos)
6254	{
6255	char buf[64];
6256	int r;
6257
6258	r = snprintf(buf, sizeof(buf), "%ld\n",
6259	ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(ptr));
6260	if (r > sizeof(buf))
6261	r = sizeof(buf);
6262	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6263	}
6264
6265	static ssize_t
6266	tracing_nsecs_write(unsigned long ptr, const char __user ubuf,
6267	size_t cnt, loff_t *ppos)
6268	{
6269	unsigned long val;
6270	int ret;
6271
6272	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6273	if (ret)
6274	return ret;
6275
6276	ptr = val 1000;
6277
6278	return cnt;
6279	}
6280
6281	static ssize_t
6282	tracing_thresh_read(struct file filp, char __user ubuf,
6283	size_t cnt, loff_t *ppos)
6284	{
6285	return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
6286	}
6287
6288	static ssize_t
6289	tracing_thresh_write(struct file filp, const char __user ubuf,
6290	size_t cnt, loff_t *ppos)
6291	{
6292	struct trace_array *tr = filp->private_data;
6293	int ret;
6294
6295	mutex_lock(&trace_types_lock);
6296	ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
6297	if (ret < 0)
6298	goto out;
6299
6300	if (tr->current_trace->update_thresh) {
6301	ret = tr->current_trace->update_thresh(tr);
6302	if (ret < 0)
6303	goto out;
6304	}
6305
6306	ret = cnt;
6307	out:
6308	mutex_unlock(&trace_types_lock);
6309
6310	return ret;
6311	}
6312
6313	#ifdef CONFIG_TRACER_MAX_TRACE
6314
6315	static ssize_t
6316	tracing_max_lat_read(struct file filp, char __user ubuf,
6317	size_t cnt, loff_t *ppos)
6318	{
6319	struct trace_array *tr = filp->private_data;
6320
6321	return tracing_nsecs_read(&tr->max_latency, ubuf, cnt, ppos);
6322	}
6323
6324	static ssize_t
6325	tracing_max_lat_write(struct file filp, const char __user ubuf,
6326	size_t cnt, loff_t *ppos)
6327	{
6328	struct trace_array *tr = filp->private_data;
6329
6330	return tracing_nsecs_write(&tr->max_latency, ubuf, cnt, ppos);
6331	}
6332
6333	#endif
6334
6335	static int open_pipe_on_cpu(struct trace_array *tr, int cpu)
6336	{
6337	if (cpu == RING_BUFFER_ALL_CPUS) {
6338	if (cpumask_empty(tr->pipe_cpumask)) {
6339	cpumask_setall(tr->pipe_cpumask);
6340	return 0;
6341	}
6342	} else if (!cpumask_test_cpu(cpu, tr->pipe_cpumask)) {
6343	cpumask_set_cpu(cpu, tr->pipe_cpumask);
6344	return 0;
6345	}
6346	return -EBUSY;
6347	}
6348
6349	static void close_pipe_on_cpu(struct trace_array *tr, int cpu)
6350	{
6351	if (cpu == RING_BUFFER_ALL_CPUS) {
6352	WARN_ON(!cpumask_full(tr->pipe_cpumask));
6353	cpumask_clear(tr->pipe_cpumask);
6354	} else {
6355	WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask));
6356	cpumask_clear_cpu(cpu, tr->pipe_cpumask);
6357	}
6358	}
6359
6360	static int tracing_open_pipe(struct inode inode, struct file filp)
6361	{
6362	struct trace_array *tr = inode->i_private;
6363	struct trace_iterator *iter;
6364	int cpu;
6365	int ret;
6366
6367	ret = tracing_check_open_get_tr(tr);
6368	if (ret)
6369	return ret;
6370
6371	mutex_lock(&trace_types_lock);
6372	cpu = tracing_get_cpu(inode);
6373	ret = open_pipe_on_cpu(tr, cpu);
6374	if (ret)
6375	goto fail_pipe_on_cpu;
6376
6377	/* create a buffer to store the information to pass to userspace */
6378	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6379	if (!iter) {
6380	ret = -ENOMEM;
6381	goto fail_alloc_iter;
6382	}
6383
6384	trace_seq_init(&iter->seq);
6385	iter->trace = tr->current_trace;
6386
6387	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
6388	ret = -ENOMEM;
6389	goto fail;
6390	}
6391
6392	/* trace pipe does not show start of buffer */
6393	cpumask_setall(iter->started);
6394
6395	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6396	iter->iter_flags \|= TRACE_FILE_LAT_FMT;
6397
6398	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
6399	if (trace_clocks[tr->clock_id].in_ns)
6400	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
6401
6402	iter->tr = tr;
6403	iter->array_buffer = &tr->array_buffer;
6404	iter->cpu_file = cpu;
6405	mutex_init(&iter->mutex);
6406	filp->private_data = iter;
6407
6408	if (iter->trace->pipe_open)
6409	iter->trace->pipe_open(iter);
6410
6411	nonseekable_open(inode, filp);
6412
6413	tr->trace_ref++;
6414
6415	mutex_unlock(&trace_types_lock);
6416	return ret;
6417
6418	fail:
6419	kfree(iter);
6420	fail_alloc_iter:
6421	close_pipe_on_cpu(tr, cpu);
6422	fail_pipe_on_cpu:
6423	__trace_array_put(tr);
6424	mutex_unlock(&trace_types_lock);
6425	return ret;
6426	}
6427
6428	static int tracing_release_pipe(struct inode inode, struct file file)
6429	{
6430	struct trace_iterator *iter = file->private_data;
6431	struct trace_array *tr = inode->i_private;
6432
6433	mutex_lock(&trace_types_lock);
6434
6435	tr->trace_ref--;
6436
6437	if (iter->trace->pipe_close)
6438	iter->trace->pipe_close(iter);
6439	close_pipe_on_cpu(tr, iter->cpu_file);
6440	mutex_unlock(&trace_types_lock);
6441
6442	free_trace_iter_content(iter);
6443	kfree(iter);
6444
6445	trace_array_put(tr);
6446
6447	return 0;
6448	}
6449
6450	static __poll_t
6451	trace_poll(struct trace_iterator iter, struct file filp, poll_table *poll_table)
6452	{
6453	struct trace_array *tr = iter->tr;
6454
6455	/* Iterators are static, they should be filled or empty */
6456	if (trace_buffer_iter(iter, iter->cpu_file))
6457	return EPOLLIN \| EPOLLRDNORM;
6458
6459	if (tr->trace_flags & TRACE_ITER_BLOCK)
6460	/*
6461	* Always select as readable when in blocking mode
6462	*/
6463	return EPOLLIN \| EPOLLRDNORM;
6464	else
6465	return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
6466	filp, poll_table, iter->tr->buffer_percent);
6467	}
6468
6469	static __poll_t
6470	tracing_poll_pipe(struct file filp, poll_table poll_table)
6471	{
6472	struct trace_iterator *iter = filp->private_data;
6473
6474	return trace_poll(iter, filp, poll_table);
6475	}
6476
6477	/* Must be called with iter->mutex held. */
6478	static int tracing_wait_pipe(struct file *filp)
6479	{
6480	struct trace_iterator *iter = filp->private_data;
6481	int ret;
6482
6483	while (trace_empty(iter)) {
6484
6485	if ((filp->f_flags & O_NONBLOCK)) {
6486	return -EAGAIN;
6487	}
6488
6489	/*
6490	* We block until we read something and tracing is disabled.
6491	* We still block if tracing is disabled, but we have never
6492	* read anything. This allows a user to cat this file, and
6493	* then enable tracing. But after we have read something,
6494	* we give an EOF when tracing is again disabled.
6495	*
6496	* iter->pos will be 0 if we haven't read anything.
6497	*/
6498	if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6499	break;
6500
6501	mutex_unlock(&iter->mutex);
6502
6503	ret = wait_on_pipe(iter, 0);
6504
6505	mutex_lock(&iter->mutex);
6506
6507	if (ret)
6508	return ret;
6509	}
6510
6511	return 1;
6512	}
6513
6514	/*
6515	* Consumer reader.
6516	*/
6517	static ssize_t
6518	tracing_read_pipe(struct file filp, char __user ubuf,
6519	size_t cnt, loff_t *ppos)
6520	{
6521	struct trace_iterator *iter = filp->private_data;
6522	ssize_t sret;
6523
6524	/*
6525	* Avoid more than one consumer on a single file descriptor
6526	* This is just a matter of traces coherency, the ring buffer itself
6527	* is protected.
6528	*/
6529	mutex_lock(&iter->mutex);
6530
6531	/* return any leftover data */
6532	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6533	if (sret != -EBUSY)
6534	goto out;
6535
6536	trace_seq_init(&iter->seq);
6537
6538	if (iter->trace->read) {
6539	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6540	if (sret)
6541	goto out;
6542	}
6543
6544	waitagain:
6545	sret = tracing_wait_pipe(filp);
6546	if (sret <= 0)
6547	goto out;
6548
6549	/* stop when tracing is finished */
6550	if (trace_empty(iter)) {
6551	sret = 0;
6552	goto out;
6553	}
6554
6555	if (cnt >= TRACE_SEQ_BUFFER_SIZE)
6556	cnt = TRACE_SEQ_BUFFER_SIZE - 1;
6557
6558	/* reset all but tr, trace, and overruns */
6559	trace_iterator_reset(iter);
6560	cpumask_clear(iter->started);
6561	trace_seq_init(&iter->seq);
6562
6563	trace_event_read_lock();
6564	trace_access_lock(iter->cpu_file);
6565	while (trace_find_next_entry_inc(iter) != NULL) {
6566	enum print_line_t ret;
6567	int save_len = iter->seq.seq.len;
6568
6569	ret = print_trace_line(iter);
6570	if (ret == TRACE_TYPE_PARTIAL_LINE) {
6571	/*
6572	* If one print_trace_line() fills entire trace_seq in one shot,
6573	* trace_seq_to_user() will returns -EBUSY because save_len == 0,
6574	* In this case, we need to consume it, otherwise, loop will peek
6575	* this event next time, resulting in an infinite loop.
6576	*/
6577	if (save_len == 0) {
6578	iter->seq.full = 0;
6579	trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
6580	trace_consume(iter);
6581	break;
6582	}
6583
6584	/* In other cases, don't print partial lines */
6585	iter->seq.seq.len = save_len;
6586	break;
6587	}
6588	if (ret != TRACE_TYPE_NO_CONSUME)
6589	trace_consume(iter);
6590
6591	if (trace_seq_used(&iter->seq) >= cnt)
6592	break;
6593
6594	/*
6595	* Setting the full flag means we reached the trace_seq buffer
6596	* size and we should leave by partial output condition above.
6597	* One of the trace_seq_* functions is not used properly.
6598	*/
6599	WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6600	iter->ent->type);
6601	}
6602	trace_access_unlock(iter->cpu_file);
6603	trace_event_read_unlock();
6604
6605	/* Now copy what we have to the user */
6606	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6607	if (iter->seq.readpos >= trace_seq_used(&iter->seq))
6608	trace_seq_init(&iter->seq);
6609
6610	/*
6611	* If there was nothing to send to user, in spite of consuming trace
6612	* entries, go back to wait for more entries.
6613	*/
6614	if (sret == -EBUSY)
6615	goto waitagain;
6616
6617	out:
6618	mutex_unlock(&iter->mutex);
6619
6620	return sret;
6621	}
6622
6623	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6624	unsigned int idx)
6625	{
6626	__free_page(spd->pages[idx]);
6627	}
6628
6629	static size_t
6630	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6631	{
6632	size_t count;
6633	int save_len;
6634	int ret;
6635
6636	/* Seq buffer is page-sized, exactly what we need. */
6637	for (;;) {
6638	save_len = iter->seq.seq.len;
6639	ret = print_trace_line(iter);
6640
6641	if (trace_seq_has_overflowed(&iter->seq)) {
6642	iter->seq.seq.len = save_len;
6643	break;
6644	}
6645
6646	/*
6647	* This should not be hit, because it should only
6648	* be set if the iter->seq overflowed. But check it
6649	* anyway to be safe.
6650	*/
6651	if (ret == TRACE_TYPE_PARTIAL_LINE) {
6652	iter->seq.seq.len = save_len;
6653	break;
6654	}
6655
6656	count = trace_seq_used(&iter->seq) - save_len;
6657	if (rem < count) {
6658	rem = 0;
6659	iter->seq.seq.len = save_len;
6660	break;
6661	}
6662
6663	if (ret != TRACE_TYPE_NO_CONSUME)
6664	trace_consume(iter);
6665	rem -= count;
6666	if (!trace_find_next_entry_inc(iter)) {
6667	rem = 0;
6668	iter->ent = NULL;
6669	break;
6670	}
6671	}
6672
6673	return rem;
6674	}
6675
6676	static ssize_t tracing_splice_read_pipe(struct file *filp,
6677	loff_t *ppos,
6678	struct pipe_inode_info *pipe,
6679	size_t len,
6680	unsigned int flags)
6681	{
6682	struct page *pages_def[PIPE_DEF_BUFFERS];
6683	struct partial_page partial_def[PIPE_DEF_BUFFERS];
6684	struct trace_iterator *iter = filp->private_data;
6685	struct splice_pipe_desc spd = {
6686	.pages = pages_def,
6687	.partial = partial_def,
6688	.nr_pages = 0, /* This gets updated below. */
6689	.nr_pages_max = PIPE_DEF_BUFFERS,
6690	.ops = &default_pipe_buf_ops,
6691	.spd_release = tracing_spd_release_pipe,
6692	};
6693	ssize_t ret;
6694	size_t rem;
6695	unsigned int i;
6696
6697	if (splice_grow_spd(pipe, &spd))
6698	return -ENOMEM;
6699
6700	mutex_lock(&iter->mutex);
6701
6702	if (iter->trace->splice_read) {
6703	ret = iter->trace->splice_read(iter, filp,
6704	ppos, pipe, len, flags);
6705	if (ret)
6706	goto out_err;
6707	}
6708
6709	ret = tracing_wait_pipe(filp);
6710	if (ret <= 0)
6711	goto out_err;
6712
6713	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6714	ret = -EFAULT;
6715	goto out_err;
6716	}
6717
6718	trace_event_read_lock();
6719	trace_access_lock(iter->cpu_file);
6720
6721	/* Fill as many pages as possible. */
6722	for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6723	spd.pages[i] = alloc_page(GFP_KERNEL);
6724	if (!spd.pages[i])
6725	break;
6726
6727	rem = tracing_fill_pipe_page(rem, iter);
6728
6729	/* Copy the data into the page, so we can start over. */
6730	ret = trace_seq_to_buffer(&iter->seq,
6731	page_address(spd.pages[i]),
6732	trace_seq_used(&iter->seq));
6733	if (ret < 0) {
6734	__free_page(spd.pages[i]);
6735	break;
6736	}
6737	spd.partial[i].offset = 0;
6738	spd.partial[i].len = trace_seq_used(&iter->seq);
6739
6740	trace_seq_init(&iter->seq);
6741	}
6742
6743	trace_access_unlock(iter->cpu_file);
6744	trace_event_read_unlock();
6745	mutex_unlock(&iter->mutex);
6746
6747	spd.nr_pages = i;
6748
6749	if (i)
6750	ret = splice_to_pipe(pipe, &spd);
6751	else
6752	ret = 0;
6753	out:
6754	splice_shrink_spd(&spd);
6755	return ret;
6756
6757	out_err:
6758	mutex_unlock(&iter->mutex);
6759	goto out;
6760	}
6761
6762	static ssize_t
6763	tracing_entries_read(struct file filp, char __user ubuf,
6764	size_t cnt, loff_t *ppos)
6765	{
6766	struct inode *inode = file_inode(filp);
6767	struct trace_array *tr = inode->i_private;
6768	int cpu = tracing_get_cpu(inode);
6769	char buf[64];
6770	int r = 0;
6771	ssize_t ret;
6772
6773	mutex_lock(&trace_types_lock);
6774
6775	if (cpu == RING_BUFFER_ALL_CPUS) {
6776	int cpu, buf_size_same;
6777	unsigned long size;
6778
6779	size = 0;
6780	buf_size_same = 1;
6781	/* check if all cpu sizes are same */
6782	for_each_tracing_cpu(cpu) {
6783	/* fill in the size from first enabled cpu */
6784	if (size == 0)
6785	size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
6786	if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
6787	buf_size_same = 0;
6788	break;
6789	}
6790	}
6791
6792	if (buf_size_same) {
6793	if (!tr->ring_buffer_expanded)
6794	r = sprintf(buf, "%lu (expanded: %lu)\n",
6795	size >> 10,
6796	trace_buf_size >> 10);
6797	else
6798	r = sprintf(buf, "%lu\n", size >> 10);
6799	} else
6800	r = sprintf(buf, "X\n");
6801	} else
6802	r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
6803
6804	mutex_unlock(&trace_types_lock);
6805
6806	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6807	return ret;
6808	}
6809
6810	static ssize_t
6811	tracing_entries_write(struct file filp, const char __user ubuf,
6812	size_t cnt, loff_t *ppos)
6813	{
6814	struct inode *inode = file_inode(filp);
6815	struct trace_array *tr = inode->i_private;
6816	unsigned long val;
6817	int ret;
6818
6819	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6820	if (ret)
6821	return ret;
6822
6823	/* must have at least 1 entry */
6824	if (!val)
6825	return -EINVAL;
6826
6827	/* value is in KB */
6828	val <<= 10;
6829	ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6830	if (ret < 0)
6831	return ret;
6832
6833	*ppos += cnt;
6834
6835	return cnt;
6836	}
6837
6838	static ssize_t
6839	tracing_total_entries_read(struct file filp, char __user ubuf,
6840	size_t cnt, loff_t *ppos)
6841	{
6842	struct trace_array *tr = filp->private_data;
6843	char buf[64];
6844	int r, cpu;
6845	unsigned long size = 0, expanded_size = 0;
6846
6847	mutex_lock(&trace_types_lock);
6848	for_each_tracing_cpu(cpu) {
6849	size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
6850	if (!tr->ring_buffer_expanded)
6851	expanded_size += trace_buf_size >> 10;
6852	}
6853	if (tr->ring_buffer_expanded)
6854	r = sprintf(buf, "%lu\n", size);
6855	else
6856	r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6857	mutex_unlock(&trace_types_lock);
6858
6859	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6860	}
6861
6862	static ssize_t
6863	tracing_last_boot_read(struct file filp, char __user ubuf, size_t cnt, loff_t *ppos)
6864	{
6865	struct trace_array *tr = filp->private_data;
6866	struct seq_buf seq;
6867	char buf[64];
6868
6869	seq_buf_init(&seq, buf, 64);
6870
6871	seq_buf_printf(&seq, "text delta:\t%ld\n", tr->text_delta);
6872	seq_buf_printf(&seq, "data delta:\t%ld\n", tr->data_delta);
6873
6874	return simple_read_from_buffer(ubuf, cnt, ppos, buf, seq_buf_used(&seq));
6875	}
6876
6877	static int tracing_buffer_meta_open(struct inode inode, struct file filp)
6878	{
6879	struct trace_array *tr = inode->i_private;
6880	int cpu = tracing_get_cpu(inode);
6881	int ret;
6882
6883	ret = tracing_check_open_get_tr(tr);
6884	if (ret)
6885	return ret;
6886
6887	ret = ring_buffer_meta_seq_init(filp, tr->array_buffer.buffer, cpu);
6888	if (ret < 0)
6889	__trace_array_put(tr);
6890	return ret;
6891	}
6892
6893	static ssize_t
6894	tracing_free_buffer_write(struct file filp, const char __user ubuf,
6895	size_t cnt, loff_t *ppos)
6896	{
6897	/*
6898	* There is no need to read what the user has written, this function
6899	* is just to make sure that there is no error when "echo" is used
6900	*/
6901
6902	*ppos += cnt;
6903
6904	return cnt;
6905	}
6906
6907	static int
6908	tracing_free_buffer_release(struct inode inode, struct file filp)
6909	{
6910	struct trace_array *tr = inode->i_private;
6911
6912	/* disable tracing ? */
6913	if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6914	tracer_tracing_off(tr);
6915	/* resize the ring buffer to 0 */
6916	tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6917
6918	trace_array_put(tr);
6919
6920	return 0;
6921	}
6922
6923	#define TRACE_MARKER_MAX_SIZE 4096
6924
6925	static ssize_t
6926	tracing_mark_write(struct file filp, const char __user ubuf,
6927	size_t cnt, loff_t *fpos)
6928	{
6929	struct trace_array *tr = filp->private_data;
6930	struct ring_buffer_event *event;
6931	enum event_trigger_type tt = ETT_NONE;
6932	struct trace_buffer *buffer;
6933	struct print_entry *entry;
6934	int meta_size;
6935	ssize_t written;
6936	size_t size;
6937	int len;
6938
6939	/* Used in tracing_mark_raw_write() as well */
6940	#define FAULTED_STR "<faulted>"
6941	#define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6942
6943	if (tracing_disabled)
6944	return -EINVAL;
6945
6946	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6947	return -EINVAL;
6948
6949	if ((ssize_t)cnt < 0)
6950	return -EINVAL;
6951
6952	if (cnt > TRACE_MARKER_MAX_SIZE)
6953	cnt = TRACE_MARKER_MAX_SIZE;
6954
6955	meta_size = sizeof(entry) + 2; / add '\0' and possible '\n' */
6956	again:
6957	size = cnt + meta_size;
6958
6959	/* If less than "<faulted>", then make sure we can still add that */
6960	if (cnt < FAULTED_SIZE)
6961	size += FAULTED_SIZE - cnt;
6962
6963	buffer = tr->array_buffer.buffer;
6964	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6965	tracing_gen_ctx());
6966	if (unlikely(!event)) {
6967	/*
6968	* If the size was greater than what was allowed, then
6969	* make it smaller and try again.
6970	*/
6971	if (size > ring_buffer_max_event_size(buffer)) {
6972	/* cnt < FAULTED size should never be bigger than max */
6973	if (WARN_ON_ONCE(cnt < FAULTED_SIZE))
6974	return -EBADF;
6975	cnt = ring_buffer_max_event_size(buffer) - meta_size;
6976	/* The above should only happen once */
6977	if (WARN_ON_ONCE(cnt + meta_size == size))
6978	return -EBADF;
6979	goto again;
6980	}
6981
6982	/* Ring buffer disabled, return as if not open for write */
6983	return -EBADF;
6984	}
6985
6986	entry = ring_buffer_event_data(event);
6987	entry->ip = _THIS_IP_;
6988
6989	len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6990	if (len) {
6991	memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6992	cnt = FAULTED_SIZE;
6993	written = -EFAULT;
6994	} else
6995	written = cnt;
6996
6997	if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6998	/* do not add \n before testing triggers, but add \0 */
6999	entry->buf[cnt] = '\0';
7000	tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event);
7001	}
7002
7003	if (entry->buf[cnt - 1] != '\n') {
7004	entry->buf[cnt] = '\n';
7005	entry->buf[cnt + 1] = '\0';
7006	} else
7007	entry->buf[cnt] = '\0';
7008
7009	if (static_branch_unlikely(&trace_marker_exports_enabled))
7010	ftrace_exports(event, TRACE_EXPORT_MARKER);
7011	__buffer_unlock_commit(buffer, event);
7012
7013	if (tt)
7014	event_triggers_post_call(tr->trace_marker_file, tt);
7015
7016	return written;
7017	}
7018
7019	static ssize_t
7020	tracing_mark_raw_write(struct file filp, const char __user ubuf,
7021	size_t cnt, loff_t *fpos)
7022	{
7023	struct trace_array *tr = filp->private_data;
7024	struct ring_buffer_event *event;
7025	struct trace_buffer *buffer;
7026	struct raw_data_entry *entry;
7027	ssize_t written;
7028	int size;
7029	int len;
7030
7031	#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
7032
7033	if (tracing_disabled)
7034	return -EINVAL;
7035
7036	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7037	return -EINVAL;
7038
7039	/* The marker must at least have a tag id */
7040	if (cnt < sizeof(unsigned int))
7041	return -EINVAL;
7042
7043	size = sizeof(*entry) + cnt;
7044	if (cnt < FAULT_SIZE_ID)
7045	size += FAULT_SIZE_ID - cnt;
7046
7047	buffer = tr->array_buffer.buffer;
7048
7049	if (size > ring_buffer_max_event_size(buffer))
7050	return -EINVAL;
7051
7052	event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
7053	tracing_gen_ctx());
7054	if (!event)
7055	/* Ring buffer disabled, return as if not open for write */
7056	return -EBADF;
7057
7058	entry = ring_buffer_event_data(event);
7059
7060	len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
7061	if (len) {
7062	entry->id = -1;
7063	memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7064	written = -EFAULT;
7065	} else
7066	written = cnt;
7067
7068	__buffer_unlock_commit(buffer, event);
7069
7070	return written;
7071	}
7072
7073	static int tracing_clock_show(struct seq_file m, void v)
7074	{
7075	struct trace_array *tr = m->private;
7076	int i;
7077
7078	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
7079	seq_printf(m,
7080	"%s%s%s%s", i ? " " : "",
7081	i == tr->clock_id ? "[" : "", trace_clocks[i].name,
7082	i == tr->clock_id ? "]" : "");
7083	seq_putc(m, '\n');
7084
7085	return 0;
7086	}
7087
7088	int tracing_set_clock(struct trace_array tr, const char clockstr)
7089	{
7090	int i;
7091
7092	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
7093	if (strcmp(trace_clocks[i].name, clockstr) == 0)
7094	break;
7095	}
7096	if (i == ARRAY_SIZE(trace_clocks))
7097	return -EINVAL;
7098
7099	mutex_lock(&trace_types_lock);
7100
7101	tr->clock_id = i;
7102
7103	ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
7104
7105	/*
7106	* New clock may not be consistent with the previous clock.
7107	* Reset the buffer so that it doesn't have incomparable timestamps.
7108	*/
7109	tracing_reset_online_cpus(&tr->array_buffer);
7110
7111	#ifdef CONFIG_TRACER_MAX_TRACE
7112	if (tr->max_buffer.buffer)
7113	ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
7114	tracing_reset_online_cpus(&tr->max_buffer);
7115	#endif
7116
7117	mutex_unlock(&trace_types_lock);
7118
7119	return 0;
7120	}
7121
7122	static ssize_t tracing_clock_write(struct file filp, const char __user ubuf,
7123	size_t cnt, loff_t *fpos)
7124	{
7125	struct seq_file *m = filp->private_data;
7126	struct trace_array *tr = m->private;
7127	char buf[64];
7128	const char *clockstr;
7129	int ret;
7130
7131	if (cnt >= sizeof(buf))
7132	return -EINVAL;
7133
7134	if (copy_from_user(buf, ubuf, cnt))
7135	return -EFAULT;
7136
7137	buf[cnt] = 0;
7138
7139	clockstr = strstrip(buf);
7140
7141	ret = tracing_set_clock(tr, clockstr);
7142	if (ret)
7143	return ret;
7144
7145	*fpos += cnt;
7146
7147	return cnt;
7148	}
7149
7150	static int tracing_clock_open(struct inode inode, struct file file)
7151	{
7152	struct trace_array *tr = inode->i_private;
7153	int ret;
7154
7155	ret = tracing_check_open_get_tr(tr);
7156	if (ret)
7157	return ret;
7158
7159	ret = single_open(file, tracing_clock_show, inode->i_private);
7160	if (ret < 0)
7161	trace_array_put(tr);
7162
7163	return ret;
7164	}
7165
7166	static int tracing_time_stamp_mode_show(struct seq_file m, void v)
7167	{
7168	struct trace_array *tr = m->private;
7169
7170	mutex_lock(&trace_types_lock);
7171
7172	if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
7173	seq_puts(m, "delta [absolute]\n");
7174	else
7175	seq_puts(m, "[delta] absolute\n");
7176
7177	mutex_unlock(&trace_types_lock);
7178
7179	return 0;
7180	}
7181
7182	static int tracing_time_stamp_mode_open(struct inode inode, struct file file)
7183	{
7184	struct trace_array *tr = inode->i_private;
7185	int ret;
7186
7187	ret = tracing_check_open_get_tr(tr);
7188	if (ret)
7189	return ret;
7190
7191	ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
7192	if (ret < 0)
7193	trace_array_put(tr);
7194
7195	return ret;
7196	}
7197
7198	u64 tracing_event_time_stamp(struct trace_buffer buffer, struct ring_buffer_event rbe)
7199	{
7200	if (rbe == this_cpu_read(trace_buffered_event))
7201	return ring_buffer_time_stamp(buffer);
7202
7203	return ring_buffer_event_time_stamp(buffer, rbe);
7204	}
7205
7206	/*
7207	* Set or disable using the per CPU trace_buffer_event when possible.
7208	*/
7209	int tracing_set_filter_buffering(struct trace_array *tr, bool set)
7210	{
7211	int ret = 0;
7212
7213	mutex_lock(&trace_types_lock);
7214
7215	if (set && tr->no_filter_buffering_ref++)
7216	goto out;
7217
7218	if (!set) {
7219	if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) {
7220	ret = -EINVAL;
7221	goto out;
7222	}
7223
7224	--tr->no_filter_buffering_ref;
7225	}
7226	out:
7227	mutex_unlock(&trace_types_lock);
7228
7229	return ret;
7230	}
7231
7232	struct ftrace_buffer_info {
7233	struct trace_iterator iter;
7234	void *spare;
7235	unsigned int spare_cpu;
7236	unsigned int spare_size;
7237	unsigned int read;
7238	};
7239
7240	#ifdef CONFIG_TRACER_SNAPSHOT
7241	static int tracing_snapshot_open(struct inode inode, struct file file)
7242	{
7243	struct trace_array *tr = inode->i_private;
7244	struct trace_iterator *iter;
7245	struct seq_file *m;
7246	int ret;
7247
7248	ret = tracing_check_open_get_tr(tr);
7249	if (ret)
7250	return ret;
7251
7252	if (file->f_mode & FMODE_READ) {
7253	iter = __tracing_open(inode, file, true);
7254	if (IS_ERR(iter))
7255	ret = PTR_ERR(iter);
7256	} else {
7257	/* Writes still need the seq_file to hold the private data */
7258	ret = -ENOMEM;
7259	m = kzalloc(sizeof(*m), GFP_KERNEL);
7260	if (!m)
7261	goto out;
7262	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
7263	if (!iter) {
7264	kfree(m);
7265	goto out;
7266	}
7267	ret = 0;
7268
7269	iter->tr = tr;
7270	iter->array_buffer = &tr->max_buffer;
7271	iter->cpu_file = tracing_get_cpu(inode);
7272	m->private = iter;
7273	file->private_data = m;
7274	}
7275	out:
7276	if (ret < 0)
7277	trace_array_put(tr);
7278
7279	return ret;
7280	}
7281
7282	static void tracing_swap_cpu_buffer(void *tr)
7283	{
7284	update_max_tr_single((struct trace_array *)tr, current, smp_processor_id());
7285	}
7286
7287	static ssize_t
7288	tracing_snapshot_write(struct file filp, const char __user ubuf, size_t cnt,
7289	loff_t *ppos)
7290	{
7291	struct seq_file *m = filp->private_data;
7292	struct trace_iterator *iter = m->private;
7293	struct trace_array *tr = iter->tr;
7294	unsigned long val;
7295	int ret;
7296
7297	ret = tracing_update_buffers(tr);
7298	if (ret < 0)
7299	return ret;
7300
7301	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7302	if (ret)
7303	return ret;
7304
7305	mutex_lock(&trace_types_lock);
7306
7307	if (tr->current_trace->use_max_tr) {
7308	ret = -EBUSY;
7309	goto out;
7310	}
7311
7312	local_irq_disable();
7313	arch_spin_lock(&tr->max_lock);
7314	if (tr->cond_snapshot)
7315	ret = -EBUSY;
7316	arch_spin_unlock(&tr->max_lock);
7317	local_irq_enable();
7318	if (ret)
7319	goto out;
7320
7321	switch (val) {
7322	case 0:
7323	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7324	ret = -EINVAL;
7325	break;
7326	}
7327	if (tr->allocated_snapshot)
7328	free_snapshot(tr);
7329	break;
7330	case 1:
7331	/* Only allow per-cpu swap if the ring buffer supports it */
7332	#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
7333	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7334	ret = -EINVAL;
7335	break;
7336	}
7337	#endif
7338	if (tr->allocated_snapshot)
7339	ret = resize_buffer_duplicate_size(&tr->max_buffer,
7340	&tr->array_buffer, iter->cpu_file);
7341
7342	ret = tracing_arm_snapshot_locked(tr);
7343	if (ret)
7344	break;
7345
7346	/* Now, we're going to swap */
7347	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
7348	local_irq_disable();
7349	update_max_tr(tr, current, smp_processor_id(), NULL);
7350	local_irq_enable();
7351	} else {
7352	smp_call_function_single(iter->cpu_file, tracing_swap_cpu_buffer,
7353	(void *)tr, 1);
7354	}
7355	tracing_disarm_snapshot(tr);
7356	break;
7357	default:
7358	if (tr->allocated_snapshot) {
7359	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7360	tracing_reset_online_cpus(&tr->max_buffer);
7361	else
7362	tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
7363	}
7364	break;
7365	}
7366
7367	if (ret >= 0) {
7368	*ppos += cnt;
7369	ret = cnt;
7370	}
7371	out:
7372	mutex_unlock(&trace_types_lock);
7373	return ret;
7374	}
7375
7376	static int tracing_snapshot_release(struct inode inode, struct file file)
7377	{
7378	struct seq_file *m = file->private_data;
7379	int ret;
7380
7381	ret = tracing_release(inode, file);
7382
7383	if (file->f_mode & FMODE_READ)
7384	return ret;
7385
7386	/* If write only, the seq_file is just a stub */
7387	if (m)
7388	kfree(m->private);
7389	kfree(m);
7390
7391	return 0;
7392	}
7393
7394	static int tracing_buffers_open(struct inode inode, struct file filp);
7395	static ssize_t tracing_buffers_read(struct file filp, char __user ubuf,
7396	size_t count, loff_t *ppos);
7397	static int tracing_buffers_release(struct inode inode, struct file file);
7398	static ssize_t tracing_buffers_splice_read(struct file file, loff_t ppos,
7399	struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7400
7401	static int snapshot_raw_open(struct inode inode, struct file filp)
7402	{
7403	struct ftrace_buffer_info *info;
7404	int ret;
7405
7406	/* The following checks for tracefs lockdown */
7407	ret = tracing_buffers_open(inode, filp);
7408	if (ret < 0)
7409	return ret;
7410
7411	info = filp->private_data;
7412
7413	if (info->iter.trace->use_max_tr) {
7414	tracing_buffers_release(inode, filp);
7415	return -EBUSY;
7416	}
7417
7418	info->iter.snapshot = true;
7419	info->iter.array_buffer = &info->iter.tr->max_buffer;
7420
7421	return ret;
7422	}
7423
7424	#endif /* CONFIG_TRACER_SNAPSHOT */
7425
7426
7427	static const struct file_operations tracing_thresh_fops = {
7428	.open = tracing_open_generic,
7429	.read = tracing_thresh_read,
7430	.write = tracing_thresh_write,
7431	.llseek = generic_file_llseek,
7432	};
7433
7434	#ifdef CONFIG_TRACER_MAX_TRACE
7435	static const struct file_operations tracing_max_lat_fops = {
7436	.open = tracing_open_generic_tr,
7437	.read = tracing_max_lat_read,
7438	.write = tracing_max_lat_write,
7439	.llseek = generic_file_llseek,
7440	.release = tracing_release_generic_tr,
7441	};
7442	#endif
7443
7444	static const struct file_operations set_tracer_fops = {
7445	.open = tracing_open_generic_tr,
7446	.read = tracing_set_trace_read,
7447	.write = tracing_set_trace_write,
7448	.llseek = generic_file_llseek,
7449	.release = tracing_release_generic_tr,
7450	};
7451
7452	static const struct file_operations tracing_pipe_fops = {
7453	.open = tracing_open_pipe,
7454	.poll = tracing_poll_pipe,
7455	.read = tracing_read_pipe,
7456	.splice_read = tracing_splice_read_pipe,
7457	.release = tracing_release_pipe,
7458	};
7459
7460	static const struct file_operations tracing_entries_fops = {
7461	.open = tracing_open_generic_tr,
7462	.read = tracing_entries_read,
7463	.write = tracing_entries_write,
7464	.llseek = generic_file_llseek,
7465	.release = tracing_release_generic_tr,
7466	};
7467
7468	static const struct file_operations tracing_buffer_meta_fops = {
7469	.open = tracing_buffer_meta_open,
7470	.read = seq_read,
7471	.llseek = seq_lseek,
7472	.release = tracing_seq_release,
7473	};
7474
7475	static const struct file_operations tracing_total_entries_fops = {
7476	.open = tracing_open_generic_tr,
7477	.read = tracing_total_entries_read,
7478	.llseek = generic_file_llseek,
7479	.release = tracing_release_generic_tr,
7480	};
7481
7482	static const struct file_operations tracing_free_buffer_fops = {
7483	.open = tracing_open_generic_tr,
7484	.write = tracing_free_buffer_write,
7485	.release = tracing_free_buffer_release,
7486	};
7487
7488	static const struct file_operations tracing_mark_fops = {
7489	.open = tracing_mark_open,
7490	.write = tracing_mark_write,
7491	.release = tracing_release_generic_tr,
7492	};
7493
7494	static const struct file_operations tracing_mark_raw_fops = {
7495	.open = tracing_mark_open,
7496	.write = tracing_mark_raw_write,
7497	.release = tracing_release_generic_tr,
7498	};
7499
7500	static const struct file_operations trace_clock_fops = {
7501	.open = tracing_clock_open,
7502	.read = seq_read,
7503	.llseek = seq_lseek,
7504	.release = tracing_single_release_tr,
7505	.write = tracing_clock_write,
7506	};
7507
7508	static const struct file_operations trace_time_stamp_mode_fops = {
7509	.open = tracing_time_stamp_mode_open,
7510	.read = seq_read,
7511	.llseek = seq_lseek,
7512	.release = tracing_single_release_tr,
7513	};
7514
7515	static const struct file_operations last_boot_fops = {
7516	.open = tracing_open_generic_tr,
7517	.read = tracing_last_boot_read,
7518	.llseek = generic_file_llseek,
7519	.release = tracing_release_generic_tr,
7520	};
7521
7522	#ifdef CONFIG_TRACER_SNAPSHOT
7523	static const struct file_operations snapshot_fops = {
7524	.open = tracing_snapshot_open,
7525	.read = seq_read,
7526	.write = tracing_snapshot_write,
7527	.llseek = tracing_lseek,
7528	.release = tracing_snapshot_release,
7529	};
7530
7531	static const struct file_operations snapshot_raw_fops = {
7532	.open = snapshot_raw_open,
7533	.read = tracing_buffers_read,
7534	.release = tracing_buffers_release,
7535	.splice_read = tracing_buffers_splice_read,
7536	};
7537
7538	#endif /* CONFIG_TRACER_SNAPSHOT */
7539
7540	/*
7541	* trace_min_max_write - Write a u64 value to a trace_min_max_param struct
7542	* @filp: The active open file structure
7543	* @ubuf: The userspace provided buffer to read value into
7544	* @cnt: The maximum number of bytes to read
7545	* @ppos: The current "file" position
7546	*
7547	* This function implements the write interface for a struct trace_min_max_param.
7548	* The filp->private_data must point to a trace_min_max_param structure that
7549	* defines where to write the value, the min and the max acceptable values,
7550	* and a lock to protect the write.
7551	*/
7552	static ssize_t
7553	trace_min_max_write(struct file filp, const char __user ubuf, size_t cnt, loff_t *ppos)
7554	{
7555	struct trace_min_max_param *param = filp->private_data;
7556	u64 val;
7557	int err;
7558
7559	if (!param)
7560	return -EFAULT;
7561
7562	err = kstrtoull_from_user(ubuf, cnt, 10, &val);
7563	if (err)
7564	return err;
7565
7566	if (param->lock)
7567	mutex_lock(param->lock);
7568
7569	if (param->min && val < *param->min)
7570	err = -EINVAL;
7571
7572	if (param->max && val > *param->max)
7573	err = -EINVAL;
7574
7575	if (!err)
7576	*param->val = val;
7577
7578	if (param->lock)
7579	mutex_unlock(param->lock);
7580
7581	if (err)
7582	return err;
7583
7584	return cnt;
7585	}
7586
7587	/*
7588	* trace_min_max_read - Read a u64 value from a trace_min_max_param struct
7589	* @filp: The active open file structure
7590	* @ubuf: The userspace provided buffer to read value into
7591	* @cnt: The maximum number of bytes to read
7592	* @ppos: The current "file" position
7593	*
7594	* This function implements the read interface for a struct trace_min_max_param.
7595	* The filp->private_data must point to a trace_min_max_param struct with valid
7596	* data.
7597	*/
7598	static ssize_t
7599	trace_min_max_read(struct file filp, char __user ubuf, size_t cnt, loff_t *ppos)
7600	{
7601	struct trace_min_max_param *param = filp->private_data;
7602	char buf[U64_STR_SIZE];
7603	int len;
7604	u64 val;
7605
7606	if (!param)
7607	return -EFAULT;
7608
7609	val = *param->val;
7610
7611	if (cnt > sizeof(buf))
7612	cnt = sizeof(buf);
7613
7614	len = snprintf(buf, sizeof(buf), "%llu\n", val);
7615
7616	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
7617	}
7618
7619	const struct file_operations trace_min_max_fops = {
7620	.open = tracing_open_generic,
7621	.read = trace_min_max_read,
7622	.write = trace_min_max_write,
7623	};
7624
7625	#define TRACING_LOG_ERRS_MAX 8
7626	#define TRACING_LOG_LOC_MAX 128
7627
7628	#define CMD_PREFIX " Command: "
7629
7630	struct err_info {
7631	const char *errs; / ptr to loc-specific array of err strings */
7632	u8 type; /* index into errs -> specific err string */
7633	u16 pos; /* caret position */
7634	u64 ts;
7635	};
7636
7637	struct tracing_log_err {
7638	struct list_head list;
7639	struct err_info info;
7640	char loc[TRACING_LOG_LOC_MAX]; /* err location */
7641	char cmd; / what caused err */
7642	};
7643
7644	static DEFINE_MUTEX(tracing_err_log_lock);
7645
7646	static struct tracing_log_err *alloc_tracing_log_err(int len)
7647	{
7648	struct tracing_log_err *err;
7649
7650	err = kzalloc(sizeof(*err), GFP_KERNEL);
7651	if (!err)
7652	return ERR_PTR(-ENOMEM);
7653
7654	err->cmd = kzalloc(len, GFP_KERNEL);
7655	if (!err->cmd) {
7656	kfree(err);
7657	return ERR_PTR(-ENOMEM);
7658	}
7659
7660	return err;
7661	}
7662
7663	static void free_tracing_log_err(struct tracing_log_err *err)
7664	{
7665	kfree(err->cmd);
7666	kfree(err);
7667	}
7668
7669	static struct tracing_log_err get_tracing_log_err(struct trace_array tr,
7670	int len)
7671	{
7672	struct tracing_log_err *err;
7673	char *cmd;
7674
7675	if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7676	err = alloc_tracing_log_err(len);
7677	if (PTR_ERR(err) != -ENOMEM)
7678	tr->n_err_log_entries++;
7679
7680	return err;
7681	}
7682	cmd = kzalloc(len, GFP_KERNEL);
7683	if (!cmd)
7684	return ERR_PTR(-ENOMEM);
7685	err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7686	kfree(err->cmd);
7687	err->cmd = cmd;
7688	list_del(&err->list);
7689
7690	return err;
7691	}
7692
7693	/**
7694	* err_pos - find the position of a string within a command for error careting
7695	* @cmd: The tracing command that caused the error
7696	* @str: The string to position the caret at within @cmd
7697	*
7698	* Finds the position of the first occurrence of @str within @cmd. The
7699	* return value can be passed to tracing_log_err() for caret placement
7700	* within @cmd.
7701	*
7702	* Returns the index within @cmd of the first occurrence of @str or 0
7703	* if @str was not found.
7704	*/
7705	unsigned int err_pos(char cmd, const char str)
7706	{
7707	char *found;
7708
7709	if (WARN_ON(!strlen(cmd)))
7710	return 0;
7711
7712	found = strstr(cmd, str);
7713	if (found)
7714	return found - cmd;
7715
7716	return 0;
7717	}
7718
7719	/**
7720	* tracing_log_err - write an error to the tracing error log
7721	* @tr: The associated trace array for the error (NULL for top level array)
7722	* @loc: A string describing where the error occurred
7723	* @cmd: The tracing command that caused the error
7724	* @errs: The array of loc-specific static error strings
7725	* @type: The index into errs[], which produces the specific static err string
7726	* @pos: The position the caret should be placed in the cmd
7727	*
7728	* Writes an error into tracing/error_log of the form:
7729	*
7730	* <loc>: error: <text>
7731	* Command: <cmd>
7732	* ^
7733	*
7734	* tracing/error_log is a small log file containing the last
7735	* TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7736	* unless there has been a tracing error, and the error log can be
7737	* cleared and have its memory freed by writing the empty string in
7738	* truncation mode to it i.e. echo > tracing/error_log.
7739	*
7740	* NOTE: the @errs array along with the @type param are used to
7741	* produce a static error string - this string is not copied and saved
7742	* when the error is logged - only a pointer to it is saved. See
7743	* existing callers for examples of how static strings are typically
7744	* defined for use with tracing_log_err().
7745	*/
7746	void tracing_log_err(struct trace_array *tr,
7747	const char loc, const char cmd,
7748	const char **errs, u8 type, u16 pos)
7749	{
7750	struct tracing_log_err *err;
7751	int len = 0;
7752
7753	if (!tr)
7754	tr = &global_trace;
7755
7756	len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1;
7757
7758	mutex_lock(&tracing_err_log_lock);
7759	err = get_tracing_log_err(tr, len);
7760	if (PTR_ERR(err) == -ENOMEM) {
7761	mutex_unlock(&tracing_err_log_lock);
7762	return;
7763	}
7764
7765	snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7766	snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd);
7767
7768	err->info.errs = errs;
7769	err->info.type = type;
7770	err->info.pos = pos;
7771	err->info.ts = local_clock();
7772
7773	list_add_tail(&err->list, &tr->err_log);
7774	mutex_unlock(&tracing_err_log_lock);
7775	}
7776
7777	static void clear_tracing_err_log(struct trace_array *tr)
7778	{
7779	struct tracing_log_err err, next;
7780
7781	mutex_lock(&tracing_err_log_lock);
7782	list_for_each_entry_safe(err, next, &tr->err_log, list) {
7783	list_del(&err->list);
7784	free_tracing_log_err(err);
7785	}
7786
7787	tr->n_err_log_entries = 0;
7788	mutex_unlock(&tracing_err_log_lock);
7789	}
7790
7791	static void tracing_err_log_seq_start(struct seq_file m, loff_t *pos)
7792	{
7793	struct trace_array *tr = m->private;
7794
7795	mutex_lock(&tracing_err_log_lock);
7796
7797	return seq_list_start(&tr->err_log, *pos);
7798	}
7799
7800	static void tracing_err_log_seq_next(struct seq_file m, void v, loff_t pos)
7801	{
7802	struct trace_array *tr = m->private;
7803
7804	return seq_list_next(v, &tr->err_log, pos);
7805	}
7806
7807	static void tracing_err_log_seq_stop(struct seq_file m, void v)
7808	{
7809	mutex_unlock(&tracing_err_log_lock);
7810	}
7811
7812	static void tracing_err_log_show_pos(struct seq_file *m, u16 pos)
7813	{
7814	u16 i;
7815
7816	for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7817	seq_putc(m, ' ');
7818	for (i = 0; i < pos; i++)
7819	seq_putc(m, ' ');
7820	seq_puts(m, "^\n");
7821	}
7822
7823	static int tracing_err_log_seq_show(struct seq_file m, void v)
7824	{
7825	struct tracing_log_err *err = v;
7826
7827	if (err) {
7828	const char *err_text = err->info.errs[err->info.type];
7829	u64 sec = err->info.ts;
7830	u32 nsec;
7831
7832	nsec = do_div(sec, NSEC_PER_SEC);
7833	seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7834	err->loc, err_text);
7835	seq_printf(m, "%s", err->cmd);
7836	tracing_err_log_show_pos(m, err->info.pos);
7837	}
7838
7839	return 0;
7840	}
7841
7842	static const struct seq_operations tracing_err_log_seq_ops = {
7843	.start = tracing_err_log_seq_start,
7844	.next = tracing_err_log_seq_next,
7845	.stop = tracing_err_log_seq_stop,
7846	.show = tracing_err_log_seq_show
7847	};
7848
7849	static int tracing_err_log_open(struct inode inode, struct file file)
7850	{
7851	struct trace_array *tr = inode->i_private;
7852	int ret = 0;
7853
7854	ret = tracing_check_open_get_tr(tr);
7855	if (ret)
7856	return ret;
7857
7858	/* If this file was opened for write, then erase contents */
7859	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7860	clear_tracing_err_log(tr);
7861
7862	if (file->f_mode & FMODE_READ) {
7863	ret = seq_open(file, &tracing_err_log_seq_ops);
7864	if (!ret) {
7865	struct seq_file *m = file->private_data;
7866	m->private = tr;
7867	} else {
7868	trace_array_put(tr);
7869	}
7870	}
7871	return ret;
7872	}
7873
7874	static ssize_t tracing_err_log_write(struct file *file,
7875	const char __user *buffer,
7876	size_t count, loff_t *ppos)
7877	{
7878	return count;
7879	}
7880
7881	static int tracing_err_log_release(struct inode inode, struct file file)
7882	{
7883	struct trace_array *tr = inode->i_private;
7884
7885	trace_array_put(tr);
7886
7887	if (file->f_mode & FMODE_READ)
7888	seq_release(inode, file);
7889
7890	return 0;
7891	}
7892
7893	static const struct file_operations tracing_err_log_fops = {
7894	.open = tracing_err_log_open,
7895	.write = tracing_err_log_write,
7896	.read = seq_read,
7897	.llseek = tracing_lseek,
7898	.release = tracing_err_log_release,
7899	};
7900
7901	static int tracing_buffers_open(struct inode inode, struct file filp)
7902	{
7903	struct trace_array *tr = inode->i_private;
7904	struct ftrace_buffer_info *info;
7905	int ret;
7906
7907	ret = tracing_check_open_get_tr(tr);
7908	if (ret)
7909	return ret;
7910
7911	info = kvzalloc(sizeof(*info), GFP_KERNEL);
7912	if (!info) {
7913	trace_array_put(tr);
7914	return -ENOMEM;
7915	}
7916
7917	mutex_lock(&trace_types_lock);
7918
7919	info->iter.tr = tr;
7920	info->iter.cpu_file = tracing_get_cpu(inode);
7921	info->iter.trace = tr->current_trace;
7922	info->iter.array_buffer = &tr->array_buffer;
7923	info->spare = NULL;
7924	/* Force reading ring buffer for first read */
7925	info->read = (unsigned int)-1;
7926
7927	filp->private_data = info;
7928
7929	tr->trace_ref++;
7930
7931	mutex_unlock(&trace_types_lock);
7932
7933	ret = nonseekable_open(inode, filp);
7934	if (ret < 0)
7935	trace_array_put(tr);
7936
7937	return ret;
7938	}
7939
7940	static __poll_t
7941	tracing_buffers_poll(struct file filp, poll_table poll_table)
7942	{
7943	struct ftrace_buffer_info *info = filp->private_data;
7944	struct trace_iterator *iter = &info->iter;
7945
7946	return trace_poll(iter, filp, poll_table);
7947	}
7948
7949	static ssize_t
7950	tracing_buffers_read(struct file filp, char __user ubuf,
7951	size_t count, loff_t *ppos)
7952	{
7953	struct ftrace_buffer_info *info = filp->private_data;
7954	struct trace_iterator *iter = &info->iter;
7955	void *trace_data;
7956	int page_size;
7957	ssize_t ret = 0;
7958	ssize_t size;
7959
7960	if (!count)
7961	return 0;
7962
7963	#ifdef CONFIG_TRACER_MAX_TRACE
7964	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7965	return -EBUSY;
7966	#endif
7967
7968	page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer);
7969
7970	/* Make sure the spare matches the current sub buffer size */
7971	if (info->spare) {
7972	if (page_size != info->spare_size) {
7973	ring_buffer_free_read_page(iter->array_buffer->buffer,
7974	info->spare_cpu, info->spare);
7975	info->spare = NULL;
7976	}
7977	}
7978
7979	if (!info->spare) {
7980	info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
7981	iter->cpu_file);
7982	if (IS_ERR(info->spare)) {
7983	ret = PTR_ERR(info->spare);
7984	info->spare = NULL;
7985	} else {
7986	info->spare_cpu = iter->cpu_file;
7987	info->spare_size = page_size;
7988	}
7989	}
7990	if (!info->spare)
7991	return ret;
7992
7993	/* Do we have previous read data to read? */
7994	if (info->read < page_size)
7995	goto read;
7996
7997	again:
7998	trace_access_lock(iter->cpu_file);
7999	ret = ring_buffer_read_page(iter->array_buffer->buffer,
8000	info->spare,
8001	count,
8002	iter->cpu_file, 0);
8003	trace_access_unlock(iter->cpu_file);
8004
8005	if (ret < 0) {
8006	if (trace_empty(iter) && !iter->closed) {
8007	if ((filp->f_flags & O_NONBLOCK))
8008	return -EAGAIN;
8009
8010	ret = wait_on_pipe(iter, 0);
8011	if (ret)
8012	return ret;
8013
8014	goto again;
8015	}
8016	return 0;
8017	}
8018
8019	info->read = 0;
8020	read:
8021	size = page_size - info->read;
8022	if (size > count)
8023	size = count;
8024	trace_data = ring_buffer_read_page_data(info->spare);
8025	ret = copy_to_user(ubuf, trace_data + info->read, size);
8026	if (ret == size)
8027	return -EFAULT;
8028
8029	size -= ret;
8030
8031	*ppos += size;
8032	info->read += size;
8033
8034	return size;
8035	}
8036
8037	static int tracing_buffers_flush(struct file *file, fl_owner_t id)
8038	{
8039	struct ftrace_buffer_info *info = file->private_data;
8040	struct trace_iterator *iter = &info->iter;
8041
8042	iter->closed = true;
8043	/* Make sure the waiters see the new wait_index */
8044	(void)atomic_fetch_inc_release(&iter->wait_index);
8045
8046	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
8047
8048	return 0;
8049	}
8050
8051	static int tracing_buffers_release(struct inode inode, struct file file)
8052	{
8053	struct ftrace_buffer_info *info = file->private_data;
8054	struct trace_iterator *iter = &info->iter;
8055
8056	mutex_lock(&trace_types_lock);
8057
8058	iter->tr->trace_ref--;
8059
8060	__trace_array_put(iter->tr);
8061
8062	if (info->spare)
8063	ring_buffer_free_read_page(iter->array_buffer->buffer,
8064	info->spare_cpu, info->spare);
8065	kvfree(info);
8066
8067	mutex_unlock(&trace_types_lock);
8068
8069	return 0;
8070	}
8071
8072	struct buffer_ref {
8073	struct trace_buffer *buffer;
8074	void *page;
8075	int cpu;
8076	refcount_t refcount;
8077	};
8078
8079	static void buffer_ref_release(struct buffer_ref *ref)
8080	{
8081	if (!refcount_dec_and_test(&ref->refcount))
8082	return;
8083	ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
8084	kfree(ref);
8085	}
8086
8087	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
8088	struct pipe_buffer *buf)
8089	{
8090	struct buffer_ref ref = (struct buffer_ref )buf->private;
8091
8092	buffer_ref_release(ref);
8093	buf->private = 0;
8094	}
8095
8096	static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
8097	struct pipe_buffer *buf)
8098	{
8099	struct buffer_ref ref = (struct buffer_ref )buf->private;
8100
8101	if (refcount_read(&ref->refcount) > INT_MAX/2)
8102	return false;
8103
8104	refcount_inc(&ref->refcount);
8105	return true;
8106	}
8107
8108	/* Pipe buffer operations for a buffer. */
8109	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
8110	.release = buffer_pipe_buf_release,
8111	.get = buffer_pipe_buf_get,
8112	};
8113
8114	/*
8115	* Callback from splice_to_pipe(), if we need to release some pages
8116	* at the end of the spd in case we error'ed out in filling the pipe.
8117	*/
8118	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
8119	{
8120	struct buffer_ref *ref =
8121	(struct buffer_ref *)spd->partial[i].private;
8122
8123	buffer_ref_release(ref);
8124	spd->partial[i].private = 0;
8125	}
8126
8127	static ssize_t
8128	tracing_buffers_splice_read(struct file file, loff_t ppos,
8129	struct pipe_inode_info *pipe, size_t len,
8130	unsigned int flags)
8131	{
8132	struct ftrace_buffer_info *info = file->private_data;
8133	struct trace_iterator *iter = &info->iter;
8134	struct partial_page partial_def[PIPE_DEF_BUFFERS];
8135	struct page *pages_def[PIPE_DEF_BUFFERS];
8136	struct splice_pipe_desc spd = {
8137	.pages = pages_def,
8138	.partial = partial_def,
8139	.nr_pages_max = PIPE_DEF_BUFFERS,
8140	.ops = &buffer_pipe_buf_ops,
8141	.spd_release = buffer_spd_release,
8142	};
8143	struct buffer_ref *ref;
8144	bool woken = false;
8145	int page_size;
8146	int entries, i;
8147	ssize_t ret = 0;
8148
8149	#ifdef CONFIG_TRACER_MAX_TRACE
8150	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8151	return -EBUSY;
8152	#endif
8153
8154	page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer);
8155	if (*ppos & (page_size - 1))
8156	return -EINVAL;
8157
8158	if (len & (page_size - 1)) {
8159	if (len < page_size)
8160	return -EINVAL;
8161	len &= (~(page_size - 1));
8162	}
8163
8164	if (splice_grow_spd(pipe, &spd))
8165	return -ENOMEM;
8166
8167	again:
8168	trace_access_lock(iter->cpu_file);
8169	entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8170
8171	for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) {
8172	struct page *page;
8173	int r;
8174
8175	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
8176	if (!ref) {
8177	ret = -ENOMEM;
8178	break;
8179	}
8180
8181	refcount_set(&ref->refcount, 1);
8182	ref->buffer = iter->array_buffer->buffer;
8183	ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
8184	if (IS_ERR(ref->page)) {
8185	ret = PTR_ERR(ref->page);
8186	ref->page = NULL;
8187	kfree(ref);
8188	break;
8189	}
8190	ref->cpu = iter->cpu_file;
8191
8192	r = ring_buffer_read_page(ref->buffer, ref->page,
8193	len, iter->cpu_file, 1);
8194	if (r < 0) {
8195	ring_buffer_free_read_page(ref->buffer, ref->cpu,
8196	ref->page);
8197	kfree(ref);
8198	break;
8199	}
8200
8201	page = virt_to_page(ring_buffer_read_page_data(ref->page));
8202
8203	spd.pages[i] = page;
8204	spd.partial[i].len = page_size;
8205	spd.partial[i].offset = 0;
8206	spd.partial[i].private = (unsigned long)ref;
8207	spd.nr_pages++;
8208	*ppos += page_size;
8209
8210	entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8211	}
8212
8213	trace_access_unlock(iter->cpu_file);
8214	spd.nr_pages = i;
8215
8216	/* did we read anything? */
8217	if (!spd.nr_pages) {
8218
8219	if (ret)
8220	goto out;
8221
8222	if (woken)
8223	goto out;
8224
8225	ret = -EAGAIN;
8226	if ((file->f_flags & O_NONBLOCK) \|\| (flags & SPLICE_F_NONBLOCK))
8227	goto out;
8228
8229	ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent);
8230	if (ret)
8231	goto out;
8232
8233	/* No need to wait after waking up when tracing is off */
8234	if (!tracer_tracing_is_on(iter->tr))
8235	goto out;
8236
8237	/* Iterate one more time to collect any new data then exit */
8238	woken = true;
8239
8240	goto again;
8241	}
8242
8243	ret = splice_to_pipe(pipe, &spd);
8244	out:
8245	splice_shrink_spd(&spd);
8246
8247	return ret;
8248	}
8249
8250	static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
8251	{
8252	struct ftrace_buffer_info *info = file->private_data;
8253	struct trace_iterator *iter = &info->iter;
8254	int err;
8255
8256	if (cmd == TRACE_MMAP_IOCTL_GET_READER) {
8257	if (!(file->f_flags & O_NONBLOCK)) {
8258	err = ring_buffer_wait(iter->array_buffer->buffer,
8259	iter->cpu_file,
8260	iter->tr->buffer_percent,
8261	NULL, NULL);
8262	if (err)
8263	return err;
8264	}
8265
8266	return ring_buffer_map_get_reader(iter->array_buffer->buffer,
8267	iter->cpu_file);
8268	} else if (cmd) {
8269	return -ENOTTY;
8270	}
8271
8272	/*
8273	* An ioctl call with cmd 0 to the ring buffer file will wake up all
8274	* waiters
8275	*/
8276	mutex_lock(&trace_types_lock);
8277
8278	/* Make sure the waiters see the new wait_index */
8279	(void)atomic_fetch_inc_release(&iter->wait_index);
8280
8281	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
8282
8283	mutex_unlock(&trace_types_lock);
8284	return 0;
8285	}
8286
8287	#ifdef CONFIG_TRACER_MAX_TRACE
8288	static int get_snapshot_map(struct trace_array *tr)
8289	{
8290	int err = 0;
8291
8292	/*
8293	* Called with mmap_lock held. lockdep would be unhappy if we would now
8294	* take trace_types_lock. Instead use the specific
8295	* snapshot_trigger_lock.
8296	*/
8297	spin_lock(&tr->snapshot_trigger_lock);
8298
8299	if (tr->snapshot \|\| tr->mapped == UINT_MAX)
8300	err = -EBUSY;
8301	else
8302	tr->mapped++;
8303
8304	spin_unlock(&tr->snapshot_trigger_lock);
8305
8306	/* Wait for update_max_tr() to observe iter->tr->mapped */
8307	if (tr->mapped == 1)
8308	synchronize_rcu();
8309
8310	return err;
8311
8312	}
8313	static void put_snapshot_map(struct trace_array *tr)
8314	{
8315	spin_lock(&tr->snapshot_trigger_lock);
8316	if (!WARN_ON(!tr->mapped))
8317	tr->mapped--;
8318	spin_unlock(&tr->snapshot_trigger_lock);
8319	}
8320	#else
8321	static inline int get_snapshot_map(struct trace_array *tr) { return 0; }
8322	static inline void put_snapshot_map(struct trace_array *tr) { }
8323	#endif
8324
8325	static void tracing_buffers_mmap_close(struct vm_area_struct *vma)
8326	{
8327	struct ftrace_buffer_info *info = vma->vm_file->private_data;
8328	struct trace_iterator *iter = &info->iter;
8329
8330	WARN_ON(ring_buffer_unmap(iter->array_buffer->buffer, iter->cpu_file));
8331	put_snapshot_map(iter->tr);
8332	}
8333
8334	static const struct vm_operations_struct tracing_buffers_vmops = {
8335	.close = tracing_buffers_mmap_close,
8336	};
8337
8338	static int tracing_buffers_mmap(struct file filp, struct vm_area_struct vma)
8339	{
8340	struct ftrace_buffer_info *info = filp->private_data;
8341	struct trace_iterator *iter = &info->iter;
8342	int ret = 0;
8343
8344	ret = get_snapshot_map(iter->tr);
8345	if (ret)
8346	return ret;
8347
8348	ret = ring_buffer_map(iter->array_buffer->buffer, iter->cpu_file, vma);
8349	if (ret)
8350	put_snapshot_map(iter->tr);
8351
8352	vma->vm_ops = &tracing_buffers_vmops;
8353
8354	return ret;
8355	}
8356
8357	static const struct file_operations tracing_buffers_fops = {
8358	.open = tracing_buffers_open,
8359	.read = tracing_buffers_read,
8360	.poll = tracing_buffers_poll,
8361	.release = tracing_buffers_release,
8362	.flush = tracing_buffers_flush,
8363	.splice_read = tracing_buffers_splice_read,
8364	.unlocked_ioctl = tracing_buffers_ioctl,
8365	.mmap = tracing_buffers_mmap,
8366	};
8367
8368	static ssize_t
8369	tracing_stats_read(struct file filp, char __user ubuf,
8370	size_t count, loff_t *ppos)
8371	{
8372	struct inode *inode = file_inode(filp);
8373	struct trace_array *tr = inode->i_private;
8374	struct array_buffer *trace_buf = &tr->array_buffer;
8375	int cpu = tracing_get_cpu(inode);
8376	struct trace_seq *s;
8377	unsigned long cnt;
8378	unsigned long long t;
8379	unsigned long usec_rem;
8380
8381	s = kmalloc(sizeof(*s), GFP_KERNEL);
8382	if (!s)
8383	return -ENOMEM;
8384
8385	trace_seq_init(s);
8386
8387	cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
8388	trace_seq_printf(s, "entries: %ld\n", cnt);
8389
8390	cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
8391	trace_seq_printf(s, "overrun: %ld\n", cnt);
8392
8393	cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
8394	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
8395
8396	cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
8397	trace_seq_printf(s, "bytes: %ld\n", cnt);
8398
8399	if (trace_clocks[tr->clock_id].in_ns) {
8400	/* local or global for trace_clock */
8401	t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8402	usec_rem = do_div(t, USEC_PER_SEC);
8403	trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
8404	t, usec_rem);
8405
8406	t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer));
8407	usec_rem = do_div(t, USEC_PER_SEC);
8408	trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
8409	} else {
8410	/* counter or tsc mode for trace_clock */
8411	trace_seq_printf(s, "oldest event ts: %llu\n",
8412	ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8413
8414	trace_seq_printf(s, "now ts: %llu\n",
8415	ring_buffer_time_stamp(trace_buf->buffer));
8416	}
8417
8418	cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
8419	trace_seq_printf(s, "dropped events: %ld\n", cnt);
8420
8421	cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
8422	trace_seq_printf(s, "read events: %ld\n", cnt);
8423
8424	count = simple_read_from_buffer(ubuf, count, ppos,
8425	s->buffer, trace_seq_used(s));
8426
8427	kfree(s);
8428
8429	return count;
8430	}
8431
8432	static const struct file_operations tracing_stats_fops = {
8433	.open = tracing_open_generic_tr,
8434	.read = tracing_stats_read,
8435	.llseek = generic_file_llseek,
8436	.release = tracing_release_generic_tr,
8437	};
8438
8439	#ifdef CONFIG_DYNAMIC_FTRACE
8440
8441	static ssize_t
8442	tracing_read_dyn_info(struct file filp, char __user ubuf,
8443	size_t cnt, loff_t *ppos)
8444	{
8445	ssize_t ret;
8446	char *buf;
8447	int r;
8448
8449	/* 512 should be plenty to hold the amount needed */
8450	#define DYN_INFO_BUF_SIZE 512
8451
8452	buf = kmalloc(DYN_INFO_BUF_SIZE, GFP_KERNEL);
8453	if (!buf)
8454	return -ENOMEM;
8455
8456	r = scnprintf(buf, DYN_INFO_BUF_SIZE,
8457	"%ld pages:%ld groups: %ld\n"
8458	"ftrace boot update time = %llu (ns)\n"
8459	"ftrace module total update time = %llu (ns)\n",
8460	ftrace_update_tot_cnt,
8461	ftrace_number_of_pages,
8462	ftrace_number_of_groups,
8463	ftrace_update_time,
8464	ftrace_total_mod_time);
8465
8466	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8467	kfree(buf);
8468	return ret;
8469	}
8470
8471	static const struct file_operations tracing_dyn_info_fops = {
8472	.open = tracing_open_generic,
8473	.read = tracing_read_dyn_info,
8474	.llseek = generic_file_llseek,
8475	};
8476	#endif /* CONFIG_DYNAMIC_FTRACE */
8477
8478	#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
8479	static void
8480	ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
8481	struct trace_array tr, struct ftrace_probe_ops ops,
8482	void *data)
8483	{
8484	tracing_snapshot_instance(tr);
8485	}
8486
8487	static void
8488	ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
8489	struct trace_array tr, struct ftrace_probe_ops ops,
8490	void *data)
8491	{
8492	struct ftrace_func_mapper *mapper = data;
8493	long *count = NULL;
8494
8495	if (mapper)
8496	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8497
8498	if (count) {
8499
8500	if (*count <= 0)
8501	return;
8502
8503	(*count)--;
8504	}
8505
8506	tracing_snapshot_instance(tr);
8507	}
8508
8509	static int
8510	ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
8511	struct ftrace_probe_ops ops, void data)
8512	{
8513	struct ftrace_func_mapper *mapper = data;
8514	long *count = NULL;
8515
8516	seq_printf(m, "%ps:", (void *)ip);
8517
8518	seq_puts(m, "snapshot");
8519
8520	if (mapper)
8521	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8522
8523	if (count)
8524	seq_printf(m, ":count=%ld\n", *count);
8525	else
8526	seq_puts(m, ":unlimited\n");
8527
8528	return 0;
8529	}
8530
8531	static int
8532	ftrace_snapshot_init(struct ftrace_probe_ops ops, struct trace_array tr,
8533	unsigned long ip, void init_data, void *data)
8534	{
8535	struct ftrace_func_mapper mapper = data;
8536
8537	if (!mapper) {
8538	mapper = allocate_ftrace_func_mapper();
8539	if (!mapper)
8540	return -ENOMEM;
8541	*data = mapper;
8542	}
8543
8544	return ftrace_func_mapper_add_ip(mapper, ip, init_data);
8545	}
8546
8547	static void
8548	ftrace_snapshot_free(struct ftrace_probe_ops ops, struct trace_array tr,
8549	unsigned long ip, void *data)
8550	{
8551	struct ftrace_func_mapper *mapper = data;
8552
8553	if (!ip) {
8554	if (!mapper)
8555	return;
8556	free_ftrace_func_mapper(mapper, NULL);
8557	return;
8558	}
8559
8560	ftrace_func_mapper_remove_ip(mapper, ip);
8561	}
8562
8563	static struct ftrace_probe_ops snapshot_probe_ops = {
8564	.func = ftrace_snapshot,
8565	.print = ftrace_snapshot_print,
8566	};
8567
8568	static struct ftrace_probe_ops snapshot_count_probe_ops = {
8569	.func = ftrace_count_snapshot,
8570	.print = ftrace_snapshot_print,
8571	.init = ftrace_snapshot_init,
8572	.free = ftrace_snapshot_free,
8573	};
8574
8575	static int
8576	ftrace_trace_snapshot_callback(struct trace_array tr, struct ftrace_hash hash,
8577	char glob, char cmd, char *param, int enable)
8578	{
8579	struct ftrace_probe_ops *ops;
8580	void count = (void )-1;
8581	char *number;
8582	int ret;
8583
8584	if (!tr)
8585	return -ENODEV;
8586
8587	/* hash funcs only work with set_ftrace_filter */
8588	if (!enable)
8589	return -EINVAL;
8590
8591	ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
8592
8593	if (glob[0] == '!') {
8594	ret = unregister_ftrace_function_probe_func(glob+1, tr, ops);
8595	if (!ret)
8596	tracing_disarm_snapshot(tr);
8597
8598	return ret;
8599	}
8600
8601	if (!param)
8602	goto out_reg;
8603
8604	number = strsep(&param, ":");
8605
8606	if (!strlen(number))
8607	goto out_reg;
8608
8609	/*
8610	* We use the callback data field (which is a pointer)
8611	* as our counter.
8612	*/
8613	ret = kstrtoul(number, 0, (unsigned long *)&count);
8614	if (ret)
8615	return ret;
8616
8617	out_reg:
8618	ret = tracing_arm_snapshot(tr);
8619	if (ret < 0)
8620	goto out;
8621
8622	ret = register_ftrace_function_probe(glob, tr, ops, count);
8623	if (ret < 0)
8624	tracing_disarm_snapshot(tr);
8625	out:
8626	return ret < 0 ? ret : 0;
8627	}
8628
8629	static struct ftrace_func_command ftrace_snapshot_cmd = {
8630	.name = "snapshot",
8631	.func = ftrace_trace_snapshot_callback,
8632	};
8633
8634	static __init int register_snapshot_cmd(void)
8635	{
8636	return register_ftrace_command(&ftrace_snapshot_cmd);
8637	}
8638	#else
8639	static inline __init int register_snapshot_cmd(void) { return 0; }
8640	#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
8641
8642	static struct dentry tracing_get_dentry(struct trace_array tr)
8643	{
8644	if (WARN_ON(!tr->dir))
8645	return ERR_PTR(-ENODEV);
8646
8647	/* Top directory uses NULL as the parent */
8648	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
8649	return NULL;
8650
8651	/* All sub buffers have a descriptor */
8652	return tr->dir;
8653	}
8654
8655	static struct dentry tracing_dentry_percpu(struct trace_array tr, int cpu)
8656	{
8657	struct dentry *d_tracer;
8658
8659	if (tr->percpu_dir)
8660	return tr->percpu_dir;
8661
8662	d_tracer = tracing_get_dentry(tr);
8663	if (IS_ERR(d_tracer))
8664	return NULL;
8665
8666	tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
8667
8668	MEM_FAIL(!tr->percpu_dir,
8669	"Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8670
8671	return tr->percpu_dir;
8672	}
8673
8674	static struct dentry *
8675	trace_create_cpu_file(const char name, umode_t mode, struct dentry parent,
8676	void data, long cpu, const struct file_operations fops)
8677	{
8678	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8679
8680	if (ret) /* See tracing_get_cpu() */
8681	d_inode(ret)->i_cdev = (void *)(cpu + 1);
8682	return ret;
8683	}
8684
8685	static void
8686	tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8687	{
8688	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8689	struct dentry *d_cpu;
8690	char cpu_dir[30]; /* 30 characters should be more than enough */
8691
8692	if (!d_percpu)
8693	return;
8694
8695	snprintf(cpu_dir, 30, "cpu%ld", cpu);
8696	d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8697	if (!d_cpu) {
8698	pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8699	return;
8700	}
8701
8702	/* per cpu trace_pipe */
8703	trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu,
8704	tr, cpu, &tracing_pipe_fops);
8705
8706	/* per cpu trace */
8707	trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu,
8708	tr, cpu, &tracing_fops);
8709
8710	trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu,
8711	tr, cpu, &tracing_buffers_fops);
8712
8713	trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu,
8714	tr, cpu, &tracing_stats_fops);
8715
8716	trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu,
8717	tr, cpu, &tracing_entries_fops);
8718
8719	if (tr->range_addr_start)
8720	trace_create_cpu_file("buffer_meta", TRACE_MODE_READ, d_cpu,
8721	tr, cpu, &tracing_buffer_meta_fops);
8722	#ifdef CONFIG_TRACER_SNAPSHOT
8723	if (!tr->range_addr_start) {
8724	trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu,
8725	tr, cpu, &snapshot_fops);
8726
8727	trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu,
8728	tr, cpu, &snapshot_raw_fops);
8729	}
8730	#endif
8731	}
8732
8733	#ifdef CONFIG_FTRACE_SELFTEST
8734	/* Let selftest have access to static functions in this file */
8735	#include "trace_selftest.c"
8736	#endif
8737
8738	static ssize_t
8739	trace_options_read(struct file filp, char __user ubuf, size_t cnt,
8740	loff_t *ppos)
8741	{
8742	struct trace_option_dentry *topt = filp->private_data;
8743	char *buf;
8744
8745	if (topt->flags->val & topt->opt->bit)
8746	buf = "1\n";
8747	else
8748	buf = "0\n";
8749
8750	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8751	}
8752
8753	static ssize_t
8754	trace_options_write(struct file filp, const char __user ubuf, size_t cnt,
8755	loff_t *ppos)
8756	{
8757	struct trace_option_dentry *topt = filp->private_data;
8758	unsigned long val;
8759	int ret;
8760
8761	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8762	if (ret)
8763	return ret;
8764
8765	if (val != 0 && val != 1)
8766	return -EINVAL;
8767
8768	if (!!(topt->flags->val & topt->opt->bit) != val) {
8769	mutex_lock(&trace_types_lock);
8770	ret = __set_tracer_option(topt->tr, topt->flags,
8771	topt->opt, !val);
8772	mutex_unlock(&trace_types_lock);
8773	if (ret)
8774	return ret;
8775	}
8776
8777	*ppos += cnt;
8778
8779	return cnt;
8780	}
8781
8782	static int tracing_open_options(struct inode inode, struct file filp)
8783	{
8784	struct trace_option_dentry *topt = inode->i_private;
8785	int ret;
8786
8787	ret = tracing_check_open_get_tr(topt->tr);
8788	if (ret)
8789	return ret;
8790
8791	filp->private_data = inode->i_private;
8792	return 0;
8793	}
8794
8795	static int tracing_release_options(struct inode inode, struct file file)
8796	{
8797	struct trace_option_dentry *topt = file->private_data;
8798
8799	trace_array_put(topt->tr);
8800	return 0;
8801	}
8802
8803	static const struct file_operations trace_options_fops = {
8804	.open = tracing_open_options,
8805	.read = trace_options_read,
8806	.write = trace_options_write,
8807	.llseek = generic_file_llseek,
8808	.release = tracing_release_options,
8809	};
8810
8811	/*
8812	* In order to pass in both the trace_array descriptor as well as the index
8813	* to the flag that the trace option file represents, the trace_array
8814	* has a character array of trace_flags_index[], which holds the index
8815	* of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8816	* The address of this character array is passed to the flag option file
8817	* read/write callbacks.
8818	*
8819	* In order to extract both the index and the trace_array descriptor,
8820	* get_tr_index() uses the following algorithm.
8821	*
8822	* idx = *ptr;
8823	*
8824	* As the pointer itself contains the address of the index (remember
8825	* index[1] == 1).
8826	*
8827	* Then to get the trace_array descriptor, by subtracting that index
8828	* from the ptr, we get to the start of the index itself.
8829	*
8830	* ptr - idx == &index[0]
8831	*
8832	* Then a simple container_of() from that pointer gets us to the
8833	* trace_array descriptor.
8834	*/
8835	static void get_tr_index(void data, struct trace_array *ptr,
8836	unsigned int *pindex)
8837	{
8838	pindex = (unsigned char *)data;
8839
8840	ptr = container_of(data - pindex, struct trace_array,
8841	trace_flags_index);
8842	}
8843
8844	static ssize_t
8845	trace_options_core_read(struct file filp, char __user ubuf, size_t cnt,
8846	loff_t *ppos)
8847	{
8848	void *tr_index = filp->private_data;
8849	struct trace_array *tr;
8850	unsigned int index;
8851	char *buf;
8852
8853	get_tr_index(tr_index, &tr, &index);
8854
8855	if (tr->trace_flags & (1 << index))
8856	buf = "1\n";
8857	else
8858	buf = "0\n";
8859
8860	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8861	}
8862
8863	static ssize_t
8864	trace_options_core_write(struct file filp, const char __user ubuf, size_t cnt,
8865	loff_t *ppos)
8866	{
8867	void *tr_index = filp->private_data;
8868	struct trace_array *tr;
8869	unsigned int index;
8870	unsigned long val;
8871	int ret;
8872
8873	get_tr_index(tr_index, &tr, &index);
8874
8875	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8876	if (ret)
8877	return ret;
8878
8879	if (val != 0 && val != 1)
8880	return -EINVAL;
8881
8882	mutex_lock(&event_mutex);
8883	mutex_lock(&trace_types_lock);
8884	ret = set_tracer_flag(tr, 1 << index, val);
8885	mutex_unlock(&trace_types_lock);
8886	mutex_unlock(&event_mutex);
8887
8888	if (ret < 0)
8889	return ret;
8890
8891	*ppos += cnt;
8892
8893	return cnt;
8894	}
8895
8896	static const struct file_operations trace_options_core_fops = {
8897	.open = tracing_open_generic,
8898	.read = trace_options_core_read,
8899	.write = trace_options_core_write,
8900	.llseek = generic_file_llseek,
8901	};
8902
8903	struct dentry trace_create_file(const char name,
8904	umode_t mode,
8905	struct dentry *parent,
8906	void *data,
8907	const struct file_operations *fops)
8908	{
8909	struct dentry *ret;
8910
8911	ret = tracefs_create_file(name, mode, parent, data, fops);
8912	if (!ret)
8913	pr_warn("Could not create tracefs '%s' entry\n", name);
8914
8915	return ret;
8916	}
8917
8918
8919	static struct dentry trace_options_init_dentry(struct trace_array tr)
8920	{
8921	struct dentry *d_tracer;
8922
8923	if (tr->options)
8924	return tr->options;
8925
8926	d_tracer = tracing_get_dentry(tr);
8927	if (IS_ERR(d_tracer))
8928	return NULL;
8929
8930	tr->options = tracefs_create_dir("options", d_tracer);
8931	if (!tr->options) {
8932	pr_warn("Could not create tracefs directory 'options'\n");
8933	return NULL;
8934	}
8935
8936	return tr->options;
8937	}
8938
8939	static void
8940	create_trace_option_file(struct trace_array *tr,
8941	struct trace_option_dentry *topt,
8942	struct tracer_flags *flags,
8943	struct tracer_opt *opt)
8944	{
8945	struct dentry *t_options;
8946
8947	t_options = trace_options_init_dentry(tr);
8948	if (!t_options)
8949	return;
8950
8951	topt->flags = flags;
8952	topt->opt = opt;
8953	topt->tr = tr;
8954
8955	topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,
8956	t_options, topt, &trace_options_fops);
8957
8958	}
8959
8960	static void
8961	create_trace_option_files(struct trace_array tr, struct tracer tracer)
8962	{
8963	struct trace_option_dentry *topts;
8964	struct trace_options *tr_topts;
8965	struct tracer_flags *flags;
8966	struct tracer_opt *opts;
8967	int cnt;
8968	int i;
8969
8970	if (!tracer)
8971	return;
8972
8973	flags = tracer->flags;
8974
8975	if (!flags \|\| !flags->opts)
8976	return;
8977
8978	/*
8979	* If this is an instance, only create flags for tracers
8980	* the instance may have.
8981	*/
8982	if (!trace_ok_for_array(tracer, tr))
8983	return;
8984
8985	for (i = 0; i < tr->nr_topts; i++) {
8986	/* Make sure there's no duplicate flags. */
8987	if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8988	return;
8989	}
8990
8991	opts = flags->opts;
8992
8993	for (cnt = 0; opts[cnt].name; cnt++)
8994	;
8995
8996	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
8997	if (!topts)
8998	return;
8999
9000	tr_topts = krealloc(tr->topts, sizeof(tr->topts) (tr->nr_topts + 1),
9001	GFP_KERNEL);
9002	if (!tr_topts) {
9003	kfree(topts);
9004	return;
9005	}
9006
9007	tr->topts = tr_topts;
9008	tr->topts[tr->nr_topts].tracer = tracer;
9009	tr->topts[tr->nr_topts].topts = topts;
9010	tr->nr_topts++;
9011
9012	for (cnt = 0; opts[cnt].name; cnt++) {
9013	create_trace_option_file(tr, &topts[cnt], flags,
9014	&opts[cnt]);
9015	MEM_FAIL(topts[cnt].entry == NULL,
9016	"Failed to create trace option: %s",
9017	opts[cnt].name);
9018	}
9019	}
9020
9021	static struct dentry *
9022	create_trace_option_core_file(struct trace_array *tr,
9023	const char *option, long index)
9024	{
9025	struct dentry *t_options;
9026
9027	t_options = trace_options_init_dentry(tr);
9028	if (!t_options)
9029	return NULL;
9030
9031	return trace_create_file(option, TRACE_MODE_WRITE, t_options,
9032	(void *)&tr->trace_flags_index[index],
9033	&trace_options_core_fops);
9034	}
9035
9036	static void create_trace_options_dir(struct trace_array *tr)
9037	{
9038	struct dentry *t_options;
9039	bool top_level = tr == &global_trace;
9040	int i;
9041
9042	t_options = trace_options_init_dentry(tr);
9043	if (!t_options)
9044	return;
9045
9046	for (i = 0; trace_options[i]; i++) {
9047	if (top_level \|\|
9048	!((1 << i) & TOP_LEVEL_TRACE_FLAGS))
9049	create_trace_option_core_file(tr, trace_options[i], i);
9050	}
9051	}
9052
9053	static ssize_t
9054	rb_simple_read(struct file filp, char __user ubuf,
9055	size_t cnt, loff_t *ppos)
9056	{
9057	struct trace_array *tr = filp->private_data;
9058	char buf[64];
9059	int r;
9060
9061	r = tracer_tracing_is_on(tr);
9062	r = sprintf(buf, "%d\n", r);
9063
9064	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
9065	}
9066
9067	static ssize_t
9068	rb_simple_write(struct file filp, const char __user ubuf,
9069	size_t cnt, loff_t *ppos)
9070	{
9071	struct trace_array *tr = filp->private_data;
9072	struct trace_buffer *buffer = tr->array_buffer.buffer;
9073	unsigned long val;
9074	int ret;
9075
9076	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9077	if (ret)
9078	return ret;
9079
9080	if (buffer) {
9081	mutex_lock(&trace_types_lock);
9082	if (!!val == tracer_tracing_is_on(tr)) {
9083	val = 0; /* do nothing */
9084	} else if (val) {
9085	tracer_tracing_on(tr);
9086	if (tr->current_trace->start)
9087	tr->current_trace->start(tr);
9088	} else {
9089	tracer_tracing_off(tr);
9090	if (tr->current_trace->stop)
9091	tr->current_trace->stop(tr);
9092	/* Wake up any waiters */
9093	ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS);
9094	}
9095	mutex_unlock(&trace_types_lock);
9096	}
9097
9098	(*ppos)++;
9099
9100	return cnt;
9101	}
9102
9103	static const struct file_operations rb_simple_fops = {
9104	.open = tracing_open_generic_tr,
9105	.read = rb_simple_read,
9106	.write = rb_simple_write,
9107	.release = tracing_release_generic_tr,
9108	.llseek = default_llseek,
9109	};
9110
9111	static ssize_t
9112	buffer_percent_read(struct file filp, char __user ubuf,
9113	size_t cnt, loff_t *ppos)
9114	{
9115	struct trace_array *tr = filp->private_data;
9116	char buf[64];
9117	int r;
9118
9119	r = tr->buffer_percent;
9120	r = sprintf(buf, "%d\n", r);
9121
9122	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
9123	}
9124
9125	static ssize_t
9126	buffer_percent_write(struct file filp, const char __user ubuf,
9127	size_t cnt, loff_t *ppos)
9128	{
9129	struct trace_array *tr = filp->private_data;
9130	unsigned long val;
9131	int ret;
9132
9133	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9134	if (ret)
9135	return ret;
9136
9137	if (val > 100)
9138	return -EINVAL;
9139
9140	tr->buffer_percent = val;
9141
9142	(*ppos)++;
9143
9144	return cnt;
9145	}
9146
9147	static const struct file_operations buffer_percent_fops = {
9148	.open = tracing_open_generic_tr,
9149	.read = buffer_percent_read,
9150	.write = buffer_percent_write,
9151	.release = tracing_release_generic_tr,
9152	.llseek = default_llseek,
9153	};
9154
9155	static ssize_t
9156	buffer_subbuf_size_read(struct file filp, char __user ubuf, size_t cnt, loff_t *ppos)
9157	{
9158	struct trace_array *tr = filp->private_data;
9159	size_t size;
9160	char buf[64];
9161	int order;
9162	int r;
9163
9164	order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
9165	size = (PAGE_SIZE << order) / 1024;
9166
9167	r = sprintf(buf, "%zd\n", size);
9168
9169	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
9170	}
9171
9172	static ssize_t
9173	buffer_subbuf_size_write(struct file filp, const char __user ubuf,
9174	size_t cnt, loff_t *ppos)
9175	{
9176	struct trace_array *tr = filp->private_data;
9177	unsigned long val;
9178	int old_order;
9179	int order;
9180	int pages;
9181	int ret;
9182
9183	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9184	if (ret)
9185	return ret;
9186
9187	val = 1024; / value passed in is in KB */
9188
9189	pages = DIV_ROUND_UP(val, PAGE_SIZE);
9190	order = fls(pages - 1);
9191
9192	/* limit between 1 and 128 system pages */
9193	if (order < 0 \|\| order > 7)
9194	return -EINVAL;
9195
9196	/* Do not allow tracing while changing the order of the ring buffer */
9197	tracing_stop_tr(tr);
9198
9199	old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
9200	if (old_order == order)
9201	goto out;
9202
9203	ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order);
9204	if (ret)
9205	goto out;
9206
9207	#ifdef CONFIG_TRACER_MAX_TRACE
9208
9209	if (!tr->allocated_snapshot)
9210	goto out_max;
9211
9212	ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order);
9213	if (ret) {
9214	/* Put back the old order */
9215	cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order);
9216	if (WARN_ON_ONCE(cnt)) {
9217	/*
9218	* AARGH! We are left with different orders!
9219	* The max buffer is our "snapshot" buffer.
9220	* When a tracer needs a snapshot (one of the
9221	* latency tracers), it swaps the max buffer
9222	* with the saved snap shot. We succeeded to
9223	* update the order of the main buffer, but failed to
9224	* update the order of the max buffer. But when we tried
9225	* to reset the main buffer to the original size, we
9226	* failed there too. This is very unlikely to
9227	* happen, but if it does, warn and kill all
9228	* tracing.
9229	*/
9230	tracing_disabled = 1;
9231	}
9232	goto out;
9233	}
9234	out_max:
9235	#endif
9236	(*ppos)++;
9237	out:
9238	if (ret)
9239	cnt = ret;
9240	tracing_start_tr(tr);
9241	return cnt;
9242	}
9243
9244	static const struct file_operations buffer_subbuf_size_fops = {
9245	.open = tracing_open_generic_tr,
9246	.read = buffer_subbuf_size_read,
9247	.write = buffer_subbuf_size_write,
9248	.release = tracing_release_generic_tr,
9249	.llseek = default_llseek,
9250	};
9251
9252	static struct dentry *trace_instance_dir;
9253
9254	static void
9255	init_tracer_tracefs(struct trace_array tr, struct dentry d_tracer);
9256
9257	static int
9258	allocate_trace_buffer(struct trace_array tr, struct array_buffer buf, int size)
9259	{
9260	enum ring_buffer_flags rb_flags;
9261
9262	rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
9263
9264	buf->tr = tr;
9265
9266	if (tr->range_addr_start && tr->range_addr_size) {
9267	buf->buffer = ring_buffer_alloc_range(size, rb_flags, 0,
9268	tr->range_addr_start,
9269	tr->range_addr_size);
9270
9271	ring_buffer_last_boot_delta(buf->buffer,
9272	&tr->text_delta, &tr->data_delta);
9273	/*
9274	* This is basically the same as a mapped buffer,
9275	* with the same restrictions.
9276	*/
9277	tr->mapped++;
9278	} else {
9279	buf->buffer = ring_buffer_alloc(size, rb_flags);
9280	}
9281	if (!buf->buffer)
9282	return -ENOMEM;
9283
9284	buf->data = alloc_percpu(struct trace_array_cpu);
9285	if (!buf->data) {
9286	ring_buffer_free(buf->buffer);
9287	buf->buffer = NULL;
9288	return -ENOMEM;
9289	}
9290
9291	/* Allocate the first page for all buffers */
9292	set_buffer_entries(&tr->array_buffer,
9293	ring_buffer_size(tr->array_buffer.buffer, 0));
9294
9295	return 0;
9296	}
9297
9298	static void free_trace_buffer(struct array_buffer *buf)
9299	{
9300	if (buf->buffer) {
9301	ring_buffer_free(buf->buffer);
9302	buf->buffer = NULL;
9303	free_percpu(buf->data);
9304	buf->data = NULL;
9305	}
9306	}
9307
9308	static int allocate_trace_buffers(struct trace_array *tr, int size)
9309	{
9310	int ret;
9311
9312	ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
9313	if (ret)
9314	return ret;
9315
9316	#ifdef CONFIG_TRACER_MAX_TRACE
9317	/* Fix mapped buffer trace arrays do not have snapshot buffers */
9318	if (tr->range_addr_start)
9319	return 0;
9320
9321	ret = allocate_trace_buffer(tr, &tr->max_buffer,
9322	allocate_snapshot ? size : 1);
9323	if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
9324	free_trace_buffer(&tr->array_buffer);
9325	return -ENOMEM;
9326	}
9327	tr->allocated_snapshot = allocate_snapshot;
9328
9329	allocate_snapshot = false;
9330	#endif
9331
9332	return 0;
9333	}
9334
9335	static void free_trace_buffers(struct trace_array *tr)
9336	{
9337	if (!tr)
9338	return;
9339
9340	free_trace_buffer(&tr->array_buffer);
9341
9342	#ifdef CONFIG_TRACER_MAX_TRACE
9343	free_trace_buffer(&tr->max_buffer);
9344	#endif
9345	}
9346
9347	static void init_trace_flags_index(struct trace_array *tr)
9348	{
9349	int i;
9350
9351	/* Used by the trace options files */
9352	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
9353	tr->trace_flags_index[i] = i;
9354	}
9355
9356	static void __update_tracer_options(struct trace_array *tr)
9357	{
9358	struct tracer *t;
9359
9360	for (t = trace_types; t; t = t->next)
9361	add_tracer_options(tr, t);
9362	}
9363
9364	static void update_tracer_options(struct trace_array *tr)
9365	{
9366	mutex_lock(&trace_types_lock);
9367	tracer_options_updated = true;
9368	__update_tracer_options(tr);
9369	mutex_unlock(&trace_types_lock);
9370	}
9371
9372	/* Must have trace_types_lock held */
9373	struct trace_array trace_array_find(const char instance)
9374	{
9375	struct trace_array tr, found = NULL;
9376
9377	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9378	if (tr->name && strcmp(tr->name, instance) == 0) {
9379	found = tr;
9380	break;
9381	}
9382	}
9383
9384	return found;
9385	}
9386
9387	struct trace_array trace_array_find_get(const char instance)
9388	{
9389	struct trace_array *tr;
9390
9391	mutex_lock(&trace_types_lock);
9392	tr = trace_array_find(instance);
9393	if (tr)
9394	tr->ref++;
9395	mutex_unlock(&trace_types_lock);
9396
9397	return tr;
9398	}
9399
9400	static int trace_array_create_dir(struct trace_array *tr)
9401	{
9402	int ret;
9403
9404	tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
9405	if (!tr->dir)
9406	return -EINVAL;
9407
9408	ret = event_trace_add_tracer(tr->dir, tr);
9409	if (ret) {
9410	tracefs_remove(tr->dir);
9411	return ret;
9412	}
9413
9414	init_tracer_tracefs(tr, tr->dir);
9415	__update_tracer_options(tr);
9416
9417	return ret;
9418	}
9419
9420	static struct trace_array *
9421	trace_array_create_systems(const char name, const char systems,
9422	unsigned long range_addr_start,
9423	unsigned long range_addr_size)
9424	{
9425	struct trace_array *tr;
9426	int ret;
9427
9428	ret = -ENOMEM;
9429	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
9430	if (!tr)
9431	return ERR_PTR(ret);
9432
9433	tr->name = kstrdup(name, GFP_KERNEL);
9434	if (!tr->name)
9435	goto out_free_tr;
9436
9437	if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
9438	goto out_free_tr;
9439
9440	if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL))
9441	goto out_free_tr;
9442
9443	if (systems) {
9444	tr->system_names = kstrdup_const(systems, GFP_KERNEL);
9445	if (!tr->system_names)
9446	goto out_free_tr;
9447	}
9448
9449	/* Only for boot up memory mapped ring buffers */
9450	tr->range_addr_start = range_addr_start;
9451	tr->range_addr_size = range_addr_size;
9452
9453	tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
9454
9455	cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
9456
9457	raw_spin_lock_init(&tr->start_lock);
9458
9459	tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9460	#ifdef CONFIG_TRACER_MAX_TRACE
9461	spin_lock_init(&tr->snapshot_trigger_lock);
9462	#endif
9463	tr->current_trace = &nop_trace;
9464
9465	INIT_LIST_HEAD(&tr->systems);
9466	INIT_LIST_HEAD(&tr->events);
9467	INIT_LIST_HEAD(&tr->hist_vars);
9468	INIT_LIST_HEAD(&tr->err_log);
9469
9470	if (allocate_trace_buffers(tr, trace_buf_size) < 0)
9471	goto out_free_tr;
9472
9473	/* The ring buffer is defaultly expanded */
9474	trace_set_ring_buffer_expanded(tr);
9475
9476	if (ftrace_allocate_ftrace_ops(tr) < 0)
9477	goto out_free_tr;
9478
9479	ftrace_init_trace_array(tr);
9480
9481	init_trace_flags_index(tr);
9482
9483	if (trace_instance_dir) {
9484	ret = trace_array_create_dir(tr);
9485	if (ret)
9486	goto out_free_tr;
9487	} else
9488	__trace_early_add_events(tr);
9489
9490	list_add(&tr->list, &ftrace_trace_arrays);
9491
9492	tr->ref++;
9493
9494	return tr;
9495
9496	out_free_tr:
9497	ftrace_free_ftrace_ops(tr);
9498	free_trace_buffers(tr);
9499	free_cpumask_var(tr->pipe_cpumask);
9500	free_cpumask_var(tr->tracing_cpumask);
9501	kfree_const(tr->system_names);
9502	kfree(tr->name);
9503	kfree(tr);
9504
9505	return ERR_PTR(ret);
9506	}
9507
9508	static struct trace_array trace_array_create(const char name)
9509	{
9510	return trace_array_create_systems(name, NULL, 0, 0);
9511	}
9512
9513	static int instance_mkdir(const char *name)
9514	{
9515	struct trace_array *tr;
9516	int ret;
9517
9518	mutex_lock(&event_mutex);
9519	mutex_lock(&trace_types_lock);
9520
9521	ret = -EEXIST;
9522	if (trace_array_find(name))
9523	goto out_unlock;
9524
9525	tr = trace_array_create(name);
9526
9527	ret = PTR_ERR_OR_ZERO(tr);
9528
9529	out_unlock:
9530	mutex_unlock(&trace_types_lock);
9531	mutex_unlock(&event_mutex);
9532	return ret;
9533	}
9534
9535	static u64 map_pages(u64 start, u64 size)
9536	{
9537	struct page **pages;
9538	phys_addr_t page_start;
9539	unsigned int page_count;
9540	unsigned int i;
9541	void *vaddr;
9542
9543	page_count = DIV_ROUND_UP(size, PAGE_SIZE);
9544
9545	page_start = start;
9546	pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL);
9547	if (!pages)
9548	return 0;
9549
9550	for (i = 0; i < page_count; i++) {
9551	phys_addr_t addr = page_start + i * PAGE_SIZE;
9552	pages[i] = pfn_to_page(addr >> PAGE_SHIFT);
9553	}
9554	vaddr = vmap(pages, page_count, VM_MAP, PAGE_KERNEL);
9555	kfree(pages);
9556
9557	return (u64)(unsigned long)vaddr;
9558	}
9559
9560	/**
9561	* trace_array_get_by_name - Create/Lookup a trace array, given its name.
9562	* @name: The name of the trace array to be looked up/created.
9563	* @systems: A list of systems to create event directories for (NULL for all)
9564	*
9565	* Returns pointer to trace array with given name.
9566	* NULL, if it cannot be created.
9567	*
9568	* NOTE: This function increments the reference counter associated with the
9569	* trace array returned. This makes sure it cannot be freed while in use.
9570	* Use trace_array_put() once the trace array is no longer needed.
9571	* If the trace_array is to be freed, trace_array_destroy() needs to
9572	* be called after the trace_array_put(), or simply let user space delete
9573	* it from the tracefs instances directory. But until the
9574	* trace_array_put() is called, user space can not delete it.
9575	*
9576	*/
9577	struct trace_array trace_array_get_by_name(const char name, const char *systems)
9578	{
9579	struct trace_array *tr;
9580
9581	mutex_lock(&event_mutex);
9582	mutex_lock(&trace_types_lock);
9583
9584	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9585	if (tr->name && strcmp(tr->name, name) == 0)
9586	goto out_unlock;
9587	}
9588
9589	tr = trace_array_create_systems(name, systems, 0, 0);
9590
9591	if (IS_ERR(tr))
9592	tr = NULL;
9593	out_unlock:
9594	if (tr)
9595	tr->ref++;
9596
9597	mutex_unlock(&trace_types_lock);
9598	mutex_unlock(&event_mutex);
9599	return tr;
9600	}
9601	EXPORT_SYMBOL_GPL(trace_array_get_by_name);
9602
9603	static int __remove_instance(struct trace_array *tr)
9604	{
9605	int i;
9606
9607	/* Reference counter for a newly created trace array = 1. */
9608	if (tr->ref > 1 \|\| (tr->current_trace && tr->trace_ref))
9609	return -EBUSY;
9610
9611	list_del(&tr->list);
9612
9613	/* Disable all the flags that were enabled coming in */
9614	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
9615	if ((1 << i) & ZEROED_TRACE_FLAGS)
9616	set_tracer_flag(tr, 1 << i, 0);
9617	}
9618
9619	if (printk_trace == tr)
9620	update_printk_trace(&global_trace);
9621
9622	tracing_set_nop(tr);
9623	clear_ftrace_function_probes(tr);
9624	event_trace_del_tracer(tr);
9625	ftrace_clear_pids(tr);
9626	ftrace_destroy_function_files(tr);
9627	tracefs_remove(tr->dir);
9628	free_percpu(tr->last_func_repeats);
9629	free_trace_buffers(tr);
9630	clear_tracing_err_log(tr);
9631
9632	for (i = 0; i < tr->nr_topts; i++) {
9633	kfree(tr->topts[i].topts);
9634	}
9635	kfree(tr->topts);
9636
9637	free_cpumask_var(tr->pipe_cpumask);
9638	free_cpumask_var(tr->tracing_cpumask);
9639	kfree_const(tr->system_names);
9640	kfree(tr->name);
9641	kfree(tr);
9642
9643	return 0;
9644	}
9645
9646	int trace_array_destroy(struct trace_array *this_tr)
9647	{
9648	struct trace_array *tr;
9649	int ret;
9650
9651	if (!this_tr)
9652	return -EINVAL;
9653
9654	mutex_lock(&event_mutex);
9655	mutex_lock(&trace_types_lock);
9656
9657	ret = -ENODEV;
9658
9659	/* Making sure trace array exists before destroying it. */
9660	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9661	if (tr == this_tr) {
9662	ret = __remove_instance(tr);
9663	break;
9664	}
9665	}
9666
9667	mutex_unlock(&trace_types_lock);
9668	mutex_unlock(&event_mutex);
9669
9670	return ret;
9671	}
9672	EXPORT_SYMBOL_GPL(trace_array_destroy);
9673
9674	static int instance_rmdir(const char *name)
9675	{
9676	struct trace_array *tr;
9677	int ret;
9678
9679	mutex_lock(&event_mutex);
9680	mutex_lock(&trace_types_lock);
9681
9682	ret = -ENODEV;
9683	tr = trace_array_find(name);
9684	if (tr)
9685	ret = __remove_instance(tr);
9686
9687	mutex_unlock(&trace_types_lock);
9688	mutex_unlock(&event_mutex);
9689
9690	return ret;
9691	}
9692
9693	static __init void create_trace_instances(struct dentry *d_tracer)
9694	{
9695	struct trace_array *tr;
9696
9697	trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
9698	instance_mkdir,
9699	instance_rmdir);
9700	if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
9701	return;
9702
9703	mutex_lock(&event_mutex);
9704	mutex_lock(&trace_types_lock);
9705
9706	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9707	if (!tr->name)
9708	continue;
9709	if (MEM_FAIL(trace_array_create_dir(tr) < 0,
9710	"Failed to create instance directory\n"))
9711	break;
9712	}
9713
9714	mutex_unlock(&trace_types_lock);
9715	mutex_unlock(&event_mutex);
9716	}
9717
9718	static void
9719	init_tracer_tracefs(struct trace_array tr, struct dentry d_tracer)
9720	{
9721	int cpu;
9722
9723	trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer,
9724	tr, &show_traces_fops);
9725
9726	trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer,
9727	tr, &set_tracer_fops);
9728
9729	trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer,
9730	tr, &tracing_cpumask_fops);
9731
9732	trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer,
9733	tr, &tracing_iter_fops);
9734
9735	trace_create_file("trace", TRACE_MODE_WRITE, d_tracer,
9736	tr, &tracing_fops);
9737
9738	trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer,
9739	tr, &tracing_pipe_fops);
9740
9741	trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer,
9742	tr, &tracing_entries_fops);
9743
9744	trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer,
9745	tr, &tracing_total_entries_fops);
9746
9747	trace_create_file("free_buffer", 0200, d_tracer,
9748	tr, &tracing_free_buffer_fops);
9749
9750	trace_create_file("trace_marker", 0220, d_tracer,
9751	tr, &tracing_mark_fops);
9752
9753	tr->trace_marker_file = __find_event_file(tr, "ftrace", "print");
9754
9755	trace_create_file("trace_marker_raw", 0220, d_tracer,
9756	tr, &tracing_mark_raw_fops);
9757
9758	trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr,
9759	&trace_clock_fops);
9760
9761	trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer,
9762	tr, &rb_simple_fops);
9763
9764	trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr,
9765	&trace_time_stamp_mode_fops);
9766
9767	tr->buffer_percent = 50;
9768
9769	trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer,
9770	tr, &buffer_percent_fops);
9771
9772	trace_create_file("buffer_subbuf_size_kb", TRACE_MODE_WRITE, d_tracer,
9773	tr, &buffer_subbuf_size_fops);
9774
9775	create_trace_options_dir(tr);
9776
9777	#ifdef CONFIG_TRACER_MAX_TRACE
9778	trace_create_maxlat_file(tr, d_tracer);
9779	#endif
9780
9781	if (ftrace_create_function_files(tr, d_tracer))
9782	MEM_FAIL(1, "Could not allocate function filter files");
9783
9784	if (tr->range_addr_start) {
9785	trace_create_file("last_boot_info", TRACE_MODE_READ, d_tracer,
9786	tr, &last_boot_fops);
9787	#ifdef CONFIG_TRACER_SNAPSHOT
9788	} else {
9789	trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer,
9790	tr, &snapshot_fops);
9791	#endif
9792	}
9793
9794	trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer,
9795	tr, &tracing_err_log_fops);
9796
9797	for_each_tracing_cpu(cpu)
9798	tracing_init_tracefs_percpu(tr, cpu);
9799
9800	ftrace_init_tracefs(tr, d_tracer);
9801	}
9802
9803	static struct vfsmount trace_automount(struct dentry mntpt, void *ingore)
9804	{
9805	struct vfsmount *mnt;
9806	struct file_system_type *type;
9807
9808	/*
9809	* To maintain backward compatibility for tools that mount
9810	* debugfs to get to the tracing facility, tracefs is automatically
9811	* mounted to the debugfs/tracing directory.
9812	*/
9813	type = get_fs_type("tracefs");
9814	if (!type)
9815	return NULL;
9816	mnt = vfs_submount(mntpt, type, "tracefs", NULL);
9817	put_filesystem(type);
9818	if (IS_ERR(mnt))
9819	return NULL;
9820	mntget(mnt);
9821
9822	return mnt;
9823	}
9824
9825	/**
9826	* tracing_init_dentry - initialize top level trace array
9827	*
9828	* This is called when creating files or directories in the tracing
9829	* directory. It is called via fs_initcall() by any of the boot up code
9830	* and expects to return the dentry of the top level tracing directory.
9831	*/
9832	int tracing_init_dentry(void)
9833	{
9834	struct trace_array *tr = &global_trace;
9835
9836	if (security_locked_down(LOCKDOWN_TRACEFS)) {
9837	pr_warn("Tracing disabled due to lockdown\n");
9838	return -EPERM;
9839	}
9840
9841	/* The top level trace array uses NULL as parent */
9842	if (tr->dir)
9843	return 0;
9844
9845	if (WARN_ON(!tracefs_initialized()))
9846	return -ENODEV;
9847
9848	/*
9849	* As there may still be users that expect the tracing
9850	* files to exist in debugfs/tracing, we must automount
9851	* the tracefs file system there, so older tools still
9852	* work with the newer kernel.
9853	*/
9854	tr->dir = debugfs_create_automount("tracing", NULL,
9855	trace_automount, NULL);
9856
9857	return 0;
9858	}
9859
9860	extern struct trace_eval_map *__start_ftrace_eval_maps[];
9861	extern struct trace_eval_map *__stop_ftrace_eval_maps[];
9862
9863	static struct workqueue_struct *eval_map_wq __initdata;
9864	static struct work_struct eval_map_work __initdata;
9865	static struct work_struct tracerfs_init_work __initdata;
9866
9867	static void __init eval_map_work_func(struct work_struct *work)
9868	{
9869	int len;
9870
9871	len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
9872	trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
9873	}
9874
9875	static int __init trace_eval_init(void)
9876	{
9877	INIT_WORK(&eval_map_work, eval_map_work_func);
9878
9879	eval_map_wq = alloc_workqueue("eval_map_wq", WQ_UNBOUND, 0);
9880	if (!eval_map_wq) {
9881	pr_err("Unable to allocate eval_map_wq\n");
9882	/* Do work here */
9883	eval_map_work_func(&eval_map_work);
9884	return -ENOMEM;
9885	}
9886
9887	queue_work(eval_map_wq, &eval_map_work);
9888	return 0;
9889	}
9890
9891	subsys_initcall(trace_eval_init);
9892
9893	static int __init trace_eval_sync(void)
9894	{
9895	/* Make sure the eval map updates are finished */
9896	if (eval_map_wq)
9897	destroy_workqueue(eval_map_wq);
9898	return 0;
9899	}
9900
9901	late_initcall_sync(trace_eval_sync);
9902
9903
9904	#ifdef CONFIG_MODULES
9905	static void trace_module_add_evals(struct module *mod)
9906	{
9907	if (!mod->num_trace_evals)
9908	return;
9909
9910	/*
9911	* Modules with bad taint do not have events created, do
9912	* not bother with enums either.
9913	*/
9914	if (trace_module_has_bad_taint(mod))
9915	return;
9916
9917	trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
9918	}
9919
9920	#ifdef CONFIG_TRACE_EVAL_MAP_FILE
9921	static void trace_module_remove_evals(struct module *mod)
9922	{
9923	union trace_eval_map_item *map;
9924	union trace_eval_map_item **last = &trace_eval_maps;
9925
9926	if (!mod->num_trace_evals)
9927	return;
9928
9929	mutex_lock(&trace_eval_mutex);
9930
9931	map = trace_eval_maps;
9932
9933	while (map) {
9934	if (map->head.mod == mod)
9935	break;
9936	map = trace_eval_jmp_to_tail(map);
9937	last = &map->tail.next;
9938	map = map->tail.next;
9939	}
9940	if (!map)
9941	goto out;
9942
9943	*last = trace_eval_jmp_to_tail(map)->tail.next;
9944	kfree(map);
9945	out:
9946	mutex_unlock(&trace_eval_mutex);
9947	}
9948	#else
9949	static inline void trace_module_remove_evals(struct module *mod) { }
9950	#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9951
9952	static int trace_module_notify(struct notifier_block *self,
9953	unsigned long val, void *data)
9954	{
9955	struct module *mod = data;
9956
9957	switch (val) {
9958	case MODULE_STATE_COMING:
9959	trace_module_add_evals(mod);
9960	break;
9961	case MODULE_STATE_GOING:
9962	trace_module_remove_evals(mod);
9963	break;
9964	}
9965
9966	return NOTIFY_OK;
9967	}
9968
9969	static struct notifier_block trace_module_nb = {
9970	.notifier_call = trace_module_notify,
9971	.priority = 0,
9972	};
9973	#endif /* CONFIG_MODULES */
9974
9975	static __init void tracer_init_tracefs_work_func(struct work_struct *work)
9976	{
9977
9978	event_trace_init();
9979
9980	init_tracer_tracefs(&global_trace, NULL);
9981	ftrace_init_tracefs_toplevel(&global_trace, NULL);
9982
9983	trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL,
9984	&global_trace, &tracing_thresh_fops);
9985
9986	trace_create_file("README", TRACE_MODE_READ, NULL,
9987	NULL, &tracing_readme_fops);
9988
9989	trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL,
9990	NULL, &tracing_saved_cmdlines_fops);
9991
9992	trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
9993	NULL, &tracing_saved_cmdlines_size_fops);
9994
9995	trace_create_file("saved_tgids", TRACE_MODE_READ, NULL,
9996	NULL, &tracing_saved_tgids_fops);
9997
9998	trace_create_eval_file(NULL);
9999
10000	#ifdef CONFIG_MODULES
10001	register_module_notifier(&trace_module_nb);
10002	#endif
10003
10004	#ifdef CONFIG_DYNAMIC_FTRACE
10005	trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
10006	NULL, &tracing_dyn_info_fops);
10007	#endif
10008
10009	create_trace_instances(NULL);
10010
10011	update_tracer_options(&global_trace);
10012	}
10013
10014	static __init int tracer_init_tracefs(void)
10015	{
10016	int ret;
10017
10018	trace_access_lock_init();
10019
10020	ret = tracing_init_dentry();
10021	if (ret)
10022	return 0;
10023
10024	if (eval_map_wq) {
10025	INIT_WORK(&tracerfs_init_work, tracer_init_tracefs_work_func);
10026	queue_work(eval_map_wq, &tracerfs_init_work);
10027	} else {
10028	tracer_init_tracefs_work_func(NULL);
10029	}
10030
10031	rv_init_interface();
10032
10033	return 0;
10034	}
10035
10036	fs_initcall(tracer_init_tracefs);
10037
10038	static int trace_die_panic_handler(struct notifier_block *self,
10039	unsigned long ev, void *unused);
10040
10041	static struct notifier_block trace_panic_notifier = {
10042	.notifier_call = trace_die_panic_handler,
10043	.priority = INT_MAX - 1,
10044	};
10045
10046	static struct notifier_block trace_die_notifier = {
10047	.notifier_call = trace_die_panic_handler,
10048	.priority = INT_MAX - 1,
10049	};
10050
10051	/*
10052	* The idea is to execute the following die/panic callback early, in order
10053	* to avoid showing irrelevant information in the trace (like other panic
10054	* notifier functions); we are the 2nd to run, after hung_task/rcu_stall
10055	* warnings get disabled (to prevent potential log flooding).
10056	*/
10057	static int trace_die_panic_handler(struct notifier_block *self,
10058	unsigned long ev, void *unused)
10059	{
10060	if (!ftrace_dump_on_oops_enabled())
10061	return NOTIFY_DONE;
10062
10063	/* The die notifier requires DIE_OOPS to trigger */
10064	if (self == &trace_die_notifier && ev != DIE_OOPS)
10065	return NOTIFY_DONE;
10066
10067	ftrace_dump(DUMP_PARAM);
10068
10069	return NOTIFY_DONE;
10070	}
10071
10072	/*
10073	* printk is set to max of 1024, we really don't need it that big.
10074	* Nothing should be printing 1000 characters anyway.
10075	*/
10076	#define TRACE_MAX_PRINT 1000
10077
10078	/*
10079	* Define here KERN_TRACE so that we have one place to modify
10080	* it if we decide to change what log level the ftrace dump
10081	* should be at.
10082	*/
10083	#define KERN_TRACE KERN_EMERG
10084
10085	void
10086	trace_printk_seq(struct trace_seq *s)
10087	{
10088	/* Probably should print a warning here. */
10089	if (s->seq.len >= TRACE_MAX_PRINT)
10090	s->seq.len = TRACE_MAX_PRINT;
10091
10092	/*
10093	* More paranoid code. Although the buffer size is set to
10094	* PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
10095	* an extra layer of protection.
10096	*/
10097	if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
10098	s->seq.len = s->seq.size - 1;
10099
10100	/* should be zero ended, but we are paranoid. */
10101	s->buffer[s->seq.len] = 0;
10102
10103	printk(KERN_TRACE "%s", s->buffer);
10104
10105	trace_seq_init(s);
10106	}
10107
10108	static void trace_init_iter(struct trace_iterator iter, struct trace_array tr)
10109	{
10110	iter->tr = tr;
10111	iter->trace = iter->tr->current_trace;
10112	iter->cpu_file = RING_BUFFER_ALL_CPUS;
10113	iter->array_buffer = &tr->array_buffer;
10114
10115	if (iter->trace && iter->trace->open)
10116	iter->trace->open(iter);
10117
10118	/* Annotate start of buffers if we had overruns */
10119	if (ring_buffer_overruns(iter->array_buffer->buffer))
10120	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
10121
10122	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
10123	if (trace_clocks[iter->tr->clock_id].in_ns)
10124	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
10125
10126	/* Can not use kmalloc for iter.temp and iter.fmt */
10127	iter->temp = static_temp_buf;
10128	iter->temp_size = STATIC_TEMP_BUF_SIZE;
10129	iter->fmt = static_fmt_buf;
10130	iter->fmt_size = STATIC_FMT_BUF_SIZE;
10131	}
10132
10133	void trace_init_global_iter(struct trace_iterator *iter)
10134	{
10135	trace_init_iter(iter, &global_trace);
10136	}
10137
10138	static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_mode)
10139	{
10140	/* use static because iter can be a bit big for the stack */
10141	static struct trace_iterator iter;
10142	unsigned int old_userobj;
10143	unsigned long flags;
10144	int cnt = 0, cpu;
10145
10146	/*
10147	* Always turn off tracing when we dump.
10148	* We don't need to show trace output of what happens
10149	* between multiple crashes.
10150	*
10151	* If the user does a sysrq-z, then they can re-enable
10152	* tracing with echo 1 > tracing_on.
10153	*/
10154	tracer_tracing_off(tr);
10155
10156	local_irq_save(flags);
10157
10158	/* Simulate the iterator */
10159	trace_init_iter(&iter, tr);
10160
10161	for_each_tracing_cpu(cpu) {
10162	atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
10163	}
10164
10165	old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
10166
10167	/* don't look at user memory in panic mode */
10168	tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
10169
10170	if (dump_mode == DUMP_ORIG)
10171	iter.cpu_file = raw_smp_processor_id();
10172	else
10173	iter.cpu_file = RING_BUFFER_ALL_CPUS;
10174
10175	if (tr == &global_trace)
10176	printk(KERN_TRACE "Dumping ftrace buffer:\n");
10177	else
10178	printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name);
10179
10180	/* Did function tracer already get disabled? */
10181	if (ftrace_is_dead()) {
10182	printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
10183	printk("# MAY BE MISSING FUNCTION EVENTS\n");
10184	}
10185
10186	/*
10187	* We need to stop all tracing on all CPUS to read
10188	* the next buffer. This is a bit expensive, but is
10189	* not done often. We fill all what we can read,
10190	* and then release the locks again.
10191	*/
10192
10193	while (!trace_empty(&iter)) {
10194
10195	if (!cnt)
10196	printk(KERN_TRACE "---------------------------------\n");
10197
10198	cnt++;
10199
10200	trace_iterator_reset(&iter);
10201	iter.iter_flags \|= TRACE_FILE_LAT_FMT;
10202
10203	if (trace_find_next_entry_inc(&iter) != NULL) {
10204	int ret;
10205
10206	ret = print_trace_line(&iter);
10207	if (ret != TRACE_TYPE_NO_CONSUME)
10208	trace_consume(&iter);
10209	}
10210	touch_nmi_watchdog();
10211
10212	trace_printk_seq(&iter.seq);
10213	}
10214
10215	if (!cnt)
10216	printk(KERN_TRACE " (ftrace buffer empty)\n");
10217	else
10218	printk(KERN_TRACE "---------------------------------\n");
10219
10220	tr->trace_flags \|= old_userobj;
10221
10222	for_each_tracing_cpu(cpu) {
10223	atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
10224	}
10225	local_irq_restore(flags);
10226	}
10227
10228	static void ftrace_dump_by_param(void)
10229	{
10230	bool first_param = true;
10231	char dump_param[MAX_TRACER_SIZE];
10232	char buf, token, *inst_name;
10233	struct trace_array *tr;
10234
10235	strscpy(dump_param, ftrace_dump_on_oops, MAX_TRACER_SIZE);
10236	buf = dump_param;
10237
10238	while ((token = strsep(&buf, ",")) != NULL) {
10239	if (first_param) {
10240	first_param = false;
10241	if (!strcmp("0", token))
10242	continue;
10243	else if (!strcmp("1", token)) {
10244	ftrace_dump_one(&global_trace, DUMP_ALL);
10245	continue;
10246	}
10247	else if (!strcmp("2", token) \|\|
10248	!strcmp("orig_cpu", token)) {
10249	ftrace_dump_one(&global_trace, DUMP_ORIG);
10250	continue;
10251	}
10252	}
10253
10254	inst_name = strsep(&token, "=");
10255	tr = trace_array_find(inst_name);
10256	if (!tr) {
10257	printk(KERN_TRACE "Instance %s not found\n", inst_name);
10258	continue;
10259	}
10260
10261	if (token && (!strcmp("2", token) \|\|
10262	!strcmp("orig_cpu", token)))
10263	ftrace_dump_one(tr, DUMP_ORIG);
10264	else
10265	ftrace_dump_one(tr, DUMP_ALL);
10266	}
10267	}
10268
10269	void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
10270	{
10271	static atomic_t dump_running;
10272
10273	/* Only allow one dump user at a time. */
10274	if (atomic_inc_return(&dump_running) != 1) {
10275	atomic_dec(&dump_running);
10276	return;
10277	}
10278
10279	switch (oops_dump_mode) {
10280	case DUMP_ALL:
10281	ftrace_dump_one(&global_trace, DUMP_ALL);
10282	break;
10283	case DUMP_ORIG:
10284	ftrace_dump_one(&global_trace, DUMP_ORIG);
10285	break;
10286	case DUMP_PARAM:
10287	ftrace_dump_by_param();
10288	break;
10289	case DUMP_NONE:
10290	break;
10291	default:
10292	printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
10293	ftrace_dump_one(&global_trace, DUMP_ALL);
10294	}
10295
10296	atomic_dec(&dump_running);
10297	}
10298	EXPORT_SYMBOL_GPL(ftrace_dump);
10299
10300	#define WRITE_BUFSIZE 4096
10301
10302	ssize_t trace_parse_run_command(struct file file, const char __user buffer,
10303	size_t count, loff_t *ppos,
10304	int (createfn)(const char ))
10305	{
10306	char kbuf, buf, *tmp;
10307	int ret = 0;
10308	size_t done = 0;
10309	size_t size;
10310
10311	kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
10312	if (!kbuf)
10313	return -ENOMEM;
10314
10315	while (done < count) {
10316	size = count - done;
10317
10318	if (size >= WRITE_BUFSIZE)
10319	size = WRITE_BUFSIZE - 1;
10320
10321	if (copy_from_user(kbuf, buffer + done, size)) {
10322	ret = -EFAULT;
10323	goto out;
10324	}
10325	kbuf[size] = '\0';
10326	buf = kbuf;
10327	do {
10328	tmp = strchr(buf, '\n');
10329	if (tmp) {
10330	*tmp = '\0';
10331	size = tmp - buf + 1;
10332	} else {
10333	size = strlen(buf);
10334	if (done + size < count) {
10335	if (buf != kbuf)
10336	break;
10337	/* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
10338	pr_warn("Line length is too long: Should be less than %d\n",
10339	WRITE_BUFSIZE - 2);
10340	ret = -EINVAL;
10341	goto out;
10342	}
10343	}
10344	done += size;
10345
10346	/* Remove comments */
10347	tmp = strchr(buf, '#');
10348
10349	if (tmp)
10350	*tmp = '\0';
10351
10352	ret = createfn(buf);
10353	if (ret)
10354	goto out;
10355	buf += size;
10356
10357	} while (done < count);
10358	}
10359	ret = done;
10360
10361	out:
10362	kfree(kbuf);
10363
10364	return ret;
10365	}
10366
10367	#ifdef CONFIG_TRACER_MAX_TRACE
10368	__init static bool tr_needs_alloc_snapshot(const char *name)
10369	{
10370	char *test;
10371	int len = strlen(name);
10372	bool ret;
10373
10374	if (!boot_snapshot_index)
10375	return false;
10376
10377	if (strncmp(name, boot_snapshot_info, len) == 0 &&
10378	boot_snapshot_info[len] == '\t')
10379	return true;
10380
10381	test = kmalloc(strlen(name) + 3, GFP_KERNEL);
10382	if (!test)
10383	return false;
10384
10385	sprintf(test, "\t%s\t", name);
10386	ret = strstr(boot_snapshot_info, test) == NULL;
10387	kfree(test);
10388	return ret;
10389	}
10390
10391	__init static void do_allocate_snapshot(const char *name)
10392	{
10393	if (!tr_needs_alloc_snapshot(name))
10394	return;
10395
10396	/*
10397	* When allocate_snapshot is set, the next call to
10398	* allocate_trace_buffers() (called by trace_array_get_by_name())
10399	* will allocate the snapshot buffer. That will alse clear
10400	* this flag.
10401	*/
10402	allocate_snapshot = true;
10403	}
10404	#else
10405	static inline void do_allocate_snapshot(const char *name) { }
10406	#endif
10407
10408	__init static void enable_instances(void)
10409	{
10410	struct trace_array *tr;
10411	char *curr_str;
10412	char *name;
10413	char *str;
10414	char *tok;
10415
10416	/* A tab is always appended */
10417	boot_instance_info[boot_instance_index - 1] = '\0';
10418	str = boot_instance_info;
10419
10420	while ((curr_str = strsep(&str, "\t"))) {
10421	phys_addr_t start = 0;
10422	phys_addr_t size = 0;
10423	unsigned long addr = 0;
10424	bool traceprintk = false;
10425	bool traceoff = false;
10426	char *flag_delim;
10427	char *addr_delim;
10428
10429	tok = strsep(&curr_str, ",");
10430
10431	flag_delim = strchr(tok, '^');
10432	addr_delim = strchr(tok, '@');
10433
10434	if (addr_delim)
10435	*addr_delim++ = '\0';
10436
10437	if (flag_delim)
10438	*flag_delim++ = '\0';
10439
10440	name = tok;
10441
10442	if (flag_delim) {
10443	char *flag;
10444
10445	while ((flag = strsep(&flag_delim, "^"))) {
10446	if (strcmp(flag, "traceoff") == 0) {
10447	traceoff = true;
10448	} else if ((strcmp(flag, "printk") == 0) \|\|
10449	(strcmp(flag, "traceprintk") == 0) \|\|
10450	(strcmp(flag, "trace_printk") == 0)) {
10451	traceprintk = true;
10452	} else {
10453	pr_info("Tracing: Invalid instance flag '%s' for %s\n",
10454	flag, name);
10455	}
10456	}
10457	}
10458
10459	tok = addr_delim;
10460	if (tok && isdigit(*tok)) {
10461	start = memparse(tok, &tok);
10462	if (!start) {
10463	pr_warn("Tracing: Invalid boot instance address for %s\n",
10464	name);
10465	continue;
10466	}
10467	if (*tok != ':') {
10468	pr_warn("Tracing: No size specified for instance %s\n", name);
10469	continue;
10470	}
10471	tok++;
10472	size = memparse(tok, &tok);
10473	if (!size) {
10474	pr_warn("Tracing: Invalid boot instance size for %s\n",
10475	name);
10476	continue;
10477	}
10478	} else if (tok) {
10479	if (!reserve_mem_find_by_name(tok, &start, &size)) {
10480	start = 0;
10481	pr_warn("Failed to map boot instance %s to %s\n", name, tok);
10482	continue;
10483	}
10484	}
10485
10486	if (start) {
10487	addr = map_pages(start, size);
10488	if (addr) {
10489	pr_info("Tracing: mapped boot instance %s at physical memory %pa of size 0x%lx\n",
10490	name, &start, (unsigned long)size);
10491	} else {
10492	pr_warn("Tracing: Failed to map boot instance %s\n", name);
10493	continue;
10494	}
10495	} else {
10496	/* Only non mapped buffers have snapshot buffers */
10497	if (IS_ENABLED(CONFIG_TRACER_MAX_TRACE))
10498	do_allocate_snapshot(name);
10499	}
10500
10501	tr = trace_array_create_systems(name, NULL, addr, size);
10502	if (IS_ERR(tr)) {
10503	pr_warn("Tracing: Failed to create instance buffer %s\n", curr_str);
10504	continue;
10505	}
10506
10507	if (traceoff)
10508	tracer_tracing_off(tr);
10509
10510	if (traceprintk)
10511	update_printk_trace(tr);
10512
10513	/*
10514	* If start is set, then this is a mapped buffer, and
10515	* cannot be deleted by user space, so keep the reference
10516	* to it.
10517	*/
10518	if (start) {
10519	tr->flags \|= TRACE_ARRAY_FL_BOOT;
10520	tr->ref++;
10521	}
10522
10523	while ((tok = strsep(&curr_str, ","))) {
10524	early_enable_events(tr, tok, true);
10525	}
10526	}
10527	}
10528
10529	__init static int tracer_alloc_buffers(void)
10530	{
10531	int ring_buf_size;
10532	int ret = -ENOMEM;
10533
10534
10535	if (security_locked_down(LOCKDOWN_TRACEFS)) {
10536	pr_warn("Tracing disabled due to lockdown\n");
10537	return -EPERM;
10538	}
10539
10540	/*
10541	* Make sure we don't accidentally add more trace options
10542	* than we have bits for.
10543	*/
10544	BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
10545
10546	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
10547	goto out;
10548
10549	if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
10550	goto out_free_buffer_mask;
10551
10552	/* Only allocate trace_printk buffers if a trace_printk exists */
10553	if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
10554	/* Must be called before global_trace.buffer is allocated */
10555	trace_printk_init_buffers();
10556
10557	/* To save memory, keep the ring buffer size to its minimum */
10558	if (global_trace.ring_buffer_expanded)
10559	ring_buf_size = trace_buf_size;
10560	else
10561	ring_buf_size = 1;
10562
10563	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
10564	cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
10565
10566	raw_spin_lock_init(&global_trace.start_lock);
10567
10568	/*
10569	* The prepare callbacks allocates some memory for the ring buffer. We
10570	* don't free the buffer if the CPU goes down. If we were to free
10571	* the buffer, then the user would lose any trace that was in the
10572	* buffer. The memory will be removed once the "instance" is removed.
10573	*/
10574	ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
10575	"trace/RB:prepare", trace_rb_cpu_prepare,
10576	NULL);
10577	if (ret < 0)
10578	goto out_free_cpumask;
10579	/* Used for event triggers */
10580	ret = -ENOMEM;
10581	temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
10582	if (!temp_buffer)
10583	goto out_rm_hp_state;
10584
10585	if (trace_create_savedcmd() < 0)
10586	goto out_free_temp_buffer;
10587
10588	if (!zalloc_cpumask_var(&global_trace.pipe_cpumask, GFP_KERNEL))
10589	goto out_free_savedcmd;
10590
10591	/* TODO: make the number of buffers hot pluggable with CPUS */
10592	if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
10593	MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
10594	goto out_free_pipe_cpumask;
10595	}
10596	if (global_trace.buffer_disabled)
10597	tracing_off();
10598
10599	if (trace_boot_clock) {
10600	ret = tracing_set_clock(&global_trace, trace_boot_clock);
10601	if (ret < 0)
10602	pr_warn("Trace clock %s not defined, going back to default\n",
10603	trace_boot_clock);
10604	}
10605
10606	/*
10607	* register_tracer() might reference current_trace, so it
10608	* needs to be set before we register anything. This is
10609	* just a bootstrap of current_trace anyway.
10610	*/
10611	global_trace.current_trace = &nop_trace;
10612
10613	global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
10614	#ifdef CONFIG_TRACER_MAX_TRACE
10615	spin_lock_init(&global_trace.snapshot_trigger_lock);
10616	#endif
10617	ftrace_init_global_array_ops(&global_trace);
10618
10619	init_trace_flags_index(&global_trace);
10620
10621	register_tracer(&nop_trace);
10622
10623	/* Function tracing may start here (via kernel command line) */
10624	init_function_trace();
10625
10626	/* All seems OK, enable tracing */
10627	tracing_disabled = 0;
10628
10629	atomic_notifier_chain_register(&panic_notifier_list,
10630	&trace_panic_notifier);
10631
10632	register_die_notifier(&trace_die_notifier);
10633
10634	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
10635
10636	INIT_LIST_HEAD(&global_trace.systems);
10637	INIT_LIST_HEAD(&global_trace.events);
10638	INIT_LIST_HEAD(&global_trace.hist_vars);
10639	INIT_LIST_HEAD(&global_trace.err_log);
10640	list_add(&global_trace.list, &ftrace_trace_arrays);
10641
10642	apply_trace_boot_options();
10643
10644	register_snapshot_cmd();
10645
10646	return 0;
10647
10648	out_free_pipe_cpumask:
10649	free_cpumask_var(global_trace.pipe_cpumask);
10650	out_free_savedcmd:
10651	trace_free_saved_cmdlines_buffer();
10652	out_free_temp_buffer:
10653	ring_buffer_free(temp_buffer);
10654	out_rm_hp_state:
10655	cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
10656	out_free_cpumask:
10657	free_cpumask_var(global_trace.tracing_cpumask);
10658	out_free_buffer_mask:
10659	free_cpumask_var(tracing_buffer_mask);
10660	out:
10661	return ret;
10662	}
10663
10664	void __init ftrace_boot_snapshot(void)
10665	{
10666	#ifdef CONFIG_TRACER_MAX_TRACE
10667	struct trace_array *tr;
10668
10669	if (!snapshot_at_boot)
10670	return;
10671
10672	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
10673	if (!tr->allocated_snapshot)
10674	continue;
10675
10676	tracing_snapshot_instance(tr);
10677	trace_array_puts(tr, " Boot snapshot taken \n");
10678	}
10679	#endif
10680	}
10681
10682	void __init early_trace_init(void)
10683	{
10684	if (tracepoint_printk) {
10685	tracepoint_print_iter =
10686	kzalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
10687	if (MEM_FAIL(!tracepoint_print_iter,
10688	"Failed to allocate trace iterator\n"))
10689	tracepoint_printk = 0;
10690	else
10691	static_key_enable(&tracepoint_printk_key.key);
10692	}
10693	tracer_alloc_buffers();
10694
10695	init_events();
10696	}
10697
10698	void __init trace_init(void)
10699	{
10700	trace_event_init();
10701
10702	if (boot_instance_index)
10703	enable_instances();
10704	}
10705
10706	__init static void clear_boot_tracer(void)
10707	{
10708	/*
10709	* The default tracer at boot buffer is an init section.
10710	* This function is called in lateinit. If we did not
10711	* find the boot tracer, then clear it out, to prevent
10712	* later registration from accessing the buffer that is
10713	* about to be freed.
10714	*/
10715	if (!default_bootup_tracer)
10716	return;
10717
10718	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
10719	default_bootup_tracer);
10720	default_bootup_tracer = NULL;
10721	}
10722
10723	#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
10724	__init static void tracing_set_default_clock(void)
10725	{
10726	/* sched_clock_stable() is determined in late_initcall */
10727	if (!trace_boot_clock && !sched_clock_stable()) {
10728	if (security_locked_down(LOCKDOWN_TRACEFS)) {
10729	pr_warn("Can not set tracing clock due to lockdown\n");
10730	return;
10731	}
10732
10733	printk(KERN_WARNING
10734	"Unstable clock detected, switching default tracing clock to \"global\"\n"
10735	"If you want to keep using the local clock, then add:\n"
10736	" \"trace_clock=local\"\n"
10737	"on the kernel command line\n");
10738	tracing_set_clock(&global_trace, "global");
10739	}
10740	}
10741	#else
10742	static inline void tracing_set_default_clock(void) { }
10743	#endif
10744
10745	__init static int late_trace_init(void)
10746	{
10747	if (tracepoint_printk && tracepoint_printk_stop_on_boot) {
10748	static_key_disable(&tracepoint_printk_key.key);
10749	tracepoint_printk = 0;
10750	}
10751
10752	tracing_set_default_clock();
10753	clear_boot_tracer();
10754	return 0;
10755	}
10756
10757	late_initcall_sync(late_trace_init);