linux - kernel/trace/trace

3186 lines · 76.7 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* OS Noise Tracer: computes the OS Noise suffered by a running thread.
4	* Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
5	*
6	* Based on "hwlat_detector" tracer by:
7	* Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <[email protected]>
8	* Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <[email protected]>
9	* With feedback from Clark Williams <[email protected]>
10	*
11	* And also based on the rtsl tracer presented on:
12	* DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13	* scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14	* (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
15	*
16	* Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <[email protected]>
17	*/
18
19	#include <linux/kthread.h>
20	#include <linux/tracefs.h>
21	#include <linux/uaccess.h>
22	#include <linux/cpumask.h>
23	#include <linux/delay.h>
24	#include <linux/sched/clock.h>
25	#include <uapi/linux/sched/types.h>
26	#include <linux/sched.h>
27	#include <linux/string.h>
28	#include "trace.h"
29
30	#ifdef CONFIG_X86_LOCAL_APIC
31	#include <asm/trace/irq_vectors.h>
32	#undef TRACE_INCLUDE_PATH
33	#undef TRACE_INCLUDE_FILE
34	#endif /* CONFIG_X86_LOCAL_APIC */
35
36	#include <trace/events/irq.h>
37	#include <trace/events/sched.h>
38
39	#define CREATE_TRACE_POINTS
40	#include <trace/events/osnoise.h>
41
42	/*
43	* Default values.
44	*/
45	#define BANNER "osnoise: "
46	#define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
47	#define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
48
49	#define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
50	#define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
51
52	/*
53	* osnoise/options entries.
54	*/
55	enum osnoise_options_index {
56	OSN_DEFAULTS = 0,
57	OSN_WORKLOAD,
58	OSN_PANIC_ON_STOP,
59	OSN_PREEMPT_DISABLE,
60	OSN_IRQ_DISABLE,
61	OSN_TIMERLAT_ALIGN,
62	OSN_MAX
63	};
64
65	static const char * const osnoise_options_str[OSN_MAX] = {
66	"DEFAULTS",
67	"OSNOISE_WORKLOAD",
68	"PANIC_ON_STOP",
69	"OSNOISE_PREEMPT_DISABLE",
70	"OSNOISE_IRQ_DISABLE",
71	"TIMERLAT_ALIGN" };
72
73	#define OSN_DEFAULT_OPTIONS 0x2
74	static unsigned long osnoise_options = OSN_DEFAULT_OPTIONS;
75
76	/*
77	* trace_array of the enabled osnoise/timerlat instances.
78	*/
79	struct osnoise_instance {
80	struct list_head list;
81	struct trace_array *tr;
82	};
83
84	static struct list_head osnoise_instances;
85
86	static void osnoise_print(const char *fmt, ...)
87	{
88	struct osnoise_instance *inst;
89	struct trace_array *tr;
90	va_list ap;
91
92	rcu_read_lock();
93	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
94	tr = inst->tr;
95	va_start(ap, fmt);
96	trace_array_vprintk(tr, _RET_IP_, fmt, ap);
97	va_end(ap);
98	}
99	rcu_read_unlock();
100	}
101
102	static bool osnoise_has_registered_instances(void)
103	{
104	return !!list_first_or_null_rcu(&osnoise_instances,
105	struct osnoise_instance,
106	list);
107	}
108
109	/*
110	* osnoise_instance_registered - check if a tr is already registered
111	*/
112	static int osnoise_instance_registered(struct trace_array *tr)
113	{
114	struct osnoise_instance *inst;
115	int found = 0;
116
117	rcu_read_lock();
118	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
119	if (inst->tr == tr)
120	found = 1;
121	}
122	rcu_read_unlock();
123
124	return found;
125	}
126
127	/*
128	* osnoise_register_instance - register a new trace instance
129	*
130	* Register a trace_array *tr in the list of instances running
131	* osnoise/timerlat tracers.
132	*/
133	static int osnoise_register_instance(struct trace_array *tr)
134	{
135	struct osnoise_instance *inst;
136
137	/*
138	* register/unregister serialization is provided by trace's
139	* trace_types_lock.
140	*/
141	lockdep_assert_held(&trace_types_lock);
142	trace_array_init_printk(tr);
143
144	inst = kmalloc_obj(*inst);
145	if (!inst)
146	return -ENOMEM;
147
148	INIT_LIST_HEAD_RCU(&inst->list);
149	inst->tr = tr;
150	list_add_tail_rcu(&inst->list, &osnoise_instances);
151
152	return 0;
153	}
154
155	/*
156	* osnoise_unregister_instance - unregister a registered trace instance
157	*
158	* Remove the trace_array *tr from the list of instances running
159	* osnoise/timerlat tracers.
160	*/
161	static void osnoise_unregister_instance(struct trace_array *tr)
162	{
163	struct osnoise_instance *inst;
164	int found = 0;
165
166	/*
167	* register/unregister serialization is provided by trace's
168	* trace_types_lock.
169	*/
170	list_for_each_entry_rcu(inst, &osnoise_instances, list,
171	lockdep_is_held(&trace_types_lock)) {
172	if (inst->tr == tr) {
173	list_del_rcu(&inst->list);
174	found = 1;
175	break;
176	}
177	}
178
179	if (!found)
180	return;
181
182	/* Do a full sync to ensure that tr remains valid, not just inst */
183	synchronize_rcu();
184	kvfree(inst);
185	}
186
187	/*
188	* NMI runtime info.
189	*/
190	struct osn_nmi {
191	u64 count;
192	u64 delta_start;
193	};
194
195	/*
196	* IRQ runtime info.
197	*/
198	struct osn_irq {
199	u64 count;
200	u64 arrival_time;
201	u64 delta_start;
202	};
203
204	#define IRQ_CONTEXT 0
205	#define THREAD_CONTEXT 1
206	#define THREAD_URET 2
207	/*
208	* sofirq runtime info.
209	*/
210	struct osn_softirq {
211	u64 count;
212	u64 arrival_time;
213	u64 delta_start;
214	};
215
216	/*
217	* thread runtime info.
218	*/
219	struct osn_thread {
220	u64 count;
221	u64 arrival_time;
222	u64 delta_start;
223	};
224
225	/*
226	* Runtime information: this structure saves the runtime information used by
227	* one sampling thread.
228	*/
229	struct osnoise_variables {
230	struct task_struct *kthread;
231	bool sampling;
232	pid_t pid;
233	struct osn_nmi nmi;
234	struct osn_irq irq;
235	struct osn_softirq softirq;
236	struct osn_thread thread;
237	local_t int_counter;
238	};
239
240	/*
241	* Per-cpu runtime information.
242	*/
243	static DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
244
245	/*
246	* this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
247	*/
248	static inline struct osnoise_variables *this_cpu_osn_var(void)
249	{
250	return this_cpu_ptr(&per_cpu_osnoise_var);
251	}
252
253	/*
254	* Protect the interface.
255	*/
256	static struct mutex interface_lock;
257
258	#ifdef CONFIG_TIMERLAT_TRACER
259	/*
260	* Runtime information for the timer mode.
261	*/
262	struct timerlat_variables {
263	struct task_struct *kthread;
264	struct hrtimer timer;
265	u64 rel_period;
266	u64 abs_period;
267	bool tracing_thread;
268	u64 count;
269	bool uthread_migrate;
270	};
271
272	static DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
273
274	/*
275	* timerlat wake-up offset for next thread with TIMERLAT_ALIGN set.
276	*/
277	static atomic64_t align_next;
278
279	/*
280	* this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
281	*/
282	static inline struct timerlat_variables *this_cpu_tmr_var(void)
283	{
284	return this_cpu_ptr(&per_cpu_timerlat_var);
285	}
286
287	/*
288	* tlat_var_reset - Reset the values of the given timerlat_variables
289	*/
290	static inline void tlat_var_reset(void)
291	{
292	struct timerlat_variables *tlat_var;
293	int cpu;
294
295	/* Synchronize with the timerlat interfaces */
296	mutex_lock(&interface_lock);
297
298	/*
299	* So far, all the values are initialized as 0, so
300	* zeroing the structure is perfect.
301	*/
302	for_each_online_cpu(cpu) {
303	tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
304	if (tlat_var->kthread)
305	hrtimer_cancel(&tlat_var->timer);
306	memset(tlat_var, 0, sizeof(*tlat_var));
307	}
308	/*
309	* Reset also align_next, to be filled by a new offset by the first timerlat
310	* thread that wakes up, if TIMERLAT_ALIGN is set.
311	*/
312	atomic64_set(&align_next, 0);
313
314	mutex_unlock(&interface_lock);
315	}
316	#else /* CONFIG_TIMERLAT_TRACER */
317	#define tlat_var_reset() do {} while (0)
318	#endif /* CONFIG_TIMERLAT_TRACER */
319
320	/*
321	* osn_var_reset - Reset the values of the given osnoise_variables
322	*/
323	static inline void osn_var_reset(void)
324	{
325	struct osnoise_variables *osn_var;
326	int cpu;
327
328	/*
329	* So far, all the values are initialized as 0, so
330	* zeroing the structure is perfect.
331	*/
332	for_each_online_cpu(cpu) {
333	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
334	memset(osn_var, 0, sizeof(*osn_var));
335	}
336	}
337
338	/*
339	* osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
340	*/
341	static inline void osn_var_reset_all(void)
342	{
343	osn_var_reset();
344	tlat_var_reset();
345	}
346
347	/*
348	* Tells NMIs to call back to the osnoise tracer to record timestamps.
349	*/
350	bool trace_osnoise_callback_enabled;
351
352	/*
353	* Tracer data.
354	*/
355	static struct osnoise_data {
356	u64 sample_period; /* total sampling period */
357	u64 sample_runtime; /* active sampling portion of period */
358	u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */
359	u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */
360	#ifdef CONFIG_TIMERLAT_TRACER
361	u64 timerlat_period; /* timerlat period */
362	u64 timerlat_align_us; /* timerlat alignment */
363	u64 print_stack; /* print IRQ stack if total > */
364	int timerlat_tracer; /* timerlat tracer */
365	#endif
366	bool tainted; /* info users and developers about a problem */
367	} osnoise_data = {
368	.sample_period = DEFAULT_SAMPLE_PERIOD,
369	.sample_runtime = DEFAULT_SAMPLE_RUNTIME,
370	.stop_tracing = 0,
371	.stop_tracing_total = 0,
372	#ifdef CONFIG_TIMERLAT_TRACER
373	.print_stack = 0,
374	.timerlat_period = DEFAULT_TIMERLAT_PERIOD,
375	.timerlat_align_us = 0,
376	.timerlat_tracer = 0,
377	#endif
378	};
379
380	#ifdef CONFIG_TIMERLAT_TRACER
381	static inline bool timerlat_enabled(void)
382	{
383	return osnoise_data.timerlat_tracer;
384	}
385
386	static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
387	{
388	struct timerlat_variables *tlat_var = this_cpu_tmr_var();
389	/*
390	* If the timerlat is enabled, but the irq handler did
391	* not run yet enabling timerlat_tracer, do not trace.
392	*/
393	if (!tlat_var->tracing_thread) {
394	osn_var->softirq.arrival_time = 0;
395	osn_var->softirq.delta_start = 0;
396	return 0;
397	}
398	return 1;
399	}
400
401	static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
402	{
403	struct timerlat_variables *tlat_var = this_cpu_tmr_var();
404	/*
405	* If the timerlat is enabled, but the irq handler did
406	* not run yet enabling timerlat_tracer, do not trace.
407	*/
408	if (!tlat_var->tracing_thread) {
409	osn_var->thread.delta_start = 0;
410	osn_var->thread.arrival_time = 0;
411	return 0;
412	}
413	return 1;
414	}
415	#else /* CONFIG_TIMERLAT_TRACER */
416	static inline bool timerlat_enabled(void)
417	{
418	return false;
419	}
420
421	static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
422	{
423	return 1;
424	}
425	static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
426	{
427	return 1;
428	}
429	#endif
430
431	#ifdef CONFIG_PREEMPT_RT
432	/*
433	* Print the osnoise header info.
434	*/
435	static void print_osnoise_headers(struct seq_file *s)
436	{
437	if (osnoise_data.tainted)
438	seq_puts(s, "# osnoise is tainted!\n");
439
440	seq_puts(s, "# _-------=> irqs-off\n");
441	seq_puts(s, "# / _------=> need-resched\n");
442	seq_puts(s, "# \| / _-----=> need-resched-lazy\n");
443	seq_puts(s, "# \|\| / _----=> hardirq/softirq\n");
444	seq_puts(s, "# \|\|\| / _---=> preempt-depth\n");
445	seq_puts(s, "# \|\|\|\| / _--=> preempt-lazy-depth\n");
446	seq_puts(s, "# \|\|\|\|\| / _-=> migrate-disable\n");
447
448	seq_puts(s, "# \|\|\|\|\|\| / ");
449	seq_puts(s, " MAX\n");
450
451	seq_puts(s, "# \|\|\|\|\| / ");
452	seq_puts(s, " SINGLE Interference counters:\n");
453
454	seq_puts(s, "# \|\|\|\|\|\|\| RUNTIME ");
455	seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
456
457	seq_puts(s, "# TASK-PID CPU# \|\|\|\|\|\|\| TIMESTAMP IN US ");
458	seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
459
460	seq_puts(s, "# \| \| \| \|\|\|\|\|\|\| \| \| ");
461	seq_puts(s, " \| \| \| \| \| \| \| \|\n");
462	}
463	#else /* CONFIG_PREEMPT_RT */
464	static void print_osnoise_headers(struct seq_file *s)
465	{
466	if (osnoise_data.tainted)
467	seq_puts(s, "# osnoise is tainted!\n");
468
469	seq_puts(s, "# _-----=> irqs-off\n");
470	seq_puts(s, "# / _----=> need-resched\n");
471	seq_puts(s, "# \| / _---=> hardirq/softirq\n");
472	seq_puts(s, "# \|\| / _--=> preempt-depth\n");
473	seq_puts(s, "# \|\|\| / _-=> migrate-disable ");
474	seq_puts(s, " MAX\n");
475	seq_puts(s, "# \|\|\|\| / delay ");
476	seq_puts(s, " SINGLE Interference counters:\n");
477
478	seq_puts(s, "# \|\|\|\|\| RUNTIME ");
479	seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
480
481	seq_puts(s, "# TASK-PID CPU# \|\|\|\|\| TIMESTAMP IN US ");
482	seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
483
484	seq_puts(s, "# \| \| \| \|\|\|\|\| \| \| ");
485	seq_puts(s, " \| \| \| \| \| \| \| \|\n");
486	}
487	#endif /* CONFIG_PREEMPT_RT */
488
489	/*
490	* osnoise_taint - report an osnoise error.
491	*/
492	#define osnoise_taint(msg) ({ \
493	osnoise_print(msg); \
494	osnoise_data.tainted = true; \
495	})
496
497	/*
498	* Record an osnoise_sample into the tracer buffer.
499	*/
500	static void
501	__record_osnoise_sample(struct osnoise_sample sample, struct trace_buffer buffer)
502	{
503	struct ring_buffer_event *event;
504	struct osnoise_entry *entry;
505
506	event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
507	tracing_gen_ctx());
508	if (!event)
509	return;
510	entry = ring_buffer_event_data(event);
511	entry->runtime = sample->runtime;
512	entry->noise = sample->noise;
513	entry->max_sample = sample->max_sample;
514	entry->hw_count = sample->hw_count;
515	entry->nmi_count = sample->nmi_count;
516	entry->irq_count = sample->irq_count;
517	entry->softirq_count = sample->softirq_count;
518	entry->thread_count = sample->thread_count;
519
520	trace_buffer_unlock_commit_nostack(buffer, event);
521	}
522
523	/*
524	* Record an osnoise_sample on all osnoise instances and fire trace event.
525	*/
526	static void record_osnoise_sample(struct osnoise_sample *sample)
527	{
528	struct osnoise_instance *inst;
529	struct trace_buffer *buffer;
530
531	trace_osnoise_sample(sample);
532
533	rcu_read_lock();
534	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
535	buffer = inst->tr->array_buffer.buffer;
536	__record_osnoise_sample(sample, buffer);
537	}
538	rcu_read_unlock();
539	}
540
541	#ifdef CONFIG_TIMERLAT_TRACER
542	/*
543	* Print the timerlat header info.
544	*/
545	#ifdef CONFIG_PREEMPT_RT
546	static void print_timerlat_headers(struct seq_file *s)
547	{
548	seq_puts(s, "# _-------=> irqs-off\n");
549	seq_puts(s, "# / _------=> need-resched\n");
550	seq_puts(s, "# \| / _-----=> need-resched-lazy\n");
551	seq_puts(s, "# \|\| / _----=> hardirq/softirq\n");
552	seq_puts(s, "# \|\|\| / _---=> preempt-depth\n");
553	seq_puts(s, "# \|\|\|\| / _--=> preempt-lazy-depth\n");
554	seq_puts(s, "# \|\|\|\|\| / _-=> migrate-disable\n");
555	seq_puts(s, "# \|\|\|\|\|\| /\n");
556	seq_puts(s, "# \|\|\|\|\|\|\| ACTIVATION\n");
557	seq_puts(s, "# TASK-PID CPU# \|\|\|\|\|\|\| TIMESTAMP ID ");
558	seq_puts(s, " CONTEXT LATENCY\n");
559	seq_puts(s, "# \| \| \| \|\|\|\|\|\|\| \| \| ");
560	seq_puts(s, " \| \|\n");
561	}
562	#else /* CONFIG_PREEMPT_RT */
563	static void print_timerlat_headers(struct seq_file *s)
564	{
565	seq_puts(s, "# _-----=> irqs-off\n");
566	seq_puts(s, "# / _----=> need-resched\n");
567	seq_puts(s, "# \| / _---=> hardirq/softirq\n");
568	seq_puts(s, "# \|\| / _--=> preempt-depth\n");
569	seq_puts(s, "# \|\|\| / _-=> migrate-disable\n");
570	seq_puts(s, "# \|\|\|\| / delay\n");
571	seq_puts(s, "# \|\|\|\|\| ACTIVATION\n");
572	seq_puts(s, "# TASK-PID CPU# \|\|\|\|\| TIMESTAMP ID ");
573	seq_puts(s, " CONTEXT LATENCY\n");
574	seq_puts(s, "# \| \| \| \|\|\|\|\| \| \| ");
575	seq_puts(s, " \| \|\n");
576	}
577	#endif /* CONFIG_PREEMPT_RT */
578
579	static void
580	__record_timerlat_sample(struct timerlat_sample sample, struct trace_buffer buffer)
581	{
582	struct ring_buffer_event *event;
583	struct timerlat_entry *entry;
584
585	event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
586	tracing_gen_ctx());
587	if (!event)
588	return;
589	entry = ring_buffer_event_data(event);
590	entry->seqnum = sample->seqnum;
591	entry->context = sample->context;
592	entry->timer_latency = sample->timer_latency;
593
594	trace_buffer_unlock_commit_nostack(buffer, event);
595	}
596
597	/*
598	* Record an timerlat_sample into the tracer buffer.
599	*/
600	static void record_timerlat_sample(struct timerlat_sample *sample)
601	{
602	struct osnoise_instance *inst;
603	struct trace_buffer *buffer;
604
605	trace_timerlat_sample(sample);
606
607	rcu_read_lock();
608	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
609	buffer = inst->tr->array_buffer.buffer;
610	__record_timerlat_sample(sample, buffer);
611	}
612	rcu_read_unlock();
613	}
614
615	#ifdef CONFIG_STACKTRACE
616
617	#define MAX_CALLS 256
618
619	/*
620	* Stack trace will take place only at IRQ level, so, no need
621	* to control nesting here.
622	*/
623	struct trace_stack {
624	int stack_size;
625	int nr_entries;
626	unsigned long calls[MAX_CALLS];
627	};
628
629	static DEFINE_PER_CPU(struct trace_stack, trace_stack);
630
631	/*
632	* timerlat_save_stack - save a stack trace without printing
633	*
634	* Save the current stack trace without printing. The
635	* stack will be printed later, after the end of the measurement.
636	*/
637	static void timerlat_save_stack(int skip)
638	{
639	unsigned int size, nr_entries;
640	struct trace_stack *fstack;
641
642	fstack = this_cpu_ptr(&trace_stack);
643
644	size = ARRAY_SIZE(fstack->calls);
645
646	nr_entries = stack_trace_save(fstack->calls, size, skip);
647
648	fstack->stack_size = nr_entries * sizeof(unsigned long);
649	fstack->nr_entries = nr_entries;
650
651	return;
652
653	}
654
655	static void
656	__timerlat_dump_stack(struct trace_buffer buffer, struct trace_stack fstack, unsigned int size)
657	{
658	struct ring_buffer_event *event;
659	struct stack_entry *entry;
660
661	event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
662	tracing_gen_ctx());
663	if (!event)
664	return;
665
666	entry = ring_buffer_event_data(event);
667
668	entry->size = fstack->nr_entries;
669	memcpy(&entry->caller, fstack->calls, size);
670
671	trace_buffer_unlock_commit_nostack(buffer, event);
672	}
673
674	/*
675	* timerlat_dump_stack - dump a stack trace previously saved
676	*/
677	static void timerlat_dump_stack(u64 latency)
678	{
679	struct osnoise_instance *inst;
680	struct trace_buffer *buffer;
681	struct trace_stack *fstack;
682	unsigned int size;
683
684	/*
685	* trace only if latency > print_stack config, if enabled.
686	*/
687	if (!osnoise_data.print_stack \|\| osnoise_data.print_stack > latency)
688	return;
689
690	preempt_disable_notrace();
691	fstack = this_cpu_ptr(&trace_stack);
692	size = fstack->stack_size;
693
694	rcu_read_lock();
695	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
696	buffer = inst->tr->array_buffer.buffer;
697	__timerlat_dump_stack(buffer, fstack, size);
698
699	}
700	rcu_read_unlock();
701	preempt_enable_notrace();
702	}
703	#else /* CONFIG_STACKTRACE */
704	#define timerlat_dump_stack(u64 latency) do {} while (0)
705	#define timerlat_save_stack(a) do {} while (0)
706	#endif /* CONFIG_STACKTRACE */
707	#endif /* CONFIG_TIMERLAT_TRACER */
708
709	/*
710	* Macros to encapsulate the time capturing infrastructure.
711	*/
712	#define time_get() trace_clock_local()
713	#define time_to_us(x) div_u64(x, 1000)
714	#define time_sub(a, b) ((a) - (b))
715
716	/*
717	* cond_move_irq_delta_start - Forward the delta_start of a running IRQ
718	*
719	* If an IRQ is preempted by an NMI, its delta_start is pushed forward
720	* to discount the NMI interference.
721	*
722	* See get_int_safe_duration().
723	*/
724	static inline void
725	cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
726	{
727	if (osn_var->irq.delta_start)
728	osn_var->irq.delta_start += duration;
729	}
730
731	#ifndef CONFIG_PREEMPT_RT
732	/*
733	* cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
734	*
735	* If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
736	* forward to discount the interference.
737	*
738	* See get_int_safe_duration().
739	*/
740	static inline void
741	cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
742	{
743	if (osn_var->softirq.delta_start)
744	osn_var->softirq.delta_start += duration;
745	}
746	#else /* CONFIG_PREEMPT_RT */
747	#define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
748	#endif
749
750	/*
751	* cond_move_thread_delta_start - Forward the delta_start of a running thread
752	*
753	* If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
754	* is pushed forward to discount the interference.
755	*
756	* See get_int_safe_duration().
757	*/
758	static inline void
759	cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
760	{
761	if (osn_var->thread.delta_start)
762	osn_var->thread.delta_start += duration;
763	}
764
765	/*
766	* get_int_safe_duration - Get the duration of a window
767	*
768	* The irq, softirq and thread variables need to have its duration without
769	* the interference from higher priority interrupts. Instead of keeping a
770	* variable to discount the interrupt interference from these variables, the
771	* starting time of these variables are pushed forward with the interrupt's
772	* duration. In this way, a single variable is used to:
773	*
774	* - Know if a given window is being measured.
775	* - Account its duration.
776	* - Discount the interference.
777	*
778	* To avoid getting inconsistent values, e.g.,:
779	*
780	* now = time_get()
781	* ---> interrupt!
782	* delta_start -= int duration;
783	* <---
784	* duration = now - delta_start;
785	*
786	* result: negative duration if the variable duration before the
787	* interrupt was smaller than the interrupt execution.
788	*
789	* A counter of interrupts is used. If the counter increased, try
790	* to capture an interference safe duration.
791	*/
792	static inline s64
793	get_int_safe_duration(struct osnoise_variables osn_var, u64 delta_start)
794	{
795	u64 int_counter, now;
796	s64 duration;
797
798	do {
799	int_counter = local_read(&osn_var->int_counter);
800	/* synchronize with interrupts */
801	barrier();
802
803	now = time_get();
804	duration = (now - *delta_start);
805
806	/* synchronize with interrupts */
807	barrier();
808	} while (int_counter != local_read(&osn_var->int_counter));
809
810	/*
811	* This is an evidence of race conditions that cause
812	* a value to be "discounted" too much.
813	*/
814	if (duration < 0)
815	osnoise_taint("Negative duration!\n");
816
817	*delta_start = 0;
818
819	return duration;
820	}
821
822	/*
823	*
824	* set_int_safe_time - Save the current time on *time, aware of interference
825	*
826	* Get the time, taking into consideration a possible interference from
827	* higher priority interrupts.
828	*
829	* See get_int_safe_duration() for an explanation.
830	*/
831	static u64
832	set_int_safe_time(struct osnoise_variables osn_var, u64 time)
833	{
834	u64 int_counter;
835
836	do {
837	int_counter = local_read(&osn_var->int_counter);
838	/* synchronize with interrupts */
839	barrier();
840
841	*time = time_get();
842
843	/* synchronize with interrupts */
844	barrier();
845	} while (int_counter != local_read(&osn_var->int_counter));
846
847	return int_counter;
848	}
849
850	#ifdef CONFIG_TIMERLAT_TRACER
851	/*
852	* copy_int_safe_time - Copy src into desc aware of interference
853	*/
854	static u64
855	copy_int_safe_time(struct osnoise_variables osn_var, u64 dst, u64 *src)
856	{
857	u64 int_counter;
858
859	do {
860	int_counter = local_read(&osn_var->int_counter);
861	/* synchronize with interrupts */
862	barrier();
863
864	dst = src;
865
866	/* synchronize with interrupts */
867	barrier();
868	} while (int_counter != local_read(&osn_var->int_counter));
869
870	return int_counter;
871	}
872	#endif /* CONFIG_TIMERLAT_TRACER */
873
874	/*
875	* trace_osnoise_callback - NMI entry/exit callback
876	*
877	* This function is called at the entry and exit NMI code. The bool enter
878	* distinguishes between either case. This function is used to note a NMI
879	* occurrence, compute the noise caused by the NMI, and to remove the noise
880	* it is potentially causing on other interference variables.
881	*/
882	void trace_osnoise_callback(bool enter)
883	{
884	struct osnoise_variables *osn_var = this_cpu_osn_var();
885	u64 duration;
886
887	if (!osn_var->sampling)
888	return;
889
890	/*
891	* Currently trace_clock_local() calls sched_clock() and the
892	* generic version is not NMI safe.
893	*/
894	if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
895	if (enter) {
896	osn_var->nmi.delta_start = time_get();
897	local_inc(&osn_var->int_counter);
898	} else {
899	duration = time_get() - osn_var->nmi.delta_start;
900
901	trace_nmi_noise(osn_var->nmi.delta_start, duration);
902
903	cond_move_irq_delta_start(osn_var, duration);
904	cond_move_softirq_delta_start(osn_var, duration);
905	cond_move_thread_delta_start(osn_var, duration);
906	}
907	}
908
909	if (enter)
910	osn_var->nmi.count++;
911	}
912
913	/*
914	* osnoise_trace_irq_entry - Note the starting of an IRQ
915	*
916	* Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
917	* it is safe to use a single variable (ons_var->irq) to save the statistics.
918	* The arrival_time is used to report... the arrival time. The delta_start
919	* is used to compute the duration at the IRQ exit handler. See
920	* cond_move_irq_delta_start().
921	*/
922	void osnoise_trace_irq_entry(int id)
923	{
924	struct osnoise_variables *osn_var = this_cpu_osn_var();
925
926	if (!osn_var->sampling)
927	return;
928	/*
929	* This value will be used in the report, but not to compute
930	* the execution time, so it is safe to get it unsafe.
931	*/
932	osn_var->irq.arrival_time = time_get();
933	set_int_safe_time(osn_var, &osn_var->irq.delta_start);
934	osn_var->irq.count++;
935
936	local_inc(&osn_var->int_counter);
937	}
938
939	/*
940	* osnoise_irq_exit - Note the end of an IRQ, sava data and trace
941	*
942	* Computes the duration of the IRQ noise, and trace it. Also discounts the
943	* interference from other sources of noise could be currently being accounted.
944	*/
945	void osnoise_trace_irq_exit(int id, const char *desc)
946	{
947	struct osnoise_variables *osn_var = this_cpu_osn_var();
948	s64 duration;
949
950	if (!osn_var->sampling)
951	return;
952
953	duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
954	trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
955	osn_var->irq.arrival_time = 0;
956	cond_move_softirq_delta_start(osn_var, duration);
957	cond_move_thread_delta_start(osn_var, duration);
958	}
959
960	/*
961	* trace_irqentry_callback - Callback to the irq:irq_entry traceevent
962	*
963	* Used to note the starting of an IRQ occurece.
964	*/
965	static void trace_irqentry_callback(void *data, int irq,
966	struct irqaction *action)
967	{
968	osnoise_trace_irq_entry(irq);
969	}
970
971	/*
972	* trace_irqexit_callback - Callback to the irq:irq_exit traceevent
973	*
974	* Used to note the end of an IRQ occurece.
975	*/
976	static void trace_irqexit_callback(void *data, int irq,
977	struct irqaction *action, int ret)
978	{
979	osnoise_trace_irq_exit(irq, action->name);
980	}
981
982	/*
983	* arch specific register function.
984	*/
985	int __weak osnoise_arch_register(void)
986	{
987	return 0;
988	}
989
990	/*
991	* arch specific unregister function.
992	*/
993	void __weak osnoise_arch_unregister(void)
994	{
995	return;
996	}
997
998	/*
999	* hook_irq_events - Hook IRQ handling events
1000	*
1001	* This function hooks the IRQ related callbacks to the respective trace
1002	* events.
1003	*/
1004	static int hook_irq_events(void)
1005	{
1006	int ret;
1007
1008	ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1009	if (ret)
1010	goto out_err;
1011
1012	ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1013	if (ret)
1014	goto out_unregister_entry;
1015
1016	ret = osnoise_arch_register();
1017	if (ret)
1018	goto out_irq_exit;
1019
1020	return 0;
1021
1022	out_irq_exit:
1023	unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1024	out_unregister_entry:
1025	unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1026	out_err:
1027	return -EINVAL;
1028	}
1029
1030	/*
1031	* unhook_irq_events - Unhook IRQ handling events
1032	*
1033	* This function unhooks the IRQ related callbacks to the respective trace
1034	* events.
1035	*/
1036	static void unhook_irq_events(void)
1037	{
1038	osnoise_arch_unregister();
1039	unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1040	unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1041	}
1042
1043	#ifndef CONFIG_PREEMPT_RT
1044	/*
1045	* trace_softirq_entry_callback - Note the starting of a softirq
1046	*
1047	* Save the starting time of a softirq. As softirqs are non-preemptive to
1048	* other softirqs, it is safe to use a single variable (ons_var->softirq)
1049	* to save the statistics. The arrival_time is used to report... the
1050	* arrival time. The delta_start is used to compute the duration at the
1051	* softirq exit handler. See cond_move_softirq_delta_start().
1052	*/
1053	static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
1054	{
1055	struct osnoise_variables *osn_var = this_cpu_osn_var();
1056
1057	if (!osn_var->sampling)
1058	return;
1059	/*
1060	* This value will be used in the report, but not to compute
1061	* the execution time, so it is safe to get it unsafe.
1062	*/
1063	osn_var->softirq.arrival_time = time_get();
1064	set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
1065	osn_var->softirq.count++;
1066
1067	local_inc(&osn_var->int_counter);
1068	}
1069
1070	/*
1071	* trace_softirq_exit_callback - Note the end of an softirq
1072	*
1073	* Computes the duration of the softirq noise, and trace it. Also discounts the
1074	* interference from other sources of noise could be currently being accounted.
1075	*/
1076	static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
1077	{
1078	struct osnoise_variables *osn_var = this_cpu_osn_var();
1079	s64 duration;
1080
1081	if (!osn_var->sampling)
1082	return;
1083
1084	if (unlikely(timerlat_enabled()))
1085	if (!timerlat_softirq_exit(osn_var))
1086	return;
1087
1088	duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
1089	trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
1090	cond_move_thread_delta_start(osn_var, duration);
1091	osn_var->softirq.arrival_time = 0;
1092	}
1093
1094	/*
1095	* hook_softirq_events - Hook softirq handling events
1096	*
1097	* This function hooks the softirq related callbacks to the respective trace
1098	* events.
1099	*/
1100	static int hook_softirq_events(void)
1101	{
1102	int ret;
1103
1104	ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1105	if (ret)
1106	goto out_err;
1107
1108	ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1109	if (ret)
1110	goto out_unreg_entry;
1111
1112	return 0;
1113
1114	out_unreg_entry:
1115	unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1116	out_err:
1117	return -EINVAL;
1118	}
1119
1120	/*
1121	* unhook_softirq_events - Unhook softirq handling events
1122	*
1123	* This function hooks the softirq related callbacks to the respective trace
1124	* events.
1125	*/
1126	static void unhook_softirq_events(void)
1127	{
1128	unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1129	unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1130	}
1131	#else /* CONFIG_PREEMPT_RT */
1132	/*
1133	* softirq are threads on the PREEMPT_RT mode.
1134	*/
1135	static int hook_softirq_events(void)
1136	{
1137	return 0;
1138	}
1139	static void unhook_softirq_events(void)
1140	{
1141	}
1142	#endif
1143
1144	/*
1145	* thread_entry - Record the starting of a thread noise window
1146	*
1147	* It saves the context switch time for a noisy thread, and increments
1148	* the interference counters.
1149	*/
1150	static void
1151	thread_entry(struct osnoise_variables osn_var, struct task_struct t)
1152	{
1153	if (!osn_var->sampling)
1154	return;
1155	/*
1156	* The arrival time will be used in the report, but not to compute
1157	* the execution time, so it is safe to get it unsafe.
1158	*/
1159	osn_var->thread.arrival_time = time_get();
1160
1161	set_int_safe_time(osn_var, &osn_var->thread.delta_start);
1162
1163	osn_var->thread.count++;
1164	local_inc(&osn_var->int_counter);
1165	}
1166
1167	/*
1168	* thread_exit - Report the end of a thread noise window
1169	*
1170	* It computes the total noise from a thread, tracing if needed.
1171	*/
1172	static void
1173	thread_exit(struct osnoise_variables osn_var, struct task_struct t)
1174	{
1175	s64 duration;
1176
1177	if (!osn_var->sampling)
1178	return;
1179
1180	if (unlikely(timerlat_enabled()))
1181	if (!timerlat_thread_exit(osn_var))
1182	return;
1183
1184	duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
1185
1186	trace_thread_noise(t, osn_var->thread.arrival_time, duration);
1187
1188	osn_var->thread.arrival_time = 0;
1189	}
1190
1191	#ifdef CONFIG_TIMERLAT_TRACER
1192	/*
1193	* osnoise_stop_exception - Stop tracing and the tracer.
1194	*/
1195	static __always_inline void osnoise_stop_exception(char *msg, int cpu)
1196	{
1197	struct osnoise_instance *inst;
1198	struct trace_array *tr;
1199
1200	rcu_read_lock();
1201	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1202	tr = inst->tr;
1203	trace_array_printk(tr, _THIS_IP_,
1204	"stop tracing hit on cpu %d due to exception: %s\n",
1205	smp_processor_id(),
1206	msg);
1207
1208	if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1209	panic("tracer hit on cpu %d due to exception: %s\n",
1210	smp_processor_id(),
1211	msg);
1212
1213	tracer_tracing_off(tr);
1214	}
1215	rcu_read_unlock();
1216	}
1217
1218	/*
1219	* trace_sched_migrate_callback - sched:sched_migrate_task trace event handler
1220	*
1221	* his function is hooked to the sched:sched_migrate_task trace event, and monitors
1222	* timerlat user-space thread migration.
1223	*/
1224	static void trace_sched_migrate_callback(void data, struct task_struct p, int dest_cpu)
1225	{
1226	struct osnoise_variables *osn_var;
1227	long cpu = task_cpu(p);
1228
1229	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
1230	if (osn_var->pid == p->pid && dest_cpu != cpu) {
1231	per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
1232	osnoise_taint("timerlat user-thread migrated\n");
1233	osnoise_stop_exception("timerlat user-thread migrated", cpu);
1234	}
1235	}
1236
1237	static bool monitor_enabled;
1238
1239	static int register_migration_monitor(void)
1240	{
1241	int ret = 0;
1242
1243	/*
1244	* Timerlat thread migration check is only required when running timerlat in user-space.
1245	* Thus, enable callback only if timerlat is set with no workload.
1246	*/
1247	if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options)) {
1248	if (WARN_ON_ONCE(monitor_enabled))
1249	return 0;
1250
1251	ret = register_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1252	if (!ret)
1253	monitor_enabled = true;
1254	}
1255
1256	return ret;
1257	}
1258
1259	static void unregister_migration_monitor(void)
1260	{
1261	if (!monitor_enabled)
1262	return;
1263
1264	unregister_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1265	monitor_enabled = false;
1266	}
1267	#else
1268	static int register_migration_monitor(void)
1269	{
1270	return 0;
1271	}
1272	static void unregister_migration_monitor(void) {}
1273	#endif
1274	/*
1275	* trace_sched_switch - sched:sched_switch trace event handler
1276	*
1277	* This function is hooked to the sched:sched_switch trace event, and it is
1278	* used to record the beginning and to report the end of a thread noise window.
1279	*/
1280	static void
1281	trace_sched_switch_callback(void *data, bool preempt,
1282	struct task_struct *p,
1283	struct task_struct *n,
1284	unsigned int prev_state)
1285	{
1286	struct osnoise_variables *osn_var = this_cpu_osn_var();
1287	int workload = test_bit(OSN_WORKLOAD, &osnoise_options);
1288
1289	if ((p->pid != osn_var->pid) \|\| !workload)
1290	thread_exit(osn_var, p);
1291
1292	if ((n->pid != osn_var->pid) \|\| !workload)
1293	thread_entry(osn_var, n);
1294	}
1295
1296	/*
1297	* hook_thread_events - Hook the instrumentation for thread noise
1298	*
1299	* Hook the osnoise tracer callbacks to handle the noise from other
1300	* threads on the necessary kernel events.
1301	*/
1302	static int hook_thread_events(void)
1303	{
1304	int ret;
1305
1306	ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1307	if (ret)
1308	return -EINVAL;
1309
1310	ret = register_migration_monitor();
1311	if (ret)
1312	goto out_unreg;
1313
1314	return 0;
1315
1316	out_unreg:
1317	unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1318	return -EINVAL;
1319	}
1320
1321	/*
1322	* unhook_thread_events - unhook the instrumentation for thread noise
1323	*
1324	* Unook the osnoise tracer callbacks to handle the noise from other
1325	* threads on the necessary kernel events.
1326	*/
1327	static void unhook_thread_events(void)
1328	{
1329	unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1330	unregister_migration_monitor();
1331	}
1332
1333	/*
1334	* save_osn_sample_stats - Save the osnoise_sample statistics
1335	*
1336	* Save the osnoise_sample statistics before the sampling phase. These
1337	* values will be used later to compute the diff betwneen the statistics
1338	* before and after the osnoise sampling.
1339	*/
1340	static void
1341	save_osn_sample_stats(struct osnoise_variables osn_var, struct osnoise_sample s)
1342	{
1343	s->nmi_count = osn_var->nmi.count;
1344	s->irq_count = osn_var->irq.count;
1345	s->softirq_count = osn_var->softirq.count;
1346	s->thread_count = osn_var->thread.count;
1347	}
1348
1349	/*
1350	* diff_osn_sample_stats - Compute the osnoise_sample statistics
1351	*
1352	* After a sample period, compute the difference on the osnoise_sample
1353	* statistics. The struct osnoise_sample *s contains the statistics saved via
1354	* save_osn_sample_stats() before the osnoise sampling.
1355	*/
1356	static void
1357	diff_osn_sample_stats(struct osnoise_variables osn_var, struct osnoise_sample s)
1358	{
1359	s->nmi_count = osn_var->nmi.count - s->nmi_count;
1360	s->irq_count = osn_var->irq.count - s->irq_count;
1361	s->softirq_count = osn_var->softirq.count - s->softirq_count;
1362	s->thread_count = osn_var->thread.count - s->thread_count;
1363	}
1364
1365	/*
1366	* osnoise_stop_tracing - Stop tracing and the tracer.
1367	*/
1368	static __always_inline void osnoise_stop_tracing(void)
1369	{
1370	struct osnoise_instance *inst;
1371	struct trace_array *tr;
1372
1373	rcu_read_lock();
1374	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1375	tr = inst->tr;
1376	trace_array_printk(tr, _THIS_IP_,
1377	"stop tracing hit on cpu %d\n", smp_processor_id());
1378
1379	if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1380	panic("tracer hit stop condition on CPU %d\n", smp_processor_id());
1381
1382	tracer_tracing_off(tr);
1383	}
1384	rcu_read_unlock();
1385	}
1386
1387	/*
1388	* osnoise_has_tracing_on - Check if there is at least one instance on
1389	*/
1390	static __always_inline int osnoise_has_tracing_on(void)
1391	{
1392	struct osnoise_instance *inst;
1393	int trace_is_on = 0;
1394
1395	rcu_read_lock();
1396	list_for_each_entry_rcu(inst, &osnoise_instances, list)
1397	trace_is_on += tracer_tracing_is_on(inst->tr);
1398	rcu_read_unlock();
1399
1400	return trace_is_on;
1401	}
1402
1403	/*
1404	* notify_new_max_latency - Notify a new max latency via fsnotify interface.
1405	*/
1406	static void notify_new_max_latency(u64 latency)
1407	{
1408	struct osnoise_instance *inst;
1409	struct trace_array *tr;
1410
1411	rcu_read_lock();
1412	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1413	tr = inst->tr;
1414	if (tracer_tracing_is_on(tr) && tr->max_latency < latency) {
1415	tr->max_latency = latency;
1416	latency_fsnotify(tr);
1417	}
1418	}
1419	rcu_read_unlock();
1420	}
1421
1422	/*
1423	* run_osnoise - Sample the time and look for osnoise
1424	*
1425	* Used to capture the time, looking for potential osnoise latency repeatedly.
1426	* Different from hwlat_detector, it is called with preemption and interrupts
1427	* enabled. This allows irqs, softirqs and threads to run, interfering on the
1428	* osnoise sampling thread, as they would do with a regular thread.
1429	*/
1430	static int run_osnoise(void)
1431	{
1432	bool disable_irq = test_bit(OSN_IRQ_DISABLE, &osnoise_options);
1433	struct osnoise_variables *osn_var = this_cpu_osn_var();
1434	u64 start, sample, last_sample;
1435	u64 last_int_count, int_count;
1436	s64 noise = 0, max_noise = 0;
1437	s64 total, last_total = 0;
1438	struct osnoise_sample s;
1439	bool disable_preemption;
1440	unsigned int threshold;
1441	u64 runtime, stop_in;
1442	u64 sum_noise = 0;
1443	int hw_count = 0;
1444	int ret = -1;
1445
1446	/*
1447	* Disabling preemption is only required if IRQs are enabled,
1448	* and the options is set on.
1449	*/
1450	disable_preemption = !disable_irq && test_bit(OSN_PREEMPT_DISABLE, &osnoise_options);
1451
1452	/*
1453	* Considers the current thread as the workload.
1454	*/
1455	osn_var->pid = current->pid;
1456
1457	/*
1458	* Save the current stats for the diff
1459	*/
1460	save_osn_sample_stats(osn_var, &s);
1461
1462	/*
1463	* if threshold is 0, use the default value of 1 us.
1464	*/
1465	threshold = tracing_thresh ? : 1000;
1466
1467	/*
1468	* Apply PREEMPT and IRQ disabled options.
1469	*/
1470	if (disable_irq)
1471	local_irq_disable();
1472
1473	if (disable_preemption)
1474	preempt_disable();
1475
1476	/*
1477	* Make sure NMIs see sampling first
1478	*/
1479	osn_var->sampling = true;
1480	barrier();
1481
1482	/*
1483	* Transform the *_us config to nanoseconds to avoid the
1484	* division on the main loop.
1485	*/
1486	runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1487	stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1488
1489	/*
1490	* Start timestamp
1491	*/
1492	start = time_get();
1493
1494	/*
1495	* "previous" loop.
1496	*/
1497	last_int_count = set_int_safe_time(osn_var, &last_sample);
1498
1499	do {
1500	/*
1501	* Get sample!
1502	*/
1503	int_count = set_int_safe_time(osn_var, &sample);
1504
1505	noise = time_sub(sample, last_sample);
1506
1507	/*
1508	* This shouldn't happen.
1509	*/
1510	if (noise < 0) {
1511	osnoise_taint("negative noise!");
1512	goto out;
1513	}
1514
1515	/*
1516	* Sample runtime.
1517	*/
1518	total = time_sub(sample, start);
1519
1520	/*
1521	* Check for possible overflows.
1522	*/
1523	if (total < last_total) {
1524	osnoise_taint("total overflow!");
1525	break;
1526	}
1527
1528	last_total = total;
1529
1530	if (noise >= threshold) {
1531	int interference = int_count - last_int_count;
1532
1533	if (noise > max_noise)
1534	max_noise = noise;
1535
1536	if (!interference)
1537	hw_count++;
1538
1539	sum_noise += noise;
1540
1541	trace_sample_threshold(last_sample, noise, interference);
1542
1543	if (osnoise_data.stop_tracing)
1544	if (noise > stop_in)
1545	osnoise_stop_tracing();
1546	}
1547
1548	/*
1549	* In some cases, notably when running on a nohz_full CPU with
1550	* a stopped tick PREEMPT_RCU or PREEMPT_LAZY have no way to
1551	* account for QSs. This will eventually cause unwarranted
1552	* noise as RCU forces preemption as the means of ending the
1553	* current grace period. We avoid this by calling
1554	* rcu_momentary_eqs(), which performs a zero duration EQS
1555	* allowing RCU to end the current grace period. This call
1556	* shouldn't be wrapped inside an RCU critical section.
1557	*
1558	* Normally QSs for other cases are handled through cond_resched().
1559	* For simplicity, however, we call rcu_momentary_eqs() for all
1560	* configurations here.
1561	*/
1562	if (!disable_irq)
1563	local_irq_disable();
1564
1565	rcu_momentary_eqs();
1566
1567	if (!disable_irq)
1568	local_irq_enable();
1569
1570	/*
1571	* For the non-preemptive kernel config: let threads runs, if
1572	* they so wish, unless set not do to so.
1573	*/
1574	if (!disable_irq && !disable_preemption)
1575	cond_resched();
1576
1577	last_sample = sample;
1578	last_int_count = int_count;
1579
1580	} while (total < runtime && !kthread_should_stop());
1581
1582	/*
1583	* Finish the above in the view for interrupts.
1584	*/
1585	barrier();
1586
1587	osn_var->sampling = false;
1588
1589	/*
1590	* Make sure sampling data is no longer updated.
1591	*/
1592	barrier();
1593
1594	/*
1595	* Return to the preemptive state.
1596	*/
1597	if (disable_preemption)
1598	preempt_enable();
1599
1600	if (disable_irq)
1601	local_irq_enable();
1602
1603	/*
1604	* Save noise info.
1605	*/
1606	s.noise = time_to_us(sum_noise);
1607	s.runtime = time_to_us(total);
1608	s.max_sample = time_to_us(max_noise);
1609	s.hw_count = hw_count;
1610
1611	/* Save interference stats info */
1612	diff_osn_sample_stats(osn_var, &s);
1613
1614	record_osnoise_sample(&s);
1615
1616	notify_new_max_latency(max_noise);
1617
1618	if (osnoise_data.stop_tracing_total)
1619	if (s.noise > osnoise_data.stop_tracing_total)
1620	osnoise_stop_tracing();
1621
1622	return 0;
1623	out:
1624	return ret;
1625	}
1626
1627	static struct cpumask osnoise_cpumask;
1628	static struct cpumask save_cpumask;
1629	static struct cpumask kthread_cpumask;
1630
1631	/*
1632	* osnoise_sleep - sleep until the next period
1633	*/
1634	static void osnoise_sleep(bool skip_period)
1635	{
1636	u64 interval;
1637	ktime_t wake_time;
1638
1639	mutex_lock(&interface_lock);
1640	if (skip_period)
1641	interval = osnoise_data.sample_period;
1642	else
1643	interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1644	mutex_unlock(&interface_lock);
1645
1646	/*
1647	* differently from hwlat_detector, the osnoise tracer can run
1648	* without a pause because preemption is on.
1649	*/
1650	if (!interval) {
1651	/* Let synchronize_rcu_tasks() make progress */
1652	cond_resched_tasks_rcu_qs();
1653	return;
1654	}
1655
1656	wake_time = ktime_add_us(ktime_get(), interval);
1657	__set_current_state(TASK_INTERRUPTIBLE);
1658
1659	while (schedule_hrtimeout(&wake_time, HRTIMER_MODE_ABS)) {
1660	if (kthread_should_stop())
1661	break;
1662	}
1663	}
1664
1665	/*
1666	* osnoise_migration_pending - checks if the task needs to migrate
1667	*
1668	* osnoise/timerlat threads are per-cpu. If there is a pending request to
1669	* migrate the thread away from the current CPU, something bad has happened.
1670	* Play the good citizen and leave.
1671	*
1672	* Returns 0 if it is safe to continue, 1 otherwise.
1673	*/
1674	static inline int osnoise_migration_pending(void)
1675	{
1676	if (!current->migration_pending)
1677	return 0;
1678
1679	/*
1680	* If migration is pending, there is a task waiting for the
1681	* tracer to enable migration. The tracer does not allow migration,
1682	* thus: taint and leave to unblock the blocked thread.
1683	*/
1684	osnoise_taint("migration requested to osnoise threads, leaving.");
1685
1686	/*
1687	* Unset this thread from the threads managed by the interface.
1688	* The tracers are responsible for cleaning their env before
1689	* exiting.
1690	*/
1691	mutex_lock(&interface_lock);
1692	this_cpu_osn_var()->kthread = NULL;
1693	cpumask_clear_cpu(smp_processor_id(), &kthread_cpumask);
1694	mutex_unlock(&interface_lock);
1695
1696	return 1;
1697	}
1698
1699	/*
1700	* osnoise_main - The osnoise detection kernel thread
1701	*
1702	* Calls run_osnoise() function to measure the osnoise for the configured runtime,
1703	* every period.
1704	*/
1705	static int osnoise_main(void *data)
1706	{
1707	unsigned long flags;
1708
1709	/*
1710	* This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1711	* The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1712	*
1713	* To work around this limitation, disable migration and remove the
1714	* flag.
1715	*/
1716	migrate_disable();
1717	raw_spin_lock_irqsave(&current->pi_lock, flags);
1718	current->flags &= ~(PF_NO_SETAFFINITY);
1719	raw_spin_unlock_irqrestore(&current->pi_lock, flags);
1720
1721	while (!kthread_should_stop()) {
1722	if (osnoise_migration_pending())
1723	break;
1724
1725	/* skip a period if tracing is off on all instances */
1726	if (!osnoise_has_tracing_on()) {
1727	osnoise_sleep(true);
1728	continue;
1729	}
1730
1731	run_osnoise();
1732	osnoise_sleep(false);
1733	}
1734
1735	migrate_enable();
1736	return 0;
1737	}
1738
1739	#ifdef CONFIG_TIMERLAT_TRACER
1740	/*
1741	* timerlat_irq - hrtimer handler for timerlat.
1742	*/
1743	static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1744	{
1745	struct osnoise_variables *osn_var = this_cpu_osn_var();
1746	struct timerlat_variables *tlat;
1747	struct timerlat_sample s;
1748	u64 now;
1749	u64 diff;
1750
1751	/*
1752	* I am not sure if the timer was armed for this CPU. So, get
1753	* the timerlat struct from the timer itself, not from this
1754	* CPU.
1755	*/
1756	tlat = container_of(timer, struct timerlat_variables, timer);
1757
1758	now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1759
1760	/*
1761	* Enable the osnoise: events for thread an softirq.
1762	*/
1763	tlat->tracing_thread = true;
1764
1765	osn_var->thread.arrival_time = time_get();
1766
1767	/*
1768	* A hardirq is running: the timer IRQ. It is for sure preempting
1769	* a thread, and potentially preempting a softirq.
1770	*
1771	* At this point, it is not interesting to know the duration of the
1772	* preempted thread (and maybe softirq), but how much time they will
1773	* delay the beginning of the execution of the timer thread.
1774	*
1775	* To get the correct (net) delay added by the softirq, its delta_start
1776	* is set as the IRQ one. In this way, at the return of the IRQ, the delta
1777	* start of the sofitrq will be zeroed, accounting then only the time
1778	* after that.
1779	*
1780	* The thread follows the same principle. However, if a softirq is
1781	* running, the thread needs to receive the softirq delta_start. The
1782	* reason being is that the softirq will be the last to be unfolded,
1783	* resseting the thread delay to zero.
1784	*
1785	* The PREEMPT_RT is a special case, though. As softirqs run as threads
1786	* on RT, moving the thread is enough.
1787	*/
1788	if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {
1789	copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1790	&osn_var->softirq.delta_start);
1791
1792	copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1793	&osn_var->irq.delta_start);
1794	} else {
1795	copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1796	&osn_var->irq.delta_start);
1797	}
1798
1799	/*
1800	* Compute the current time with the expected time.
1801	*/
1802	diff = now - tlat->abs_period;
1803
1804	tlat->count++;
1805	s.seqnum = tlat->count;
1806	s.timer_latency = diff;
1807	s.context = IRQ_CONTEXT;
1808
1809	record_timerlat_sample(&s);
1810
1811	if (osnoise_data.stop_tracing) {
1812	if (time_to_us(diff) >= osnoise_data.stop_tracing) {
1813
1814	/*
1815	* At this point, if stop_tracing is set and <= print_stack,
1816	* print_stack is set and would be printed in the thread handler.
1817	*
1818	* Thus, print the stack trace as it is helpful to define the
1819	* root cause of an IRQ latency.
1820	*/
1821	if (osnoise_data.stop_tracing <= osnoise_data.print_stack) {
1822	timerlat_save_stack(0);
1823	timerlat_dump_stack(time_to_us(diff));
1824	}
1825
1826	osnoise_stop_tracing();
1827	notify_new_max_latency(diff);
1828
1829	wake_up_process(tlat->kthread);
1830
1831	return HRTIMER_NORESTART;
1832	}
1833	}
1834
1835	wake_up_process(tlat->kthread);
1836
1837	if (osnoise_data.print_stack)
1838	timerlat_save_stack(0);
1839
1840	return HRTIMER_NORESTART;
1841	}
1842
1843	/*
1844	* wait_next_period - Wait for the next period for timerlat
1845	*/
1846	static int wait_next_period(struct timerlat_variables *tlat)
1847	{
1848	ktime_t next_abs_period, now;
1849	u64 rel_period = osnoise_data.timerlat_period * 1000;
1850
1851	now = hrtimer_cb_get_time(&tlat->timer);
1852	next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1853
1854	/*
1855	* Save the next abs_period.
1856	*/
1857	tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1858
1859	/*
1860	* Align thread in the first cycle on each CPU to the set alignment
1861	* if TIMERLAT_ALIGN is set.
1862	*
1863	* This is done by using an atomic64_t to store the next absolute period.
1864	* The first thread that wakes up will set the atomic64_t to its
1865	* absolute period, and the other threads will increment it by
1866	* the alignment value.
1867	*/
1868	if (test_bit(OSN_TIMERLAT_ALIGN, &osnoise_options) && !tlat->count
1869	&& atomic64_cmpxchg_relaxed(&align_next, 0, tlat->abs_period)) {
1870	/*
1871	* A thread has already set align_next, use it and increment it
1872	* to be used by the next thread that wakes up after this one.
1873	*/
1874	tlat->abs_period = atomic64_add_return_relaxed(
1875	osnoise_data.timerlat_align_us * 1000, &align_next);
1876	next_abs_period = ns_to_ktime(tlat->abs_period);
1877	}
1878
1879	/*
1880	* If the new abs_period is in the past, skip the activation.
1881	*/
1882	while (ktime_compare(now, next_abs_period) > 0) {
1883	next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1884	tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1885	}
1886
1887	set_current_state(TASK_INTERRUPTIBLE);
1888
1889	hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1890	schedule();
1891	return 1;
1892	}
1893
1894	/*
1895	* timerlat_main- Timerlat main
1896	*/
1897	static int timerlat_main(void *data)
1898	{
1899	struct osnoise_variables *osn_var = this_cpu_osn_var();
1900	struct timerlat_variables *tlat = this_cpu_tmr_var();
1901	struct timerlat_sample s;
1902	struct sched_param sp;
1903	unsigned long flags;
1904	u64 now, diff;
1905
1906	/*
1907	* Make the thread RT, that is how cyclictest is usually used.
1908	*/
1909	sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1910	sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1911
1912	/*
1913	* This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1914	* The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1915	*
1916	* To work around this limitation, disable migration and remove the
1917	* flag.
1918	*/
1919	migrate_disable();
1920	raw_spin_lock_irqsave(&current->pi_lock, flags);
1921	current->flags &= ~(PF_NO_SETAFFINITY);
1922	raw_spin_unlock_irqrestore(&current->pi_lock, flags);
1923
1924	tlat->count = 0;
1925	tlat->tracing_thread = false;
1926
1927	hrtimer_setup(&tlat->timer, timerlat_irq, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1928	tlat->kthread = current;
1929	osn_var->pid = current->pid;
1930	/*
1931	* Annotate the arrival time.
1932	*/
1933	tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1934
1935	wait_next_period(tlat);
1936
1937	osn_var->sampling = 1;
1938
1939	while (!kthread_should_stop()) {
1940
1941	now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1942	diff = now - tlat->abs_period;
1943
1944	s.seqnum = tlat->count;
1945	s.timer_latency = diff;
1946	s.context = THREAD_CONTEXT;
1947
1948	record_timerlat_sample(&s);
1949
1950	notify_new_max_latency(diff);
1951
1952	timerlat_dump_stack(time_to_us(diff));
1953
1954	tlat->tracing_thread = false;
1955	if (osnoise_data.stop_tracing_total)
1956	if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1957	osnoise_stop_tracing();
1958
1959	if (osnoise_migration_pending())
1960	break;
1961
1962	wait_next_period(tlat);
1963	}
1964
1965	hrtimer_cancel(&tlat->timer);
1966	migrate_enable();
1967	return 0;
1968	}
1969	#else /* CONFIG_TIMERLAT_TRACER */
1970	static int timerlat_main(void *data)
1971	{
1972	return 0;
1973	}
1974	#endif /* CONFIG_TIMERLAT_TRACER */
1975
1976	/*
1977	* stop_kthread - stop a workload thread
1978	*/
1979	static void stop_kthread(unsigned int cpu)
1980	{
1981	struct task_struct *kthread;
1982
1983	kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL);
1984	if (kthread) {
1985	if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask) &&
1986	!WARN_ON(!test_bit(OSN_WORKLOAD, &osnoise_options))) {
1987	kthread_stop(kthread);
1988	} else if (!WARN_ON(test_bit(OSN_WORKLOAD, &osnoise_options))) {
1989	/*
1990	* This is a user thread waiting on the timerlat_fd. We need
1991	* to close all users, and the best way to guarantee this is
1992	* by killing the thread. NOTE: this is a purpose specific file.
1993	*/
1994	kill_pid(kthread->thread_pid, SIGKILL, 1);
1995	put_task_struct(kthread);
1996	}
1997	} else {
1998	/* if no workload, just return */
1999	if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2000	/*
2001	* This is set in the osnoise tracer case.
2002	*/
2003	per_cpu(per_cpu_osnoise_var, cpu).sampling = false;
2004	barrier();
2005	}
2006	}
2007	}
2008
2009	/*
2010	* stop_per_cpu_kthread - Stop per-cpu threads
2011	*
2012	* Stop the osnoise sampling htread. Use this on unload and at system
2013	* shutdown.
2014	*/
2015	static void stop_per_cpu_kthreads(void)
2016	{
2017	int cpu;
2018
2019	cpus_read_lock();
2020
2021	for_each_online_cpu(cpu)
2022	stop_kthread(cpu);
2023
2024	cpus_read_unlock();
2025	}
2026
2027	/*
2028	* start_kthread - Start a workload thread
2029	*/
2030	static int start_kthread(unsigned int cpu)
2031	{
2032	struct task_struct *kthread;
2033	void *main = osnoise_main;
2034	char comm[24];
2035
2036	/* Do not start a new thread if it is already running */
2037	if (per_cpu(per_cpu_osnoise_var, cpu).kthread)
2038	return 0;
2039
2040	if (timerlat_enabled()) {
2041	snprintf(comm, 24, "timerlat/%d", cpu);
2042	main = timerlat_main;
2043	} else {
2044	/* if no workload, just return */
2045	if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2046	per_cpu(per_cpu_osnoise_var, cpu).sampling = true;
2047	barrier();
2048	return 0;
2049	}
2050	snprintf(comm, 24, "osnoise/%d", cpu);
2051	}
2052
2053	kthread = kthread_run_on_cpu(main, NULL, cpu, comm);
2054
2055	if (IS_ERR(kthread)) {
2056	pr_err(BANNER "could not start sampling thread\n");
2057	return -ENOMEM;
2058	}
2059
2060	per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
2061	cpumask_set_cpu(cpu, &kthread_cpumask);
2062
2063	return 0;
2064	}
2065
2066	/*
2067	* start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
2068	*
2069	* This starts the kernel thread that will look for osnoise on many
2070	* cpus.
2071	*/
2072	static int start_per_cpu_kthreads(void)
2073	{
2074	struct cpumask *current_mask = &save_cpumask;
2075	int retval = 0;
2076	int cpu;
2077
2078	if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2079	if (timerlat_enabled())
2080	return 0;
2081	}
2082
2083	cpus_read_lock();
2084	/*
2085	* Run only on online CPUs in which osnoise is allowed to run.
2086	*/
2087	cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask);
2088
2089	for_each_possible_cpu(cpu) {
2090	if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask)) {
2091	struct task_struct *kthread;
2092
2093	kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL);
2094	if (!WARN_ON(!kthread))
2095	kthread_stop(kthread);
2096	}
2097	}
2098
2099	for_each_cpu(cpu, current_mask) {
2100	retval = start_kthread(cpu);
2101	if (retval) {
2102	cpus_read_unlock();
2103	stop_per_cpu_kthreads();
2104	return retval;
2105	}
2106	}
2107
2108	cpus_read_unlock();
2109
2110	return retval;
2111	}
2112
2113	#ifdef CONFIG_HOTPLUG_CPU
2114	static void osnoise_hotplug_workfn(struct work_struct *dummy)
2115	{
2116	unsigned int cpu = smp_processor_id();
2117
2118	guard(mutex)(&trace_types_lock);
2119
2120	if (!osnoise_has_registered_instances())
2121	return;
2122
2123	guard(cpus_read_lock)();
2124	guard(mutex)(&interface_lock);
2125
2126	if (!cpu_online(cpu))
2127	return;
2128
2129	if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
2130	return;
2131
2132	start_kthread(cpu);
2133	}
2134
2135	static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
2136
2137	/*
2138	* osnoise_cpu_init - CPU hotplug online callback function
2139	*/
2140	static int osnoise_cpu_init(unsigned int cpu)
2141	{
2142	schedule_work_on(cpu, &osnoise_hotplug_work);
2143	return 0;
2144	}
2145
2146	/*
2147	* osnoise_cpu_die - CPU hotplug offline callback function
2148	*/
2149	static int osnoise_cpu_die(unsigned int cpu)
2150	{
2151	stop_kthread(cpu);
2152	return 0;
2153	}
2154
2155	static void osnoise_init_hotplug_support(void)
2156	{
2157	int ret;
2158
2159	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
2160	osnoise_cpu_init, osnoise_cpu_die);
2161	if (ret < 0)
2162	pr_warn(BANNER "Error to init cpu hotplug support\n");
2163
2164	return;
2165	}
2166	#else /* CONFIG_HOTPLUG_CPU */
2167	static void osnoise_init_hotplug_support(void)
2168	{
2169	return;
2170	}
2171	#endif /* CONFIG_HOTPLUG_CPU */
2172
2173	/*
2174	* seq file functions for the osnoise/options file.
2175	*/
2176	static void s_options_start(struct seq_file s, loff_t *pos)
2177	{
2178	int option = *pos;
2179
2180	mutex_lock(&interface_lock);
2181
2182	if (option >= OSN_MAX)
2183	return NULL;
2184
2185	return pos;
2186	}
2187
2188	static void s_options_next(struct seq_file s, void v, loff_t pos)
2189	{
2190	int option = ++(*pos);
2191
2192	if (option >= OSN_MAX)
2193	return NULL;
2194
2195	return pos;
2196	}
2197
2198	static int s_options_show(struct seq_file s, void v)
2199	{
2200	loff_t *pos = v;
2201	int option = *pos;
2202
2203	if (option == OSN_DEFAULTS) {
2204	if (osnoise_options == OSN_DEFAULT_OPTIONS)
2205	seq_printf(s, "%s", osnoise_options_str[option]);
2206	else
2207	seq_printf(s, "NO_%s", osnoise_options_str[option]);
2208	goto out;
2209	}
2210
2211	if (test_bit(option, &osnoise_options))
2212	seq_printf(s, "%s", osnoise_options_str[option]);
2213	else
2214	seq_printf(s, "NO_%s", osnoise_options_str[option]);
2215
2216	out:
2217	if (option != OSN_MAX)
2218	seq_puts(s, " ");
2219
2220	return 0;
2221	}
2222
2223	static void s_options_stop(struct seq_file s, void v)
2224	{
2225	seq_puts(s, "\n");
2226	mutex_unlock(&interface_lock);
2227	}
2228
2229	static const struct seq_operations osnoise_options_seq_ops = {
2230	.start = s_options_start,
2231	.next = s_options_next,
2232	.show = s_options_show,
2233	.stop = s_options_stop
2234	};
2235
2236	static int osnoise_options_open(struct inode inode, struct file file)
2237	{
2238	return seq_open(file, &osnoise_options_seq_ops);
2239	};
2240
2241	/**
2242	* osnoise_options_write - Write function for "options" entry
2243	* @filp: The active open file structure
2244	* @ubuf: The user buffer that contains the value to write
2245	* @cnt: The maximum number of bytes to write to "file"
2246	* @ppos: The current position in @file
2247	*
2248	* Writing the option name sets the option, writing the "NO_"
2249	* prefix in front of the option name disables it.
2250	*
2251	* Writing "DEFAULTS" resets the option values to the default ones.
2252	*/
2253	static ssize_t osnoise_options_write(struct file filp, const char __user ubuf,
2254	size_t cnt, loff_t *ppos)
2255	{
2256	int running, option, enable, retval;
2257	char buf[256], *option_str;
2258
2259	if (cnt >= 256)
2260	return -EINVAL;
2261
2262	if (copy_from_user(buf, ubuf, cnt))
2263	return -EFAULT;
2264
2265	buf[cnt] = 0;
2266
2267	if (strncmp(buf, "NO_", 3)) {
2268	option_str = strstrip(buf);
2269	enable = true;
2270	} else {
2271	option_str = strstrip(&buf[3]);
2272	enable = false;
2273	}
2274
2275	option = match_string(osnoise_options_str, OSN_MAX, option_str);
2276	if (option < 0)
2277	return -EINVAL;
2278
2279	/*
2280	* trace_types_lock is taken to avoid concurrency on start/stop.
2281	*/
2282	mutex_lock(&trace_types_lock);
2283	running = osnoise_has_registered_instances();
2284	if (running)
2285	stop_per_cpu_kthreads();
2286
2287	/*
2288	* avoid CPU hotplug operations that might read options.
2289	*/
2290	cpus_read_lock();
2291	mutex_lock(&interface_lock);
2292
2293	retval = cnt;
2294
2295	if (enable) {
2296	if (option == OSN_DEFAULTS)
2297	osnoise_options = OSN_DEFAULT_OPTIONS;
2298	else
2299	set_bit(option, &osnoise_options);
2300	} else {
2301	if (option == OSN_DEFAULTS)
2302	retval = -EINVAL;
2303	else
2304	clear_bit(option, &osnoise_options);
2305	}
2306
2307	mutex_unlock(&interface_lock);
2308	cpus_read_unlock();
2309
2310	if (running)
2311	start_per_cpu_kthreads();
2312	mutex_unlock(&trace_types_lock);
2313
2314	return retval;
2315	}
2316
2317	/*
2318	* osnoise_cpus_read - Read function for reading the "cpus" file
2319	* @filp: The active open file structure
2320	* @ubuf: The userspace provided buffer to read value into
2321	* @cnt: The maximum number of bytes to read
2322	* @ppos: The current "file" position
2323	*
2324	* Prints the "cpus" output into the user-provided buffer.
2325	*/
2326	static ssize_t
2327	osnoise_cpus_read(struct file filp, char __user ubuf, size_t count,
2328	loff_t *ppos)
2329	{
2330	char *mask_str __free(kfree) = NULL;
2331	int len;
2332
2333	guard(mutex)(&interface_lock);
2334
2335	len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
2336	mask_str = kmalloc(len, GFP_KERNEL);
2337	if (!mask_str)
2338	return -ENOMEM;
2339
2340	len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
2341	if (len >= count)
2342	return -EINVAL;
2343
2344	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
2345
2346	return count;
2347	}
2348
2349	/*
2350	* osnoise_cpus_write - Write function for "cpus" entry
2351	* @filp: The active open file structure
2352	* @ubuf: The user buffer that contains the value to write
2353	* @count: The maximum number of bytes to write to "file"
2354	* @ppos: The current position in @file
2355	*
2356	* This function provides a write implementation for the "cpus"
2357	* interface to the osnoise trace. By default, it lists all CPUs,
2358	* in this way, allowing osnoise threads to run on any online CPU
2359	* of the system. It serves to restrict the execution of osnoise to the
2360	* set of CPUs writing via this interface. Why not use "tracing_cpumask"?
2361	* Because the user might be interested in tracing what is running on
2362	* other CPUs. For instance, one might run osnoise in one HT CPU
2363	* while observing what is running on the sibling HT CPU.
2364	*/
2365	static ssize_t
2366	osnoise_cpus_write(struct file filp, const char __user ubuf, size_t count,
2367	loff_t *ppos)
2368	{
2369	cpumask_var_t osnoise_cpumask_new;
2370	int running, err;
2371	char *buf __free(kfree) = NULL;
2372
2373	if (count < 1)
2374	return 0;
2375
2376	buf = memdup_user_nul(ubuf, count);
2377	if (IS_ERR(buf))
2378	return PTR_ERR(buf);
2379
2380	if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
2381	return -ENOMEM;
2382
2383	err = cpulist_parse(buf, osnoise_cpumask_new);
2384	if (err)
2385	goto err_free;
2386
2387	/*
2388	* trace_types_lock is taken to avoid concurrency on start/stop.
2389	*/
2390	mutex_lock(&trace_types_lock);
2391	running = osnoise_has_registered_instances();
2392	if (running)
2393	stop_per_cpu_kthreads();
2394
2395	/*
2396	* osnoise_cpumask is read by CPU hotplug operations.
2397	*/
2398	cpus_read_lock();
2399	mutex_lock(&interface_lock);
2400
2401	cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
2402
2403	mutex_unlock(&interface_lock);
2404	cpus_read_unlock();
2405
2406	if (running)
2407	start_per_cpu_kthreads();
2408	mutex_unlock(&trace_types_lock);
2409
2410	free_cpumask_var(osnoise_cpumask_new);
2411	return count;
2412
2413	err_free:
2414	free_cpumask_var(osnoise_cpumask_new);
2415
2416	return err;
2417	}
2418
2419	#ifdef CONFIG_TIMERLAT_TRACER
2420	static int timerlat_fd_open(struct inode inode, struct file file)
2421	{
2422	struct osnoise_variables *osn_var;
2423	struct timerlat_variables *tlat;
2424	long cpu = (long) inode->i_cdev;
2425
2426	mutex_lock(&interface_lock);
2427
2428	/*
2429	* This file is accessible only if timerlat is enabled, and
2430	* NO_OSNOISE_WORKLOAD is set.
2431	*/
2432	if (!timerlat_enabled() \|\| test_bit(OSN_WORKLOAD, &osnoise_options)) {
2433	mutex_unlock(&interface_lock);
2434	return -EINVAL;
2435	}
2436
2437	migrate_disable();
2438
2439	osn_var = this_cpu_osn_var();
2440
2441	/*
2442	* The osn_var->pid holds the single access to this file.
2443	*/
2444	if (osn_var->pid) {
2445	mutex_unlock(&interface_lock);
2446	migrate_enable();
2447	return -EBUSY;
2448	}
2449
2450	/*
2451	* timerlat tracer is a per-cpu tracer. Check if the user-space too
2452	* is pinned to a single CPU. The tracer laters monitor if the task
2453	* migrates and then disables tracer if it does. However, it is
2454	* worth doing this basic acceptance test to avoid obviusly wrong
2455	* setup.
2456	*/
2457	if (current->nr_cpus_allowed > 1 \|\| cpu != smp_processor_id()) {
2458	mutex_unlock(&interface_lock);
2459	migrate_enable();
2460	return -EPERM;
2461	}
2462
2463	/*
2464	* From now on, it is good to go.
2465	*/
2466	file->private_data = inode->i_cdev;
2467
2468	get_task_struct(current);
2469
2470	osn_var->kthread = current;
2471	osn_var->pid = current->pid;
2472
2473	/*
2474	* Setup is done.
2475	*/
2476	mutex_unlock(&interface_lock);
2477
2478	tlat = this_cpu_tmr_var();
2479	tlat->count = 0;
2480
2481	hrtimer_setup(&tlat->timer, timerlat_irq, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
2482
2483	migrate_enable();
2484	return 0;
2485	};
2486
2487	/*
2488	* timerlat_fd_read - Read function for "timerlat_fd" file
2489	* @file: The active open file structure
2490	* @ubuf: The userspace provided buffer to read value into
2491	* @cnt: The maximum number of bytes to read
2492	* @ppos: The current "file" position
2493	*
2494	* Prints 1 on timerlat, the number of interferences on osnoise, -1 on error.
2495	*/
2496	static ssize_t
2497	timerlat_fd_read(struct file file, char __user ubuf, size_t count,
2498	loff_t *ppos)
2499	{
2500	long cpu = (long) file->private_data;
2501	struct osnoise_variables *osn_var;
2502	struct timerlat_variables *tlat;
2503	struct timerlat_sample s;
2504	s64 diff;
2505	u64 now;
2506
2507	migrate_disable();
2508
2509	tlat = this_cpu_tmr_var();
2510
2511	/*
2512	* While in user-space, the thread is migratable. There is nothing
2513	* we can do about it.
2514	* So, if the thread is running on another CPU, stop the machinery.
2515	*/
2516	if (cpu == smp_processor_id()) {
2517	if (tlat->uthread_migrate) {
2518	migrate_enable();
2519	return -EINVAL;
2520	}
2521	} else {
2522	per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
2523	osnoise_taint("timerlat user thread migrate\n");
2524	osnoise_stop_tracing();
2525	migrate_enable();
2526	return -EINVAL;
2527	}
2528
2529	osn_var = this_cpu_osn_var();
2530
2531	/*
2532	* The timerlat in user-space runs in a different order:
2533	* the read() starts from the execution of the previous occurrence,
2534	* sleeping for the next occurrence.
2535	*
2536	* So, skip if we are entering on read() before the first wakeup
2537	* from timerlat IRQ:
2538	*/
2539	if (likely(osn_var->sampling)) {
2540	now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2541	diff = now - tlat->abs_period;
2542
2543	/*
2544	* it was not a timer firing, but some other signal?
2545	*/
2546	if (diff < 0)
2547	goto out;
2548
2549	s.seqnum = tlat->count;
2550	s.timer_latency = diff;
2551	s.context = THREAD_URET;
2552
2553	record_timerlat_sample(&s);
2554
2555	notify_new_max_latency(diff);
2556
2557	tlat->tracing_thread = false;
2558	if (osnoise_data.stop_tracing_total) {
2559	if (time_to_us(diff) >= osnoise_data.stop_tracing_total) {
2560	timerlat_dump_stack(time_to_us(diff));
2561	osnoise_stop_tracing();
2562	}
2563	}
2564	} else {
2565	tlat->tracing_thread = false;
2566	tlat->kthread = current;
2567
2568	/* Annotate now to drift new period */
2569	tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
2570
2571	osn_var->sampling = 1;
2572	}
2573
2574	/* wait for the next period */
2575	wait_next_period(tlat);
2576
2577	/* This is the wakeup from this cycle */
2578	now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2579	diff = now - tlat->abs_period;
2580
2581	/*
2582	* it was not a timer firing, but some other signal?
2583	*/
2584	if (diff < 0)
2585	goto out;
2586
2587	s.seqnum = tlat->count;
2588	s.timer_latency = diff;
2589	s.context = THREAD_CONTEXT;
2590
2591	record_timerlat_sample(&s);
2592
2593	if (osnoise_data.stop_tracing_total) {
2594	if (time_to_us(diff) >= osnoise_data.stop_tracing_total) {
2595	timerlat_dump_stack(time_to_us(diff));
2596	notify_new_max_latency(diff);
2597	osnoise_stop_tracing();
2598	}
2599	}
2600
2601	out:
2602	migrate_enable();
2603	return 0;
2604	}
2605
2606	static int timerlat_fd_release(struct inode inode, struct file file)
2607	{
2608	struct osnoise_variables *osn_var;
2609	struct timerlat_variables *tlat_var;
2610	long cpu = (long) file->private_data;
2611
2612	migrate_disable();
2613	mutex_lock(&interface_lock);
2614
2615	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
2616	tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
2617
2618	if (tlat_var->kthread)
2619	hrtimer_cancel(&tlat_var->timer);
2620	memset(tlat_var, 0, sizeof(*tlat_var));
2621
2622	osn_var->sampling = 0;
2623	osn_var->pid = 0;
2624
2625	/*
2626	* We are leaving, not being stopped... see stop_kthread();
2627	*/
2628	if (osn_var->kthread) {
2629	put_task_struct(osn_var->kthread);
2630	osn_var->kthread = NULL;
2631	}
2632
2633	mutex_unlock(&interface_lock);
2634	migrate_enable();
2635	return 0;
2636	}
2637	#endif
2638
2639	/*
2640	* osnoise/runtime_us: cannot be greater than the period.
2641	*/
2642	static struct trace_min_max_param osnoise_runtime = {
2643	.lock = &interface_lock,
2644	.val = &osnoise_data.sample_runtime,
2645	.max = &osnoise_data.sample_period,
2646	.min = NULL,
2647	};
2648
2649	/*
2650	* osnoise/period_us: cannot be smaller than the runtime.
2651	*/
2652	static struct trace_min_max_param osnoise_period = {
2653	.lock = &interface_lock,
2654	.val = &osnoise_data.sample_period,
2655	.max = NULL,
2656	.min = &osnoise_data.sample_runtime,
2657	};
2658
2659	/*
2660	* osnoise/stop_tracing_us: no limit.
2661	*/
2662	static struct trace_min_max_param osnoise_stop_tracing_in = {
2663	.lock = &interface_lock,
2664	.val = &osnoise_data.stop_tracing,
2665	.max = NULL,
2666	.min = NULL,
2667	};
2668
2669	/*
2670	* osnoise/stop_tracing_total_us: no limit.
2671	*/
2672	static struct trace_min_max_param osnoise_stop_tracing_total = {
2673	.lock = &interface_lock,
2674	.val = &osnoise_data.stop_tracing_total,
2675	.max = NULL,
2676	.min = NULL,
2677	};
2678
2679	#ifdef CONFIG_TIMERLAT_TRACER
2680	/*
2681	* osnoise/print_stack: print the stacktrace of the IRQ handler if the total
2682	* latency is higher than val.
2683	*/
2684	static struct trace_min_max_param osnoise_print_stack = {
2685	.lock = &interface_lock,
2686	.val = &osnoise_data.print_stack,
2687	.max = NULL,
2688	.min = NULL,
2689	};
2690
2691	/*
2692	* osnoise/timerlat_period: min 100 us, max 1 s
2693	*/
2694	static u64 timerlat_min_period = 100;
2695	static u64 timerlat_max_period = 1000000;
2696	static struct trace_min_max_param timerlat_period = {
2697	.lock = &interface_lock,
2698	.val = &osnoise_data.timerlat_period,
2699	.max = &timerlat_max_period,
2700	.min = &timerlat_min_period,
2701	};
2702
2703	/*
2704	* osnoise/timerlat_align_us: align the first wakeup of all timerlat
2705	* threads to a common boundary (in us). 0 means disabled.
2706	*/
2707	static struct trace_min_max_param timerlat_align_us = {
2708	.lock = &interface_lock,
2709	.val = &osnoise_data.timerlat_align_us,
2710	.max = NULL,
2711	.min = NULL,
2712	};
2713
2714	static const struct file_operations timerlat_fd_fops = {
2715	.open = timerlat_fd_open,
2716	.read = timerlat_fd_read,
2717	.release = timerlat_fd_release,
2718	.llseek = generic_file_llseek,
2719	};
2720	#endif
2721
2722	static const struct file_operations cpus_fops = {
2723	.open = tracing_open_generic,
2724	.read = osnoise_cpus_read,
2725	.write = osnoise_cpus_write,
2726	.llseek = generic_file_llseek,
2727	};
2728
2729	static const struct file_operations osnoise_options_fops = {
2730	.open = osnoise_options_open,
2731	.read = seq_read,
2732	.llseek = seq_lseek,
2733	.release = seq_release,
2734	.write = osnoise_options_write
2735	};
2736
2737	#ifdef CONFIG_TIMERLAT_TRACER
2738	#ifdef CONFIG_STACKTRACE
2739	static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2740	{
2741	struct dentry *tmp;
2742
2743	tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
2744	&osnoise_print_stack, &trace_min_max_fops);
2745	if (!tmp)
2746	return -ENOMEM;
2747
2748	return 0;
2749	}
2750	#else /* CONFIG_STACKTRACE */
2751	static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2752	{
2753	return 0;
2754	}
2755	#endif /* CONFIG_STACKTRACE */
2756
2757	static int osnoise_create_cpu_timerlat_fd(struct dentry *top_dir)
2758	{
2759	struct dentry *timerlat_fd;
2760	struct dentry *per_cpu;
2761	struct dentry *cpu_dir;
2762	char cpu_str[30]; /* see trace.c: tracing_init_tracefs_percpu() */
2763	long cpu;
2764
2765	/*
2766	* Why not using tracing instance per_cpu/ dir?
2767	*
2768	* Because osnoise/timerlat have a single workload, having
2769	* multiple files like these are waste of memory.
2770	*/
2771	per_cpu = tracefs_create_dir("per_cpu", top_dir);
2772	if (!per_cpu)
2773	return -ENOMEM;
2774
2775	for_each_possible_cpu(cpu) {
2776	snprintf(cpu_str, 30, "cpu%ld", cpu);
2777	cpu_dir = tracefs_create_dir(cpu_str, per_cpu);
2778	if (!cpu_dir)
2779	goto out_clean;
2780
2781	timerlat_fd = trace_create_file("timerlat_fd", TRACE_MODE_READ,
2782	cpu_dir, NULL, &timerlat_fd_fops);
2783	if (!timerlat_fd)
2784	goto out_clean;
2785
2786	/* Record the CPU */
2787	d_inode(timerlat_fd)->i_cdev = (void *)(cpu);
2788	}
2789
2790	return 0;
2791
2792	out_clean:
2793	tracefs_remove(per_cpu);
2794	return -ENOMEM;
2795	}
2796
2797	/*
2798	* init_timerlat_tracefs - A function to initialize the timerlat interface files
2799	*/
2800	static int init_timerlat_tracefs(struct dentry *top_dir)
2801	{
2802	struct dentry *tmp;
2803	int retval;
2804
2805	tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
2806	&timerlat_period, &trace_min_max_fops);
2807	if (!tmp)
2808	return -ENOMEM;
2809
2810	tmp = tracefs_create_file("timerlat_align_us", TRACE_MODE_WRITE, top_dir,
2811	&timerlat_align_us, &trace_min_max_fops);
2812	if (!tmp)
2813	return -ENOMEM;
2814
2815	retval = osnoise_create_cpu_timerlat_fd(top_dir);
2816	if (retval)
2817	return retval;
2818
2819	return init_timerlat_stack_tracefs(top_dir);
2820	}
2821	#else /* CONFIG_TIMERLAT_TRACER */
2822	static int init_timerlat_tracefs(struct dentry *top_dir)
2823	{
2824	return 0;
2825	}
2826	#endif /* CONFIG_TIMERLAT_TRACER */
2827
2828	/*
2829	* init_tracefs - A function to initialize the tracefs interface files
2830	*
2831	* This function creates entries in tracefs for "osnoise" and "timerlat".
2832	* It creates these directories in the tracing directory, and within that
2833	* directory the use can change and view the configs.
2834	*/
2835	static int init_tracefs(void)
2836	{
2837	struct dentry *top_dir;
2838	struct dentry *tmp;
2839	int ret;
2840
2841	ret = tracing_init_dentry();
2842	if (ret)
2843	return -ENOMEM;
2844
2845	top_dir = tracefs_create_dir("osnoise", NULL);
2846	if (!top_dir)
2847	return 0;
2848
2849	tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
2850	&osnoise_period, &trace_min_max_fops);
2851	if (!tmp)
2852	goto err;
2853
2854	tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
2855	&osnoise_runtime, &trace_min_max_fops);
2856	if (!tmp)
2857	goto err;
2858
2859	tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
2860	&osnoise_stop_tracing_in, &trace_min_max_fops);
2861	if (!tmp)
2862	goto err;
2863
2864	tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
2865	&osnoise_stop_tracing_total, &trace_min_max_fops);
2866	if (!tmp)
2867	goto err;
2868
2869	tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
2870	if (!tmp)
2871	goto err;
2872
2873	tmp = trace_create_file("options", TRACE_MODE_WRITE, top_dir, NULL,
2874	&osnoise_options_fops);
2875	if (!tmp)
2876	goto err;
2877
2878	ret = init_timerlat_tracefs(top_dir);
2879	if (ret)
2880	goto err;
2881
2882	return 0;
2883
2884	err:
2885	tracefs_remove(top_dir);
2886	return -ENOMEM;
2887	}
2888
2889	static int osnoise_hook_events(void)
2890	{
2891	int retval;
2892
2893	/*
2894	* Trace is already hooked, we are re-enabling from
2895	* a stop_tracing_*.
2896	*/
2897	if (trace_osnoise_callback_enabled)
2898	return 0;
2899
2900	retval = hook_irq_events();
2901	if (retval)
2902	return -EINVAL;
2903
2904	retval = hook_softirq_events();
2905	if (retval)
2906	goto out_unhook_irq;
2907
2908	retval = hook_thread_events();
2909	/*
2910	* All fine!
2911	*/
2912	if (!retval)
2913	return 0;
2914
2915	unhook_softirq_events();
2916	out_unhook_irq:
2917	unhook_irq_events();
2918	return -EINVAL;
2919	}
2920
2921	static void osnoise_unhook_events(void)
2922	{
2923	unhook_thread_events();
2924	unhook_softirq_events();
2925	unhook_irq_events();
2926	}
2927
2928	/*
2929	* osnoise_workload_start - start the workload and hook to events
2930	*/
2931	static int osnoise_workload_start(void)
2932	{
2933	int retval;
2934
2935	/*
2936	* Instances need to be registered after calling workload
2937	* start. Hence, if there is already an instance, the
2938	* workload was already registered. Otherwise, this
2939	* code is on the way to register the first instance,
2940	* and the workload will start.
2941	*/
2942	if (osnoise_has_registered_instances())
2943	return 0;
2944
2945	osn_var_reset_all();
2946
2947	retval = osnoise_hook_events();
2948	if (retval)
2949	return retval;
2950
2951	/*
2952	* Make sure that ftrace_nmi_enter/exit() see reset values
2953	* before enabling trace_osnoise_callback_enabled.
2954	*/
2955	barrier();
2956	trace_osnoise_callback_enabled = true;
2957
2958	retval = start_per_cpu_kthreads();
2959	if (retval) {
2960	trace_osnoise_callback_enabled = false;
2961	/*
2962	* Make sure that ftrace_nmi_enter/exit() see
2963	* trace_osnoise_callback_enabled as false before continuing.
2964	*/
2965	barrier();
2966
2967	osnoise_unhook_events();
2968	return retval;
2969	}
2970
2971	return 0;
2972	}
2973
2974	/*
2975	* osnoise_workload_stop - stop the workload and unhook the events
2976	*/
2977	static void osnoise_workload_stop(void)
2978	{
2979	/*
2980	* Instances need to be unregistered before calling
2981	* stop. Hence, if there is a registered instance, more
2982	* than one instance is running, and the workload will not
2983	* yet stop. Otherwise, this code is on the way to disable
2984	* the last instance, and the workload can stop.
2985	*/
2986	if (osnoise_has_registered_instances())
2987	return;
2988
2989	/*
2990	* If callbacks were already disabled in a previous stop
2991	* call, there is no need to disable then again.
2992	*
2993	* For instance, this happens when tracing is stopped via:
2994	* echo 0 > tracing_on
2995	* echo nop > current_tracer.
2996	*/
2997	if (!trace_osnoise_callback_enabled)
2998	return;
2999
3000	trace_osnoise_callback_enabled = false;
3001	/*
3002	* Make sure that ftrace_nmi_enter/exit() see
3003	* trace_osnoise_callback_enabled as false before continuing.
3004	*/
3005	barrier();
3006
3007	stop_per_cpu_kthreads();
3008
3009	osnoise_unhook_events();
3010	}
3011
3012	static void osnoise_tracer_start(struct trace_array *tr)
3013	{
3014	int retval;
3015
3016	/*
3017	* If the instance is already registered, there is no need to
3018	* register it again.
3019	*/
3020	if (osnoise_instance_registered(tr))
3021	return;
3022
3023	retval = osnoise_workload_start();
3024	if (retval)
3025	pr_err(BANNER "Error starting osnoise tracer\n");
3026
3027	osnoise_register_instance(tr);
3028	}
3029
3030	static void osnoise_tracer_stop(struct trace_array *tr)
3031	{
3032	osnoise_unregister_instance(tr);
3033	osnoise_workload_stop();
3034	}
3035
3036	static int osnoise_tracer_init(struct trace_array *tr)
3037	{
3038	/*
3039	* Only allow osnoise tracer if timerlat tracer is not running
3040	* already.
3041	*/
3042	if (timerlat_enabled())
3043	return -EBUSY;
3044
3045	tr->max_latency = 0;
3046
3047	osnoise_tracer_start(tr);
3048	return 0;
3049	}
3050
3051	static void osnoise_tracer_reset(struct trace_array *tr)
3052	{
3053	osnoise_tracer_stop(tr);
3054	}
3055
3056	static struct tracer osnoise_tracer __read_mostly = {
3057	.name = "osnoise",
3058	.init = osnoise_tracer_init,
3059	.reset = osnoise_tracer_reset,
3060	.start = osnoise_tracer_start,
3061	.stop = osnoise_tracer_stop,
3062	.print_header = print_osnoise_headers,
3063	.allow_instances = true,
3064	};
3065
3066	#ifdef CONFIG_TIMERLAT_TRACER
3067	static void timerlat_tracer_start(struct trace_array *tr)
3068	{
3069	int retval;
3070
3071	/*
3072	* If the instance is already registered, there is no need to
3073	* register it again.
3074	*/
3075	if (osnoise_instance_registered(tr))
3076	return;
3077
3078	retval = osnoise_workload_start();
3079	if (retval)
3080	pr_err(BANNER "Error starting timerlat tracer\n");
3081
3082	osnoise_register_instance(tr);
3083
3084	return;
3085	}
3086
3087	static void timerlat_tracer_stop(struct trace_array *tr)
3088	{
3089	int cpu;
3090
3091	osnoise_unregister_instance(tr);
3092
3093	/*
3094	* Instruct the threads to stop only if this is the last instance.
3095	*/
3096	if (!osnoise_has_registered_instances()) {
3097	for_each_online_cpu(cpu)
3098	per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
3099	}
3100
3101	osnoise_workload_stop();
3102	}
3103
3104	static int timerlat_tracer_init(struct trace_array *tr)
3105	{
3106	/*
3107	* Only allow timerlat tracer if osnoise tracer is not running already.
3108	*/
3109	if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer)
3110	return -EBUSY;
3111
3112	/*
3113	* If this is the first instance, set timerlat_tracer to block
3114	* osnoise tracer start.
3115	*/
3116	if (!osnoise_has_registered_instances())
3117	osnoise_data.timerlat_tracer = 1;
3118
3119	tr->max_latency = 0;
3120	timerlat_tracer_start(tr);
3121
3122	return 0;
3123	}
3124
3125	static void timerlat_tracer_reset(struct trace_array *tr)
3126	{
3127	timerlat_tracer_stop(tr);
3128
3129	/*
3130	* If this is the last instance, reset timerlat_tracer allowing
3131	* osnoise to be started.
3132	*/
3133	if (!osnoise_has_registered_instances())
3134	osnoise_data.timerlat_tracer = 0;
3135	}
3136
3137	static struct tracer timerlat_tracer __read_mostly = {
3138	.name = "timerlat",
3139	.init = timerlat_tracer_init,
3140	.reset = timerlat_tracer_reset,
3141	.start = timerlat_tracer_start,
3142	.stop = timerlat_tracer_stop,
3143	.print_header = print_timerlat_headers,
3144	.allow_instances = true,
3145	};
3146
3147	__init static int init_timerlat_tracer(void)
3148	{
3149	return register_tracer(&timerlat_tracer);
3150	}
3151	#else /* CONFIG_TIMERLAT_TRACER */
3152	__init static int init_timerlat_tracer(void)
3153	{
3154	return 0;
3155	}
3156	#endif /* CONFIG_TIMERLAT_TRACER */
3157
3158	__init static int init_osnoise_tracer(void)
3159	{
3160	int ret;
3161
3162	mutex_init(&interface_lock);
3163
3164	cpumask_copy(&osnoise_cpumask, cpu_all_mask);
3165
3166	ret = register_tracer(&osnoise_tracer);
3167	if (ret) {
3168	pr_err(BANNER "Error registering osnoise!\n");
3169	return ret;
3170	}
3171
3172	ret = init_timerlat_tracer();
3173	if (ret) {
3174	pr_err(BANNER "Error registering timerlat!\n");
3175	return ret;
3176	}
3177
3178	osnoise_init_hotplug_support();
3179
3180	INIT_LIST_HEAD_RCU(&osnoise_instances);
3181
3182	init_tracefs();
3183
3184	return 0;
3185	}
3186	late_initcall(init_osnoise_tracer);