linux - kernel/events/core.c

15607 lines · 380.4 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Performance events core code:
4	*
5	* Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <[email protected]>
6	* Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7	* Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8	* Copyright © 2009 Paul Mackerras, IBM Corp. <[email protected]>
9	*/
10
11	#include <linux/fs.h>
12	#include <linux/mm.h>
13	#include <linux/cpu.h>
14	#include <linux/smp.h>
15	#include <linux/idr.h>
16	#include <linux/file.h>
17	#include <linux/poll.h>
18	#include <linux/slab.h>
19	#include <linux/hash.h>
20	#include <linux/tick.h>
21	#include <linux/sysfs.h>
22	#include <linux/dcache.h>
23	#include <linux/percpu.h>
24	#include <linux/ptrace.h>
25	#include <linux/reboot.h>
26	#include <linux/vmstat.h>
27	#include <linux/device.h>
28	#include <linux/export.h>
29	#include <linux/vmalloc.h>
30	#include <linux/hardirq.h>
31	#include <linux/hugetlb.h>
32	#include <linux/rculist.h>
33	#include <linux/uaccess.h>
34	#include <linux/syscalls.h>
35	#include <linux/anon_inodes.h>
36	#include <linux/kernel_stat.h>
37	#include <linux/cgroup.h>
38	#include <linux/perf_event.h>
39	#include <linux/trace_events.h>
40	#include <linux/hw_breakpoint.h>
41	#include <linux/mm_types.h>
42	#include <linux/module.h>
43	#include <linux/mman.h>
44	#include <linux/compat.h>
45	#include <linux/bpf.h>
46	#include <linux/filter.h>
47	#include <linux/namei.h>
48	#include <linux/parser.h>
49	#include <linux/sched/clock.h>
50	#include <linux/sched/mm.h>
51	#include <linux/proc_ns.h>
52	#include <linux/mount.h>
53	#include <linux/min_heap.h>
54	#include <linux/highmem.h>
55	#include <linux/pgtable.h>
56	#include <linux/buildid.h>
57	#include <linux/task_work.h>
58	#include <linux/percpu-rwsem.h>
59	#include <linux/unwind_deferred.h>
60	#include <linux/kvm_types.h>
61	#include <linux/seq_file.h>
62
63	#include "internal.h"
64
65	#include <asm/irq_regs.h>
66
67	typedef int (remote_function_f)(void );
68
69	struct remote_function_call {
70	struct task_struct *p;
71	remote_function_f func;
72	void *info;
73	int ret;
74	};
75
76	static void remote_function(void *data)
77	{
78	struct remote_function_call *tfc = data;
79	struct task_struct *p = tfc->p;
80
81	if (p) {
82	/* -EAGAIN */
83	if (task_cpu(p) != smp_processor_id())
84	return;
85
86	/*
87	* Now that we're on right CPU with IRQs disabled, we can test
88	* if we hit the right task without races.
89	*/
90
91	tfc->ret = -ESRCH; /* No such (running) process */
92	if (p != current)
93	return;
94	}
95
96	tfc->ret = tfc->func(tfc->info);
97	}
98
99	/**
100	* task_function_call - call a function on the cpu on which a task runs
101	* @p: the task to evaluate
102	* @func: the function to be called
103	* @info: the function call argument
104	*
105	* Calls the function @func when the task is currently running. This might
106	* be on the current CPU, which just calls the function directly. This will
107	* retry due to any failures in smp_call_function_single(), such as if the
108	* task_cpu() goes offline concurrently.
109	*
110	* returns @func return value or -ESRCH or -ENXIO when the process isn't running
111	*/
112	static int
113	task_function_call(struct task_struct p, remote_function_f func, void info)
114	{
115	struct remote_function_call data = {
116	.p = p,
117	.func = func,
118	.info = info,
119	.ret = -EAGAIN,
120	};
121	int ret;
122
123	for (;;) {
124	ret = smp_call_function_single(task_cpu(p), remote_function,
125	&data, 1);
126	if (!ret)
127	ret = data.ret;
128
129	if (ret != -EAGAIN)
130	break;
131
132	cond_resched();
133	}
134
135	return ret;
136	}
137
138	/**
139	* cpu_function_call - call a function on the cpu
140	* @cpu: target cpu to queue this function
141	* @func: the function to be called
142	* @info: the function call argument
143	*
144	* Calls the function @func on the remote cpu.
145	*
146	* returns: @func return value or -ENXIO when the cpu is offline
147	*/
148	static int cpu_function_call(int cpu, remote_function_f func, void *info)
149	{
150	struct remote_function_call data = {
151	.p = NULL,
152	.func = func,
153	.info = info,
154	.ret = -ENXIO, /* No such CPU */
155	};
156
157	smp_call_function_single(cpu, remote_function, &data, 1);
158
159	return data.ret;
160	}
161
162	enum event_type_t {
163	EVENT_FLEXIBLE = 0x01,
164	EVENT_PINNED = 0x02,
165	EVENT_TIME = 0x04,
166	EVENT_FROZEN = 0x08,
167	/* see ctx_resched() for details */
168	EVENT_CPU = 0x10,
169	EVENT_CGROUP = 0x20,
170
171	/*
172	* EVENT_GUEST is set when scheduling in/out events between the host
173	* and a guest with a mediated vPMU. Among other things, EVENT_GUEST
174	* is used:
175	*
176	* - In for_each_epc() to skip PMUs that don't support events in a
177	* MEDIATED_VPMU guest, i.e. don't need to be context switched.
178	* - To indicate the start/end point of the events in a guest. Guest
179	* running time is deducted for host-only (exclude_guest) events.
180	*/
181	EVENT_GUEST = 0x40,
182	EVENT_FLAGS = EVENT_CGROUP \| EVENT_GUEST,
183	/* compound helpers */
184	EVENT_ALL = EVENT_FLEXIBLE \| EVENT_PINNED,
185	EVENT_TIME_FROZEN = EVENT_TIME \| EVENT_FROZEN,
186	};
187
188	static inline void __perf_ctx_lock(struct perf_event_context *ctx)
189	{
190	raw_spin_lock(&ctx->lock);
191	WARN_ON_ONCE(ctx->is_active & EVENT_FROZEN);
192	}
193
194	static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
195	struct perf_event_context *ctx)
196	{
197	__perf_ctx_lock(&cpuctx->ctx);
198	if (ctx)
199	__perf_ctx_lock(ctx);
200	}
201
202	static inline void __perf_ctx_unlock(struct perf_event_context *ctx)
203	{
204	/*
205	* If ctx_sched_in() didn't again set any ALL flags, clean up
206	* after ctx_sched_out() by clearing is_active.
207	*/
208	if (ctx->is_active & EVENT_FROZEN) {
209	if (!(ctx->is_active & EVENT_ALL))
210	ctx->is_active = 0;
211	else
212	ctx->is_active &= ~EVENT_FROZEN;
213	}
214	raw_spin_unlock(&ctx->lock);
215	}
216
217	static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
218	struct perf_event_context *ctx)
219	{
220	if (ctx)
221	__perf_ctx_unlock(ctx);
222	__perf_ctx_unlock(&cpuctx->ctx);
223	}
224
225	typedef struct {
226	struct perf_cpu_context *cpuctx;
227	struct perf_event_context *ctx;
228	} class_perf_ctx_lock_t;
229
230	static inline void class_perf_ctx_lock_destructor(class_perf_ctx_lock_t *_T)
231	{ perf_ctx_unlock(_T->cpuctx, _T->ctx); }
232
233	static inline class_perf_ctx_lock_t
234	class_perf_ctx_lock_constructor(struct perf_cpu_context *cpuctx,
235	struct perf_event_context *ctx)
236	{ perf_ctx_lock(cpuctx, ctx); return (class_perf_ctx_lock_t){ cpuctx, ctx }; }
237
238	#define TASK_TOMBSTONE ((void *)-1L)
239
240	static bool is_kernel_event(struct perf_event *event)
241	{
242	return READ_ONCE(event->owner) == TASK_TOMBSTONE;
243	}
244
245	static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
246
247	struct perf_event_context *perf_cpu_task_ctx(void)
248	{
249	lockdep_assert_irqs_disabled();
250	return this_cpu_ptr(&perf_cpu_context)->task_ctx;
251	}
252
253	/*
254	* On task ctx scheduling...
255	*
256	* When !ctx->nr_events a task context will not be scheduled. This means
257	* we can disable the scheduler hooks (for performance) without leaving
258	* pending task ctx state.
259	*
260	* This however results in two special cases:
261	*
262	* - removing the last event from a task ctx; this is relatively straight
263	* forward and is done in __perf_remove_from_context.
264	*
265	* - adding the first event to a task ctx; this is tricky because we cannot
266	* rely on ctx->is_active and therefore cannot use event_function_call().
267	* See perf_install_in_context().
268	*
269	* If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
270	*/
271
272	typedef void (event_f)(struct perf_event , struct perf_cpu_context *,
273	struct perf_event_context , void );
274
275	struct event_function_struct {
276	struct perf_event *event;
277	event_f func;
278	void *data;
279	};
280
281	static int event_function(void *info)
282	{
283	struct event_function_struct *efs = info;
284	struct perf_event *event = efs->event;
285	struct perf_event_context *ctx = event->ctx;
286	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
287	struct perf_event_context *task_ctx = cpuctx->task_ctx;
288	int ret = 0;
289
290	lockdep_assert_irqs_disabled();
291
292	perf_ctx_lock(cpuctx, task_ctx);
293	/*
294	* Since we do the IPI call without holding ctx->lock things can have
295	* changed, double check we hit the task we set out to hit.
296	*/
297	if (ctx->task) {
298	if (ctx->task != current) {
299	ret = -ESRCH;
300	goto unlock;
301	}
302
303	/*
304	* We only use event_function_call() on established contexts,
305	* and event_function() is only ever called when active (or
306	* rather, we'll have bailed in task_function_call() or the
307	* above ctx->task != current test), therefore we must have
308	* ctx->is_active here.
309	*/
310	WARN_ON_ONCE(!ctx->is_active);
311	/*
312	* And since we have ctx->is_active, cpuctx->task_ctx must
313	* match.
314	*/
315	WARN_ON_ONCE(task_ctx != ctx);
316	} else {
317	WARN_ON_ONCE(&cpuctx->ctx != ctx);
318	}
319
320	efs->func(event, cpuctx, ctx, efs->data);
321	unlock:
322	perf_ctx_unlock(cpuctx, task_ctx);
323
324	return ret;
325	}
326
327	static void event_function_call(struct perf_event event, event_f func, void data)
328	{
329	struct perf_event_context *ctx = event->ctx;
330	struct task_struct task = READ_ONCE(ctx->task); / verified in event_function */
331	struct perf_cpu_context *cpuctx;
332	struct event_function_struct efs = {
333	.event = event,
334	.func = func,
335	.data = data,
336	};
337
338	if (!event->parent) {
339	/*
340	* If this is a !child event, we must hold ctx::mutex to
341	* stabilize the event->ctx relation. See
342	* perf_event_ctx_lock().
343	*/
344	lockdep_assert_held(&ctx->mutex);
345	}
346
347	if (!task) {
348	cpu_function_call(event->cpu, event_function, &efs);
349	return;
350	}
351
352	if (task == TASK_TOMBSTONE)
353	return;
354
355	again:
356	if (!task_function_call(task, event_function, &efs))
357	return;
358
359	local_irq_disable();
360	cpuctx = this_cpu_ptr(&perf_cpu_context);
361	perf_ctx_lock(cpuctx, ctx);
362	/*
363	* Reload the task pointer, it might have been changed by
364	* a concurrent perf_event_context_sched_out().
365	*/
366	task = ctx->task;
367	if (task == TASK_TOMBSTONE)
368	goto unlock;
369	if (ctx->is_active) {
370	perf_ctx_unlock(cpuctx, ctx);
371	local_irq_enable();
372	goto again;
373	}
374	func(event, NULL, ctx, data);
375	unlock:
376	perf_ctx_unlock(cpuctx, ctx);
377	local_irq_enable();
378	}
379
380	/*
381	* Similar to event_function_call() + event_function(), but hard assumes IRQs
382	* are already disabled and we're on the right CPU.
383	*/
384	static void event_function_local(struct perf_event event, event_f func, void data)
385	{
386	struct perf_event_context *ctx = event->ctx;
387	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
388	struct task_struct *task = READ_ONCE(ctx->task);
389	struct perf_event_context *task_ctx = NULL;
390
391	lockdep_assert_irqs_disabled();
392
393	if (task) {
394	if (task == TASK_TOMBSTONE)
395	return;
396
397	task_ctx = ctx;
398	}
399
400	perf_ctx_lock(cpuctx, task_ctx);
401
402	task = ctx->task;
403	if (task == TASK_TOMBSTONE)
404	goto unlock;
405
406	if (task) {
407	/*
408	* We must be either inactive or active and the right task,
409	* otherwise we're screwed, since we cannot IPI to somewhere
410	* else.
411	*/
412	if (ctx->is_active) {
413	if (WARN_ON_ONCE(task != current))
414	goto unlock;
415
416	if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))
417	goto unlock;
418	}
419	} else {
420	WARN_ON_ONCE(&cpuctx->ctx != ctx);
421	}
422
423	func(event, cpuctx, ctx, data);
424	unlock:
425	perf_ctx_unlock(cpuctx, task_ctx);
426	}
427
428	#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP \|\
429	PERF_FLAG_FD_OUTPUT \|\
430	PERF_FLAG_PID_CGROUP \|\
431	PERF_FLAG_FD_CLOEXEC)
432
433	/*
434	* branch priv levels that need permission checks
435	*/
436	#define PERF_SAMPLE_BRANCH_PERM_PLM \
437	(PERF_SAMPLE_BRANCH_KERNEL \|\
438	PERF_SAMPLE_BRANCH_HV)
439
440	/*
441	* perf_sched_events : >0 events exist
442	*/
443
444	static void perf_sched_delayed(struct work_struct *work);
445	DEFINE_STATIC_KEY_FALSE(perf_sched_events);
446	static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
447	static DEFINE_MUTEX(perf_sched_mutex);
448	static atomic_t perf_sched_count;
449
450	static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
451
452	static atomic_t nr_mmap_events __read_mostly;
453	static atomic_t nr_comm_events __read_mostly;
454	static atomic_t nr_namespaces_events __read_mostly;
455	static atomic_t nr_task_events __read_mostly;
456	static atomic_t nr_freq_events __read_mostly;
457	static atomic_t nr_switch_events __read_mostly;
458	static atomic_t nr_ksymbol_events __read_mostly;
459	static atomic_t nr_bpf_events __read_mostly;
460	static atomic_t nr_cgroup_events __read_mostly;
461	static atomic_t nr_text_poke_events __read_mostly;
462	static atomic_t nr_build_id_events __read_mostly;
463
464	static LIST_HEAD(pmus);
465	static DEFINE_MUTEX(pmus_lock);
466	static struct srcu_struct pmus_srcu;
467	static cpumask_var_t perf_online_mask;
468	static cpumask_var_t perf_online_core_mask;
469	static cpumask_var_t perf_online_die_mask;
470	static cpumask_var_t perf_online_cluster_mask;
471	static cpumask_var_t perf_online_pkg_mask;
472	static cpumask_var_t perf_online_sys_mask;
473	static struct kmem_cache *perf_event_cache;
474
475	#ifdef CONFIG_PERF_GUEST_MEDIATED_PMU
476	static DEFINE_PER_CPU(bool, guest_ctx_loaded);
477
478	static __always_inline bool is_guest_mediated_pmu_loaded(void)
479	{
480	return __this_cpu_read(guest_ctx_loaded);
481	}
482	#else
483	static __always_inline bool is_guest_mediated_pmu_loaded(void)
484	{
485	return false;
486	}
487	#endif
488
489	/*
490	* perf event paranoia level:
491	* -1 - not paranoid at all
492	* 0 - disallow raw tracepoint access for unpriv
493	* 1 - disallow cpu events for unpriv
494	* 2 - disallow kernel profiling for unpriv
495	*/
496	int sysctl_perf_event_paranoid __read_mostly = 2;
497
498	/* Minimum for 512 kiB + 1 user control page. 'free' kiB per user. */
499	static int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024);
500
501	/*
502	* max perf event sample rate
503	*/
504	#define DEFAULT_MAX_SAMPLE_RATE 100000
505	#define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
506	#define DEFAULT_CPU_TIME_MAX_PERCENT 25
507
508	int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
509	static int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
510
511	static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
512	static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS;
513
514	static int perf_sample_allowed_ns __read_mostly =
515	DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
516
517	static void update_perf_cpu_limits(void)
518	{
519	u64 tmp = perf_sample_period_ns;
520
521	tmp *= sysctl_perf_cpu_time_max_percent;
522	tmp = div_u64(tmp, 100);
523	if (!tmp)
524	tmp = 1;
525
526	WRITE_ONCE(perf_sample_allowed_ns, tmp);
527	}
528
529	static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc);
530
531	static int perf_event_max_sample_rate_handler(const struct ctl_table *table, int write,
532	void buffer, size_t lenp, loff_t *ppos)
533	{
534	int ret;
535	int perf_cpu = sysctl_perf_cpu_time_max_percent;
536	/*
537	* If throttling is disabled don't allow the write:
538	*/
539	if (write && (perf_cpu == 100 \|\| perf_cpu == 0))
540	return -EINVAL;
541
542	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
543	if (ret \|\| !write)
544	return ret;
545
546	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
547	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
548	update_perf_cpu_limits();
549
550	return 0;
551	}
552
553	static int perf_cpu_time_max_percent_handler(const struct ctl_table *table, int write,
554	void buffer, size_t lenp, loff_t *ppos)
555	{
556	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
557
558	if (ret \|\| !write)
559	return ret;
560
561	if (sysctl_perf_cpu_time_max_percent == 100 \|\|
562	sysctl_perf_cpu_time_max_percent == 0) {
563	printk(KERN_WARNING
564	"perf: Dynamic interrupt throttling disabled, can hang your system!\n");
565	WRITE_ONCE(perf_sample_allowed_ns, 0);
566	} else {
567	update_perf_cpu_limits();
568	}
569
570	return 0;
571	}
572
573	static const struct ctl_table events_core_sysctl_table[] = {
574	/*
575	* User-space relies on this file as a feature check for
576	* perf_events being enabled. It's an ABI, do not remove!
577	*/
578	{
579	.procname = "perf_event_paranoid",
580	.data = &sysctl_perf_event_paranoid,
581	.maxlen = sizeof(sysctl_perf_event_paranoid),
582	.mode = 0644,
583	.proc_handler = proc_dointvec,
584	},
585	{
586	.procname = "perf_event_mlock_kb",
587	.data = &sysctl_perf_event_mlock,
588	.maxlen = sizeof(sysctl_perf_event_mlock),
589	.mode = 0644,
590	.proc_handler = proc_dointvec,
591	},
592	{
593	.procname = "perf_event_max_sample_rate",
594	.data = &sysctl_perf_event_sample_rate,
595	.maxlen = sizeof(sysctl_perf_event_sample_rate),
596	.mode = 0644,
597	.proc_handler = perf_event_max_sample_rate_handler,
598	.extra1 = SYSCTL_ONE,
599	},
600	{
601	.procname = "perf_cpu_time_max_percent",
602	.data = &sysctl_perf_cpu_time_max_percent,
603	.maxlen = sizeof(sysctl_perf_cpu_time_max_percent),
604	.mode = 0644,
605	.proc_handler = perf_cpu_time_max_percent_handler,
606	.extra1 = SYSCTL_ZERO,
607	.extra2 = SYSCTL_ONE_HUNDRED,
608	},
609	};
610
611	static int __init init_events_core_sysctls(void)
612	{
613	register_sysctl_init("kernel", events_core_sysctl_table);
614	return 0;
615	}
616	core_initcall(init_events_core_sysctls);
617
618
619	/*
620	* perf samples are done in some very critical code paths (NMIs).
621	* If they take too much CPU time, the system can lock up and not
622	* get any real work done. This will drop the sample rate when
623	* we detect that events are taking too long.
624	*/
625	#define NR_ACCUMULATED_SAMPLES 128
626	static DEFINE_PER_CPU(u64, running_sample_length);
627
628	static u64 __report_avg;
629	static u64 __report_allowed;
630
631	static void perf_duration_warn(struct irq_work *w)
632	{
633	printk_ratelimited(KERN_INFO
634	"perf: interrupt took too long (%lld > %lld), lowering "
635	"kernel.perf_event_max_sample_rate to %d\n",
636	__report_avg, __report_allowed,
637	sysctl_perf_event_sample_rate);
638	}
639
640	static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
641
642	void perf_sample_event_took(u64 sample_len_ns)
643	{
644	u64 max_len = READ_ONCE(perf_sample_allowed_ns);
645	u64 running_len;
646	u64 avg_len;
647	u32 max;
648
649	if (max_len == 0)
650	return;
651
652	/* Decay the counter by 1 average sample. */
653	running_len = __this_cpu_read(running_sample_length);
654	running_len -= running_len/NR_ACCUMULATED_SAMPLES;
655	running_len += sample_len_ns;
656	__this_cpu_write(running_sample_length, running_len);
657
658	/*
659	* Note: this will be biased artificially low until we have
660	* seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
661	* from having to maintain a count.
662	*/
663	avg_len = running_len/NR_ACCUMULATED_SAMPLES;
664	if (avg_len <= max_len)
665	return;
666
667	__report_avg = avg_len;
668	__report_allowed = max_len;
669
670	/*
671	* Compute a throttle threshold 25% below the current duration.
672	*/
673	avg_len += avg_len / 4;
674	max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent;
675	if (avg_len < max)
676	max /= (u32)avg_len;
677	else
678	max = 1;
679
680	WRITE_ONCE(perf_sample_allowed_ns, avg_len);
681	WRITE_ONCE(max_samples_per_tick, max);
682
683	sysctl_perf_event_sample_rate = max * HZ;
684	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
685
686	if (!irq_work_queue(&perf_duration_work)) {
687	early_printk("perf: interrupt took too long (%lld > %lld), lowering "
688	"kernel.perf_event_max_sample_rate to %d\n",
689	__report_avg, __report_allowed,
690	sysctl_perf_event_sample_rate);
691	}
692	}
693
694	static atomic64_t perf_event_id;
695
696	static void update_context_time(struct perf_event_context *ctx);
697	static u64 perf_event_time(struct perf_event *event);
698
699	void __weak perf_event_print_debug(void) { }
700
701	static inline u64 perf_clock(void)
702	{
703	return local_clock();
704	}
705
706	static inline u64 perf_event_clock(struct perf_event *event)
707	{
708	return event->clock();
709	}
710
711	/*
712	* State based event timekeeping...
713	*
714	* The basic idea is to use event->state to determine which (if any) time
715	* fields to increment with the current delta. This means we only need to
716	* update timestamps when we change state or when they are explicitly requested
717	* (read).
718	*
719	* Event groups make things a little more complicated, but not terribly so. The
720	* rules for a group are that if the group leader is OFF the entire group is
721	* OFF, irrespective of what the group member states are. This results in
722	* __perf_effective_state().
723	*
724	* A further ramification is that when a group leader flips between OFF and
725	* !OFF, we need to update all group member times.
726	*
727	*
728	* NOTE: perf_event_time() is based on the (cgroup) context time, and thus we
729	* need to make sure the relevant context time is updated before we try and
730	* update our timestamps.
731	*/
732
733	static __always_inline enum perf_event_state
734	__perf_effective_state(struct perf_event *event)
735	{
736	struct perf_event *leader = event->group_leader;
737
738	if (leader->state <= PERF_EVENT_STATE_OFF)
739	return leader->state;
740
741	return event->state;
742	}
743
744	static __always_inline void
745	__perf_update_times(struct perf_event event, u64 now, u64 enabled, u64 *running)
746	{
747	enum perf_event_state state = __perf_effective_state(event);
748	u64 delta = now - event->tstamp;
749
750	*enabled = event->total_time_enabled;
751	if (state >= PERF_EVENT_STATE_INACTIVE)
752	*enabled += delta;
753
754	*running = event->total_time_running;
755	if (state >= PERF_EVENT_STATE_ACTIVE)
756	*running += delta;
757	}
758
759	static void perf_event_update_time(struct perf_event *event)
760	{
761	u64 now = perf_event_time(event);
762
763	__perf_update_times(event, now, &event->total_time_enabled,
764	&event->total_time_running);
765	event->tstamp = now;
766	}
767
768	static void perf_event_update_sibling_time(struct perf_event *leader)
769	{
770	struct perf_event *sibling;
771
772	for_each_sibling_event(sibling, leader)
773	perf_event_update_time(sibling);
774	}
775
776	static void
777	perf_event_set_state(struct perf_event *event, enum perf_event_state state)
778	{
779	if (event->state == state)
780	return;
781
782	perf_event_update_time(event);
783	/*
784	* If a group leader gets enabled/disabled all its siblings
785	* are affected too.
786	*/
787	if ((event->state < 0) ^ (state < 0))
788	perf_event_update_sibling_time(event);
789
790	WRITE_ONCE(event->state, state);
791	}
792
793	/*
794	* UP store-release, load-acquire
795	*/
796
797	#define __store_release(ptr, val) \
798	do { \
799	barrier(); \
800	WRITE_ONCE(*(ptr), (val)); \
801	} while (0)
802
803	#define __load_acquire(ptr) \
804	({ \
805	__unqual_scalar_typeof((ptr)) ___p = READ_ONCE((ptr)); \
806	barrier(); \
807	___p; \
808	})
809
810	static bool perf_skip_pmu_ctx(struct perf_event_pmu_context *pmu_ctx,
811	enum event_type_t event_type)
812	{
813	if ((event_type & EVENT_CGROUP) && !pmu_ctx->nr_cgroups)
814	return true;
815	if ((event_type & EVENT_GUEST) &&
816	!(pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU))
817	return true;
818	return false;
819	}
820
821	#define for_each_epc(_epc, _ctx, _pmu, _event_type) \
822	list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \
823	if (perf_skip_pmu_ctx(_epc, _event_type)) \
824	continue; \
825	else if (_pmu && _epc->pmu != _pmu) \
826	continue; \
827	else
828
829	static void perf_ctx_disable(struct perf_event_context *ctx,
830	enum event_type_t event_type)
831	{
832	struct perf_event_pmu_context *pmu_ctx;
833
834	for_each_epc(pmu_ctx, ctx, NULL, event_type)
835	perf_pmu_disable(pmu_ctx->pmu);
836	}
837
838	static void perf_ctx_enable(struct perf_event_context *ctx,
839	enum event_type_t event_type)
840	{
841	struct perf_event_pmu_context *pmu_ctx;
842
843	for_each_epc(pmu_ctx, ctx, NULL, event_type)
844	perf_pmu_enable(pmu_ctx->pmu);
845	}
846
847	static void ctx_sched_out(struct perf_event_context ctx, struct pmu pmu, enum event_type_t event_type);
848	static void ctx_sched_in(struct perf_event_context ctx, struct pmu pmu, enum event_type_t event_type);
849
850	static inline void update_perf_time_ctx(struct perf_time_ctx *time, u64 now, bool adv)
851	{
852	if (adv)
853	time->time += now - time->stamp;
854	time->stamp = now;
855
856	/*
857	* The above: time' = time + (now - timestamp), can be re-arranged
858	* into: time` = now + (time - timestamp), which gives a single value
859	* offset to compute future time without locks on.
860	*
861	* See perf_event_time_now(), which can be used from NMI context where
862	* it's (obviously) not possible to acquire ctx->lock in order to read
863	* both the above values in a consistent manner.
864	*/
865	WRITE_ONCE(time->offset, time->time - time->stamp);
866	}
867
868	static_assert(offsetof(struct perf_event_context, timeguest) -
869	offsetof(struct perf_event_context, time) ==
870	sizeof(struct perf_time_ctx));
871
872	#define T_TOTAL 0
873	#define T_GUEST 1
874
875	static inline u64 __perf_event_time_ctx(struct perf_event *event,
876	struct perf_time_ctx *times)
877	{
878	u64 time = times[T_TOTAL].time;
879
880	if (event->attr.exclude_guest)
881	time -= times[T_GUEST].time;
882
883	return time;
884	}
885
886	static inline u64 __perf_event_time_ctx_now(struct perf_event *event,
887	struct perf_time_ctx *times,
888	u64 now)
889	{
890	if (is_guest_mediated_pmu_loaded() && event->attr.exclude_guest) {
891	/*
892	* (now + times[total].offset) - (now + times[guest].offset) :=
893	* times[total].offset - times[guest].offset
894	*/
895	return READ_ONCE(times[T_TOTAL].offset) - READ_ONCE(times[T_GUEST].offset);
896	}
897
898	return now + READ_ONCE(times[T_TOTAL].offset);
899	}
900
901	#ifdef CONFIG_CGROUP_PERF
902
903	static inline bool
904	perf_cgroup_match(struct perf_event *event)
905	{
906	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
907
908	/* @event doesn't care about cgroup */
909	if (!event->cgrp)
910	return true;
911
912	/* wants specific cgroup scope but @cpuctx isn't associated with any */
913	if (!cpuctx->cgrp)
914	return false;
915
916	/*
917	* Cgroup scoping is recursive. An event enabled for a cgroup is
918	* also enabled for all its descendant cgroups. If @cpuctx's
919	* cgroup is a descendant of @event's (the test covers identity
920	* case), it's a match.
921	*/
922	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
923	event->cgrp->css.cgroup);
924	}
925
926	static inline void perf_detach_cgroup(struct perf_event *event)
927	{
928	css_put(&event->cgrp->css);
929	event->cgrp = NULL;
930	}
931
932	static inline int is_cgroup_event(struct perf_event *event)
933	{
934	return event->cgrp != NULL;
935	}
936
937	static_assert(offsetof(struct perf_cgroup_info, timeguest) -
938	offsetof(struct perf_cgroup_info, time) ==
939	sizeof(struct perf_time_ctx));
940
941	static inline u64 perf_cgroup_event_time(struct perf_event *event)
942	{
943	struct perf_cgroup_info *t;
944
945	t = per_cpu_ptr(event->cgrp->info, event->cpu);
946	return __perf_event_time_ctx(event, &t->time);
947	}
948
949	static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
950	{
951	struct perf_cgroup_info *t;
952
953	t = per_cpu_ptr(event->cgrp->info, event->cpu);
954	if (!__load_acquire(&t->active))
955	return __perf_event_time_ctx(event, &t->time);
956
957	return __perf_event_time_ctx_now(event, &t->time, now);
958	}
959
960	static inline void __update_cgrp_guest_time(struct perf_cgroup_info *info, u64 now, bool adv)
961	{
962	update_perf_time_ctx(&info->timeguest, now, adv);
963	}
964
965	static inline void update_cgrp_time(struct perf_cgroup_info *info, u64 now)
966	{
967	update_perf_time_ctx(&info->time, now, true);
968	if (is_guest_mediated_pmu_loaded())
969	__update_cgrp_guest_time(info, now, true);
970	}
971
972	static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
973	{
974	struct perf_cgroup *cgrp = cpuctx->cgrp;
975	struct cgroup_subsys_state *css;
976	struct perf_cgroup_info *info;
977
978	if (cgrp) {
979	u64 now = perf_clock();
980
981	for (css = &cgrp->css; css; css = css->parent) {
982	cgrp = container_of(css, struct perf_cgroup, css);
983	info = this_cpu_ptr(cgrp->info);
984
985	update_cgrp_time(info, now);
986	if (final)
987	__store_release(&info->active, 0);
988	}
989	}
990	}
991
992	static inline void update_cgrp_time_from_event(struct perf_event *event)
993	{
994	struct perf_cgroup_info *info;
995
996	/*
997	* ensure we access cgroup data only when needed and
998	* when we know the cgroup is pinned (css_get)
999	*/
1000	if (!is_cgroup_event(event))
1001	return;
1002
1003	info = this_cpu_ptr(event->cgrp->info);
1004	/*
1005	* Do not update time when cgroup is not active
1006	*/
1007	if (info->active)
1008	update_cgrp_time(info, perf_clock());
1009	}
1010
1011	static inline void
1012	perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest)
1013	{
1014	struct perf_event_context *ctx = &cpuctx->ctx;
1015	struct perf_cgroup *cgrp = cpuctx->cgrp;
1016	struct perf_cgroup_info *info;
1017	struct cgroup_subsys_state *css;
1018
1019	/*
1020	* ctx->lock held by caller
1021	* ensure we do not access cgroup data
1022	* unless we have the cgroup pinned (css_get)
1023	*/
1024	if (!cgrp)
1025	return;
1026
1027	WARN_ON_ONCE(!ctx->nr_cgroups);
1028
1029	for (css = &cgrp->css; css; css = css->parent) {
1030	cgrp = container_of(css, struct perf_cgroup, css);
1031	info = this_cpu_ptr(cgrp->info);
1032	if (guest) {
1033	__update_cgrp_guest_time(info, ctx->time.stamp, false);
1034	} else {
1035	update_perf_time_ctx(&info->time, ctx->time.stamp, false);
1036	__store_release(&info->active, 1);
1037	}
1038	}
1039	}
1040
1041	/*
1042	* reschedule events based on the cgroup constraint of task.
1043	*/
1044	static void perf_cgroup_switch(struct task_struct *task)
1045	{
1046	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
1047	struct perf_cgroup *cgrp;
1048
1049	/*
1050	* cpuctx->cgrp is set when the first cgroup event enabled,
1051	* and is cleared when the last cgroup event disabled.
1052	*/
1053	if (READ_ONCE(cpuctx->cgrp) == NULL)
1054	return;
1055
1056	cgrp = perf_cgroup_from_task(task, NULL);
1057	if (READ_ONCE(cpuctx->cgrp) == cgrp)
1058	return;
1059
1060	guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx);
1061	/*
1062	* Re-check, could've raced vs perf_remove_from_context().
1063	*/
1064	if (READ_ONCE(cpuctx->cgrp) == NULL)
1065	return;
1066
1067	WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
1068	perf_ctx_disable(&cpuctx->ctx, EVENT_CGROUP);
1069
1070	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL\|EVENT_CGROUP);
1071	/*
1072	* must not be done before ctxswout due
1073	* to update_cgrp_time_from_cpuctx() in
1074	* ctx_sched_out()
1075	*/
1076	cpuctx->cgrp = cgrp;
1077	/*
1078	* set cgrp before ctxsw in to allow
1079	* perf_cgroup_set_timestamp() in ctx_sched_in()
1080	* to not have to pass task around
1081	*/
1082	ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL\|EVENT_CGROUP);
1083
1084	perf_ctx_enable(&cpuctx->ctx, EVENT_CGROUP);
1085	}
1086
1087	static int perf_cgroup_ensure_storage(struct perf_event *event,
1088	struct cgroup_subsys_state *css)
1089	{
1090	struct perf_cpu_context *cpuctx;
1091	struct perf_event **storage;
1092	int cpu, heap_size, ret = 0;
1093
1094	/*
1095	* Allow storage to have sufficient space for an iterator for each
1096	* possibly nested cgroup plus an iterator for events with no cgroup.
1097	*/
1098	for (heap_size = 1; css; css = css->parent)
1099	heap_size++;
1100
1101	for_each_possible_cpu(cpu) {
1102	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
1103	if (heap_size <= cpuctx->heap_size)
1104	continue;
1105
1106	storage = kmalloc_node(heap_size * sizeof(struct perf_event *),
1107	GFP_KERNEL, cpu_to_node(cpu));
1108	if (!storage) {
1109	ret = -ENOMEM;
1110	break;
1111	}
1112
1113	raw_spin_lock_irq(&cpuctx->ctx.lock);
1114	if (cpuctx->heap_size < heap_size) {
1115	swap(cpuctx->heap, storage);
1116	if (storage == cpuctx->heap_default)
1117	storage = NULL;
1118	cpuctx->heap_size = heap_size;
1119	}
1120	raw_spin_unlock_irq(&cpuctx->ctx.lock);
1121
1122	kfree(storage);
1123	}
1124
1125	return ret;
1126	}
1127
1128	static inline int perf_cgroup_connect(int fd, struct perf_event *event,
1129	struct perf_event_attr *attr,
1130	struct perf_event *group_leader)
1131	{
1132	struct perf_cgroup *cgrp;
1133	struct cgroup_subsys_state *css;
1134	CLASS(fd, f)(fd);
1135	int ret = 0;
1136
1137	if (fd_empty(f))
1138	return -EBADF;
1139
1140	css = css_tryget_online_from_dir(fd_file(f)->f_path.dentry,
1141	&perf_event_cgrp_subsys);
1142	if (IS_ERR(css))
1143	return PTR_ERR(css);
1144
1145	ret = perf_cgroup_ensure_storage(event, css);
1146	if (ret)
1147	return ret;
1148
1149	cgrp = container_of(css, struct perf_cgroup, css);
1150	event->cgrp = cgrp;
1151
1152	/*
1153	* all events in a group must monitor
1154	* the same cgroup because a task belongs
1155	* to only one perf cgroup at a time
1156	*/
1157	if (group_leader && group_leader->cgrp != cgrp) {
1158	perf_detach_cgroup(event);
1159	ret = -EINVAL;
1160	}
1161	return ret;
1162	}
1163
1164	static inline void
1165	perf_cgroup_event_enable(struct perf_event event, struct perf_event_context ctx)
1166	{
1167	struct perf_cpu_context *cpuctx;
1168
1169	if (!is_cgroup_event(event))
1170	return;
1171
1172	event->pmu_ctx->nr_cgroups++;
1173
1174	/*
1175	* Because cgroup events are always per-cpu events,
1176	* @ctx == &cpuctx->ctx.
1177	*/
1178	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1179
1180	if (ctx->nr_cgroups++)
1181	return;
1182
1183	cpuctx->cgrp = perf_cgroup_from_task(current, ctx);
1184	}
1185
1186	static inline void
1187	perf_cgroup_event_disable(struct perf_event event, struct perf_event_context ctx)
1188	{
1189	struct perf_cpu_context *cpuctx;
1190
1191	if (!is_cgroup_event(event))
1192	return;
1193
1194	event->pmu_ctx->nr_cgroups--;
1195
1196	/*
1197	* Because cgroup events are always per-cpu events,
1198	* @ctx == &cpuctx->ctx.
1199	*/
1200	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1201
1202	if (--ctx->nr_cgroups)
1203	return;
1204
1205	cpuctx->cgrp = NULL;
1206	}
1207
1208	#else /* !CONFIG_CGROUP_PERF */
1209
1210	static inline bool
1211	perf_cgroup_match(struct perf_event *event)
1212	{
1213	return true;
1214	}
1215
1216	static inline void perf_detach_cgroup(struct perf_event *event)
1217	{}
1218
1219	static inline int is_cgroup_event(struct perf_event *event)
1220	{
1221	return 0;
1222	}
1223
1224	static inline void update_cgrp_time_from_event(struct perf_event *event)
1225	{
1226	}
1227
1228	static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx,
1229	bool final)
1230	{
1231	}
1232
1233	static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
1234	struct perf_event_attr *attr,
1235	struct perf_event *group_leader)
1236	{
1237	return -EINVAL;
1238	}
1239
1240	static inline void
1241	perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest)
1242	{
1243	}
1244
1245	static inline u64 perf_cgroup_event_time(struct perf_event *event)
1246	{
1247	return 0;
1248	}
1249
1250	static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
1251	{
1252	return 0;
1253	}
1254
1255	static inline void
1256	perf_cgroup_event_enable(struct perf_event event, struct perf_event_context ctx)
1257	{
1258	}
1259
1260	static inline void
1261	perf_cgroup_event_disable(struct perf_event event, struct perf_event_context ctx)
1262	{
1263	}
1264
1265	static void perf_cgroup_switch(struct task_struct *task)
1266	{
1267	}
1268	#endif
1269
1270	/*
1271	* set default to be dependent on timer tick just
1272	* like original code
1273	*/
1274	#define PERF_CPU_HRTIMER (1000 / HZ)
1275	/*
1276	* function must be called with interrupts disabled
1277	*/
1278	static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
1279	{
1280	struct perf_cpu_pmu_context *cpc;
1281	bool rotations;
1282
1283	lockdep_assert_irqs_disabled();
1284
1285	cpc = container_of(hr, struct perf_cpu_pmu_context, hrtimer);
1286	rotations = perf_rotate_context(cpc);
1287
1288	raw_spin_lock(&cpc->hrtimer_lock);
1289	if (rotations)
1290	hrtimer_forward_now(hr, cpc->hrtimer_interval);
1291	else
1292	cpc->hrtimer_active = 0;
1293	raw_spin_unlock(&cpc->hrtimer_lock);
1294
1295	return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
1296	}
1297
1298	static void __perf_mux_hrtimer_init(struct perf_cpu_pmu_context *cpc, int cpu)
1299	{
1300	struct hrtimer *timer = &cpc->hrtimer;
1301	struct pmu *pmu = cpc->epc.pmu;
1302	u64 interval;
1303
1304	/*
1305	* check default is sane, if not set then force to
1306	* default interval (1/tick)
1307	*/
1308	interval = pmu->hrtimer_interval_ms;
1309	if (interval < 1)
1310	interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
1311
1312	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
1313
1314	raw_spin_lock_init(&cpc->hrtimer_lock);
1315	hrtimer_setup(timer, perf_mux_hrtimer_handler, CLOCK_MONOTONIC,
1316	HRTIMER_MODE_ABS_PINNED_HARD);
1317	}
1318
1319	static int perf_mux_hrtimer_restart(struct perf_cpu_pmu_context *cpc)
1320	{
1321	struct hrtimer *timer = &cpc->hrtimer;
1322	unsigned long flags;
1323
1324	raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags);
1325	if (!cpc->hrtimer_active) {
1326	cpc->hrtimer_active = 1;
1327	hrtimer_forward_now(timer, cpc->hrtimer_interval);
1328	hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
1329	}
1330	raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags);
1331
1332	return 0;
1333	}
1334
1335	static int perf_mux_hrtimer_restart_ipi(void *arg)
1336	{
1337	return perf_mux_hrtimer_restart(arg);
1338	}
1339
1340	static __always_inline struct perf_cpu_pmu_context this_cpc(struct pmu pmu)
1341	{
1342	return *this_cpu_ptr(pmu->cpu_pmu_context);
1343	}
1344
1345	void perf_pmu_disable(struct pmu *pmu)
1346	{
1347	int *count = &this_cpc(pmu)->pmu_disable_count;
1348	if (!(*count)++)
1349	pmu->pmu_disable(pmu);
1350	}
1351
1352	void perf_pmu_enable(struct pmu *pmu)
1353	{
1354	int *count = &this_cpc(pmu)->pmu_disable_count;
1355	if (!--(*count))
1356	pmu->pmu_enable(pmu);
1357	}
1358
1359	static void perf_assert_pmu_disabled(struct pmu *pmu)
1360	{
1361	int *count = &this_cpc(pmu)->pmu_disable_count;
1362	WARN_ON_ONCE(*count == 0);
1363	}
1364
1365	static inline void perf_pmu_read(struct perf_event *event)
1366	{
1367	if (event->state == PERF_EVENT_STATE_ACTIVE)
1368	event->pmu->read(event);
1369	}
1370
1371	static void get_ctx(struct perf_event_context *ctx)
1372	{
1373	refcount_inc(&ctx->refcount);
1374	}
1375
1376	static void free_ctx(struct rcu_head *head)
1377	{
1378	struct perf_event_context *ctx;
1379
1380	ctx = container_of(head, struct perf_event_context, rcu_head);
1381	kfree(ctx);
1382	}
1383
1384	static void put_ctx(struct perf_event_context *ctx)
1385	{
1386	if (refcount_dec_and_test(&ctx->refcount)) {
1387	if (ctx->parent_ctx)
1388	put_ctx(ctx->parent_ctx);
1389	if (ctx->task && ctx->task != TASK_TOMBSTONE)
1390	put_task_struct(ctx->task);
1391	call_rcu(&ctx->rcu_head, free_ctx);
1392	} else {
1393	smp_mb__after_atomic(); /* pairs with wait_var_event() */
1394	if (ctx->task == TASK_TOMBSTONE)
1395	wake_up_var(&ctx->refcount);
1396	}
1397	}
1398
1399	/*
1400	* Because of perf_event::ctx migration in sys_perf_event_open::move_group and
1401	* perf_pmu_migrate_context() we need some magic.
1402	*
1403	* Those places that change perf_event::ctx will hold both
1404	* perf_event_ctx::mutex of the 'old' and 'new' ctx value.
1405	*
1406	* Lock ordering is by mutex address. There are two other sites where
1407	* perf_event_context::mutex nests and those are:
1408	*
1409	* - perf_event_exit_task_context() [ child , 0 ]
1410	* perf_event_exit_event()
1411	* put_event() [ parent, 1 ]
1412	*
1413	* - perf_event_init_context() [ parent, 0 ]
1414	* inherit_task_group()
1415	* inherit_group()
1416	* inherit_event()
1417	* perf_event_alloc()
1418	* perf_init_event()
1419	* perf_try_init_event() [ child , 1 ]
1420	*
1421	* While it appears there is an obvious deadlock here -- the parent and child
1422	* nesting levels are inverted between the two. This is in fact safe because
1423	* life-time rules separate them. That is an exiting task cannot fork, and a
1424	* spawning task cannot (yet) exit.
1425	*
1426	* But remember that these are parent<->child context relations, and
1427	* migration does not affect children, therefore these two orderings should not
1428	* interact.
1429	*
1430	* The change in perf_event::ctx does not affect children (as claimed above)
1431	* because the sys_perf_event_open() case will install a new event and break
1432	* the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1433	* concerned with cpuctx and that doesn't have children.
1434	*
1435	* The places that change perf_event::ctx will issue:
1436	*
1437	* perf_remove_from_context();
1438	* synchronize_rcu();
1439	* perf_install_in_context();
1440	*
1441	* to affect the change. The remove_from_context() + synchronize_rcu() should
1442	* quiesce the event, after which we can install it in the new location. This
1443	* means that only external vectors (perf_fops, prctl) can perturb the event
1444	* while in transit. Therefore all such accessors should also acquire
1445	* perf_event_context::mutex to serialize against this.
1446	*
1447	* However; because event->ctx can change while we're waiting to acquire
1448	* ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1449	* function.
1450	*
1451	* Lock order:
1452	* exec_update_lock
1453	* task_struct::perf_event_mutex
1454	* perf_event_context::mutex
1455	* perf_event::child_mutex;
1456	* perf_event_context::lock
1457	* mmap_lock
1458	* perf_event::mmap_mutex
1459	* perf_buffer::aux_mutex
1460	* perf_addr_filters_head::lock
1461	*
1462	* cpu_hotplug_lock
1463	* pmus_lock
1464	* cpuctx->mutex / perf_event_context::mutex
1465	*/
1466	static struct perf_event_context *
1467	perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
1468	{
1469	struct perf_event_context *ctx;
1470
1471	again:
1472	rcu_read_lock();
1473	ctx = READ_ONCE(event->ctx);
1474	if (!refcount_inc_not_zero(&ctx->refcount)) {
1475	rcu_read_unlock();
1476	goto again;
1477	}
1478	rcu_read_unlock();
1479
1480	mutex_lock_nested(&ctx->mutex, nesting);
1481	if (event->ctx != ctx) {
1482	mutex_unlock(&ctx->mutex);
1483	put_ctx(ctx);
1484	goto again;
1485	}
1486
1487	return ctx;
1488	}
1489
1490	static inline struct perf_event_context *
1491	perf_event_ctx_lock(struct perf_event *event)
1492	{
1493	return perf_event_ctx_lock_nested(event, 0);
1494	}
1495
1496	static void perf_event_ctx_unlock(struct perf_event *event,
1497	struct perf_event_context *ctx)
1498	{
1499	mutex_unlock(&ctx->mutex);
1500	put_ctx(ctx);
1501	}
1502
1503	/*
1504	* This must be done under the ctx->lock, such as to serialize against
1505	* context_equiv(), therefore we cannot call put_ctx() since that might end up
1506	* calling scheduler related locks and ctx->lock nests inside those.
1507	*/
1508	static __must_check struct perf_event_context *
1509	unclone_ctx(struct perf_event_context *ctx)
1510	{
1511	struct perf_event_context *parent_ctx = ctx->parent_ctx;
1512
1513	lockdep_assert_held(&ctx->lock);
1514
1515	if (parent_ctx)
1516	ctx->parent_ctx = NULL;
1517	ctx->generation++;
1518
1519	return parent_ctx;
1520	}
1521
1522	static u32 perf_event_pid_type(struct perf_event event, struct task_struct p,
1523	enum pid_type type)
1524	{
1525	u32 nr;
1526	/*
1527	* only top level events have the pid namespace they were created in
1528	*/
1529	if (event->parent)
1530	event = event->parent;
1531
1532	nr = __task_pid_nr_ns(p, type, event->ns);
1533	/* avoid -1 if it is idle thread or runs in another ns */
1534	if (!nr && !pid_alive(p))
1535	nr = -1;
1536	return nr;
1537	}
1538
1539	static u32 perf_event_pid(struct perf_event event, struct task_struct p)
1540	{
1541	return perf_event_pid_type(event, p, PIDTYPE_TGID);
1542	}
1543
1544	static u32 perf_event_tid(struct perf_event event, struct task_struct p)
1545	{
1546	return perf_event_pid_type(event, p, PIDTYPE_PID);
1547	}
1548
1549	/*
1550	* If we inherit events we want to return the parent event id
1551	* to userspace.
1552	*/
1553	static u64 primary_event_id(struct perf_event *event)
1554	{
1555	u64 id = event->id;
1556
1557	if (event->parent)
1558	id = event->parent->id;
1559
1560	return id;
1561	}
1562
1563	/*
1564	* Get the perf_event_context for a task and lock it.
1565	*
1566	* This has to cope with the fact that until it is locked,
1567	* the context could get moved to another task.
1568	*/
1569	static struct perf_event_context *
1570	perf_lock_task_context(struct task_struct task, unsigned long flags)
1571	{
1572	struct perf_event_context *ctx;
1573
1574	retry:
1575	/*
1576	* One of the few rules of preemptible RCU is that one cannot do
1577	* rcu_read_unlock() while holding a scheduler (or nested) lock when
1578	* part of the read side critical section was irqs-enabled -- see
1579	* rcu_read_unlock_special().
1580	*
1581	* Since ctx->lock nests under rq->lock we must ensure the entire read
1582	* side critical section has interrupts disabled.
1583	*/
1584	local_irq_save(*flags);
1585	rcu_read_lock();
1586	ctx = rcu_dereference(task->perf_event_ctxp);
1587	if (ctx) {
1588	/*
1589	* If this context is a clone of another, it might
1590	* get swapped for another underneath us by
1591	* perf_event_task_sched_out, though the
1592	* rcu_read_lock() protects us from any context
1593	* getting freed. Lock the context and check if it
1594	* got swapped before we could get the lock, and retry
1595	* if so. If we locked the right context, then it
1596	* can't get swapped on us any more.
1597	*/
1598	raw_spin_lock(&ctx->lock);
1599	if (ctx != rcu_dereference(task->perf_event_ctxp)) {
1600	raw_spin_unlock(&ctx->lock);
1601	rcu_read_unlock();
1602	local_irq_restore(*flags);
1603	goto retry;
1604	}
1605
1606	if (ctx->task == TASK_TOMBSTONE \|\|
1607	!refcount_inc_not_zero(&ctx->refcount)) {
1608	raw_spin_unlock(&ctx->lock);
1609	ctx = NULL;
1610	} else {
1611	WARN_ON_ONCE(ctx->task != task);
1612	}
1613	}
1614	rcu_read_unlock();
1615	if (!ctx)
1616	local_irq_restore(*flags);
1617	return ctx;
1618	}
1619
1620	/*
1621	* Get the context for a task and increment its pin_count so it
1622	* can't get swapped to another task. This also increments its
1623	* reference count so that the context can't get freed.
1624	*/
1625	static struct perf_event_context *
1626	perf_pin_task_context(struct task_struct *task)
1627	{
1628	struct perf_event_context *ctx;
1629	unsigned long flags;
1630
1631	ctx = perf_lock_task_context(task, &flags);
1632	if (ctx) {
1633	++ctx->pin_count;
1634	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1635	}
1636	return ctx;
1637	}
1638
1639	static void perf_unpin_context(struct perf_event_context *ctx)
1640	{
1641	unsigned long flags;
1642
1643	raw_spin_lock_irqsave(&ctx->lock, flags);
1644	--ctx->pin_count;
1645	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1646	}
1647
1648	/*
1649	* Update the record of the current time in a context.
1650	*/
1651	static void __update_context_time(struct perf_event_context *ctx, bool adv)
1652	{
1653	lockdep_assert_held(&ctx->lock);
1654
1655	update_perf_time_ctx(&ctx->time, perf_clock(), adv);
1656	}
1657
1658	static void __update_context_guest_time(struct perf_event_context *ctx, bool adv)
1659	{
1660	lockdep_assert_held(&ctx->lock);
1661
1662	/* must be called after __update_context_time(); */
1663	update_perf_time_ctx(&ctx->timeguest, ctx->time.stamp, adv);
1664	}
1665
1666	static void update_context_time(struct perf_event_context *ctx)
1667	{
1668	__update_context_time(ctx, true);
1669	if (is_guest_mediated_pmu_loaded())
1670	__update_context_guest_time(ctx, true);
1671	}
1672
1673	static u64 perf_event_time(struct perf_event *event)
1674	{
1675	struct perf_event_context *ctx = event->ctx;
1676
1677	if (unlikely(!ctx))
1678	return 0;
1679
1680	if (is_cgroup_event(event))
1681	return perf_cgroup_event_time(event);
1682
1683	return __perf_event_time_ctx(event, &ctx->time);
1684	}
1685
1686	static u64 perf_event_time_now(struct perf_event *event, u64 now)
1687	{
1688	struct perf_event_context *ctx = event->ctx;
1689
1690	if (unlikely(!ctx))
1691	return 0;
1692
1693	if (is_cgroup_event(event))
1694	return perf_cgroup_event_time_now(event, now);
1695
1696	if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
1697	return __perf_event_time_ctx(event, &ctx->time);
1698
1699	return __perf_event_time_ctx_now(event, &ctx->time, now);
1700	}
1701
1702	static enum event_type_t get_event_type(struct perf_event *event)
1703	{
1704	struct perf_event_context *ctx = event->ctx;
1705	enum event_type_t event_type;
1706
1707	lockdep_assert_held(&ctx->lock);
1708
1709	/*
1710	* It's 'group type', really, because if our group leader is
1711	* pinned, so are we.
1712	*/
1713	if (event->group_leader != event)
1714	event = event->group_leader;
1715
1716	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;
1717	if (!ctx->task)
1718	event_type \|= EVENT_CPU;
1719
1720	return event_type;
1721	}
1722
1723	/*
1724	* Helper function to initialize event group nodes.
1725	*/
1726	static void init_event_group(struct perf_event *event)
1727	{
1728	RB_CLEAR_NODE(&event->group_node);
1729	event->group_index = 0;
1730	}
1731
1732	/*
1733	* Extract pinned or flexible groups from the context
1734	* based on event attrs bits.
1735	*/
1736	static struct perf_event_groups *
1737	get_event_groups(struct perf_event event, struct perf_event_context ctx)
1738	{
1739	if (event->attr.pinned)
1740	return &ctx->pinned_groups;
1741	else
1742	return &ctx->flexible_groups;
1743	}
1744
1745	/*
1746	* Helper function to initializes perf_event_group trees.
1747	*/
1748	static void perf_event_groups_init(struct perf_event_groups *groups)
1749	{
1750	groups->tree = RB_ROOT;
1751	groups->index = 0;
1752	}
1753
1754	static inline struct cgroup event_cgroup(const struct perf_event event)
1755	{
1756	struct cgroup *cgroup = NULL;
1757
1758	#ifdef CONFIG_CGROUP_PERF
1759	if (event->cgrp)
1760	cgroup = event->cgrp->css.cgroup;
1761	#endif
1762
1763	return cgroup;
1764	}
1765
1766	/*
1767	* Compare function for event groups;
1768	*
1769	* Implements complex key that first sorts by CPU and then by virtual index
1770	* which provides ordering when rotating groups for the same CPU.
1771	*/
1772	static __always_inline int
1773	perf_event_groups_cmp(const int left_cpu, const struct pmu *left_pmu,
1774	const struct cgroup *left_cgroup, const u64 left_group_index,
1775	const struct perf_event *right)
1776	{
1777	if (left_cpu < right->cpu)
1778	return -1;
1779	if (left_cpu > right->cpu)
1780	return 1;
1781
1782	if (left_pmu) {
1783	if (left_pmu < right->pmu_ctx->pmu)
1784	return -1;
1785	if (left_pmu > right->pmu_ctx->pmu)
1786	return 1;
1787	}
1788
1789	#ifdef CONFIG_CGROUP_PERF
1790	{
1791	const struct cgroup *right_cgroup = event_cgroup(right);
1792
1793	if (left_cgroup != right_cgroup) {
1794	if (!left_cgroup) {
1795	/*
1796	* Left has no cgroup but right does, no
1797	* cgroups come first.
1798	*/
1799	return -1;
1800	}
1801	if (!right_cgroup) {
1802	/*
1803	* Right has no cgroup but left does, no
1804	* cgroups come first.
1805	*/
1806	return 1;
1807	}
1808	/* Two dissimilar cgroups, order by id. */
1809	if (cgroup_id(left_cgroup) < cgroup_id(right_cgroup))
1810	return -1;
1811
1812	return 1;
1813	}
1814	}
1815	#endif
1816
1817	if (left_group_index < right->group_index)
1818	return -1;
1819	if (left_group_index > right->group_index)
1820	return 1;
1821
1822	return 0;
1823	}
1824
1825	#define __node_2_pe(node) \
1826	rb_entry((node), struct perf_event, group_node)
1827
1828	static inline bool __group_less(struct rb_node a, const struct rb_node b)
1829	{
1830	struct perf_event *e = __node_2_pe(a);
1831	return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e),
1832	e->group_index, __node_2_pe(b)) < 0;
1833	}
1834
1835	struct __group_key {
1836	int cpu;
1837	struct pmu *pmu;
1838	struct cgroup *cgroup;
1839	};
1840
1841	static inline int __group_cmp(const void key, const struct rb_node node)
1842	{
1843	const struct __group_key *a = key;
1844	const struct perf_event *b = __node_2_pe(node);
1845
1846	/* partial/subtree match: @cpu, @pmu, @cgroup; ignore: @group_index */
1847	return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b);
1848	}
1849
1850	static inline int
1851	__group_cmp_ignore_cgroup(const void key, const struct rb_node node)
1852	{
1853	const struct __group_key *a = key;
1854	const struct perf_event *b = __node_2_pe(node);
1855
1856	/* partial/subtree match: @cpu, @pmu, ignore: @cgroup, @group_index */
1857	return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b),
1858	b->group_index, b);
1859	}
1860
1861	/*
1862	* Insert @event into @groups' tree; using
1863	* {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
1864	* as key. This places it last inside the {cpu,pmu,cgroup} subtree.
1865	*/
1866	static void
1867	perf_event_groups_insert(struct perf_event_groups *groups,
1868	struct perf_event *event)
1869	{
1870	event->group_index = ++groups->index;
1871
1872	rb_add(&event->group_node, &groups->tree, __group_less);
1873	}
1874
1875	/*
1876	* Helper function to insert event into the pinned or flexible groups.
1877	*/
1878	static void
1879	add_event_to_groups(struct perf_event event, struct perf_event_context ctx)
1880	{
1881	struct perf_event_groups *groups;
1882
1883	groups = get_event_groups(event, ctx);
1884	perf_event_groups_insert(groups, event);
1885	}
1886
1887	/*
1888	* Delete a group from a tree.
1889	*/
1890	static void
1891	perf_event_groups_delete(struct perf_event_groups *groups,
1892	struct perf_event *event)
1893	{
1894	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) \|\|
1895	RB_EMPTY_ROOT(&groups->tree));
1896
1897	rb_erase(&event->group_node, &groups->tree);
1898	init_event_group(event);
1899	}
1900
1901	/*
1902	* Helper function to delete event from its groups.
1903	*/
1904	static void
1905	del_event_from_groups(struct perf_event event, struct perf_event_context ctx)
1906	{
1907	struct perf_event_groups *groups;
1908
1909	groups = get_event_groups(event, ctx);
1910	perf_event_groups_delete(groups, event);
1911	}
1912
1913	/*
1914	* Get the leftmost event in the {cpu,pmu,cgroup} subtree.
1915	*/
1916	static struct perf_event *
1917	perf_event_groups_first(struct perf_event_groups *groups, int cpu,
1918	struct pmu pmu, struct cgroup cgrp)
1919	{
1920	struct __group_key key = {
1921	.cpu = cpu,
1922	.pmu = pmu,
1923	.cgroup = cgrp,
1924	};
1925	struct rb_node *node;
1926
1927	node = rb_find_first(&key, &groups->tree, __group_cmp);
1928	if (node)
1929	return __node_2_pe(node);
1930
1931	return NULL;
1932	}
1933
1934	static struct perf_event *
1935	perf_event_groups_next(struct perf_event event, struct pmu pmu)
1936	{
1937	struct __group_key key = {
1938	.cpu = event->cpu,
1939	.pmu = pmu,
1940	.cgroup = event_cgroup(event),
1941	};
1942	struct rb_node *next;
1943
1944	next = rb_next_match(&key, &event->group_node, __group_cmp);
1945	if (next)
1946	return __node_2_pe(next);
1947
1948	return NULL;
1949	}
1950
1951	#define perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu) \
1952	for (event = perf_event_groups_first(groups, cpu, pmu, NULL); \
1953	event; event = perf_event_groups_next(event, pmu))
1954
1955	/*
1956	* Iterate through the whole groups tree.
1957	*/
1958	#define perf_event_groups_for_each(event, groups) \
1959	for (event = rb_entry_safe(rb_first(&((groups)->tree)), \
1960	typeof(*event), group_node); event; \
1961	event = rb_entry_safe(rb_next(&event->group_node), \
1962	typeof(*event), group_node))
1963
1964	/*
1965	* Does the event attribute request inherit with PERF_SAMPLE_READ
1966	*/
1967	static inline bool has_inherit_and_sample_read(struct perf_event_attr *attr)
1968	{
1969	return attr->inherit && (attr->sample_type & PERF_SAMPLE_READ);
1970	}
1971
1972	/*
1973	* Add an event from the lists for its context.
1974	* Must be called with ctx->mutex and ctx->lock held.
1975	*/
1976	static void
1977	list_add_event(struct perf_event event, struct perf_event_context ctx)
1978	{
1979	lockdep_assert_held(&ctx->lock);
1980
1981	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1982	event->attach_state \|= PERF_ATTACH_CONTEXT;
1983
1984	event->tstamp = perf_event_time(event);
1985
1986	/*
1987	* If we're a stand alone event or group leader, we go to the context
1988	* list, group events are kept attached to the group so that
1989	* perf_group_detach can, at all times, locate all siblings.
1990	*/
1991	if (event->group_leader == event) {
1992	event->group_caps = event->event_caps;
1993	add_event_to_groups(event, ctx);
1994	}
1995
1996	list_add_rcu(&event->event_entry, &ctx->event_list);
1997	ctx->nr_events++;
1998	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
1999	ctx->nr_user++;
2000	if (event->attr.inherit_stat)
2001	ctx->nr_stat++;
2002	if (has_inherit_and_sample_read(&event->attr))
2003	local_inc(&ctx->nr_no_switch_fast);
2004
2005	if (event->state > PERF_EVENT_STATE_OFF)
2006	perf_cgroup_event_enable(event, ctx);
2007
2008	ctx->generation++;
2009	event->pmu_ctx->nr_events++;
2010	}
2011
2012	/*
2013	* Initialize event state based on the perf_event_attr::disabled.
2014	*/
2015	static inline void perf_event__state_init(struct perf_event *event)
2016	{
2017	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
2018	PERF_EVENT_STATE_INACTIVE;
2019	}
2020
2021	static int __perf_event_read_size(u64 read_format, int nr_siblings)
2022	{
2023	int entry = sizeof(u64); /* value */
2024	int size = 0;
2025	int nr = 1;
2026
2027	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2028	size += sizeof(u64);
2029
2030	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2031	size += sizeof(u64);
2032
2033	if (read_format & PERF_FORMAT_ID)
2034	entry += sizeof(u64);
2035
2036	if (read_format & PERF_FORMAT_LOST)
2037	entry += sizeof(u64);
2038
2039	if (read_format & PERF_FORMAT_GROUP) {
2040	nr += nr_siblings;
2041	size += sizeof(u64);
2042	}
2043
2044	/*
2045	* Since perf_event_validate_size() limits this to 16k and inhibits
2046	* adding more siblings, this will never overflow.
2047	*/
2048	return size + nr * entry;
2049	}
2050
2051	static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
2052	{
2053	struct perf_sample_data *data;
2054	u16 size = 0;
2055
2056	if (sample_type & PERF_SAMPLE_IP)
2057	size += sizeof(data->ip);
2058
2059	if (sample_type & PERF_SAMPLE_ADDR)
2060	size += sizeof(data->addr);
2061
2062	if (sample_type & PERF_SAMPLE_PERIOD)
2063	size += sizeof(data->period);
2064
2065	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
2066	size += sizeof(data->weight.full);
2067
2068	if (sample_type & PERF_SAMPLE_READ)
2069	size += event->read_size;
2070
2071	if (sample_type & PERF_SAMPLE_DATA_SRC)
2072	size += sizeof(data->data_src.val);
2073
2074	if (sample_type & PERF_SAMPLE_TRANSACTION)
2075	size += sizeof(data->txn);
2076
2077	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
2078	size += sizeof(data->phys_addr);
2079
2080	if (sample_type & PERF_SAMPLE_CGROUP)
2081	size += sizeof(data->cgroup);
2082
2083	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
2084	size += sizeof(data->data_page_size);
2085
2086	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
2087	size += sizeof(data->code_page_size);
2088
2089	event->header_size = size;
2090	}
2091
2092	/*
2093	* Called at perf_event creation and when events are attached/detached from a
2094	* group.
2095	*/
2096	static void perf_event__header_size(struct perf_event *event)
2097	{
2098	event->read_size =
2099	__perf_event_read_size(event->attr.read_format,
2100	event->group_leader->nr_siblings);
2101	__perf_event_header_size(event, event->attr.sample_type);
2102	}
2103
2104	static void perf_event__id_header_size(struct perf_event *event)
2105	{
2106	struct perf_sample_data *data;
2107	u64 sample_type = event->attr.sample_type;
2108	u16 size = 0;
2109
2110	if (sample_type & PERF_SAMPLE_TID)
2111	size += sizeof(data->tid_entry);
2112
2113	if (sample_type & PERF_SAMPLE_TIME)
2114	size += sizeof(data->time);
2115
2116	if (sample_type & PERF_SAMPLE_IDENTIFIER)
2117	size += sizeof(data->id);
2118
2119	if (sample_type & PERF_SAMPLE_ID)
2120	size += sizeof(data->id);
2121
2122	if (sample_type & PERF_SAMPLE_STREAM_ID)
2123	size += sizeof(data->stream_id);
2124
2125	if (sample_type & PERF_SAMPLE_CPU)
2126	size += sizeof(data->cpu_entry);
2127
2128	event->id_header_size = size;
2129	}
2130
2131	/*
2132	* Check that adding an event to the group does not result in anybody
2133	* overflowing the 64k event limit imposed by the output buffer.
2134	*
2135	* Specifically, check that the read_size for the event does not exceed 16k,
2136	* read_size being the one term that grows with groups size. Since read_size
2137	* depends on per-event read_format, also (re)check the existing events.
2138	*
2139	* This leaves 48k for the constant size fields and things like callchains,
2140	* branch stacks and register sets.
2141	*/
2142	static bool perf_event_validate_size(struct perf_event *event)
2143	{
2144	struct perf_event sibling, group_leader = event->group_leader;
2145
2146	if (__perf_event_read_size(event->attr.read_format,
2147	group_leader->nr_siblings + 1) > 16*1024)
2148	return false;
2149
2150	if (__perf_event_read_size(group_leader->attr.read_format,
2151	group_leader->nr_siblings + 1) > 16*1024)
2152	return false;
2153
2154	/*
2155	* When creating a new group leader, group_leader->ctx is initialized
2156	* after the size has been validated, but we cannot safely use
2157	* for_each_sibling_event() until group_leader->ctx is set. A new group
2158	* leader cannot have any siblings yet, so we can safely skip checking
2159	* the non-existent siblings.
2160	*/
2161	if (event == group_leader)
2162	return true;
2163
2164	for_each_sibling_event(sibling, group_leader) {
2165	if (__perf_event_read_size(sibling->attr.read_format,
2166	group_leader->nr_siblings + 1) > 16*1024)
2167	return false;
2168	}
2169
2170	return true;
2171	}
2172
2173	static void perf_group_attach(struct perf_event *event)
2174	{
2175	struct perf_event group_leader = event->group_leader, pos;
2176
2177	lockdep_assert_held(&event->ctx->lock);
2178
2179	/*
2180	* We can have double attach due to group movement (move_group) in
2181	* perf_event_open().
2182	*/
2183	if (event->attach_state & PERF_ATTACH_GROUP)
2184	return;
2185
2186	event->attach_state \|= PERF_ATTACH_GROUP;
2187
2188	if (group_leader == event)
2189	return;
2190
2191	WARN_ON_ONCE(group_leader->ctx != event->ctx);
2192
2193	group_leader->group_caps &= event->event_caps;
2194
2195	list_add_tail(&event->sibling_list, &group_leader->sibling_list);
2196	group_leader->nr_siblings++;
2197	group_leader->group_generation++;
2198
2199	perf_event__header_size(group_leader);
2200
2201	for_each_sibling_event(pos, group_leader)
2202	perf_event__header_size(pos);
2203	}
2204
2205	/*
2206	* Remove an event from the lists for its context.
2207	* Must be called with ctx->mutex and ctx->lock held.
2208	*/
2209	static void
2210	list_del_event(struct perf_event event, struct perf_event_context ctx)
2211	{
2212	WARN_ON_ONCE(event->ctx != ctx);
2213	lockdep_assert_held(&ctx->lock);
2214
2215	/*
2216	* We can have double detach due to exit/hot-unplug + close.
2217	*/
2218	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
2219	return;
2220
2221	event->attach_state &= ~PERF_ATTACH_CONTEXT;
2222
2223	ctx->nr_events--;
2224	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
2225	ctx->nr_user--;
2226	if (event->attr.inherit_stat)
2227	ctx->nr_stat--;
2228	if (has_inherit_and_sample_read(&event->attr))
2229	local_dec(&ctx->nr_no_switch_fast);
2230
2231	list_del_rcu(&event->event_entry);
2232
2233	if (event->group_leader == event)
2234	del_event_from_groups(event, ctx);
2235
2236	ctx->generation++;
2237	event->pmu_ctx->nr_events--;
2238	}
2239
2240	static int
2241	perf_aux_output_match(struct perf_event event, struct perf_event aux_event)
2242	{
2243	if (!has_aux(aux_event))
2244	return 0;
2245
2246	if (!event->pmu->aux_output_match)
2247	return 0;
2248
2249	return event->pmu->aux_output_match(aux_event);
2250	}
2251
2252	static void put_event(struct perf_event *event);
2253	static void __event_disable(struct perf_event *event,
2254	struct perf_event_context *ctx,
2255	enum perf_event_state state);
2256
2257	static void perf_put_aux_event(struct perf_event *event)
2258	{
2259	struct perf_event_context *ctx = event->ctx;
2260	struct perf_event *iter;
2261
2262	/*
2263	* If event uses aux_event tear down the link
2264	*/
2265	if (event->aux_event) {
2266	iter = event->aux_event;
2267	event->aux_event = NULL;
2268	put_event(iter);
2269	return;
2270	}
2271
2272	/*
2273	* If the event is an aux_event, tear down all links to
2274	* it from other events.
2275	*/
2276	for_each_sibling_event(iter, event) {
2277	if (iter->aux_event != event)
2278	continue;
2279
2280	iter->aux_event = NULL;
2281	put_event(event);
2282
2283	/*
2284	* If it's ACTIVE, schedule it out and put it into ERROR
2285	* state so that we don't try to schedule it again. Note
2286	* that perf_event_enable() will clear the ERROR status.
2287	*/
2288	__event_disable(iter, ctx, PERF_EVENT_STATE_ERROR);
2289	}
2290	}
2291
2292	static bool perf_need_aux_event(struct perf_event *event)
2293	{
2294	return event->attr.aux_output \|\| has_aux_action(event);
2295	}
2296
2297	static int perf_get_aux_event(struct perf_event *event,
2298	struct perf_event *group_leader)
2299	{
2300	/*
2301	* Our group leader must be an aux event if we want to be
2302	* an aux_output. This way, the aux event will precede its
2303	* aux_output events in the group, and therefore will always
2304	* schedule first.
2305	*/
2306	if (!group_leader)
2307	return 0;
2308
2309	/*
2310	* aux_output and aux_sample_size are mutually exclusive.
2311	*/
2312	if (event->attr.aux_output && event->attr.aux_sample_size)
2313	return 0;
2314
2315	if (event->attr.aux_output &&
2316	!perf_aux_output_match(event, group_leader))
2317	return 0;
2318
2319	if ((event->attr.aux_pause \|\| event->attr.aux_resume) &&
2320	!(group_leader->pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE))
2321	return 0;
2322
2323	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)
2324	return 0;
2325
2326	if (!atomic_long_inc_not_zero(&group_leader->refcount))
2327	return 0;
2328
2329	/*
2330	* Link aux_outputs to their aux event; this is undone in
2331	* perf_group_detach() by perf_put_aux_event(). When the
2332	* group in torn down, the aux_output events loose their
2333	* link to the aux_event and can't schedule any more.
2334	*/
2335	event->aux_event = group_leader;
2336
2337	return 1;
2338	}
2339
2340	static inline struct list_head get_event_list(struct perf_event event)
2341	{
2342	return event->attr.pinned ? &event->pmu_ctx->pinned_active :
2343	&event->pmu_ctx->flexible_active;
2344	}
2345
2346	/* @sibling must already be unlinked from its old leader's sibling_list. */
2347	static void perf_promote_sibling_to_leader(struct perf_event *sibling,
2348	struct perf_event_context *ctx,
2349	int group_caps)
2350	{
2351	/*
2352	* Events that have PERF_EV_CAP_SIBLING require being part of
2353	* a group and cannot exist on their own, schedule them out
2354	* and move them into the ERROR state. Also see
2355	* _perf_event_enable(), it will not be able to recover this
2356	* ERROR state.
2357	*/
2358	if (sibling->event_caps & PERF_EV_CAP_SIBLING)
2359	__event_disable(sibling, ctx, PERF_EVENT_STATE_ERROR);
2360
2361	sibling->group_leader = sibling;
2362	sibling->group_caps = group_caps;
2363
2364	if (sibling->attach_state & PERF_ATTACH_CONTEXT) {
2365	add_event_to_groups(sibling, ctx);
2366
2367	if (sibling->state == PERF_EVENT_STATE_ACTIVE)
2368	list_add_tail(&sibling->active_list, get_event_list(sibling));
2369	}
2370
2371	perf_event__header_size(sibling);
2372	}
2373
2374	static void perf_group_detach(struct perf_event *event)
2375	{
2376	struct perf_event *leader = event->group_leader;
2377	struct perf_event sibling, tmp;
2378	struct perf_event_context *ctx = event->ctx;
2379
2380	lockdep_assert_held(&ctx->lock);
2381
2382	/*
2383	* We can have double detach due to exit/hot-unplug + close.
2384	*/
2385	if (!(event->attach_state & PERF_ATTACH_GROUP))
2386	return;
2387
2388	event->attach_state &= ~PERF_ATTACH_GROUP;
2389
2390	perf_put_aux_event(event);
2391
2392	/*
2393	* If this is a sibling, remove it from its group.
2394	*/
2395	if (leader != event) {
2396	list_del_init(&event->sibling_list);
2397	leader->nr_siblings--;
2398	leader->group_generation++;
2399	perf_promote_sibling_to_leader(event, ctx, event->event_caps);
2400	goto out;
2401	}
2402
2403	/*
2404	* If this was a group event with sibling events then
2405	* upgrade the siblings to singleton events by adding them
2406	* to whatever list we are on.
2407	*/
2408	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {
2409	list_del_init(&sibling->sibling_list);
2410
2411	/* Inherit group flags from the previous leader */
2412	perf_promote_sibling_to_leader(sibling, ctx, event->group_caps);
2413
2414	WARN_ON_ONCE(sibling->ctx != event->ctx);
2415	}
2416	event->nr_siblings = 0;
2417
2418	out:
2419	for_each_sibling_event(tmp, leader)
2420	perf_event__header_size(tmp);
2421
2422	perf_event__header_size(leader);
2423	}
2424
2425	static void perf_child_detach(struct perf_event *event)
2426	{
2427	struct perf_event *parent_event = event->parent;
2428
2429	if (!(event->attach_state & PERF_ATTACH_CHILD))
2430	return;
2431
2432	event->attach_state &= ~PERF_ATTACH_CHILD;
2433
2434	if (WARN_ON_ONCE(!parent_event))
2435	return;
2436
2437	/*
2438	* Can't check this from an IPI, the holder is likey another CPU.
2439	*
2440	lockdep_assert_held(&parent_event->child_mutex);
2441	*/
2442
2443	list_del_init(&event->child_list);
2444	}
2445
2446	static bool is_orphaned_event(struct perf_event *event)
2447	{
2448	return event->state == PERF_EVENT_STATE_DEAD;
2449	}
2450
2451	static inline int
2452	event_filter_match(struct perf_event *event)
2453	{
2454	return (event->cpu == -1 \|\| event->cpu == smp_processor_id()) &&
2455	perf_cgroup_match(event);
2456	}
2457
2458	static inline bool is_event_in_freq_mode(struct perf_event *event)
2459	{
2460	return event->attr.freq && event->attr.sample_freq;
2461	}
2462
2463	static void
2464	event_sched_out(struct perf_event event, struct perf_event_context ctx)
2465	{
2466	struct perf_event_pmu_context *epc = event->pmu_ctx;
2467	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);
2468	enum perf_event_state state = PERF_EVENT_STATE_INACTIVE;
2469
2470	// XXX cpc serialization, probably per-cpu IRQ disabled
2471
2472	WARN_ON_ONCE(event->ctx != ctx);
2473	lockdep_assert_held(&ctx->lock);
2474
2475	if (event->state != PERF_EVENT_STATE_ACTIVE)
2476	return;
2477
2478	/*
2479	* Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but
2480	* we can schedule events _OUT_ individually through things like
2481	* __perf_remove_from_context().
2482	*/
2483	list_del_init(&event->active_list);
2484
2485	perf_pmu_disable(event->pmu);
2486
2487	event->pmu->del(event, 0);
2488	event->oncpu = -1;
2489
2490	if (event->pending_disable) {
2491	event->pending_disable = 0;
2492	perf_cgroup_event_disable(event, ctx);
2493	state = PERF_EVENT_STATE_OFF;
2494	}
2495
2496	perf_event_set_state(event, state);
2497
2498	if (!is_software_event(event))
2499	cpc->active_oncpu--;
2500	if (is_event_in_freq_mode(event)) {
2501	ctx->nr_freq--;
2502	epc->nr_freq--;
2503	}
2504	if (event->attr.exclusive \|\| !cpc->active_oncpu)
2505	cpc->exclusive = 0;
2506
2507	perf_pmu_enable(event->pmu);
2508	}
2509
2510	static void
2511	group_sched_out(struct perf_event group_event, struct perf_event_context ctx)
2512	{
2513	struct perf_event *event;
2514
2515	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
2516	return;
2517
2518	perf_assert_pmu_disabled(group_event->pmu_ctx->pmu);
2519
2520	event_sched_out(group_event, ctx);
2521
2522	/*
2523	* Schedule out siblings (if any):
2524	*/
2525	for_each_sibling_event(event, group_event)
2526	event_sched_out(event, ctx);
2527	}
2528
2529	static inline void
2530	__ctx_time_update(struct perf_cpu_context cpuctx, struct perf_event_context ctx,
2531	bool final, enum event_type_t event_type)
2532	{
2533	if (ctx->is_active & EVENT_TIME) {
2534	if (ctx->is_active & EVENT_FROZEN)
2535	return;
2536
2537	update_context_time(ctx);
2538	/* vPMU should not stop time */
2539	update_cgrp_time_from_cpuctx(cpuctx, !(event_type & EVENT_GUEST) && final);
2540	}
2541	}
2542
2543	static inline void
2544	ctx_time_update(struct perf_cpu_context cpuctx, struct perf_event_context ctx)
2545	{
2546	__ctx_time_update(cpuctx, ctx, false, 0);
2547	}
2548
2549	/*
2550	* To be used inside perf_ctx_lock() / perf_ctx_unlock(). Lasts until perf_ctx_unlock().
2551	*/
2552	static inline void
2553	ctx_time_freeze(struct perf_cpu_context cpuctx, struct perf_event_context ctx)
2554	{
2555	ctx_time_update(cpuctx, ctx);
2556	if (ctx->is_active & EVENT_TIME)
2557	ctx->is_active \|= EVENT_FROZEN;
2558	}
2559
2560	static inline void
2561	ctx_time_update_event(struct perf_event_context ctx, struct perf_event event)
2562	{
2563	if (ctx->is_active & EVENT_TIME) {
2564	if (ctx->is_active & EVENT_FROZEN)
2565	return;
2566	update_context_time(ctx);
2567	update_cgrp_time_from_event(event);
2568	}
2569	}
2570
2571	#define DETACH_GROUP 0x01UL
2572	#define DETACH_CHILD 0x02UL
2573	#define DETACH_EXIT 0x04UL
2574	#define DETACH_REVOKE 0x08UL
2575	#define DETACH_DEAD 0x10UL
2576
2577	/*
2578	* Cross CPU call to remove a performance event
2579	*
2580	* We disable the event on the hardware level first. After that we
2581	* remove it from the context list.
2582	*/
2583	static void
2584	__perf_remove_from_context(struct perf_event *event,
2585	struct perf_cpu_context *cpuctx,
2586	struct perf_event_context *ctx,
2587	void *info)
2588	{
2589	struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;
2590	enum perf_event_state state = PERF_EVENT_STATE_OFF;
2591	unsigned long flags = (unsigned long)info;
2592
2593	ctx_time_update(cpuctx, ctx);
2594
2595	/*
2596	* Ensure event_sched_out() switches to OFF, at the very least
2597	* this avoids raising perf_pending_task() at this time.
2598	*/
2599	if (flags & DETACH_EXIT)
2600	state = PERF_EVENT_STATE_EXIT;
2601	if (flags & DETACH_REVOKE)
2602	state = PERF_EVENT_STATE_REVOKED;
2603	if (flags & DETACH_DEAD)
2604	state = PERF_EVENT_STATE_DEAD;
2605
2606	__event_disable(event, ctx, state);
2607
2608	if (flags & DETACH_GROUP)
2609	perf_group_detach(event);
2610	if (flags & DETACH_CHILD)
2611	perf_child_detach(event);
2612	list_del_event(event, ctx);
2613
2614	if (!pmu_ctx->nr_events) {
2615	pmu_ctx->rotate_necessary = 0;
2616
2617	if (ctx->task && ctx->is_active) {
2618	struct perf_cpu_pmu_context *cpc = this_cpc(pmu_ctx->pmu);
2619
2620	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
2621	cpc->task_epc = NULL;
2622	}
2623	}
2624
2625	if (!ctx->nr_events && ctx->is_active) {
2626	if (ctx == &cpuctx->ctx)
2627	update_cgrp_time_from_cpuctx(cpuctx, true);
2628
2629	ctx->is_active = 0;
2630	if (ctx->task) {
2631	WARN_ON_ONCE(cpuctx->task_ctx != ctx);
2632	cpuctx->task_ctx = NULL;
2633	}
2634	}
2635	}
2636
2637	/*
2638	* Remove the event from a task's (or a CPU's) list of events.
2639	*
2640	* If event->ctx is a cloned context, callers must make sure that
2641	* every task struct that event->ctx->task could possibly point to
2642	* remains valid. This is OK when called from perf_release since
2643	* that only calls us on the top-level context, which can't be a clone.
2644	* When called from perf_event_exit_task, it's OK because the
2645	* context has been detached from its task.
2646	*/
2647	static void perf_remove_from_context(struct perf_event *event, unsigned long flags)
2648	{
2649	struct perf_event_context *ctx = event->ctx;
2650
2651	lockdep_assert_held(&ctx->mutex);
2652
2653	/*
2654	* Because of perf_event_exit_task(), perf_remove_from_context() ought
2655	* to work in the face of TASK_TOMBSTONE, unlike every other
2656	* event_function_call() user.
2657	*/
2658	raw_spin_lock_irq(&ctx->lock);
2659	if (!ctx->is_active) {
2660	__perf_remove_from_context(event, this_cpu_ptr(&perf_cpu_context),
2661	ctx, (void *)flags);
2662	raw_spin_unlock_irq(&ctx->lock);
2663	return;
2664	}
2665	raw_spin_unlock_irq(&ctx->lock);
2666
2667	event_function_call(event, __perf_remove_from_context, (void *)flags);
2668	}
2669
2670	static void __event_disable(struct perf_event *event,
2671	struct perf_event_context *ctx,
2672	enum perf_event_state state)
2673	{
2674	event_sched_out(event, ctx);
2675	if (event->state > PERF_EVENT_STATE_OFF)
2676	perf_cgroup_event_disable(event, ctx);
2677	perf_event_set_state(event, min(event->state, state));
2678	}
2679
2680	/*
2681	* Cross CPU call to disable a performance event
2682	*/
2683	static void __perf_event_disable(struct perf_event *event,
2684	struct perf_cpu_context *cpuctx,
2685	struct perf_event_context *ctx,
2686	void *info)
2687	{
2688	if (event->state < PERF_EVENT_STATE_INACTIVE)
2689	return;
2690
2691	perf_pmu_disable(event->pmu_ctx->pmu);
2692	ctx_time_update_event(ctx, event);
2693
2694	/*
2695	* When disabling a group leader, the whole group becomes ineligible
2696	* to run, so schedule out the full group.
2697	*/
2698	if (event == event->group_leader)
2699	group_sched_out(event, ctx);
2700
2701	/*
2702	* But only mark the leader OFF; the siblings will remain
2703	* INACTIVE.
2704	*/
2705	__event_disable(event, ctx, PERF_EVENT_STATE_OFF);
2706
2707	perf_pmu_enable(event->pmu_ctx->pmu);
2708	}
2709
2710	/*
2711	* Disable an event.
2712	*
2713	* If event->ctx is a cloned context, callers must make sure that
2714	* every task struct that event->ctx->task could possibly point to
2715	* remains valid. This condition is satisfied when called through
2716	* perf_event_for_each_child or perf_event_for_each because they
2717	* hold the top-level event's child_mutex, so any descendant that
2718	* goes to exit will block in perf_event_exit_event().
2719	*
2720	* When called from perf_pending_disable it's OK because event->ctx
2721	* is the current context on this CPU and preemption is disabled,
2722	* hence we can't get into perf_event_task_sched_out for this context.
2723	*/
2724	static void _perf_event_disable(struct perf_event *event)
2725	{
2726	struct perf_event_context *ctx = event->ctx;
2727
2728	raw_spin_lock_irq(&ctx->lock);
2729	if (event->state <= PERF_EVENT_STATE_OFF) {
2730	raw_spin_unlock_irq(&ctx->lock);
2731	return;
2732	}
2733	raw_spin_unlock_irq(&ctx->lock);
2734
2735	event_function_call(event, __perf_event_disable, NULL);
2736	}
2737
2738	void perf_event_disable_local(struct perf_event *event)
2739	{
2740	event_function_local(event, __perf_event_disable, NULL);
2741	}
2742
2743	/*
2744	* Strictly speaking kernel users cannot create groups and therefore this
2745	* interface does not need the perf_event_ctx_lock() magic.
2746	*/
2747	void perf_event_disable(struct perf_event *event)
2748	{
2749	struct perf_event_context *ctx;
2750
2751	ctx = perf_event_ctx_lock(event);
2752	_perf_event_disable(event);
2753	perf_event_ctx_unlock(event, ctx);
2754	}
2755	EXPORT_SYMBOL_GPL(perf_event_disable);
2756
2757	void perf_event_disable_inatomic(struct perf_event *event)
2758	{
2759	event->pending_disable = 1;
2760	irq_work_queue(&event->pending_disable_irq);
2761	}
2762
2763	#define MAX_INTERRUPTS (~0ULL)
2764
2765	static void perf_log_throttle(struct perf_event *event, int enable);
2766	static void perf_log_itrace_start(struct perf_event *event);
2767
2768	static void perf_event_unthrottle(struct perf_event *event, bool start)
2769	{
2770	if (event->state != PERF_EVENT_STATE_ACTIVE)
2771	return;
2772
2773	event->hw.interrupts = 0;
2774	if (start)
2775	event->pmu->start(event, 0);
2776	if (event == event->group_leader)
2777	perf_log_throttle(event, 1);
2778	}
2779
2780	static void perf_event_throttle(struct perf_event *event)
2781	{
2782	if (event->state != PERF_EVENT_STATE_ACTIVE)
2783	return;
2784
2785	event->hw.interrupts = MAX_INTERRUPTS;
2786	event->pmu->stop(event, 0);
2787	if (event == event->group_leader)
2788	perf_log_throttle(event, 0);
2789	}
2790
2791	static void perf_event_unthrottle_group(struct perf_event *event, bool skip_start_event)
2792	{
2793	struct perf_event sibling, leader = event->group_leader;
2794
2795	perf_event_unthrottle(leader, skip_start_event ? leader != event : true);
2796	for_each_sibling_event(sibling, leader)
2797	perf_event_unthrottle(sibling, skip_start_event ? sibling != event : true);
2798	}
2799
2800	static void perf_event_throttle_group(struct perf_event *event)
2801	{
2802	struct perf_event sibling, leader = event->group_leader;
2803
2804	perf_event_throttle(leader);
2805	for_each_sibling_event(sibling, leader)
2806	perf_event_throttle(sibling);
2807	}
2808
2809	static int
2810	event_sched_in(struct perf_event event, struct perf_event_context ctx)
2811	{
2812	struct perf_event_pmu_context *epc = event->pmu_ctx;
2813	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);
2814	int ret = 0;
2815
2816	WARN_ON_ONCE(event->ctx != ctx);
2817
2818	lockdep_assert_held(&ctx->lock);
2819
2820	if (event->state <= PERF_EVENT_STATE_OFF)
2821	return 0;
2822
2823	WRITE_ONCE(event->oncpu, smp_processor_id());
2824	/*
2825	* Order event::oncpu write to happen before the ACTIVE state is
2826	* visible. This allows perf_event_{stop,read}() to observe the correct
2827	* ->oncpu if it sees ACTIVE.
2828	*/
2829	smp_wmb();
2830	perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE);
2831
2832	/*
2833	* Unthrottle events, since we scheduled we might have missed several
2834	* ticks already, also for a heavily scheduling task there is little
2835	* guarantee it'll get a tick in a timely manner.
2836	*/
2837	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS))
2838	perf_event_unthrottle(event, false);
2839
2840	perf_pmu_disable(event->pmu);
2841
2842	perf_log_itrace_start(event);
2843
2844	if (event->pmu->add(event, PERF_EF_START)) {
2845	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2846	event->oncpu = -1;
2847	ret = -EAGAIN;
2848	goto out;
2849	}
2850
2851	if (!is_software_event(event))
2852	cpc->active_oncpu++;
2853	if (is_event_in_freq_mode(event)) {
2854	ctx->nr_freq++;
2855	epc->nr_freq++;
2856	}
2857	if (event->attr.exclusive)
2858	cpc->exclusive = 1;
2859
2860	out:
2861	perf_pmu_enable(event->pmu);
2862
2863	return ret;
2864	}
2865
2866	static int
2867	group_sched_in(struct perf_event group_event, struct perf_event_context ctx)
2868	{
2869	struct perf_event event, partial_group = NULL;
2870	struct pmu *pmu = group_event->pmu_ctx->pmu;
2871
2872	if (group_event->state == PERF_EVENT_STATE_OFF)
2873	return 0;
2874
2875	pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
2876
2877	if (event_sched_in(group_event, ctx))
2878	goto error;
2879
2880	/*
2881	* Schedule in siblings as one group (if any):
2882	*/
2883	for_each_sibling_event(event, group_event) {
2884	if (event_sched_in(event, ctx)) {
2885	partial_group = event;
2886	goto group_error;
2887	}
2888	}
2889
2890	if (!pmu->commit_txn(pmu))
2891	return 0;
2892
2893	group_error:
2894	/*
2895	* Groups can be scheduled in as one unit only, so undo any
2896	* partial group before returning:
2897	* The events up to the failed event are scheduled out normally.
2898	*/
2899	for_each_sibling_event(event, group_event) {
2900	if (event == partial_group)
2901	break;
2902
2903	event_sched_out(event, ctx);
2904	}
2905	event_sched_out(group_event, ctx);
2906
2907	error:
2908	pmu->cancel_txn(pmu);
2909	return -EAGAIN;
2910	}
2911
2912	/*
2913	* Work out whether we can put this event group on the CPU now.
2914	*/
2915	static int group_can_go_on(struct perf_event *event, int can_add_hw)
2916	{
2917	struct perf_event_pmu_context *epc = event->pmu_ctx;
2918	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);
2919
2920	/*
2921	* Groups consisting entirely of software events can always go on.
2922	*/
2923	if (event->group_caps & PERF_EV_CAP_SOFTWARE)
2924	return 1;
2925	/*
2926	* If an exclusive group is already on, no other hardware
2927	* events can go on.
2928	*/
2929	if (cpc->exclusive)
2930	return 0;
2931	/*
2932	* If this group is exclusive and there are already
2933	* events on the CPU, it can't go on.
2934	*/
2935	if (event->attr.exclusive && !list_empty(get_event_list(event)))
2936	return 0;
2937	/*
2938	* Otherwise, try to add it if all previous groups were able
2939	* to go on.
2940	*/
2941	return can_add_hw;
2942	}
2943
2944	static void add_event_to_ctx(struct perf_event *event,
2945	struct perf_event_context *ctx)
2946	{
2947	list_add_event(event, ctx);
2948	perf_group_attach(event);
2949	}
2950
2951	static void task_ctx_sched_out(struct perf_event_context *ctx,
2952	struct pmu *pmu,
2953	enum event_type_t event_type)
2954	{
2955	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
2956
2957	if (!cpuctx->task_ctx)
2958	return;
2959
2960	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2961	return;
2962
2963	ctx_sched_out(ctx, pmu, event_type);
2964	}
2965
2966	static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
2967	struct perf_event_context *ctx,
2968	struct pmu *pmu,
2969	enum event_type_t event_type)
2970	{
2971	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED \| event_type);
2972	if (ctx)
2973	ctx_sched_in(ctx, pmu, EVENT_PINNED \| event_type);
2974	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE \| event_type);
2975	if (ctx)
2976	ctx_sched_in(ctx, pmu, EVENT_FLEXIBLE \| event_type);
2977	}
2978
2979	/*
2980	* We want to maintain the following priority of scheduling:
2981	* - CPU pinned (EVENT_CPU \| EVENT_PINNED)
2982	* - task pinned (EVENT_PINNED)
2983	* - CPU flexible (EVENT_CPU \| EVENT_FLEXIBLE)
2984	* - task flexible (EVENT_FLEXIBLE).
2985	*
2986	* In order to avoid unscheduling and scheduling back in everything every
2987	* time an event is added, only do it for the groups of equal priority and
2988	* below.
2989	*
2990	* This can be called after a batch operation on task events, in which case
2991	* event_type is a bit mask of the types of events involved. For CPU events,
2992	* event_type is only either EVENT_PINNED or EVENT_FLEXIBLE.
2993	*/
2994	static void ctx_resched(struct perf_cpu_context *cpuctx,
2995	struct perf_event_context *task_ctx,
2996	struct pmu *pmu, enum event_type_t event_type)
2997	{
2998	bool cpu_event = !!(event_type & EVENT_CPU);
2999	struct perf_event_pmu_context *epc;
3000
3001	/*
3002	* If pinned groups are involved, flexible groups also need to be
3003	* scheduled out.
3004	*/
3005	if (event_type & EVENT_PINNED)
3006	event_type \|= EVENT_FLEXIBLE;
3007
3008	event_type &= EVENT_ALL;
3009
3010	for_each_epc(epc, &cpuctx->ctx, pmu, 0)
3011	perf_pmu_disable(epc->pmu);
3012
3013	if (task_ctx) {
3014	for_each_epc(epc, task_ctx, pmu, 0)
3015	perf_pmu_disable(epc->pmu);
3016
3017	task_ctx_sched_out(task_ctx, pmu, event_type);
3018	}
3019
3020	/*
3021	* Decide which cpu ctx groups to schedule out based on the types
3022	* of events that caused rescheduling:
3023	* - EVENT_CPU: schedule out corresponding groups;
3024	* - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;
3025	* - otherwise, do nothing more.
3026	*/
3027	if (cpu_event)
3028	ctx_sched_out(&cpuctx->ctx, pmu, event_type);
3029	else if (event_type & EVENT_PINNED)
3030	ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);
3031
3032	perf_event_sched_in(cpuctx, task_ctx, pmu, 0);
3033
3034	for_each_epc(epc, &cpuctx->ctx, pmu, 0)
3035	perf_pmu_enable(epc->pmu);
3036
3037	if (task_ctx) {
3038	for_each_epc(epc, task_ctx, pmu, 0)
3039	perf_pmu_enable(epc->pmu);
3040	}
3041	}
3042
3043	void perf_pmu_resched(struct pmu *pmu)
3044	{
3045	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3046	struct perf_event_context *task_ctx = cpuctx->task_ctx;
3047
3048	perf_ctx_lock(cpuctx, task_ctx);
3049	ctx_resched(cpuctx, task_ctx, pmu, EVENT_ALL\|EVENT_CPU);
3050	perf_ctx_unlock(cpuctx, task_ctx);
3051	}
3052
3053	/*
3054	* Cross CPU call to install and enable a performance event
3055	*
3056	* Very similar to remote_function() + event_function() but cannot assume that
3057	* things like ctx->is_active and cpuctx->task_ctx are set.
3058	*/
3059	static int __perf_install_in_context(void *info)
3060	{
3061	struct perf_event *event = info;
3062	struct perf_event_context *ctx = event->ctx;
3063	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3064	struct perf_event_context *task_ctx = cpuctx->task_ctx;
3065	bool reprogram = true;
3066	int ret = 0;
3067
3068	raw_spin_lock(&cpuctx->ctx.lock);
3069	if (ctx->task) {
3070	raw_spin_lock(&ctx->lock);
3071	task_ctx = ctx;
3072
3073	reprogram = (ctx->task == current);
3074
3075	/*
3076	* If the task is running, it must be running on this CPU,
3077	* otherwise we cannot reprogram things.
3078	*
3079	* If its not running, we don't care, ctx->lock will
3080	* serialize against it becoming runnable.
3081	*/
3082	if (task_curr(ctx->task) && !reprogram) {
3083	ret = -ESRCH;
3084	goto unlock;
3085	}
3086
3087	WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);
3088	} else if (task_ctx) {
3089	raw_spin_lock(&task_ctx->lock);
3090	}
3091
3092	#ifdef CONFIG_CGROUP_PERF
3093	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
3094	/*
3095	* If the current cgroup doesn't match the event's
3096	* cgroup, we should not try to schedule it.
3097	*/
3098	struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
3099	reprogram = cgroup_is_descendant(cgrp->css.cgroup,
3100	event->cgrp->css.cgroup);
3101	}
3102	#endif
3103
3104	if (reprogram) {
3105	ctx_time_freeze(cpuctx, ctx);
3106	add_event_to_ctx(event, ctx);
3107	ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu,
3108	get_event_type(event));
3109	} else {
3110	add_event_to_ctx(event, ctx);
3111	}
3112
3113	unlock:
3114	perf_ctx_unlock(cpuctx, task_ctx);
3115
3116	return ret;
3117	}
3118
3119	static bool exclusive_event_installable(struct perf_event *event,
3120	struct perf_event_context *ctx);
3121
3122	/*
3123	* Attach a performance event to a context.
3124	*
3125	* Very similar to event_function_call, see comment there.
3126	*/
3127	static void
3128	perf_install_in_context(struct perf_event_context *ctx,
3129	struct perf_event *event,
3130	int cpu)
3131	{
3132	struct task_struct *task = READ_ONCE(ctx->task);
3133
3134	lockdep_assert_held(&ctx->mutex);
3135
3136	WARN_ON_ONCE(!exclusive_event_installable(event, ctx));
3137
3138	if (event->cpu != -1)
3139	WARN_ON_ONCE(event->cpu != cpu);
3140
3141	/*
3142	* Ensures that if we can observe event->ctx, both the event and ctx
3143	* will be 'complete'. See perf_iterate_sb_cpu().
3144	*/
3145	smp_store_release(&event->ctx, ctx);
3146
3147	/*
3148	* perf_event_attr::disabled events will not run and can be initialized
3149	* without IPI. Except when this is the first event for the context, in
3150	* that case we need the magic of the IPI to set ctx->is_active.
3151	*
3152	* The IOC_ENABLE that is sure to follow the creation of a disabled
3153	* event will issue the IPI and reprogram the hardware.
3154	*/
3155	if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF &&
3156	ctx->nr_events && !is_cgroup_event(event)) {
3157	raw_spin_lock_irq(&ctx->lock);
3158	if (ctx->task == TASK_TOMBSTONE) {
3159	raw_spin_unlock_irq(&ctx->lock);
3160	return;
3161	}
3162	add_event_to_ctx(event, ctx);
3163	raw_spin_unlock_irq(&ctx->lock);
3164	return;
3165	}
3166
3167	if (!task) {
3168	cpu_function_call(cpu, __perf_install_in_context, event);
3169	return;
3170	}
3171
3172	/*
3173	* Should not happen, we validate the ctx is still alive before calling.
3174	*/
3175	if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
3176	return;
3177
3178	/*
3179	* Installing events is tricky because we cannot rely on ctx->is_active
3180	* to be set in case this is the nr_events 0 -> 1 transition.
3181	*
3182	* Instead we use task_curr(), which tells us if the task is running.
3183	* However, since we use task_curr() outside of rq::lock, we can race
3184	* against the actual state. This means the result can be wrong.
3185	*
3186	* If we get a false positive, we retry, this is harmless.
3187	*
3188	* If we get a false negative, things are complicated. If we are after
3189	* perf_event_context_sched_in() ctx::lock will serialize us, and the
3190	* value must be correct. If we're before, it doesn't matter since
3191	* perf_event_context_sched_in() will program the counter.
3192	*
3193	* However, this hinges on the remote context switch having observed
3194	* our task->perf_event_ctxp[] store, such that it will in fact take
3195	* ctx::lock in perf_event_context_sched_in().
3196	*
3197	* We do this by task_function_call(), if the IPI fails to hit the task
3198	* we know any future context switch of task must see the
3199	* perf_event_ctpx[] store.
3200	*/
3201
3202	/*
3203	* This smp_mb() orders the task->perf_event_ctxp[] store with the
3204	* task_cpu() load, such that if the IPI then does not find the task
3205	* running, a future context switch of that task must observe the
3206	* store.
3207	*/
3208	smp_mb();
3209	again:
3210	if (!task_function_call(task, __perf_install_in_context, event))
3211	return;
3212
3213	raw_spin_lock_irq(&ctx->lock);
3214	task = ctx->task;
3215	if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
3216	/*
3217	* Cannot happen because we already checked above (which also
3218	* cannot happen), and we hold ctx->mutex, which serializes us
3219	* against perf_event_exit_task_context().
3220	*/
3221	raw_spin_unlock_irq(&ctx->lock);
3222	return;
3223	}
3224	/*
3225	* If the task is not running, ctx->lock will avoid it becoming so,
3226	* thus we can safely install the event.
3227	*/
3228	if (task_curr(task)) {
3229	raw_spin_unlock_irq(&ctx->lock);
3230	goto again;
3231	}
3232	add_event_to_ctx(event, ctx);
3233	raw_spin_unlock_irq(&ctx->lock);
3234	}
3235
3236	/*
3237	* Cross CPU call to enable a performance event
3238	*/
3239	static void __perf_event_enable(struct perf_event *event,
3240	struct perf_cpu_context *cpuctx,
3241	struct perf_event_context *ctx,
3242	void *info)
3243	{
3244	struct perf_event *leader = event->group_leader;
3245	struct perf_event_context *task_ctx;
3246
3247	if (event->state >= PERF_EVENT_STATE_INACTIVE \|\|
3248	event->state <= PERF_EVENT_STATE_ERROR)
3249	return;
3250
3251	ctx_time_freeze(cpuctx, ctx);
3252
3253	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
3254	perf_cgroup_event_enable(event, ctx);
3255
3256	if (!ctx->is_active)
3257	return;
3258
3259	if (!event_filter_match(event))
3260	return;
3261
3262	/*
3263	* If the event is in a group and isn't the group leader,
3264	* then don't put it on unless the group is on.
3265	*/
3266	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
3267	return;
3268
3269	task_ctx = cpuctx->task_ctx;
3270	if (ctx->task)
3271	WARN_ON_ONCE(task_ctx != ctx);
3272
3273	ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, get_event_type(event));
3274	}
3275
3276	/*
3277	* Enable an event.
3278	*
3279	* If event->ctx is a cloned context, callers must make sure that
3280	* every task struct that event->ctx->task could possibly point to
3281	* remains valid. This condition is satisfied when called through
3282	* perf_event_for_each_child or perf_event_for_each as described
3283	* for perf_event_disable.
3284	*/
3285	static void _perf_event_enable(struct perf_event *event)
3286	{
3287	struct perf_event_context *ctx = event->ctx;
3288
3289	raw_spin_lock_irq(&ctx->lock);
3290	if (event->state >= PERF_EVENT_STATE_INACTIVE \|\|
3291	event->state < PERF_EVENT_STATE_ERROR) {
3292	out:
3293	raw_spin_unlock_irq(&ctx->lock);
3294	return;
3295	}
3296
3297	/*
3298	* If the event is in error state, clear that first.
3299	*
3300	* That way, if we see the event in error state below, we know that it
3301	* has gone back into error state, as distinct from the task having
3302	* been scheduled away before the cross-call arrived.
3303	*/
3304	if (event->state == PERF_EVENT_STATE_ERROR) {
3305	/*
3306	* Detached SIBLING events cannot leave ERROR state.
3307	*/
3308	if (event->event_caps & PERF_EV_CAP_SIBLING &&
3309	event->group_leader == event)
3310	goto out;
3311
3312	event->state = PERF_EVENT_STATE_OFF;
3313	}
3314	raw_spin_unlock_irq(&ctx->lock);
3315
3316	event_function_call(event, __perf_event_enable, NULL);
3317	}
3318
3319	/*
3320	* See perf_event_disable();
3321	*/
3322	void perf_event_enable(struct perf_event *event)
3323	{
3324	struct perf_event_context *ctx;
3325
3326	ctx = perf_event_ctx_lock(event);
3327	_perf_event_enable(event);
3328	perf_event_ctx_unlock(event, ctx);
3329	}
3330	EXPORT_SYMBOL_GPL(perf_event_enable);
3331
3332	struct stop_event_data {
3333	struct perf_event *event;
3334	unsigned int restart;
3335	};
3336
3337	static int __perf_event_stop(void *info)
3338	{
3339	struct stop_event_data *sd = info;
3340	struct perf_event *event = sd->event;
3341
3342	/* if it's already INACTIVE, do nothing */
3343	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3344	return 0;
3345
3346	/* matches smp_wmb() in event_sched_in() */
3347	smp_rmb();
3348
3349	/*
3350	* There is a window with interrupts enabled before we get here,
3351	* so we need to check again lest we try to stop another CPU's event.
3352	*/
3353	if (READ_ONCE(event->oncpu) != smp_processor_id())
3354	return -EAGAIN;
3355
3356	event->pmu->stop(event, PERF_EF_UPDATE);
3357
3358	/*
3359	* May race with the actual stop (through perf_pmu_output_stop()),
3360	* but it is only used for events with AUX ring buffer, and such
3361	* events will refuse to restart because of rb::aux_mmap_count==0,
3362	* see comments in perf_aux_output_begin().
3363	*
3364	* Since this is happening on an event-local CPU, no trace is lost
3365	* while restarting.
3366	*/
3367	if (sd->restart)
3368	event->pmu->start(event, 0);
3369
3370	return 0;
3371	}
3372
3373	static int perf_event_stop(struct perf_event *event, int restart)
3374	{
3375	struct stop_event_data sd = {
3376	.event = event,
3377	.restart = restart,
3378	};
3379	int ret = 0;
3380
3381	do {
3382	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3383	return 0;
3384
3385	/* matches smp_wmb() in event_sched_in() */
3386	smp_rmb();
3387
3388	/*
3389	* We only want to restart ACTIVE events, so if the event goes
3390	* inactive here (event->oncpu==-1), there's nothing more to do;
3391	* fall through with ret==-ENXIO.
3392	*/
3393	ret = cpu_function_call(READ_ONCE(event->oncpu),
3394	__perf_event_stop, &sd);
3395	} while (ret == -EAGAIN);
3396
3397	return ret;
3398	}
3399
3400	/*
3401	* In order to contain the amount of racy and tricky in the address filter
3402	* configuration management, it is a two part process:
3403	*
3404	* (p1) when userspace mappings change as a result of (1) or (2) or (3) below,
3405	* we update the addresses of corresponding vmas in
3406	* event::addr_filter_ranges array and bump the event::addr_filters_gen;
3407	* (p2) when an event is scheduled in (pmu::add), it calls
3408	* perf_event_addr_filters_sync() which calls pmu::addr_filters_sync()
3409	* if the generation has changed since the previous call.
3410	*
3411	* If (p1) happens while the event is active, we restart it to force (p2).
3412	*
3413	* (1) perf_addr_filters_apply(): adjusting filters' offsets based on
3414	* pre-existing mappings, called once when new filters arrive via SET_FILTER
3415	* ioctl;
3416	* (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly
3417	* registered mapping, called for every new mmap(), with mm::mmap_lock down
3418	* for reading;
3419	* (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process
3420	* of exec.
3421	*/
3422	void perf_event_addr_filters_sync(struct perf_event *event)
3423	{
3424	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
3425
3426	if (!has_addr_filter(event))
3427	return;
3428
3429	raw_spin_lock(&ifh->lock);
3430	if (event->addr_filters_gen != event->hw.addr_filters_gen) {
3431	event->pmu->addr_filters_sync(event);
3432	event->hw.addr_filters_gen = event->addr_filters_gen;
3433	}
3434	raw_spin_unlock(&ifh->lock);
3435	}
3436	EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync);
3437
3438	static int _perf_event_refresh(struct perf_event *event, int refresh)
3439	{
3440	/*
3441	* not supported on inherited events
3442	*/
3443	if (event->attr.inherit \|\| !is_sampling_event(event))
3444	return -EINVAL;
3445
3446	atomic_add(refresh, &event->event_limit);
3447	_perf_event_enable(event);
3448
3449	return 0;
3450	}
3451
3452	/*
3453	* See perf_event_disable()
3454	*/
3455	int perf_event_refresh(struct perf_event *event, int refresh)
3456	{
3457	struct perf_event_context *ctx;
3458	int ret;
3459
3460	ctx = perf_event_ctx_lock(event);
3461	ret = _perf_event_refresh(event, refresh);
3462	perf_event_ctx_unlock(event, ctx);
3463
3464	return ret;
3465	}
3466	EXPORT_SYMBOL_GPL(perf_event_refresh);
3467
3468	static int perf_event_modify_breakpoint(struct perf_event *bp,
3469	struct perf_event_attr *attr)
3470	{
3471	int err;
3472
3473	_perf_event_disable(bp);
3474
3475	err = modify_user_hw_breakpoint_check(bp, attr, true);
3476
3477	if (!bp->attr.disabled)
3478	_perf_event_enable(bp);
3479
3480	return err;
3481	}
3482
3483	/*
3484	* Copy event-type-independent attributes that may be modified.
3485	*/
3486	static void perf_event_modify_copy_attr(struct perf_event_attr *to,
3487	const struct perf_event_attr *from)
3488	{
3489	to->sig_data = from->sig_data;
3490	}
3491
3492	static int perf_event_modify_attr(struct perf_event *event,
3493	struct perf_event_attr *attr)
3494	{
3495	int (func)(struct perf_event , struct perf_event_attr *);
3496	struct perf_event *child;
3497	int err;
3498
3499	if (event->attr.type != attr->type)
3500	return -EINVAL;
3501
3502	switch (event->attr.type) {
3503	case PERF_TYPE_BREAKPOINT:
3504	func = perf_event_modify_breakpoint;
3505	break;
3506	default:
3507	/* Place holder for future additions. */
3508	return -EOPNOTSUPP;
3509	}
3510
3511	WARN_ON_ONCE(event->ctx->parent_ctx);
3512
3513	mutex_lock(&event->child_mutex);
3514	/*
3515	* Event-type-independent attributes must be copied before event-type
3516	* modification, which will validate that final attributes match the
3517	* source attributes after all relevant attributes have been copied.
3518	*/
3519	perf_event_modify_copy_attr(&event->attr, attr);
3520	err = func(event, attr);
3521	if (err)
3522	goto out;
3523	list_for_each_entry(child, &event->child_list, child_list) {
3524	perf_event_modify_copy_attr(&child->attr, attr);
3525	err = func(child, attr);
3526	if (err)
3527	goto out;
3528	}
3529	out:
3530	mutex_unlock(&event->child_mutex);
3531	return err;
3532	}
3533
3534	static void __pmu_ctx_sched_out(struct perf_event_pmu_context *pmu_ctx,
3535	enum event_type_t event_type)
3536	{
3537	struct perf_event_context *ctx = pmu_ctx->ctx;
3538	struct perf_event event, tmp;
3539	struct pmu *pmu = pmu_ctx->pmu;
3540
3541	if (ctx->task && !(ctx->is_active & EVENT_ALL)) {
3542	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);
3543
3544	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
3545	cpc->task_epc = NULL;
3546	}
3547
3548	if (!(event_type & EVENT_ALL))
3549	return;
3550
3551	perf_pmu_disable(pmu);
3552	if (event_type & EVENT_PINNED) {
3553	list_for_each_entry_safe(event, tmp,
3554	&pmu_ctx->pinned_active,
3555	active_list)
3556	group_sched_out(event, ctx);
3557	}
3558
3559	if (event_type & EVENT_FLEXIBLE) {
3560	list_for_each_entry_safe(event, tmp,
3561	&pmu_ctx->flexible_active,
3562	active_list)
3563	group_sched_out(event, ctx);
3564	/*
3565	* Since we cleared EVENT_FLEXIBLE, also clear
3566	* rotate_necessary, is will be reset by
3567	* ctx_flexible_sched_in() when needed.
3568	*/
3569	pmu_ctx->rotate_necessary = 0;
3570	}
3571	perf_pmu_enable(pmu);
3572	}
3573
3574	/*
3575	* Be very careful with the @pmu argument since this will change ctx state.
3576	* The @pmu argument works for ctx_resched(), because that is symmetric in
3577	* ctx_sched_out() / ctx_sched_in() usage and the ctx state ends up invariant.
3578	*
3579	* However, if you were to be asymmetrical, you could end up with messed up
3580	* state, eg. ctx->is_active cleared even though most EPCs would still actually
3581	* be active.
3582	*/
3583	static void
3584	ctx_sched_out(struct perf_event_context ctx, struct pmu pmu, enum event_type_t event_type)
3585	{
3586	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3587	enum event_type_t active_type = event_type & ~EVENT_FLAGS;
3588	struct perf_event_pmu_context *pmu_ctx;
3589	int is_active = ctx->is_active;
3590
3591
3592	lockdep_assert_held(&ctx->lock);
3593
3594	if (likely(!ctx->nr_events)) {
3595	/*
3596	* See __perf_remove_from_context().
3597	*/
3598	WARN_ON_ONCE(ctx->is_active);
3599	if (ctx->task)
3600	WARN_ON_ONCE(cpuctx->task_ctx);
3601	return;
3602	}
3603
3604	/*
3605	* Always update time if it was set; not only when it changes.
3606	* Otherwise we can 'forget' to update time for any but the last
3607	* context we sched out. For example:
3608	*
3609	* ctx_sched_out(.event_type = EVENT_FLEXIBLE)
3610	* ctx_sched_out(.event_type = EVENT_PINNED)
3611	*
3612	* would only update time for the pinned events.
3613	*/
3614	__ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx, event_type);
3615
3616	/*
3617	* CPU-release for the below ->is_active store,
3618	* see __load_acquire() in perf_event_time_now()
3619	*/
3620	barrier();
3621	ctx->is_active &= ~active_type;
3622
3623	if (!(ctx->is_active & EVENT_ALL)) {
3624	/*
3625	* For FROZEN, preserve TIME\|FROZEN such that perf_event_time_now()
3626	* does not observe a hole. perf_ctx_unlock() will clean up.
3627	*/
3628	if (ctx->is_active & EVENT_FROZEN)
3629	ctx->is_active &= EVENT_TIME_FROZEN;
3630	else
3631	ctx->is_active = 0;
3632	}
3633
3634	if (ctx->task) {
3635	WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3636	if (!(ctx->is_active & EVENT_ALL))
3637	cpuctx->task_ctx = NULL;
3638	}
3639
3640	if (event_type & EVENT_GUEST) {
3641	/*
3642	* Schedule out all exclude_guest events of PMU
3643	* with PERF_PMU_CAP_MEDIATED_VPMU.
3644	*/
3645	is_active = EVENT_ALL;
3646	__update_context_guest_time(ctx, false);
3647	perf_cgroup_set_timestamp(cpuctx, true);
3648	barrier();
3649	} else {
3650	is_active ^= ctx->is_active; /* changed bits */
3651	}
3652
3653	for_each_epc(pmu_ctx, ctx, pmu, event_type)
3654	__pmu_ctx_sched_out(pmu_ctx, is_active);
3655	}
3656
3657	/*
3658	* Test whether two contexts are equivalent, i.e. whether they have both been
3659	* cloned from the same version of the same context.
3660	*
3661	* Equivalence is measured using a generation number in the context that is
3662	* incremented on each modification to it; see unclone_ctx(), list_add_event()
3663	* and list_del_event().
3664	*/
3665	static int context_equiv(struct perf_event_context *ctx1,
3666	struct perf_event_context *ctx2)
3667	{
3668	lockdep_assert_held(&ctx1->lock);
3669	lockdep_assert_held(&ctx2->lock);
3670
3671	/* Pinning disables the swap optimization */
3672	if (ctx1->pin_count \|\| ctx2->pin_count)
3673	return 0;
3674
3675	/* If ctx1 is the parent of ctx2 */
3676	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
3677	return 1;
3678
3679	/* If ctx2 is the parent of ctx1 */
3680	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
3681	return 1;
3682
3683	/*
3684	* If ctx1 and ctx2 have the same parent; we flatten the parent
3685	* hierarchy, see perf_event_init_context().
3686	*/
3687	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
3688	ctx1->parent_gen == ctx2->parent_gen)
3689	return 1;
3690
3691	/* Unmatched */
3692	return 0;
3693	}
3694
3695	static void __perf_event_sync_stat(struct perf_event *event,
3696	struct perf_event *next_event)
3697	{
3698	u64 value;
3699
3700	if (!event->attr.inherit_stat)
3701	return;
3702
3703	/*
3704	* Update the event value, we cannot use perf_event_read()
3705	* because we're in the middle of a context switch and have IRQs
3706	* disabled, which upsets smp_call_function_single(), however
3707	* we know the event must be on the current CPU, therefore we
3708	* don't need to use it.
3709	*/
3710	perf_pmu_read(event);
3711
3712	perf_event_update_time(event);
3713
3714	/*
3715	* In order to keep per-task stats reliable we need to flip the event
3716	* values when we flip the contexts.
3717	*/
3718	value = local64_read(&next_event->count);
3719	value = local64_xchg(&event->count, value);
3720	local64_set(&next_event->count, value);
3721
3722	swap(event->total_time_enabled, next_event->total_time_enabled);
3723	swap(event->total_time_running, next_event->total_time_running);
3724
3725	/*
3726	* Since we swizzled the values, update the user visible data too.
3727	*/
3728	perf_event_update_userpage(event);
3729	perf_event_update_userpage(next_event);
3730	}
3731
3732	static void perf_event_sync_stat(struct perf_event_context *ctx,
3733	struct perf_event_context *next_ctx)
3734	{
3735	struct perf_event event, next_event;
3736
3737	if (!ctx->nr_stat)
3738	return;
3739
3740	update_context_time(ctx);
3741
3742	event = list_first_entry(&ctx->event_list,
3743	struct perf_event, event_entry);
3744
3745	next_event = list_first_entry(&next_ctx->event_list,
3746	struct perf_event, event_entry);
3747
3748	while (&event->event_entry != &ctx->event_list &&
3749	&next_event->event_entry != &next_ctx->event_list) {
3750
3751	__perf_event_sync_stat(event, next_event);
3752
3753	event = list_next_entry(event, event_entry);
3754	next_event = list_next_entry(next_event, event_entry);
3755	}
3756	}
3757
3758	static void perf_ctx_sched_task_cb(struct perf_event_context *ctx,
3759	struct task_struct *task, bool sched_in)
3760	{
3761	struct perf_event_pmu_context *pmu_ctx;
3762	struct perf_cpu_pmu_context *cpc;
3763
3764	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
3765	cpc = this_cpc(pmu_ctx->pmu);
3766
3767	if (cpc->task_epc != pmu_ctx)
3768	continue;
3769
3770	if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)
3771	pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in);
3772	}
3773	}
3774
3775	static void
3776	perf_event_context_sched_out(struct task_struct task, struct task_struct next)
3777	{
3778	struct perf_event_context *ctx = task->perf_event_ctxp;
3779	struct perf_event_context *next_ctx;
3780	struct perf_event_context parent, next_parent;
3781	int do_switch = 1;
3782
3783	if (likely(!ctx))
3784	return;
3785
3786	rcu_read_lock();
3787	next_ctx = rcu_dereference(next->perf_event_ctxp);
3788	if (!next_ctx)
3789	goto unlock;
3790
3791	parent = rcu_dereference(ctx->parent_ctx);
3792	next_parent = rcu_dereference(next_ctx->parent_ctx);
3793
3794	/* If neither context have a parent context; they cannot be clones. */
3795	if (!parent && !next_parent)
3796	goto unlock;
3797
3798	if (next_parent == ctx \|\| next_ctx == parent \|\| next_parent == parent) {
3799	/*
3800	* Looks like the two contexts are clones, so we might be
3801	* able to optimize the context switch. We lock both
3802	* contexts and check that they are clones under the
3803	* lock (including re-checking that neither has been
3804	* uncloned in the meantime). It doesn't matter which
3805	* order we take the locks because no other cpu could
3806	* be trying to lock both of these tasks.
3807	*/
3808	raw_spin_lock(&ctx->lock);
3809	raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
3810	if (context_equiv(ctx, next_ctx)) {
3811
3812	perf_ctx_disable(ctx, 0);
3813
3814	/* PMIs are disabled; ctx->nr_no_switch_fast is stable. */
3815	if (local_read(&ctx->nr_no_switch_fast) \|\|
3816	local_read(&next_ctx->nr_no_switch_fast)) {
3817	/*
3818	* Must not swap out ctx when there's pending
3819	* events that rely on the ctx->task relation.
3820	*
3821	* Likewise, when a context contains inherit +
3822	* SAMPLE_READ events they should be switched
3823	* out using the slow path so that they are
3824	* treated as if they were distinct contexts.
3825	*/
3826	raw_spin_unlock(&next_ctx->lock);
3827	rcu_read_unlock();
3828	goto inside_switch;
3829	}
3830
3831	WRITE_ONCE(ctx->task, next);
3832	WRITE_ONCE(next_ctx->task, task);
3833
3834	perf_ctx_sched_task_cb(ctx, task, false);
3835
3836	perf_ctx_enable(ctx, 0);
3837
3838	/*
3839	* RCU_INIT_POINTER here is safe because we've not
3840	* modified the ctx and the above modification of
3841	* ctx->task is immaterial since this value is
3842	* always verified under ctx->lock which we're now
3843	* holding.
3844	*/
3845	RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx);
3846	RCU_INIT_POINTER(next->perf_event_ctxp, ctx);
3847
3848	do_switch = 0;
3849
3850	perf_event_sync_stat(ctx, next_ctx);
3851	}
3852	raw_spin_unlock(&next_ctx->lock);
3853	raw_spin_unlock(&ctx->lock);
3854	}
3855	unlock:
3856	rcu_read_unlock();
3857
3858	if (do_switch) {
3859	raw_spin_lock(&ctx->lock);
3860	perf_ctx_disable(ctx, 0);
3861
3862	inside_switch:
3863	perf_ctx_sched_task_cb(ctx, task, false);
3864	task_ctx_sched_out(ctx, NULL, EVENT_ALL);
3865
3866	perf_ctx_enable(ctx, 0);
3867	raw_spin_unlock(&ctx->lock);
3868	}
3869	}
3870
3871	static DEFINE_PER_CPU(struct list_head, sched_cb_list);
3872	static DEFINE_PER_CPU(int, perf_sched_cb_usages);
3873
3874	void perf_sched_cb_dec(struct pmu *pmu)
3875	{
3876	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);
3877
3878	this_cpu_dec(perf_sched_cb_usages);
3879	barrier();
3880
3881	if (!--cpc->sched_cb_usage)
3882	list_del(&cpc->sched_cb_entry);
3883	}
3884
3885
3886	void perf_sched_cb_inc(struct pmu *pmu)
3887	{
3888	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);
3889
3890	if (!cpc->sched_cb_usage++)
3891	list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
3892
3893	barrier();
3894	this_cpu_inc(perf_sched_cb_usages);
3895	}
3896
3897	/*
3898	* This function provides the context switch callback to the lower code
3899	* layer. It is invoked ONLY when the context switch callback is enabled.
3900	*
3901	* This callback is relevant even to per-cpu events; for example multi event
3902	* PEBS requires this to provide PID/TID information. This requires we flush
3903	* all queued PEBS records before we context switch to a new task.
3904	*/
3905	static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc,
3906	struct task_struct *task, bool sched_in)
3907	{
3908	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3909	struct pmu *pmu;
3910
3911	pmu = cpc->epc.pmu;
3912
3913	/* software PMUs will not have sched_task */
3914	if (WARN_ON_ONCE(!pmu->sched_task))
3915	return;
3916
3917	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
3918	perf_pmu_disable(pmu);
3919
3920	pmu->sched_task(&cpc->epc, task, sched_in);
3921
3922	perf_pmu_enable(pmu);
3923	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
3924	}
3925
3926	static void perf_pmu_sched_task(struct task_struct *prev,
3927	struct task_struct *next,
3928	bool sched_in)
3929	{
3930	struct perf_cpu_pmu_context cpc, cpc2;
3931
3932	if (prev == next)
3933	return;
3934
3935	list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {
3936	if (cpc->task_epc)
3937	continue;
3938
3939	__perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in);
3940	}
3941	}
3942
3943	static void perf_event_switch(struct task_struct *task,
3944	struct task_struct *next_prev, bool sched_in);
3945
3946	/*
3947	* Called from scheduler to remove the events of the current task,
3948	* with interrupts disabled.
3949	*
3950	* We stop each event and update the event value in event->count.
3951	*
3952	* This does not protect us against NMI, but disable()
3953	* sets the disabled bit in the control field of event _before_
3954	* accessing the event control register. If a NMI hits, then it will
3955	* not restart the event.
3956	*/
3957	void __perf_event_task_sched_out(struct task_struct *task,
3958	struct task_struct *next)
3959	{
3960	if (__this_cpu_read(perf_sched_cb_usages))
3961	perf_pmu_sched_task(task, next, false);
3962
3963	if (atomic_read(&nr_switch_events))
3964	perf_event_switch(task, next, false);
3965
3966	perf_event_context_sched_out(task, next);
3967
3968	/*
3969	* if cgroup events exist on this CPU, then we need
3970	* to check if we have to switch out PMU state.
3971	* cgroup event are system-wide mode only
3972	*/
3973	perf_cgroup_switch(next);
3974	}
3975
3976	static bool perf_less_group_idx(const void l, const void r, void __always_unused *args)
3977	{
3978	const struct perf_event le = (const struct perf_event **)l;
3979	const struct perf_event re = (const struct perf_event **)r;
3980
3981	return le->group_index < re->group_index;
3982	}
3983
3984	DEFINE_MIN_HEAP(struct perf_event *, perf_event_min_heap);
3985
3986	static const struct min_heap_callbacks perf_min_heap = {
3987	.less = perf_less_group_idx,
3988	.swp = NULL,
3989	};
3990
3991	static void __heap_add(struct perf_event_min_heap heap, struct perf_event event)
3992	{
3993	struct perf_event **itrs = heap->data;
3994
3995	if (event) {
3996	itrs[heap->nr] = event;
3997	heap->nr++;
3998	}
3999	}
4000
4001	static void __link_epc(struct perf_event_pmu_context *pmu_ctx)
4002	{
4003	struct perf_cpu_pmu_context *cpc;
4004
4005	if (!pmu_ctx->ctx->task)
4006	return;
4007
4008	cpc = this_cpc(pmu_ctx->pmu);
4009	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
4010	cpc->task_epc = pmu_ctx;
4011	}
4012
4013	static noinline int visit_groups_merge(struct perf_event_context *ctx,
4014	struct perf_event_groups *groups, int cpu,
4015	struct pmu *pmu,
4016	int (func)(struct perf_event , void *),
4017	void *data)
4018	{
4019	#ifdef CONFIG_CGROUP_PERF
4020	struct cgroup_subsys_state *css = NULL;
4021	#endif
4022	struct perf_cpu_context *cpuctx = NULL;
4023	/* Space for per CPU and/or any CPU event iterators. */
4024	struct perf_event *itrs[2];
4025	struct perf_event_min_heap event_heap;
4026	struct perf_event **evt;
4027	int ret;
4028
4029	if (pmu->filter && pmu->filter(pmu, cpu))
4030	return 0;
4031
4032	if (!ctx->task) {
4033	cpuctx = this_cpu_ptr(&perf_cpu_context);
4034	event_heap = (struct perf_event_min_heap){
4035	.data = cpuctx->heap,
4036	.nr = 0,
4037	.size = cpuctx->heap_size,
4038	};
4039
4040	lockdep_assert_held(&cpuctx->ctx.lock);
4041
4042	#ifdef CONFIG_CGROUP_PERF
4043	if (cpuctx->cgrp)
4044	css = &cpuctx->cgrp->css;
4045	#endif
4046	} else {
4047	event_heap = (struct perf_event_min_heap){
4048	.data = itrs,
4049	.nr = 0,
4050	.size = ARRAY_SIZE(itrs),
4051	};
4052	/* Events not within a CPU context may be on any CPU. */
4053	__heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL));
4054	}
4055	evt = event_heap.data;
4056
4057	__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, NULL));
4058
4059	#ifdef CONFIG_CGROUP_PERF
4060	for (; css; css = css->parent)
4061	__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup));
4062	#endif
4063
4064	if (event_heap.nr) {
4065	__link_epc((*evt)->pmu_ctx);
4066	perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu);
4067	}
4068
4069	min_heapify_all_inline(&event_heap, &perf_min_heap, NULL);
4070
4071	while (event_heap.nr) {
4072	ret = func(*evt, data);
4073	if (ret)
4074	return ret;
4075
4076	evt = perf_event_groups_next(evt, pmu);
4077	if (*evt)
4078	min_heap_sift_down_inline(&event_heap, 0, &perf_min_heap, NULL);
4079	else
4080	min_heap_pop_inline(&event_heap, &perf_min_heap, NULL);
4081	}
4082
4083	return 0;
4084	}
4085
4086	/*
4087	* Because the userpage is strictly per-event (there is no concept of context,
4088	* so there cannot be a context indirection), every userpage must be updated
4089	* when context time starts :-(
4090	*
4091	* IOW, we must not miss EVENT_TIME edges.
4092	*/
4093	static inline bool event_update_userpage(struct perf_event *event)
4094	{
4095	if (likely(!refcount_read(&event->mmap_count)))
4096	return false;
4097
4098	perf_event_update_time(event);
4099	perf_event_update_userpage(event);
4100
4101	return true;
4102	}
4103
4104	static inline void group_update_userpage(struct perf_event *group_event)
4105	{
4106	struct perf_event *event;
4107
4108	if (!event_update_userpage(group_event))
4109	return;
4110
4111	for_each_sibling_event(event, group_event)
4112	event_update_userpage(event);
4113	}
4114
4115	struct merge_sched_data {
4116	int can_add_hw;
4117	enum event_type_t event_type;
4118	};
4119
4120	static int merge_sched_in(struct perf_event event, void data)
4121	{
4122	struct perf_event_context *ctx = event->ctx;
4123	struct merge_sched_data *msd = data;
4124
4125	if (event->state <= PERF_EVENT_STATE_OFF)
4126	return 0;
4127
4128	if (!event_filter_match(event))
4129	return 0;
4130
4131	/*
4132	* Don't schedule in any host events from PMU with
4133	* PERF_PMU_CAP_MEDIATED_VPMU, while a guest is running.
4134	*/
4135	if (is_guest_mediated_pmu_loaded() &&
4136	event->pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU &&
4137	!(msd->event_type & EVENT_GUEST))
4138	return 0;
4139
4140	if (group_can_go_on(event, msd->can_add_hw)) {
4141	if (!group_sched_in(event, ctx))
4142	list_add_tail(&event->active_list, get_event_list(event));
4143	}
4144
4145	if (event->state == PERF_EVENT_STATE_INACTIVE) {
4146	msd->can_add_hw = 0;
4147	if (event->attr.pinned) {
4148	perf_cgroup_event_disable(event, ctx);
4149	perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
4150
4151	if (*perf_event_fasync(event))
4152	event->pending_kill = POLL_ERR;
4153
4154	event->pending_wakeup = 1;
4155	irq_work_queue(&event->pending_irq);
4156	} else {
4157	struct perf_cpu_pmu_context *cpc = this_cpc(event->pmu_ctx->pmu);
4158
4159	event->pmu_ctx->rotate_necessary = 1;
4160	perf_mux_hrtimer_restart(cpc);
4161	group_update_userpage(event);
4162	}
4163	}
4164
4165	return 0;
4166	}
4167
4168	static void pmu_groups_sched_in(struct perf_event_context *ctx,
4169	struct perf_event_groups *groups,
4170	struct pmu *pmu,
4171	enum event_type_t event_type)
4172	{
4173	struct merge_sched_data msd = {
4174	.can_add_hw = 1,
4175	.event_type = event_type,
4176	};
4177	visit_groups_merge(ctx, groups, smp_processor_id(), pmu,
4178	merge_sched_in, &msd);
4179	}
4180
4181	static void __pmu_ctx_sched_in(struct perf_event_pmu_context *pmu_ctx,
4182	enum event_type_t event_type)
4183	{
4184	struct perf_event_context *ctx = pmu_ctx->ctx;
4185
4186	if (event_type & EVENT_PINNED)
4187	pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu, event_type);
4188	if (event_type & EVENT_FLEXIBLE)
4189	pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu, event_type);
4190	}
4191
4192	static void
4193	ctx_sched_in(struct perf_event_context ctx, struct pmu pmu, enum event_type_t event_type)
4194	{
4195	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4196	enum event_type_t active_type = event_type & ~EVENT_FLAGS;
4197	struct perf_event_pmu_context *pmu_ctx;
4198	int is_active = ctx->is_active;
4199
4200	lockdep_assert_held(&ctx->lock);
4201
4202	if (likely(!ctx->nr_events))
4203	return;
4204
4205	if (!(is_active & EVENT_TIME)) {
4206	/* EVENT_TIME should be active while the guest runs */
4207	WARN_ON_ONCE(event_type & EVENT_GUEST);
4208	/* start ctx time */
4209	__update_context_time(ctx, false);
4210	perf_cgroup_set_timestamp(cpuctx, false);
4211	/*
4212	* CPU-release for the below ->is_active store,
4213	* see __load_acquire() in perf_event_time_now()
4214	*/
4215	barrier();
4216	}
4217
4218	ctx->is_active \|= active_type \| EVENT_TIME;
4219	if (ctx->task) {
4220	if (!(is_active & EVENT_ALL))
4221	cpuctx->task_ctx = ctx;
4222	else
4223	WARN_ON_ONCE(cpuctx->task_ctx != ctx);
4224	}
4225
4226	if (event_type & EVENT_GUEST) {
4227	/*
4228	* Schedule in the required exclude_guest events of PMU
4229	* with PERF_PMU_CAP_MEDIATED_VPMU.
4230	*/
4231	is_active = event_type & EVENT_ALL;
4232
4233	/*
4234	* Update ctx time to set the new start time for
4235	* the exclude_guest events.
4236	*/
4237	update_context_time(ctx);
4238	update_cgrp_time_from_cpuctx(cpuctx, false);
4239	barrier();
4240	} else {
4241	is_active ^= ctx->is_active; /* changed bits */
4242	}
4243
4244	/*
4245	* First go through the list and put on any pinned groups
4246	* in order to give them the best chance of going on.
4247	*/
4248	if (is_active & EVENT_PINNED) {
4249	for_each_epc(pmu_ctx, ctx, pmu, event_type)
4250	__pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED \| (event_type & EVENT_GUEST));
4251	}
4252
4253	/* Then walk through the lower prio flexible groups */
4254	if (is_active & EVENT_FLEXIBLE) {
4255	for_each_epc(pmu_ctx, ctx, pmu, event_type)
4256	__pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE \| (event_type & EVENT_GUEST));
4257	}
4258	}
4259
4260	static void perf_event_context_sched_in(struct task_struct *task)
4261	{
4262	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4263	struct perf_event_context *ctx;
4264
4265	rcu_read_lock();
4266	ctx = rcu_dereference(task->perf_event_ctxp);
4267	if (!ctx)
4268	goto rcu_unlock;
4269
4270	if (cpuctx->task_ctx == ctx) {
4271	perf_ctx_lock(cpuctx, ctx);
4272	perf_ctx_disable(ctx, 0);
4273
4274	perf_ctx_sched_task_cb(ctx, task, true);
4275
4276	perf_ctx_enable(ctx, 0);
4277	perf_ctx_unlock(cpuctx, ctx);
4278	goto rcu_unlock;
4279	}
4280
4281	perf_ctx_lock(cpuctx, ctx);
4282	/*
4283	* We must check ctx->nr_events while holding ctx->lock, such
4284	* that we serialize against perf_install_in_context().
4285	*/
4286	if (!ctx->nr_events)
4287	goto unlock;
4288
4289	perf_ctx_disable(ctx, 0);
4290	/*
4291	* We want to keep the following priority order:
4292	* cpu pinned (that don't need to move), task pinned,
4293	* cpu flexible, task flexible.
4294	*
4295	* However, if task's ctx is not carrying any pinned
4296	* events, no need to flip the cpuctx's events around.
4297	*/
4298	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {
4299	perf_ctx_disable(&cpuctx->ctx, 0);
4300	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE);
4301	}
4302
4303	perf_event_sched_in(cpuctx, ctx, NULL, 0);
4304
4305	perf_ctx_sched_task_cb(cpuctx->task_ctx, task, true);
4306
4307	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
4308	perf_ctx_enable(&cpuctx->ctx, 0);
4309
4310	perf_ctx_enable(ctx, 0);
4311
4312	unlock:
4313	perf_ctx_unlock(cpuctx, ctx);
4314	rcu_unlock:
4315	rcu_read_unlock();
4316	}
4317
4318	/*
4319	* Called from scheduler to add the events of the current task
4320	* with interrupts disabled.
4321	*
4322	* We restore the event value and then enable it.
4323	*
4324	* This does not protect us against NMI, but enable()
4325	* sets the enabled bit in the control field of event _before_
4326	* accessing the event control register. If a NMI hits, then it will
4327	* keep the event running.
4328	*/
4329	void __perf_event_task_sched_in(struct task_struct *prev,
4330	struct task_struct *task)
4331	{
4332	perf_event_context_sched_in(task);
4333
4334	if (atomic_read(&nr_switch_events))
4335	perf_event_switch(task, prev, true);
4336
4337	if (__this_cpu_read(perf_sched_cb_usages))
4338	perf_pmu_sched_task(prev, task, true);
4339	}
4340
4341	static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
4342	{
4343	u64 frequency = event->attr.sample_freq;
4344	u64 sec = NSEC_PER_SEC;
4345	u64 divisor, dividend;
4346
4347	int count_fls, nsec_fls, frequency_fls, sec_fls;
4348
4349	count_fls = fls64(count);
4350	nsec_fls = fls64(nsec);
4351	frequency_fls = fls64(frequency);
4352	sec_fls = 30;
4353
4354	/*
4355	* We got @count in @nsec, with a target of sample_freq HZ
4356	* the target period becomes:
4357	*
4358	* @count * 10^9
4359	* period = -------------------
4360	* @nsec * sample_freq
4361	*
4362	*/
4363
4364	/*
4365	* Reduce accuracy by one bit such that @a and @b converge
4366	* to a similar magnitude.
4367	*/
4368	#define REDUCE_FLS(a, b) \
4369	do { \
4370	if (a##_fls > b##_fls) { \
4371	a >>= 1; \
4372	a##_fls--; \
4373	} else { \
4374	b >>= 1; \
4375	b##_fls--; \
4376	} \
4377	} while (0)
4378
4379	/*
4380	* Reduce accuracy until either term fits in a u64, then proceed with
4381	* the other, so that finally we can do a u64/u64 division.
4382	*/
4383	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
4384	REDUCE_FLS(nsec, frequency);
4385	REDUCE_FLS(sec, count);
4386	}
4387
4388	if (count_fls + sec_fls > 64) {
4389	divisor = nsec * frequency;
4390
4391	while (count_fls + sec_fls > 64) {
4392	REDUCE_FLS(count, sec);
4393	divisor >>= 1;
4394	}
4395
4396	dividend = count * sec;
4397	} else {
4398	dividend = count * sec;
4399
4400	while (nsec_fls + frequency_fls > 64) {
4401	REDUCE_FLS(nsec, frequency);
4402	dividend >>= 1;
4403	}
4404
4405	divisor = nsec * frequency;
4406	}
4407
4408	if (!divisor)
4409	return dividend;
4410
4411	return div64_u64(dividend, divisor);
4412	}
4413
4414	static DEFINE_PER_CPU(int, perf_throttled_count);
4415	static DEFINE_PER_CPU(u64, perf_throttled_seq);
4416
4417	static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
4418	{
4419	struct hw_perf_event *hwc = &event->hw;
4420	s64 period, sample_period;
4421	s64 delta;
4422
4423	period = perf_calculate_period(event, nsec, count);
4424
4425	delta = (s64)(period - hwc->sample_period);
4426	if (delta >= 0)
4427	delta += 7;
4428	else
4429	delta -= 7;
4430	delta /= 8; /* low pass filter */
4431
4432	sample_period = hwc->sample_period + delta;
4433
4434	if (!sample_period)
4435	sample_period = 1;
4436
4437	hwc->sample_period = sample_period;
4438
4439	if (local64_read(&hwc->period_left) > 8*sample_period) {
4440	if (disable)
4441	event->pmu->stop(event, PERF_EF_UPDATE);
4442
4443	local64_set(&hwc->period_left, 0);
4444
4445	if (disable)
4446	event->pmu->start(event, PERF_EF_RELOAD);
4447	}
4448	}
4449
4450	static void perf_adjust_freq_unthr_events(struct list_head *event_list)
4451	{
4452	struct perf_event *event;
4453	struct hw_perf_event *hwc;
4454	u64 now, period = TICK_NSEC;
4455	s64 delta;
4456
4457	list_for_each_entry(event, event_list, active_list) {
4458	if (event->state != PERF_EVENT_STATE_ACTIVE)
4459	continue;
4460
4461	// XXX use visit thingy to avoid the -1,cpu match
4462	if (!event_filter_match(event))
4463	continue;
4464
4465	hwc = &event->hw;
4466
4467	if (hwc->interrupts == MAX_INTERRUPTS)
4468	perf_event_unthrottle_group(event, is_event_in_freq_mode(event));
4469
4470	if (!is_event_in_freq_mode(event))
4471	continue;
4472
4473	/*
4474	* stop the event and update event->count
4475	*/
4476	event->pmu->stop(event, PERF_EF_UPDATE);
4477
4478	now = local64_read(&event->count);
4479	delta = now - hwc->freq_count_stamp;
4480	hwc->freq_count_stamp = now;
4481
4482	/*
4483	* restart the event
4484	* reload only if value has changed
4485	* we have stopped the event so tell that
4486	* to perf_adjust_period() to avoid stopping it
4487	* twice.
4488	*/
4489	if (delta > 0)
4490	perf_adjust_period(event, period, delta, false);
4491
4492	event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
4493	}
4494	}
4495
4496	/*
4497	* combine freq adjustment with unthrottling to avoid two passes over the
4498	* events. At the same time, make sure, having freq events does not change
4499	* the rate of unthrottling as that would introduce bias.
4500	*/
4501	static void
4502	perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
4503	{
4504	struct perf_event_pmu_context *pmu_ctx;
4505
4506	/*
4507	* only need to iterate over all events iff:
4508	* - context have events in frequency mode (needs freq adjust)
4509	* - there are events to unthrottle on this cpu
4510	*/
4511	if (!(ctx->nr_freq \|\| unthrottle))
4512	return;
4513
4514	raw_spin_lock(&ctx->lock);
4515
4516	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
4517	if (!(pmu_ctx->nr_freq \|\| unthrottle))
4518	continue;
4519	if (!perf_pmu_ctx_is_active(pmu_ctx))
4520	continue;
4521	if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)
4522	continue;
4523
4524	perf_pmu_disable(pmu_ctx->pmu);
4525	perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);
4526	perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);
4527	perf_pmu_enable(pmu_ctx->pmu);
4528	}
4529
4530	raw_spin_unlock(&ctx->lock);
4531	}
4532
4533	/*
4534	* Move @event to the tail of the @ctx's elegible events.
4535	*/
4536	static void rotate_ctx(struct perf_event_context ctx, struct perf_event event)
4537	{
4538	/*
4539	* Rotate the first entry last of non-pinned groups. Rotation might be
4540	* disabled by the inheritance code.
4541	*/
4542	if (ctx->rotate_disable)
4543	return;
4544
4545	perf_event_groups_delete(&ctx->flexible_groups, event);
4546	perf_event_groups_insert(&ctx->flexible_groups, event);
4547	}
4548
4549	/* pick an event from the flexible_groups to rotate */
4550	static inline struct perf_event *
4551	ctx_event_to_rotate(struct perf_event_pmu_context *pmu_ctx)
4552	{
4553	struct perf_event *event;
4554	struct rb_node *node;
4555	struct rb_root *tree;
4556	struct __group_key key = {
4557	.pmu = pmu_ctx->pmu,
4558	};
4559
4560	/* pick the first active flexible event */
4561	event = list_first_entry_or_null(&pmu_ctx->flexible_active,
4562	struct perf_event, active_list);
4563	if (event)
4564	goto out;
4565
4566	/* if no active flexible event, pick the first event */
4567	tree = &pmu_ctx->ctx->flexible_groups.tree;
4568
4569	if (!pmu_ctx->ctx->task) {
4570	key.cpu = smp_processor_id();
4571
4572	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
4573	if (node)
4574	event = __node_2_pe(node);
4575	goto out;
4576	}
4577
4578	key.cpu = -1;
4579	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
4580	if (node) {
4581	event = __node_2_pe(node);
4582	goto out;
4583	}
4584
4585	key.cpu = smp_processor_id();
4586	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
4587	if (node)
4588	event = __node_2_pe(node);
4589
4590	out:
4591	/*
4592	* Unconditionally clear rotate_necessary; if ctx_flexible_sched_in()
4593	* finds there are unschedulable events, it will set it again.
4594	*/
4595	pmu_ctx->rotate_necessary = 0;
4596
4597	return event;
4598	}
4599
4600	static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc)
4601	{
4602	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4603	struct perf_event_pmu_context cpu_epc, task_epc = NULL;
4604	struct perf_event cpu_event = NULL, task_event = NULL;
4605	int cpu_rotate, task_rotate;
4606	struct pmu *pmu;
4607
4608	/*
4609	* Since we run this from IRQ context, nobody can install new
4610	* events, thus the event count values are stable.
4611	*/
4612
4613	cpu_epc = &cpc->epc;
4614	pmu = cpu_epc->pmu;
4615	task_epc = cpc->task_epc;
4616
4617	cpu_rotate = cpu_epc->rotate_necessary;
4618	task_rotate = task_epc ? task_epc->rotate_necessary : 0;
4619
4620	if (!(cpu_rotate \|\| task_rotate))
4621	return false;
4622
4623	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
4624	perf_pmu_disable(pmu);
4625
4626	if (task_rotate)
4627	task_event = ctx_event_to_rotate(task_epc);
4628	if (cpu_rotate)
4629	cpu_event = ctx_event_to_rotate(cpu_epc);
4630
4631	/*
4632	* As per the order given at ctx_resched() first 'pop' task flexible
4633	* and then, if needed CPU flexible.
4634	*/
4635	if (task_event \|\| (task_epc && cpu_event)) {
4636	update_context_time(task_epc->ctx);
4637	__pmu_ctx_sched_out(task_epc, EVENT_FLEXIBLE);
4638	}
4639
4640	if (cpu_event) {
4641	update_context_time(&cpuctx->ctx);
4642	update_cgrp_time_from_cpuctx(cpuctx, false);
4643	__pmu_ctx_sched_out(cpu_epc, EVENT_FLEXIBLE);
4644	rotate_ctx(&cpuctx->ctx, cpu_event);
4645	__pmu_ctx_sched_in(cpu_epc, EVENT_FLEXIBLE);
4646	}
4647
4648	if (task_event)
4649	rotate_ctx(task_epc->ctx, task_event);
4650
4651	if (task_event \|\| (task_epc && cpu_event))
4652	__pmu_ctx_sched_in(task_epc, EVENT_FLEXIBLE);
4653
4654	perf_pmu_enable(pmu);
4655	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
4656
4657	return true;
4658	}
4659
4660	void perf_event_task_tick(void)
4661	{
4662	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4663	struct perf_event_context *ctx;
4664	int throttled;
4665
4666	lockdep_assert_irqs_disabled();
4667
4668	__this_cpu_inc(perf_throttled_seq);
4669	throttled = __this_cpu_xchg(perf_throttled_count, 0);
4670	tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
4671
4672	perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled);
4673
4674	rcu_read_lock();
4675	ctx = rcu_dereference(current->perf_event_ctxp);
4676	if (ctx)
4677	perf_adjust_freq_unthr_context(ctx, !!throttled);
4678	rcu_read_unlock();
4679	}
4680
4681	static int event_enable_on_exec(struct perf_event *event,
4682	struct perf_event_context *ctx)
4683	{
4684	if (!event->attr.enable_on_exec)
4685	return 0;
4686
4687	event->attr.enable_on_exec = 0;
4688	if (event->state >= PERF_EVENT_STATE_INACTIVE)
4689	return 0;
4690
4691	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
4692
4693	return 1;
4694	}
4695
4696	/*
4697	* Enable all of a task's events that have been marked enable-on-exec.
4698	* This expects task == current.
4699	*/
4700	static void perf_event_enable_on_exec(struct perf_event_context *ctx)
4701	{
4702	struct perf_event_context *clone_ctx = NULL;
4703	enum event_type_t event_type = 0;
4704	struct perf_cpu_context *cpuctx;
4705	struct perf_event *event;
4706	unsigned long flags;
4707	int enabled = 0;
4708
4709	local_irq_save(flags);
4710	if (WARN_ON_ONCE(current->perf_event_ctxp != ctx))
4711	goto out;
4712
4713	if (!ctx->nr_events)
4714	goto out;
4715
4716	cpuctx = this_cpu_ptr(&perf_cpu_context);
4717	perf_ctx_lock(cpuctx, ctx);
4718	ctx_time_freeze(cpuctx, ctx);
4719
4720	list_for_each_entry(event, &ctx->event_list, event_entry) {
4721	enabled \|= event_enable_on_exec(event, ctx);
4722	event_type \|= get_event_type(event);
4723	}
4724
4725	/*
4726	* Unclone and reschedule this context if we enabled any event.
4727	*/
4728	if (enabled) {
4729	clone_ctx = unclone_ctx(ctx);
4730	ctx_resched(cpuctx, ctx, NULL, event_type);
4731	}
4732	perf_ctx_unlock(cpuctx, ctx);
4733
4734	out:
4735	local_irq_restore(flags);
4736
4737	if (clone_ctx)
4738	put_ctx(clone_ctx);
4739	}
4740
4741	static void perf_remove_from_owner(struct perf_event *event);
4742	static void perf_event_exit_event(struct perf_event *event,
4743	struct perf_event_context *ctx,
4744	struct task_struct *task,
4745	unsigned long detach_flags);
4746
4747	/*
4748	* Removes all events from the current task that have been marked
4749	* remove-on-exec, and feeds their values back to parent events.
4750	*/
4751	static void perf_event_remove_on_exec(struct perf_event_context *ctx)
4752	{
4753	struct perf_event_context *clone_ctx = NULL;
4754	struct perf_event event, next;
4755	unsigned long flags;
4756	bool modified = false;
4757
4758	mutex_lock(&ctx->mutex);
4759
4760	if (WARN_ON_ONCE(ctx->task != current))
4761	goto unlock;
4762
4763	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {
4764	if (!event->attr.remove_on_exec)
4765	continue;
4766
4767	if (!is_kernel_event(event))
4768	perf_remove_from_owner(event);
4769
4770	modified = true;
4771
4772	perf_event_exit_event(event, ctx, ctx->task, DETACH_GROUP);
4773	}
4774
4775	raw_spin_lock_irqsave(&ctx->lock, flags);
4776	if (modified)
4777	clone_ctx = unclone_ctx(ctx);
4778	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4779
4780	unlock:
4781	mutex_unlock(&ctx->mutex);
4782
4783	if (clone_ctx)
4784	put_ctx(clone_ctx);
4785	}
4786
4787	struct perf_read_data {
4788	struct perf_event *event;
4789	bool group;
4790	int ret;
4791	};
4792
4793	static inline const struct cpumask *perf_scope_cpu_topology_cpumask(unsigned int scope, int cpu);
4794
4795	static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)
4796	{
4797	int local_cpu = smp_processor_id();
4798	u16 local_pkg, event_pkg;
4799
4800	if ((unsigned)event_cpu >= nr_cpu_ids)
4801	return event_cpu;
4802
4803	if (event->group_caps & PERF_EV_CAP_READ_SCOPE) {
4804	const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(event->pmu->scope, event_cpu);
4805
4806	if (cpumask && cpumask_test_cpu(local_cpu, cpumask))
4807	return local_cpu;
4808	}
4809
4810	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {
4811	event_pkg = topology_physical_package_id(event_cpu);
4812	local_pkg = topology_physical_package_id(local_cpu);
4813
4814	if (event_pkg == local_pkg)
4815	return local_cpu;
4816	}
4817
4818	return event_cpu;
4819	}
4820
4821	/*
4822	* Cross CPU call to read the hardware event
4823	*/
4824	static void __perf_event_read(void *info)
4825	{
4826	struct perf_read_data *data = info;
4827	struct perf_event sub, event = data->event;
4828	struct perf_event_context *ctx = event->ctx;
4829	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4830	struct pmu *pmu;
4831
4832	/*
4833	* If this is a task context, we need to check whether it is
4834	* the current task context of this cpu. If not it has been
4835	* scheduled out before the smp call arrived. In that case
4836	* event->count would have been updated to a recent sample
4837	* when the event was scheduled out.
4838	*/
4839	if (ctx->task && cpuctx->task_ctx != ctx)
4840	return;
4841
4842	guard(raw_spinlock)(&ctx->lock);
4843	ctx_time_update_event(ctx, event);
4844
4845	perf_event_update_time(event);
4846	if (data->group)
4847	perf_event_update_sibling_time(event);
4848
4849	if (event->state != PERF_EVENT_STATE_ACTIVE)
4850	return;
4851
4852	if (!data->group) {
4853	perf_pmu_read(event);
4854	data->ret = 0;
4855	return;
4856	}
4857
4858	pmu = event->pmu_ctx->pmu;
4859	pmu->start_txn(pmu, PERF_PMU_TXN_READ);
4860
4861	perf_pmu_read(event);
4862	for_each_sibling_event(sub, event)
4863	perf_pmu_read(sub);
4864
4865	data->ret = pmu->commit_txn(pmu);
4866	}
4867
4868	static inline u64 perf_event_count(struct perf_event *event, bool self)
4869	{
4870	if (self)
4871	return local64_read(&event->count);
4872
4873	return local64_read(&event->count) + atomic64_read(&event->child_count);
4874	}
4875
4876	static void calc_timer_values(struct perf_event *event,
4877	u64 *now,
4878	u64 *enabled,
4879	u64 *running)
4880	{
4881	u64 ctx_time;
4882
4883	*now = perf_clock();
4884	ctx_time = perf_event_time_now(event, *now);
4885	__perf_update_times(event, ctx_time, enabled, running);
4886	}
4887
4888	/*
4889	* NMI-safe method to read a local event, that is an event that
4890	* is:
4891	* - either for the current task, or for this CPU
4892	* - does not have inherit set, for inherited task events
4893	* will not be local and we cannot read them atomically
4894	* - must not have a pmu::count method
4895	*/
4896	int perf_event_read_local(struct perf_event event, u64 value,
4897	u64 enabled, u64 running)
4898	{
4899	unsigned long flags;
4900	int event_oncpu;
4901	int event_cpu;
4902	int ret = 0;
4903
4904	/*
4905	* Disabling interrupts avoids all counter scheduling (context
4906	* switches, timer based rotation and IPIs).
4907	*/
4908	local_irq_save(flags);
4909
4910	/*
4911	* It must not be an event with inherit set, we cannot read
4912	* all child counters from atomic context.
4913	*/
4914	if (event->attr.inherit) {
4915	ret = -EOPNOTSUPP;
4916	goto out;
4917	}
4918
4919	/* If this is a per-task event, it must be for current */
4920	if ((event->attach_state & PERF_ATTACH_TASK) &&
4921	event->hw.target != current) {
4922	ret = -EINVAL;
4923	goto out;
4924	}
4925
4926	/*
4927	* Get the event CPU numbers, and adjust them to local if the event is
4928	* a per-package event that can be read locally
4929	*/
4930	event_oncpu = __perf_event_read_cpu(event, event->oncpu);
4931	event_cpu = __perf_event_read_cpu(event, event->cpu);
4932
4933	/* If this is a per-CPU event, it must be for this CPU */
4934	if (!(event->attach_state & PERF_ATTACH_TASK) &&
4935	event_cpu != smp_processor_id()) {
4936	ret = -EINVAL;
4937	goto out;
4938	}
4939
4940	/* If this is a pinned event it must be running on this CPU */
4941	if (event->attr.pinned && event_oncpu != smp_processor_id()) {
4942	ret = -EBUSY;
4943	goto out;
4944	}
4945
4946	/*
4947	* If the event is currently on this CPU, its either a per-task event,
4948	* or local to this CPU. Furthermore it means its ACTIVE (otherwise
4949	* oncpu == -1).
4950	*/
4951	if (event_oncpu == smp_processor_id())
4952	event->pmu->read(event);
4953
4954	*value = local64_read(&event->count);
4955	if (enabled \|\| running) {
4956	u64 __enabled, __running, __now;
4957
4958	calc_timer_values(event, &__now, &__enabled, &__running);
4959	if (enabled)
4960	*enabled = __enabled;
4961	if (running)
4962	*running = __running;
4963	}
4964	out:
4965	local_irq_restore(flags);
4966
4967	return ret;
4968	}
4969
4970	static int perf_event_read(struct perf_event *event, bool group)
4971	{
4972	enum perf_event_state state = READ_ONCE(event->state);
4973	int event_cpu, ret = 0;
4974
4975	/*
4976	* If event is enabled and currently active on a CPU, update the
4977	* value in the event structure:
4978	*/
4979	again:
4980	if (state == PERF_EVENT_STATE_ACTIVE) {
4981	struct perf_read_data data;
4982
4983	/*
4984	* Orders the ->state and ->oncpu loads such that if we see
4985	* ACTIVE we must also see the right ->oncpu.
4986	*
4987	* Matches the smp_wmb() from event_sched_in().
4988	*/
4989	smp_rmb();
4990
4991	event_cpu = READ_ONCE(event->oncpu);
4992	if ((unsigned)event_cpu >= nr_cpu_ids)
4993	return 0;
4994
4995	data = (struct perf_read_data){
4996	.event = event,
4997	.group = group,
4998	.ret = 0,
4999	};
5000
5001	preempt_disable();
5002	event_cpu = __perf_event_read_cpu(event, event_cpu);
5003
5004	/*
5005	* Purposely ignore the smp_call_function_single() return
5006	* value.
5007	*
5008	* If event_cpu isn't a valid CPU it means the event got
5009	* scheduled out and that will have updated the event count.
5010	*
5011	* Therefore, either way, we'll have an up-to-date event count
5012	* after this.
5013	*/
5014	(void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1);
5015	preempt_enable();
5016	ret = data.ret;
5017
5018	} else if (state == PERF_EVENT_STATE_INACTIVE) {
5019	struct perf_event_context *ctx = event->ctx;
5020	unsigned long flags;
5021
5022	raw_spin_lock_irqsave(&ctx->lock, flags);
5023	state = event->state;
5024	if (state != PERF_EVENT_STATE_INACTIVE) {
5025	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5026	goto again;
5027	}
5028
5029	/*
5030	* May read while context is not active (e.g., thread is
5031	* blocked), in that case we cannot update context time
5032	*/
5033	ctx_time_update_event(ctx, event);
5034
5035	perf_event_update_time(event);
5036	if (group)
5037	perf_event_update_sibling_time(event);
5038	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5039	}
5040
5041	return ret;
5042	}
5043
5044	/*
5045	* Initialize the perf_event context in a task_struct:
5046	*/
5047	static void __perf_event_init_context(struct perf_event_context *ctx)
5048	{
5049	raw_spin_lock_init(&ctx->lock);
5050	mutex_init(&ctx->mutex);
5051	INIT_LIST_HEAD(&ctx->pmu_ctx_list);
5052	perf_event_groups_init(&ctx->pinned_groups);
5053	perf_event_groups_init(&ctx->flexible_groups);
5054	INIT_LIST_HEAD(&ctx->event_list);
5055	refcount_set(&ctx->refcount, 1);
5056	}
5057
5058	static void
5059	__perf_init_event_pmu_context(struct perf_event_pmu_context epc, struct pmu pmu)
5060	{
5061	epc->pmu = pmu;
5062	INIT_LIST_HEAD(&epc->pmu_ctx_entry);
5063	INIT_LIST_HEAD(&epc->pinned_active);
5064	INIT_LIST_HEAD(&epc->flexible_active);
5065	atomic_set(&epc->refcount, 1);
5066	}
5067
5068	static struct perf_event_context *
5069	alloc_perf_context(struct task_struct *task)
5070	{
5071	struct perf_event_context *ctx;
5072
5073	ctx = kzalloc_obj(struct perf_event_context);
5074	if (!ctx)
5075	return NULL;
5076
5077	__perf_event_init_context(ctx);
5078	if (task)
5079	ctx->task = get_task_struct(task);
5080
5081	return ctx;
5082	}
5083
5084	static struct task_struct *
5085	find_lively_task_by_vpid(pid_t vpid)
5086	{
5087	struct task_struct *task;
5088
5089	rcu_read_lock();
5090	if (!vpid)
5091	task = current;
5092	else
5093	task = find_task_by_vpid(vpid);
5094	if (task)
5095	get_task_struct(task);
5096	rcu_read_unlock();
5097
5098	if (!task)
5099	return ERR_PTR(-ESRCH);
5100
5101	return task;
5102	}
5103
5104	/*
5105	* Returns a matching context with refcount and pincount.
5106	*/
5107	static struct perf_event_context *
5108	find_get_context(struct task_struct task, struct perf_event event)
5109	{
5110	struct perf_event_context ctx, clone_ctx = NULL;
5111	struct perf_cpu_context *cpuctx;
5112	unsigned long flags;
5113	int err;
5114
5115	if (!task) {
5116	/* Must be root to operate on a CPU event: */
5117	err = perf_allow_cpu();
5118	if (err)
5119	return ERR_PTR(err);
5120
5121	cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
5122	ctx = &cpuctx->ctx;
5123	get_ctx(ctx);
5124	raw_spin_lock_irqsave(&ctx->lock, flags);
5125	++ctx->pin_count;
5126	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5127
5128	return ctx;
5129	}
5130
5131	err = -EINVAL;
5132	retry:
5133	ctx = perf_lock_task_context(task, &flags);
5134	if (ctx) {
5135	clone_ctx = unclone_ctx(ctx);
5136	++ctx->pin_count;
5137
5138	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5139
5140	if (clone_ctx)
5141	put_ctx(clone_ctx);
5142	} else {
5143	ctx = alloc_perf_context(task);
5144	err = -ENOMEM;
5145	if (!ctx)
5146	goto errout;
5147
5148	err = 0;
5149	mutex_lock(&task->perf_event_mutex);
5150	/*
5151	* If it has already passed perf_event_exit_task().
5152	* we must see PF_EXITING, it takes this mutex too.
5153	*/
5154	if (task->flags & PF_EXITING)
5155	err = -ESRCH;
5156	else if (task->perf_event_ctxp)
5157	err = -EAGAIN;
5158	else {
5159	get_ctx(ctx);
5160	++ctx->pin_count;
5161	rcu_assign_pointer(task->perf_event_ctxp, ctx);
5162	}
5163	mutex_unlock(&task->perf_event_mutex);
5164
5165	if (unlikely(err)) {
5166	put_ctx(ctx);
5167
5168	if (err == -EAGAIN)
5169	goto retry;
5170	goto errout;
5171	}
5172	}
5173
5174	return ctx;
5175
5176	errout:
5177	return ERR_PTR(err);
5178	}
5179
5180	static struct perf_event_pmu_context *
5181	find_get_pmu_context(struct pmu pmu, struct perf_event_context ctx,
5182	struct perf_event *event)
5183	{
5184	struct perf_event_pmu_context new = NULL, pos = NULL, *epc;
5185
5186	if (!ctx->task) {
5187	/*
5188	* perf_pmu_migrate_context() / __perf_pmu_install_event()
5189	* relies on the fact that find_get_pmu_context() cannot fail
5190	* for CPU contexts.
5191	*/
5192	struct perf_cpu_pmu_context *cpc;
5193
5194	cpc = *per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);
5195	epc = &cpc->epc;
5196	raw_spin_lock_irq(&ctx->lock);
5197	if (!epc->ctx) {
5198	/*
5199	* One extra reference for the pmu; see perf_pmu_free().
5200	*/
5201	atomic_set(&epc->refcount, 2);
5202	epc->embedded = 1;
5203	list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);
5204	epc->ctx = ctx;
5205	} else {
5206	WARN_ON_ONCE(epc->ctx != ctx);
5207	atomic_inc(&epc->refcount);
5208	}
5209	raw_spin_unlock_irq(&ctx->lock);
5210	return epc;
5211	}
5212
5213	new = kzalloc_obj(*epc);
5214	if (!new)
5215	return ERR_PTR(-ENOMEM);
5216
5217	__perf_init_event_pmu_context(new, pmu);
5218
5219	/*
5220	* XXX
5221	*
5222	* lockdep_assert_held(&ctx->mutex);
5223	*
5224	* can't because perf_event_init_task() doesn't actually hold the
5225	* child_ctx->mutex.
5226	*/
5227
5228	raw_spin_lock_irq(&ctx->lock);
5229	list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) {
5230	if (epc->pmu == pmu) {
5231	WARN_ON_ONCE(epc->ctx != ctx);
5232	atomic_inc(&epc->refcount);
5233	goto found_epc;
5234	}
5235	/* Make sure the pmu_ctx_list is sorted by PMU type: */
5236	if (!pos && epc->pmu->type > pmu->type)
5237	pos = epc;
5238	}
5239
5240	epc = new;
5241	new = NULL;
5242
5243	if (!pos)
5244	list_add_tail(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);
5245	else
5246	list_add(&epc->pmu_ctx_entry, pos->pmu_ctx_entry.prev);
5247
5248	epc->ctx = ctx;
5249
5250	found_epc:
5251	raw_spin_unlock_irq(&ctx->lock);
5252	kfree(new);
5253
5254	return epc;
5255	}
5256
5257	static void get_pmu_ctx(struct perf_event_pmu_context *epc)
5258	{
5259	WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount));
5260	}
5261
5262	static void free_cpc_rcu(struct rcu_head *head)
5263	{
5264	struct perf_cpu_pmu_context *cpc =
5265	container_of(head, typeof(*cpc), epc.rcu_head);
5266
5267	kfree(cpc);
5268	}
5269
5270	static void free_epc_rcu(struct rcu_head *head)
5271	{
5272	struct perf_event_pmu_context epc = container_of(head, typeof(epc), rcu_head);
5273
5274	kfree(epc);
5275	}
5276
5277	static void put_pmu_ctx(struct perf_event_pmu_context *epc)
5278	{
5279	struct perf_event_context *ctx = epc->ctx;
5280	unsigned long flags;
5281
5282	/*
5283	* XXX
5284	*
5285	* lockdep_assert_held(&ctx->mutex);
5286	*
5287	* can't because of the call-site in _free_event()/put_event()
5288	* which isn't always called under ctx->mutex.
5289	*/
5290	if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags))
5291	return;
5292
5293	WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry));
5294
5295	list_del_init(&epc->pmu_ctx_entry);
5296	epc->ctx = NULL;
5297
5298	WARN_ON_ONCE(!list_empty(&epc->pinned_active));
5299	WARN_ON_ONCE(!list_empty(&epc->flexible_active));
5300
5301	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5302
5303	if (epc->embedded) {
5304	call_rcu(&epc->rcu_head, free_cpc_rcu);
5305	return;
5306	}
5307
5308	call_rcu(&epc->rcu_head, free_epc_rcu);
5309	}
5310
5311	static void perf_event_free_filter(struct perf_event *event);
5312
5313	static void free_event_rcu(struct rcu_head *head)
5314	{
5315	struct perf_event event = container_of(head, typeof(event), rcu_head);
5316
5317	if (event->ns)
5318	put_pid_ns(event->ns);
5319	perf_event_free_filter(event);
5320	kfree(event->addr_filter_ranges);
5321	kmem_cache_free(perf_event_cache, event);
5322	}
5323
5324	static void ring_buffer_attach(struct perf_event *event,
5325	struct perf_buffer *rb);
5326
5327	static void detach_sb_event(struct perf_event *event)
5328	{
5329	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
5330
5331	raw_spin_lock(&pel->lock);
5332	list_del_rcu(&event->sb_list);
5333	raw_spin_unlock(&pel->lock);
5334	}
5335
5336	static bool is_sb_event(struct perf_event *event)
5337	{
5338	struct perf_event_attr *attr = &event->attr;
5339
5340	if (event->parent)
5341	return false;
5342
5343	if (event->attach_state & PERF_ATTACH_TASK)
5344	return false;
5345
5346	if (attr->mmap \|\| attr->mmap_data \|\| attr->mmap2 \|\|
5347	attr->comm \|\| attr->comm_exec \|\|
5348	attr->task \|\| attr->ksymbol \|\|
5349	attr->context_switch \|\| attr->text_poke \|\|
5350	attr->bpf_event)
5351	return true;
5352
5353	return false;
5354	}
5355
5356	static void unaccount_pmu_sb_event(struct perf_event *event)
5357	{
5358	if (is_sb_event(event))
5359	detach_sb_event(event);
5360	}
5361
5362	#ifdef CONFIG_NO_HZ_FULL
5363	static DEFINE_SPINLOCK(nr_freq_lock);
5364	#endif
5365
5366	static void unaccount_freq_event_nohz(void)
5367	{
5368	#ifdef CONFIG_NO_HZ_FULL
5369	spin_lock(&nr_freq_lock);
5370	if (atomic_dec_and_test(&nr_freq_events))
5371	tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
5372	spin_unlock(&nr_freq_lock);
5373	#endif
5374	}
5375
5376	static void unaccount_freq_event(void)
5377	{
5378	if (tick_nohz_full_enabled())
5379	unaccount_freq_event_nohz();
5380	else
5381	atomic_dec(&nr_freq_events);
5382	}
5383
5384
5385	static struct perf_ctx_data *
5386	alloc_perf_ctx_data(struct kmem_cache *ctx_cache, bool global, gfp_t gfp_flags)
5387	{
5388	struct perf_ctx_data *cd;
5389
5390	cd = kzalloc_obj(*cd, gfp_flags);
5391	if (!cd)
5392	return NULL;
5393
5394	cd->data = kmem_cache_zalloc(ctx_cache, gfp_flags);
5395	if (!cd->data) {
5396	kfree(cd);
5397	return NULL;
5398	}
5399
5400	cd->global = global;
5401	cd->ctx_cache = ctx_cache;
5402	refcount_set(&cd->refcount, 1);
5403
5404	return cd;
5405	}
5406
5407	static void free_perf_ctx_data(struct perf_ctx_data *cd)
5408	{
5409	kmem_cache_free(cd->ctx_cache, cd->data);
5410	kfree(cd);
5411	}
5412
5413	static void __free_perf_ctx_data_rcu(struct rcu_head *rcu_head)
5414	{
5415	struct perf_ctx_data *cd;
5416
5417	cd = container_of(rcu_head, struct perf_ctx_data, rcu_head);
5418	free_perf_ctx_data(cd);
5419	}
5420
5421	static inline void perf_free_ctx_data_rcu(struct perf_ctx_data *cd)
5422	{
5423	call_rcu(&cd->rcu_head, __free_perf_ctx_data_rcu);
5424	}
5425
5426	static int
5427	attach_task_ctx_data(struct task_struct task, struct kmem_cache ctx_cache,
5428	bool global, gfp_t gfp_flags)
5429	{
5430	struct perf_ctx_data cd, old = NULL;
5431
5432	cd = alloc_perf_ctx_data(ctx_cache, global, gfp_flags);
5433	if (!cd)
5434	return -ENOMEM;
5435
5436	for (;;) {
5437	if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) {
5438	if (old)
5439	perf_free_ctx_data_rcu(old);
5440	/*
5441	* Above try_cmpxchg() pairs with try_cmpxchg() from
5442	* detach_task_ctx_data() such that
5443	* if we race with perf_event_exit_task(), we must
5444	* observe PF_EXITING.
5445	*/
5446	if (task->flags & PF_EXITING) {
5447	/* detach_task_ctx_data() may free it already */
5448	if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL))
5449	perf_free_ctx_data_rcu(cd);
5450	}
5451	return 0;
5452	}
5453
5454	if (!old) {
5455	/*
5456	* After seeing a dead @old, we raced with
5457	* removal and lost, try again to install @cd.
5458	*/
5459	continue;
5460	}
5461
5462	if (refcount_inc_not_zero(&old->refcount)) {
5463	if (global)
5464	old->global = true;
5465	free_perf_ctx_data(cd); /* unused */
5466	return 0;
5467	}
5468
5469	/*
5470	* @old is a dead object, refcount==0 is stable, try and
5471	* replace it with @cd.
5472	*/
5473	}
5474	return 0;
5475	}
5476
5477	static void __detach_global_ctx_data(void);
5478	DEFINE_STATIC_PERCPU_RWSEM(global_ctx_data_rwsem);
5479	static refcount_t global_ctx_data_ref;
5480
5481	static int
5482	attach_global_ctx_data(struct kmem_cache *ctx_cache)
5483	{
5484	struct task_struct g, p;
5485	struct perf_ctx_data *cd;
5486	int ret;
5487
5488	if (refcount_inc_not_zero(&global_ctx_data_ref))
5489	return 0;
5490
5491	guard(percpu_write)(&global_ctx_data_rwsem);
5492	if (refcount_inc_not_zero(&global_ctx_data_ref))
5493	return 0;
5494	again:
5495	/* Allocate everything */
5496	scoped_guard (rcu) {
5497	for_each_process_thread(g, p) {
5498	if (p->flags & PF_EXITING)
5499	continue;
5500	cd = rcu_dereference(p->perf_ctx_data);
5501	if (cd && !cd->global) {
5502	cd->global = 1;
5503	if (!refcount_inc_not_zero(&cd->refcount))
5504	cd = NULL;
5505	}
5506	if (!cd) {
5507	/*
5508	* Try to allocate context quickly before
5509	* traversing the whole thread list again.
5510	*/
5511	if (!attach_task_ctx_data(p, ctx_cache, true, GFP_NOWAIT))
5512	continue;
5513	get_task_struct(p);
5514	goto alloc;
5515	}
5516	}
5517	}
5518
5519	refcount_set(&global_ctx_data_ref, 1);
5520
5521	return 0;
5522	alloc:
5523	ret = attach_task_ctx_data(p, ctx_cache, true, GFP_KERNEL);
5524	put_task_struct(p);
5525	if (ret) {
5526	__detach_global_ctx_data();
5527	return ret;
5528	}
5529	goto again;
5530	}
5531
5532	static int
5533	attach_perf_ctx_data(struct perf_event *event)
5534	{
5535	struct task_struct *task = event->hw.target;
5536	struct kmem_cache *ctx_cache = event->pmu->task_ctx_cache;
5537	int ret;
5538
5539	if (!ctx_cache)
5540	return -ENOMEM;
5541
5542	if (task)
5543	return attach_task_ctx_data(task, ctx_cache, false, GFP_KERNEL);
5544
5545	ret = attach_global_ctx_data(ctx_cache);
5546	if (ret)
5547	return ret;
5548
5549	event->attach_state \|= PERF_ATTACH_GLOBAL_DATA;
5550	return 0;
5551	}
5552
5553	static void
5554	detach_task_ctx_data(struct task_struct *p)
5555	{
5556	struct perf_ctx_data *cd;
5557
5558	scoped_guard (rcu) {
5559	cd = rcu_dereference(p->perf_ctx_data);
5560	if (!cd \|\| !refcount_dec_and_test(&cd->refcount))
5561	return;
5562	}
5563
5564	/*
5565	* The old ctx_data may be lost because of the race.
5566	* Nothing is required to do for the case.
5567	* See attach_task_ctx_data().
5568	*/
5569	if (try_cmpxchg((struct perf_ctx_data **)&p->perf_ctx_data, &cd, NULL))
5570	perf_free_ctx_data_rcu(cd);
5571	}
5572
5573	static void __detach_global_ctx_data(void)
5574	{
5575	struct task_struct g, p;
5576	struct perf_ctx_data *cd;
5577
5578	scoped_guard (rcu) {
5579	for_each_process_thread(g, p) {
5580	cd = rcu_dereference(p->perf_ctx_data);
5581	if (cd && cd->global) {
5582	cd->global = 0;
5583	detach_task_ctx_data(p);
5584	}
5585	}
5586	}
5587	}
5588
5589	static void detach_global_ctx_data(void)
5590	{
5591	if (refcount_dec_not_one(&global_ctx_data_ref))
5592	return;
5593
5594	guard(percpu_write)(&global_ctx_data_rwsem);
5595	if (!refcount_dec_and_test(&global_ctx_data_ref))
5596	return;
5597
5598	/* remove everything */
5599	__detach_global_ctx_data();
5600	}
5601
5602	static void detach_perf_ctx_data(struct perf_event *event)
5603	{
5604	struct task_struct *task = event->hw.target;
5605
5606	event->attach_state &= ~PERF_ATTACH_TASK_DATA;
5607
5608	if (task)
5609	return detach_task_ctx_data(task);
5610
5611	if (event->attach_state & PERF_ATTACH_GLOBAL_DATA) {
5612	detach_global_ctx_data();
5613	event->attach_state &= ~PERF_ATTACH_GLOBAL_DATA;
5614	}
5615	}
5616
5617	static void unaccount_event(struct perf_event *event)
5618	{
5619	bool dec = false;
5620
5621	if (event->parent)
5622	return;
5623
5624	if (event->attach_state & (PERF_ATTACH_TASK \| PERF_ATTACH_SCHED_CB))
5625	dec = true;
5626	if (event->attr.mmap \|\| event->attr.mmap_data)
5627	atomic_dec(&nr_mmap_events);
5628	if (event->attr.build_id)
5629	atomic_dec(&nr_build_id_events);
5630	if (event->attr.comm)
5631	atomic_dec(&nr_comm_events);
5632	if (event->attr.namespaces)
5633	atomic_dec(&nr_namespaces_events);
5634	if (event->attr.cgroup)
5635	atomic_dec(&nr_cgroup_events);
5636	if (event->attr.task)
5637	atomic_dec(&nr_task_events);
5638	if (event->attr.freq)
5639	unaccount_freq_event();
5640	if (event->attr.context_switch) {
5641	dec = true;
5642	atomic_dec(&nr_switch_events);
5643	}
5644	if (is_cgroup_event(event))
5645	dec = true;
5646	if (has_branch_stack(event))
5647	dec = true;
5648	if (event->attr.ksymbol)
5649	atomic_dec(&nr_ksymbol_events);
5650	if (event->attr.bpf_event)
5651	atomic_dec(&nr_bpf_events);
5652	if (event->attr.text_poke)
5653	atomic_dec(&nr_text_poke_events);
5654
5655	if (dec) {
5656	if (!atomic_add_unless(&perf_sched_count, -1, 1))
5657	schedule_delayed_work(&perf_sched_work, HZ);
5658	}
5659
5660	unaccount_pmu_sb_event(event);
5661	}
5662
5663	static void perf_sched_delayed(struct work_struct *work)
5664	{
5665	mutex_lock(&perf_sched_mutex);
5666	if (atomic_dec_and_test(&perf_sched_count))
5667	static_branch_disable(&perf_sched_events);
5668	mutex_unlock(&perf_sched_mutex);
5669	}
5670
5671	/*
5672	* The following implement mutual exclusion of events on "exclusive" pmus
5673	* (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
5674	* at a time, so we disallow creating events that might conflict, namely:
5675	*
5676	* 1) cpu-wide events in the presence of per-task events,
5677	* 2) per-task events in the presence of cpu-wide events,
5678	* 3) two matching events on the same perf_event_context.
5679	*
5680	* The former two cases are handled in the allocation path (perf_event_alloc(),
5681	* _free_event()), the latter -- before the first perf_install_in_context().
5682	*/
5683	static int exclusive_event_init(struct perf_event *event)
5684	{
5685	struct pmu *pmu = event->pmu;
5686
5687	if (!is_exclusive_pmu(pmu))
5688	return 0;
5689
5690	/*
5691	* Prevent co-existence of per-task and cpu-wide events on the
5692	* same exclusive pmu.
5693	*
5694	* Negative pmu::exclusive_cnt means there are cpu-wide
5695	* events on this "exclusive" pmu, positive means there are
5696	* per-task events.
5697	*
5698	* Since this is called in perf_event_alloc() path, event::ctx
5699	* doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
5700	* to mean "per-task event", because unlike other attach states it
5701	* never gets cleared.
5702	*/
5703	if (event->attach_state & PERF_ATTACH_TASK) {
5704	if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
5705	return -EBUSY;
5706	} else {
5707	if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
5708	return -EBUSY;
5709	}
5710
5711	event->attach_state \|= PERF_ATTACH_EXCLUSIVE;
5712
5713	return 0;
5714	}
5715
5716	static void exclusive_event_destroy(struct perf_event *event)
5717	{
5718	struct pmu *pmu = event->pmu;
5719
5720	/* see comment in exclusive_event_init() */
5721	if (event->attach_state & PERF_ATTACH_TASK)
5722	atomic_dec(&pmu->exclusive_cnt);
5723	else
5724	atomic_inc(&pmu->exclusive_cnt);
5725
5726	event->attach_state &= ~PERF_ATTACH_EXCLUSIVE;
5727	}
5728
5729	static bool exclusive_event_match(struct perf_event e1, struct perf_event e2)
5730	{
5731	if ((e1->pmu == e2->pmu) &&
5732	(e1->cpu == e2->cpu \|\|
5733	e1->cpu == -1 \|\|
5734	e2->cpu == -1))
5735	return true;
5736	return false;
5737	}
5738
5739	static bool exclusive_event_installable(struct perf_event *event,
5740	struct perf_event_context *ctx)
5741	{
5742	struct perf_event *iter_event;
5743	struct pmu *pmu = event->pmu;
5744
5745	lockdep_assert_held(&ctx->mutex);
5746
5747	if (!is_exclusive_pmu(pmu))
5748	return true;
5749
5750	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
5751	if (exclusive_event_match(iter_event, event))
5752	return false;
5753	}
5754
5755	return true;
5756	}
5757
5758	static void perf_free_addr_filters(struct perf_event *event);
5759
5760	/* vs perf_event_alloc() error */
5761	static void __free_event(struct perf_event *event)
5762	{
5763	struct pmu *pmu = event->pmu;
5764
5765	security_perf_event_free(event);
5766
5767	if (event->attach_state & PERF_ATTACH_CALLCHAIN)
5768	put_callchain_buffers();
5769
5770	if (event->attach_state & PERF_ATTACH_EXCLUSIVE)
5771	exclusive_event_destroy(event);
5772
5773	if (is_cgroup_event(event))
5774	perf_detach_cgroup(event);
5775
5776	if (event->attach_state & PERF_ATTACH_TASK_DATA)
5777	detach_perf_ctx_data(event);
5778
5779	if (event->destroy)
5780	event->destroy(event);
5781
5782	/*
5783	* Must be after ->destroy(), due to uprobe_perf_close() using
5784	* hw.target.
5785	*/
5786	if (event->hw.target)
5787	put_task_struct(event->hw.target);
5788
5789	if (event->pmu_ctx) {
5790	/*
5791	* put_pmu_ctx() needs an event->ctx reference, because of
5792	* epc->ctx.
5793	*/
5794	WARN_ON_ONCE(!pmu);
5795	WARN_ON_ONCE(!event->ctx);
5796	WARN_ON_ONCE(event->pmu_ctx->ctx != event->ctx);
5797	put_pmu_ctx(event->pmu_ctx);
5798	}
5799
5800	/*
5801	* perf_event_free_task() relies on put_ctx() being 'last', in
5802	* particular all task references must be cleaned up.
5803	*/
5804	if (event->ctx)
5805	put_ctx(event->ctx);
5806
5807	if (pmu) {
5808	module_put(pmu->module);
5809	scoped_guard (spinlock, &pmu->events_lock) {
5810	list_del(&event->pmu_list);
5811	wake_up_var(pmu);
5812	}
5813	}
5814
5815	call_rcu(&event->rcu_head, free_event_rcu);
5816	}
5817
5818	static void mediated_pmu_unaccount_event(struct perf_event *event);
5819
5820	DEFINE_FREE(__free_event, struct perf_event *, if (_T) __free_event(_T))
5821
5822	/* vs perf_event_alloc() success */
5823	static void _free_event(struct perf_event *event)
5824	{
5825	irq_work_sync(&event->pending_irq);
5826	irq_work_sync(&event->pending_disable_irq);
5827
5828	unaccount_event(event);
5829	mediated_pmu_unaccount_event(event);
5830
5831	if (event->rb) {
5832	/*
5833	* Can happen when we close an event with re-directed output.
5834	*
5835	* Since we have a 0 refcount, perf_mmap_close() will skip
5836	* over us; possibly making our ring_buffer_put() the last.
5837	*/
5838	mutex_lock(&event->mmap_mutex);
5839	ring_buffer_attach(event, NULL);
5840	mutex_unlock(&event->mmap_mutex);
5841	}
5842
5843	perf_event_free_bpf_prog(event);
5844	perf_free_addr_filters(event);
5845
5846	__free_event(event);
5847	}
5848
5849	/*
5850	* Used to free events which have a known refcount of 1, such as in error paths
5851	* of inherited events.
5852	*/
5853	static void free_event(struct perf_event *event)
5854	{
5855	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
5856	"unexpected event refcount: %ld; ptr=%p\n",
5857	atomic_long_read(&event->refcount), event)) {
5858	/* leak to avoid use-after-free */
5859	return;
5860	}
5861
5862	_free_event(event);
5863	}
5864
5865	/*
5866	* Remove user event from the owner task.
5867	*/
5868	static void perf_remove_from_owner(struct perf_event *event)
5869	{
5870	struct task_struct *owner;
5871
5872	rcu_read_lock();
5873	/*
5874	* Matches the smp_store_release() in perf_event_exit_task(). If we
5875	* observe !owner it means the list deletion is complete and we can
5876	* indeed free this event, otherwise we need to serialize on
5877	* owner->perf_event_mutex.
5878	*/
5879	owner = READ_ONCE(event->owner);
5880	if (owner) {
5881	/*
5882	* Since delayed_put_task_struct() also drops the last
5883	* task reference we can safely take a new reference
5884	* while holding the rcu_read_lock().
5885	*/
5886	get_task_struct(owner);
5887	}
5888	rcu_read_unlock();
5889
5890	if (owner) {
5891	/*
5892	* If we're here through perf_event_exit_task() we're already
5893	* holding ctx->mutex which would be an inversion wrt. the
5894	* normal lock order.
5895	*
5896	* However we can safely take this lock because its the child
5897	* ctx->mutex.
5898	*/
5899	mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
5900
5901	/*
5902	* We have to re-check the event->owner field, if it is cleared
5903	* we raced with perf_event_exit_task(), acquiring the mutex
5904	* ensured they're done, and we can proceed with freeing the
5905	* event.
5906	*/
5907	if (event->owner) {
5908	list_del_init(&event->owner_entry);
5909	smp_store_release(&event->owner, NULL);
5910	}
5911	mutex_unlock(&owner->perf_event_mutex);
5912	put_task_struct(owner);
5913	}
5914	}
5915
5916	static void put_event(struct perf_event *event)
5917	{
5918	struct perf_event *parent;
5919
5920	if (!atomic_long_dec_and_test(&event->refcount))
5921	return;
5922
5923	parent = event->parent;
5924	_free_event(event);
5925
5926	/* Matches the refcount bump in inherit_event() */
5927	if (parent)
5928	put_event(parent);
5929	}
5930
5931	/*
5932	* Kill an event dead; while event:refcount will preserve the event
5933	* object, it will not preserve its functionality. Once the last 'user'
5934	* gives up the object, we'll destroy the thing.
5935	*/
5936	int perf_event_release_kernel(struct perf_event *event)
5937	{
5938	struct perf_event_context *ctx = event->ctx;
5939	struct perf_event child, tmp;
5940
5941	/*
5942	* If we got here through err_alloc: free_event(event); we will not
5943	* have attached to a context yet.
5944	*/
5945	if (!ctx) {
5946	WARN_ON_ONCE(event->attach_state &
5947	(PERF_ATTACH_CONTEXT\|PERF_ATTACH_GROUP));
5948	goto no_ctx;
5949	}
5950
5951	if (!is_kernel_event(event))
5952	perf_remove_from_owner(event);
5953
5954	ctx = perf_event_ctx_lock(event);
5955	WARN_ON_ONCE(ctx->parent_ctx);
5956
5957	/*
5958	* Mark this event as STATE_DEAD, there is no external reference to it
5959	* anymore.
5960	*
5961	* Anybody acquiring event->child_mutex after the below loop _must_
5962	* also see this, most importantly inherit_event() which will avoid
5963	* placing more children on the list.
5964	*
5965	* Thus this guarantees that we will in fact observe and kill _ALL_
5966	* child events.
5967	*/
5968	if (event->state > PERF_EVENT_STATE_REVOKED) {
5969	perf_remove_from_context(event, DETACH_GROUP\|DETACH_DEAD);
5970	} else {
5971	event->state = PERF_EVENT_STATE_DEAD;
5972	}
5973
5974	perf_event_ctx_unlock(event, ctx);
5975
5976	again:
5977	mutex_lock(&event->child_mutex);
5978	list_for_each_entry(child, &event->child_list, child_list) {
5979	/*
5980	* Cannot change, child events are not migrated, see the
5981	* comment with perf_event_ctx_lock_nested().
5982	*/
5983	ctx = READ_ONCE(child->ctx);
5984	/*
5985	* Since child_mutex nests inside ctx::mutex, we must jump
5986	* through hoops. We start by grabbing a reference on the ctx.
5987	*
5988	* Since the event cannot get freed while we hold the
5989	* child_mutex, the context must also exist and have a !0
5990	* reference count.
5991	*/
5992	get_ctx(ctx);
5993
5994	/*
5995	* Now that we have a ctx ref, we can drop child_mutex, and
5996	* acquire ctx::mutex without fear of it going away. Then we
5997	* can re-acquire child_mutex.
5998	*/
5999	mutex_unlock(&event->child_mutex);
6000	mutex_lock(&ctx->mutex);
6001	mutex_lock(&event->child_mutex);
6002
6003	/*
6004	* Now that we hold ctx::mutex and child_mutex, revalidate our
6005	* state, if child is still the first entry, it didn't get freed
6006	* and we can continue doing so.
6007	*/
6008	tmp = list_first_entry_or_null(&event->child_list,
6009	struct perf_event, child_list);
6010	if (tmp == child) {
6011	perf_remove_from_context(child, DETACH_GROUP \| DETACH_CHILD);
6012	} else {
6013	child = NULL;
6014	}
6015
6016	mutex_unlock(&event->child_mutex);
6017	mutex_unlock(&ctx->mutex);
6018
6019	if (child) {
6020	/* Last reference unless ->pending_task work is pending */
6021	put_event(child);
6022	}
6023	put_ctx(ctx);
6024
6025	goto again;
6026	}
6027	mutex_unlock(&event->child_mutex);
6028
6029	no_ctx:
6030	/*
6031	* Last reference unless ->pending_task work is pending on this event
6032	* or any of its children.
6033	*/
6034	put_event(event);
6035	return 0;
6036	}
6037	EXPORT_SYMBOL_GPL(perf_event_release_kernel);
6038
6039	/*
6040	* Called when the last reference to the file is gone.
6041	*/
6042	static int perf_release(struct inode inode, struct file file)
6043	{
6044	perf_event_release_kernel(file->private_data);
6045	return 0;
6046	}
6047
6048	static u64 __perf_event_read_value(struct perf_event event, u64 enabled, u64 *running)
6049	{
6050	struct perf_event *child;
6051	u64 total = 0;
6052
6053	*enabled = 0;
6054	*running = 0;
6055
6056	mutex_lock(&event->child_mutex);
6057
6058	(void)perf_event_read(event, false);
6059	total += perf_event_count(event, false);
6060
6061	*enabled += event->total_time_enabled +
6062	atomic64_read(&event->child_total_time_enabled);
6063	*running += event->total_time_running +
6064	atomic64_read(&event->child_total_time_running);
6065
6066	list_for_each_entry(child, &event->child_list, child_list) {
6067	(void)perf_event_read(child, false);
6068	total += perf_event_count(child, false);
6069	*enabled += child->total_time_enabled;
6070	*running += child->total_time_running;
6071	}
6072	mutex_unlock(&event->child_mutex);
6073
6074	return total;
6075	}
6076
6077	u64 perf_event_read_value(struct perf_event event, u64 enabled, u64 *running)
6078	{
6079	struct perf_event_context *ctx;
6080	u64 count;
6081
6082	ctx = perf_event_ctx_lock(event);
6083	count = __perf_event_read_value(event, enabled, running);
6084	perf_event_ctx_unlock(event, ctx);
6085
6086	return count;
6087	}
6088	EXPORT_SYMBOL_GPL(perf_event_read_value);
6089
6090	static int __perf_read_group_add(struct perf_event *leader,
6091	u64 read_format, u64 *values)
6092	{
6093	struct perf_event_context *ctx = leader->ctx;
6094	struct perf_event sub, parent;
6095	unsigned long flags;
6096	int n = 1; /* skip @nr */
6097	int ret;
6098
6099	ret = perf_event_read(leader, true);
6100	if (ret)
6101	return ret;
6102
6103	raw_spin_lock_irqsave(&ctx->lock, flags);
6104	/*
6105	* Verify the grouping between the parent and child (inherited)
6106	* events is still in tact.
6107	*
6108	* Specifically:
6109	* - leader->ctx->lock pins leader->sibling_list
6110	* - parent->child_mutex pins parent->child_list
6111	* - parent->ctx->mutex pins parent->sibling_list
6112	*
6113	* Because parent->ctx != leader->ctx (and child_list nests inside
6114	* ctx->mutex), group destruction is not atomic between children, also
6115	* see perf_event_release_kernel(). Additionally, parent can grow the
6116	* group.
6117	*
6118	* Therefore it is possible to have parent and child groups in a
6119	* different configuration and summing over such a beast makes no sense
6120	* what so ever.
6121	*
6122	* Reject this.
6123	*/
6124	parent = leader->parent;
6125	if (parent &&
6126	(parent->group_generation != leader->group_generation \|\|
6127	parent->nr_siblings != leader->nr_siblings)) {
6128	ret = -ECHILD;
6129	goto unlock;
6130	}
6131
6132	/*
6133	* Since we co-schedule groups, {enabled,running} times of siblings
6134	* will be identical to those of the leader, so we only publish one
6135	* set.
6136	*/
6137	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
6138	values[n++] += leader->total_time_enabled +
6139	atomic64_read(&leader->child_total_time_enabled);
6140	}
6141
6142	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
6143	values[n++] += leader->total_time_running +
6144	atomic64_read(&leader->child_total_time_running);
6145	}
6146
6147	/*
6148	* Write {count,id} tuples for every sibling.
6149	*/
6150	values[n++] += perf_event_count(leader, false);
6151	if (read_format & PERF_FORMAT_ID)
6152	values[n++] = primary_event_id(leader);
6153	if (read_format & PERF_FORMAT_LOST)
6154	values[n++] = atomic64_read(&leader->lost_samples);
6155
6156	for_each_sibling_event(sub, leader) {
6157	values[n++] += perf_event_count(sub, false);
6158	if (read_format & PERF_FORMAT_ID)
6159	values[n++] = primary_event_id(sub);
6160	if (read_format & PERF_FORMAT_LOST)
6161	values[n++] = atomic64_read(&sub->lost_samples);
6162	}
6163
6164	unlock:
6165	raw_spin_unlock_irqrestore(&ctx->lock, flags);
6166	return ret;
6167	}
6168
6169	static int perf_read_group(struct perf_event *event,
6170	u64 read_format, char __user *buf)
6171	{
6172	struct perf_event leader = event->group_leader, child;
6173	struct perf_event_context *ctx = leader->ctx;
6174	int ret;
6175	u64 *values;
6176
6177	lockdep_assert_held(&ctx->mutex);
6178
6179	values = kzalloc(event->read_size, GFP_KERNEL);
6180	if (!values)
6181	return -ENOMEM;
6182
6183	values[0] = 1 + leader->nr_siblings;
6184
6185	mutex_lock(&leader->child_mutex);
6186
6187	ret = __perf_read_group_add(leader, read_format, values);
6188	if (ret)
6189	goto unlock;
6190
6191	list_for_each_entry(child, &leader->child_list, child_list) {
6192	ret = __perf_read_group_add(child, read_format, values);
6193	if (ret)
6194	goto unlock;
6195	}
6196
6197	mutex_unlock(&leader->child_mutex);
6198
6199	ret = event->read_size;
6200	if (copy_to_user(buf, values, event->read_size))
6201	ret = -EFAULT;
6202	goto out;
6203
6204	unlock:
6205	mutex_unlock(&leader->child_mutex);
6206	out:
6207	kfree(values);
6208	return ret;
6209	}
6210
6211	static int perf_read_one(struct perf_event *event,
6212	u64 read_format, char __user *buf)
6213	{
6214	u64 enabled, running;
6215	u64 values[5];
6216	int n = 0;
6217
6218	values[n++] = __perf_event_read_value(event, &enabled, &running);
6219	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
6220	values[n++] = enabled;
6221	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
6222	values[n++] = running;
6223	if (read_format & PERF_FORMAT_ID)
6224	values[n++] = primary_event_id(event);
6225	if (read_format & PERF_FORMAT_LOST)
6226	values[n++] = atomic64_read(&event->lost_samples);
6227
6228	if (copy_to_user(buf, values, n * sizeof(u64)))
6229	return -EFAULT;
6230
6231	return n * sizeof(u64);
6232	}
6233
6234	static bool is_event_hup(struct perf_event *event)
6235	{
6236	bool no_children;
6237
6238	if (event->state > PERF_EVENT_STATE_EXIT)
6239	return false;
6240
6241	mutex_lock(&event->child_mutex);
6242	no_children = list_empty(&event->child_list);
6243	mutex_unlock(&event->child_mutex);
6244	return no_children;
6245	}
6246
6247	/*
6248	* Read the performance event - simple non blocking version for now
6249	*/
6250	static ssize_t
6251	__perf_read(struct perf_event event, char __user buf, size_t count)
6252	{
6253	u64 read_format = event->attr.read_format;
6254	int ret;
6255
6256	/*
6257	* Return end-of-file for a read on an event that is in
6258	* error state (i.e. because it was pinned but it couldn't be
6259	* scheduled on to the CPU at some point).
6260	*/
6261	if (event->state == PERF_EVENT_STATE_ERROR)
6262	return 0;
6263
6264	if (count < event->read_size)
6265	return -ENOSPC;
6266
6267	WARN_ON_ONCE(event->ctx->parent_ctx);
6268	if (read_format & PERF_FORMAT_GROUP)
6269	ret = perf_read_group(event, read_format, buf);
6270	else
6271	ret = perf_read_one(event, read_format, buf);
6272
6273	return ret;
6274	}
6275
6276	static ssize_t
6277	perf_read(struct file file, char __user buf, size_t count, loff_t *ppos)
6278	{
6279	struct perf_event *event = file->private_data;
6280	struct perf_event_context *ctx;
6281	int ret;
6282
6283	ret = security_perf_event_read(event);
6284	if (ret)
6285	return ret;
6286
6287	ctx = perf_event_ctx_lock(event);
6288	ret = __perf_read(event, buf, count);
6289	perf_event_ctx_unlock(event, ctx);
6290
6291	return ret;
6292	}
6293
6294	static __poll_t perf_poll(struct file file, poll_table wait)
6295	{
6296	struct perf_event *event = file->private_data;
6297	struct perf_buffer *rb;
6298	__poll_t events = EPOLLHUP;
6299
6300	if (event->state <= PERF_EVENT_STATE_REVOKED)
6301	return EPOLLERR;
6302
6303	poll_wait(file, &event->waitq, wait);
6304
6305	if (event->state <= PERF_EVENT_STATE_REVOKED)
6306	return EPOLLERR;
6307
6308	if (is_event_hup(event))
6309	return events;
6310
6311	if (unlikely(READ_ONCE(event->state) == PERF_EVENT_STATE_ERROR &&
6312	event->attr.pinned))
6313	return EPOLLERR;
6314
6315	/*
6316	* Pin the event->rb by taking event->mmap_mutex; otherwise
6317	* perf_event_set_output() can swizzle our rb and make us miss wakeups.
6318	*/
6319	mutex_lock(&event->mmap_mutex);
6320	rb = event->rb;
6321	if (rb)
6322	events = atomic_xchg(&rb->poll, 0);
6323	mutex_unlock(&event->mmap_mutex);
6324	return events;
6325	}
6326
6327	static void _perf_event_reset(struct perf_event *event)
6328	{
6329	(void)perf_event_read(event, false);
6330	local64_set(&event->count, 0);
6331	perf_event_update_userpage(event);
6332	}
6333
6334	/* Assume it's not an event with inherit set. */
6335	u64 perf_event_pause(struct perf_event *event, bool reset)
6336	{
6337	struct perf_event_context *ctx;
6338	u64 count;
6339
6340	ctx = perf_event_ctx_lock(event);
6341	WARN_ON_ONCE(event->attr.inherit);
6342	_perf_event_disable(event);
6343	count = local64_read(&event->count);
6344	if (reset)
6345	local64_set(&event->count, 0);
6346	perf_event_ctx_unlock(event, ctx);
6347
6348	return count;
6349	}
6350	EXPORT_SYMBOL_GPL(perf_event_pause);
6351
6352	#ifdef CONFIG_PERF_GUEST_MEDIATED_PMU
6353	static atomic_t nr_include_guest_events __read_mostly;
6354
6355	static atomic_t nr_mediated_pmu_vms __read_mostly;
6356	static DEFINE_MUTEX(perf_mediated_pmu_mutex);
6357
6358	/* !exclude_guest event of PMU with PERF_PMU_CAP_MEDIATED_VPMU */
6359	static inline bool is_include_guest_event(struct perf_event *event)
6360	{
6361	if (!event->pmu)
6362	return false;
6363
6364	if ((event->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) &&
6365	!event->attr.exclude_guest)
6366	return true;
6367
6368	return false;
6369	}
6370
6371	static int mediated_pmu_account_event(struct perf_event *event)
6372	{
6373	if (!is_include_guest_event(event))
6374	return 0;
6375
6376	if (atomic_inc_not_zero(&nr_include_guest_events))
6377	return 0;
6378
6379	guard(mutex)(&perf_mediated_pmu_mutex);
6380	if (atomic_read(&nr_mediated_pmu_vms))
6381	return -EOPNOTSUPP;
6382
6383	atomic_inc(&nr_include_guest_events);
6384	return 0;
6385	}
6386
6387	static void mediated_pmu_unaccount_event(struct perf_event *event)
6388	{
6389	if (!is_include_guest_event(event))
6390	return;
6391
6392	if (WARN_ON_ONCE(!atomic_read(&nr_include_guest_events)))
6393	return;
6394
6395	atomic_dec(&nr_include_guest_events);
6396	}
6397
6398	/*
6399	* Currently invoked at VM creation to
6400	* - Check whether there are existing !exclude_guest events of PMU with
6401	* PERF_PMU_CAP_MEDIATED_VPMU
6402	* - Set nr_mediated_pmu_vms to prevent !exclude_guest event creation on
6403	* PMUs with PERF_PMU_CAP_MEDIATED_VPMU
6404	*
6405	* No impact for the PMU without PERF_PMU_CAP_MEDIATED_VPMU. The perf
6406	* still owns all the PMU resources.
6407	*/
6408	int perf_create_mediated_pmu(void)
6409	{
6410	if (atomic_inc_not_zero(&nr_mediated_pmu_vms))
6411	return 0;
6412
6413	guard(mutex)(&perf_mediated_pmu_mutex);
6414	if (atomic_read(&nr_include_guest_events))
6415	return -EBUSY;
6416
6417	atomic_inc(&nr_mediated_pmu_vms);
6418	return 0;
6419	}
6420	EXPORT_SYMBOL_FOR_KVM(perf_create_mediated_pmu);
6421
6422	void perf_release_mediated_pmu(void)
6423	{
6424	if (WARN_ON_ONCE(!atomic_read(&nr_mediated_pmu_vms)))
6425	return;
6426
6427	atomic_dec(&nr_mediated_pmu_vms);
6428	}
6429	EXPORT_SYMBOL_FOR_KVM(perf_release_mediated_pmu);
6430
6431	/* When loading a guest's mediated PMU, schedule out all exclude_guest events. */
6432	void perf_load_guest_context(void)
6433	{
6434	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
6435
6436	lockdep_assert_irqs_disabled();
6437
6438	guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx);
6439
6440	if (WARN_ON_ONCE(__this_cpu_read(guest_ctx_loaded)))
6441	return;
6442
6443	perf_ctx_disable(&cpuctx->ctx, EVENT_GUEST);
6444	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_GUEST);
6445	if (cpuctx->task_ctx) {
6446	perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST);
6447	task_ctx_sched_out(cpuctx->task_ctx, NULL, EVENT_GUEST);
6448	}
6449
6450	perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST);
6451	if (cpuctx->task_ctx)
6452	perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST);
6453
6454	__this_cpu_write(guest_ctx_loaded, true);
6455	}
6456	EXPORT_SYMBOL_GPL(perf_load_guest_context);
6457
6458	void perf_put_guest_context(void)
6459	{
6460	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
6461
6462	lockdep_assert_irqs_disabled();
6463
6464	guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx);
6465
6466	if (WARN_ON_ONCE(!__this_cpu_read(guest_ctx_loaded)))
6467	return;
6468
6469	perf_ctx_disable(&cpuctx->ctx, EVENT_GUEST);
6470	if (cpuctx->task_ctx)
6471	perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST);
6472
6473	perf_event_sched_in(cpuctx, cpuctx->task_ctx, NULL, EVENT_GUEST);
6474
6475	if (cpuctx->task_ctx)
6476	perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST);
6477	perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST);
6478
6479	__this_cpu_write(guest_ctx_loaded, false);
6480	}
6481	EXPORT_SYMBOL_GPL(perf_put_guest_context);
6482	#else
6483	static int mediated_pmu_account_event(struct perf_event *event) { return 0; }
6484	static void mediated_pmu_unaccount_event(struct perf_event *event) {}
6485	#endif
6486
6487	/*
6488	* Holding the top-level event's child_mutex means that any
6489	* descendant process that has inherited this event will block
6490	* in perf_event_exit_event() if it goes to exit, thus satisfying the
6491	* task existence requirements of perf_event_enable/disable.
6492	*/
6493	static void perf_event_for_each_child(struct perf_event *event,
6494	void (func)(struct perf_event ))
6495	{
6496	struct perf_event *child;
6497
6498	WARN_ON_ONCE(event->ctx->parent_ctx);
6499
6500	mutex_lock(&event->child_mutex);
6501	func(event);
6502	list_for_each_entry(child, &event->child_list, child_list)
6503	func(child);
6504	mutex_unlock(&event->child_mutex);
6505	}
6506
6507	static void perf_event_for_each(struct perf_event *event,
6508	void (func)(struct perf_event ))
6509	{
6510	struct perf_event_context *ctx = event->ctx;
6511	struct perf_event *sibling;
6512
6513	lockdep_assert_held(&ctx->mutex);
6514
6515	event = event->group_leader;
6516
6517	perf_event_for_each_child(event, func);
6518	for_each_sibling_event(sibling, event)
6519	perf_event_for_each_child(sibling, func);
6520	}
6521
6522	static void __perf_event_period(struct perf_event *event,
6523	struct perf_cpu_context *cpuctx,
6524	struct perf_event_context *ctx,
6525	void *info)
6526	{
6527	u64 value = ((u64 )info);
6528	bool active;
6529
6530	if (event->attr.freq) {
6531	event->attr.sample_freq = value;
6532	} else {
6533	event->attr.sample_period = value;
6534	event->hw.sample_period = value;
6535	}
6536
6537	active = (event->state == PERF_EVENT_STATE_ACTIVE);
6538	if (active) {
6539	perf_pmu_disable(event->pmu);
6540	event->pmu->stop(event, PERF_EF_UPDATE);
6541	}
6542
6543	local64_set(&event->hw.period_left, 0);
6544
6545	if (active) {
6546	event->pmu->start(event, PERF_EF_RELOAD);
6547	/*
6548	* Once the period is force-reset, the event starts immediately.
6549	* But the event/group could be throttled. Unthrottle the
6550	* event/group now to avoid the next tick trying to unthrottle
6551	* while we already re-started the event/group.
6552	*/
6553	if (event->hw.interrupts == MAX_INTERRUPTS)
6554	perf_event_unthrottle_group(event, true);
6555	perf_pmu_enable(event->pmu);
6556	}
6557	}
6558
6559	static int perf_event_check_period(struct perf_event *event, u64 value)
6560	{
6561	return event->pmu->check_period(event, value);
6562	}
6563
6564	static int _perf_event_period(struct perf_event *event, u64 value)
6565	{
6566	if (!is_sampling_event(event))
6567	return -EINVAL;
6568
6569	if (!value)
6570	return -EINVAL;
6571
6572	if (event->attr.freq) {
6573	if (value > sysctl_perf_event_sample_rate)
6574	return -EINVAL;
6575	} else {
6576	if (perf_event_check_period(event, value))
6577	return -EINVAL;
6578	if (value & (1ULL << 63))
6579	return -EINVAL;
6580	}
6581
6582	event_function_call(event, __perf_event_period, &value);
6583
6584	return 0;
6585	}
6586
6587	int perf_event_period(struct perf_event *event, u64 value)
6588	{
6589	struct perf_event_context *ctx;
6590	int ret;
6591
6592	ctx = perf_event_ctx_lock(event);
6593	ret = _perf_event_period(event, value);
6594	perf_event_ctx_unlock(event, ctx);
6595
6596	return ret;
6597	}
6598	EXPORT_SYMBOL_GPL(perf_event_period);
6599
6600	static const struct file_operations perf_fops;
6601
6602	static inline bool is_perf_file(struct fd f)
6603	{
6604	return !fd_empty(f) && fd_file(f)->f_op == &perf_fops;
6605	}
6606
6607	static int perf_event_set_output(struct perf_event *event,
6608	struct perf_event *output_event);
6609	static int perf_event_set_filter(struct perf_event event, void __user arg);
6610	static int perf_copy_attr(struct perf_event_attr __user *uattr,
6611	struct perf_event_attr *attr);
6612	static int __perf_event_set_bpf_prog(struct perf_event *event,
6613	struct bpf_prog *prog,
6614	u64 bpf_cookie);
6615
6616	static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
6617	{
6618	void (func)(struct perf_event );
6619	u32 flags = arg;
6620
6621	if (event->state <= PERF_EVENT_STATE_REVOKED)
6622	return -ENODEV;
6623
6624	switch (cmd) {
6625	case PERF_EVENT_IOC_ENABLE:
6626	func = _perf_event_enable;
6627	break;
6628	case PERF_EVENT_IOC_DISABLE:
6629	func = _perf_event_disable;
6630	break;
6631	case PERF_EVENT_IOC_RESET:
6632	func = _perf_event_reset;
6633	break;
6634
6635	case PERF_EVENT_IOC_REFRESH:
6636	return _perf_event_refresh(event, arg);
6637
6638	case PERF_EVENT_IOC_PERIOD:
6639	{
6640	u64 value;
6641
6642	if (copy_from_user(&value, (u64 __user *)arg, sizeof(value)))
6643	return -EFAULT;
6644
6645	return _perf_event_period(event, value);
6646	}
6647	case PERF_EVENT_IOC_ID:
6648	{
6649	u64 id = primary_event_id(event);
6650
6651	if (copy_to_user((void __user *)arg, &id, sizeof(id)))
6652	return -EFAULT;
6653	return 0;
6654	}
6655
6656	case PERF_EVENT_IOC_SET_OUTPUT:
6657	{
6658	CLASS(fd, output)(arg); // arg == -1 => empty
6659	struct perf_event *output_event = NULL;
6660	if (arg != -1) {
6661	if (!is_perf_file(output))
6662	return -EBADF;
6663	output_event = fd_file(output)->private_data;
6664	}
6665	return perf_event_set_output(event, output_event);
6666	}
6667
6668	case PERF_EVENT_IOC_SET_FILTER:
6669	return perf_event_set_filter(event, (void __user *)arg);
6670
6671	case PERF_EVENT_IOC_SET_BPF:
6672	{
6673	struct bpf_prog *prog;
6674	int err;
6675
6676	prog = bpf_prog_get(arg);
6677	if (IS_ERR(prog))
6678	return PTR_ERR(prog);
6679
6680	err = __perf_event_set_bpf_prog(event, prog, 0);
6681	if (err) {
6682	bpf_prog_put(prog);
6683	return err;
6684	}
6685
6686	return 0;
6687	}
6688
6689	case PERF_EVENT_IOC_PAUSE_OUTPUT: {
6690	struct perf_buffer *rb;
6691
6692	rcu_read_lock();
6693	rb = rcu_dereference(event->rb);
6694	if (!rb \|\| !rb->nr_pages) {
6695	rcu_read_unlock();
6696	return -EINVAL;
6697	}
6698	rb_toggle_paused(rb, !!arg);
6699	rcu_read_unlock();
6700	return 0;
6701	}
6702
6703	case PERF_EVENT_IOC_QUERY_BPF:
6704	return perf_event_query_prog_array(event, (void __user *)arg);
6705
6706	case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: {
6707	struct perf_event_attr new_attr;
6708	int err = perf_copy_attr((struct perf_event_attr __user *)arg,
6709	&new_attr);
6710
6711	if (err)
6712	return err;
6713
6714	return perf_event_modify_attr(event, &new_attr);
6715	}
6716	default:
6717	return -ENOTTY;
6718	}
6719
6720	if (flags & PERF_IOC_FLAG_GROUP)
6721	perf_event_for_each(event, func);
6722	else
6723	perf_event_for_each_child(event, func);
6724
6725	return 0;
6726	}
6727
6728	static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
6729	{
6730	struct perf_event *event = file->private_data;
6731	struct perf_event_context *ctx;
6732	long ret;
6733
6734	/* Treat ioctl like writes as it is likely a mutating operation. */
6735	ret = security_perf_event_write(event);
6736	if (ret)
6737	return ret;
6738
6739	ctx = perf_event_ctx_lock(event);
6740	ret = _perf_ioctl(event, cmd, arg);
6741	perf_event_ctx_unlock(event, ctx);
6742
6743	return ret;
6744	}
6745
6746	#ifdef CONFIG_COMPAT
6747	static long perf_compat_ioctl(struct file *file, unsigned int cmd,
6748	unsigned long arg)
6749	{
6750	switch (_IOC_NR(cmd)) {
6751	case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
6752	case _IOC_NR(PERF_EVENT_IOC_ID):
6753	case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF):
6754	case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES):
6755	/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
6756	if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
6757	cmd &= ~IOCSIZE_MASK;
6758	cmd \|= sizeof(void *) << IOCSIZE_SHIFT;
6759	}
6760	break;
6761	}
6762	return perf_ioctl(file, cmd, arg);
6763	}
6764	#else
6765	# define perf_compat_ioctl NULL
6766	#endif
6767
6768	int perf_event_task_enable(void)
6769	{
6770	struct perf_event_context *ctx;
6771	struct perf_event *event;
6772
6773	mutex_lock(&current->perf_event_mutex);
6774	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
6775	ctx = perf_event_ctx_lock(event);
6776	perf_event_for_each_child(event, _perf_event_enable);
6777	perf_event_ctx_unlock(event, ctx);
6778	}
6779	mutex_unlock(&current->perf_event_mutex);
6780
6781	return 0;
6782	}
6783
6784	int perf_event_task_disable(void)
6785	{
6786	struct perf_event_context *ctx;
6787	struct perf_event *event;
6788
6789	mutex_lock(&current->perf_event_mutex);
6790	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
6791	ctx = perf_event_ctx_lock(event);
6792	perf_event_for_each_child(event, _perf_event_disable);
6793	perf_event_ctx_unlock(event, ctx);
6794	}
6795	mutex_unlock(&current->perf_event_mutex);
6796
6797	return 0;
6798	}
6799
6800	static int perf_event_index(struct perf_event *event)
6801	{
6802	if (event->hw.state & PERF_HES_STOPPED)
6803	return 0;
6804
6805	if (event->state != PERF_EVENT_STATE_ACTIVE)
6806	return 0;
6807
6808	return event->pmu->event_idx(event);
6809	}
6810
6811	static void perf_event_init_userpage(struct perf_event *event)
6812	{
6813	struct perf_event_mmap_page *userpg;
6814	struct perf_buffer *rb;
6815
6816	rcu_read_lock();
6817	rb = rcu_dereference(event->rb);
6818	if (!rb)
6819	goto unlock;
6820
6821	userpg = rb->user_page;
6822
6823	/* Allow new userspace to detect that bit 0 is deprecated */
6824	userpg->cap_bit0_is_deprecated = 1;
6825	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
6826	userpg->data_offset = PAGE_SIZE;
6827	userpg->data_size = perf_data_size(rb);
6828
6829	unlock:
6830	rcu_read_unlock();
6831	}
6832
6833	void __weak arch_perf_update_userpage(
6834	struct perf_event event, struct perf_event_mmap_page userpg, u64 now)
6835	{
6836	}
6837
6838	/*
6839	* Callers need to ensure there can be no nesting of this function, otherwise
6840	* the seqlock logic goes bad. We can not serialize this because the arch
6841	* code calls this from NMI context.
6842	*/
6843	void perf_event_update_userpage(struct perf_event *event)
6844	{
6845	struct perf_event_mmap_page *userpg;
6846	struct perf_buffer *rb;
6847	u64 enabled, running, now;
6848
6849	rcu_read_lock();
6850	rb = rcu_dereference(event->rb);
6851	if (!rb)
6852	goto unlock;
6853
6854	/*
6855	* Disable preemption to guarantee consistent time stamps are stored to
6856	* the user page.
6857	*/
6858	preempt_disable();
6859
6860	/*
6861	* Compute total_time_enabled, total_time_running based on snapshot
6862	* values taken when the event was last scheduled in.
6863	*
6864	* We cannot simply call update_context_time() because doing so would
6865	* lead to deadlock when called from NMI context.
6866	*/
6867	calc_timer_values(event, &now, &enabled, &running);
6868
6869	userpg = rb->user_page;
6870
6871	++userpg->lock;
6872	barrier();
6873	userpg->index = perf_event_index(event);
6874	userpg->offset = perf_event_count(event, false);
6875	if (userpg->index)
6876	userpg->offset -= local64_read(&event->hw.prev_count);
6877
6878	userpg->time_enabled = enabled +
6879	atomic64_read(&event->child_total_time_enabled);
6880
6881	userpg->time_running = running +
6882	atomic64_read(&event->child_total_time_running);
6883
6884	arch_perf_update_userpage(event, userpg, now);
6885
6886	barrier();
6887	++userpg->lock;
6888	preempt_enable();
6889	unlock:
6890	rcu_read_unlock();
6891	}
6892	EXPORT_SYMBOL_GPL(perf_event_update_userpage);
6893
6894	static void ring_buffer_attach(struct perf_event *event,
6895	struct perf_buffer *rb)
6896	{
6897	struct perf_buffer *old_rb = NULL;
6898	unsigned long flags;
6899
6900	WARN_ON_ONCE(event->parent);
6901
6902	if (event->rb) {
6903	/*
6904	* Should be impossible, we set this when removing
6905	* event->rb_entry and wait/clear when adding event->rb_entry.
6906	*/
6907	WARN_ON_ONCE(event->rcu_pending);
6908
6909	old_rb = event->rb;
6910	spin_lock_irqsave(&old_rb->event_lock, flags);
6911	list_del_rcu(&event->rb_entry);
6912	spin_unlock_irqrestore(&old_rb->event_lock, flags);
6913
6914	event->rcu_batches = get_state_synchronize_rcu();
6915	event->rcu_pending = 1;
6916	}
6917
6918	if (rb) {
6919	if (event->rcu_pending) {
6920	cond_synchronize_rcu(event->rcu_batches);
6921	event->rcu_pending = 0;
6922	}
6923
6924	spin_lock_irqsave(&rb->event_lock, flags);
6925	list_add_rcu(&event->rb_entry, &rb->event_list);
6926	spin_unlock_irqrestore(&rb->event_lock, flags);
6927	}
6928
6929	/*
6930	* Avoid racing with perf_mmap_close(AUX): stop the event
6931	* before swizzling the event::rb pointer; if it's getting
6932	* unmapped, its aux_mmap_count will be 0 and it won't
6933	* restart. See the comment in __perf_pmu_output_stop().
6934	*
6935	* Data will inevitably be lost when set_output is done in
6936	* mid-air, but then again, whoever does it like this is
6937	* not in for the data anyway.
6938	*/
6939	if (has_aux(event))
6940	perf_event_stop(event, 0);
6941
6942	rcu_assign_pointer(event->rb, rb);
6943
6944	if (old_rb) {
6945	ring_buffer_put(old_rb);
6946	/*
6947	* Since we detached before setting the new rb, so that we
6948	* could attach the new rb, we could have missed a wakeup.
6949	* Provide it now.
6950	*/
6951	wake_up_all(&event->waitq);
6952	}
6953	}
6954
6955	static void ring_buffer_wakeup(struct perf_event *event)
6956	{
6957	struct perf_buffer *rb;
6958
6959	if (event->parent)
6960	event = event->parent;
6961
6962	rcu_read_lock();
6963	rb = rcu_dereference(event->rb);
6964	if (rb) {
6965	list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
6966	wake_up_all(&event->waitq);
6967	}
6968	rcu_read_unlock();
6969	}
6970
6971	struct perf_buffer ring_buffer_get(struct perf_event event)
6972	{
6973	struct perf_buffer *rb;
6974
6975	if (event->parent)
6976	event = event->parent;
6977
6978	rcu_read_lock();
6979	rb = rcu_dereference(event->rb);
6980	if (rb) {
6981	if (!refcount_inc_not_zero(&rb->refcount))
6982	rb = NULL;
6983	}
6984	rcu_read_unlock();
6985
6986	return rb;
6987	}
6988
6989	void ring_buffer_put(struct perf_buffer *rb)
6990	{
6991	if (!refcount_dec_and_test(&rb->refcount))
6992	return;
6993
6994	WARN_ON_ONCE(!list_empty(&rb->event_list));
6995
6996	call_rcu(&rb->rcu_head, rb_free_rcu);
6997	}
6998
6999	typedef void (mapped_f)(struct perf_event event, struct mm_struct *mm);
7000
7001	#define get_mapped(event, func) \
7002	({ struct pmu *pmu; \
7003	mapped_f f = NULL; \
7004	guard(rcu)(); \
7005	pmu = READ_ONCE(event->pmu); \
7006	if (pmu) \
7007	f = pmu->func; \
7008	f; \
7009	})
7010
7011	static void perf_mmap_open(struct vm_area_struct *vma)
7012	{
7013	struct perf_event *event = vma->vm_file->private_data;
7014	mapped_f mapped = get_mapped(event, event_mapped);
7015
7016	refcount_inc(&event->mmap_count);
7017	refcount_inc(&event->rb->mmap_count);
7018
7019	if (vma_start_pgoff(vma))
7020	refcount_inc(&event->rb->aux_mmap_count);
7021
7022	if (mapped)
7023	mapped(event, vma->vm_mm);
7024	}
7025
7026	static void perf_pmu_output_stop(struct perf_event *event);
7027	static void perf_mmap_unaccount(struct vm_area_struct vma, struct perf_buffer rb);
7028
7029	/*
7030	* A buffer can be mmap()ed multiple times; either directly through the same
7031	* event, or through other events by use of perf_event_set_output().
7032	*
7033	* In order to undo the VM accounting done by perf_mmap() we need to destroy
7034	* the buffer here, where we still have a VM context. This means we need
7035	* to detach all events redirecting to us.
7036	*/
7037	static void perf_mmap_close(struct vm_area_struct *vma)
7038	{
7039	struct perf_event *event = vma->vm_file->private_data;
7040	mapped_f unmapped = get_mapped(event, event_unmapped);
7041	struct perf_buffer *rb = ring_buffer_get(event);
7042	struct user_struct *mmap_user = rb->mmap_user;
7043
7044	/* FIXIES vs perf_pmu_unregister() */
7045	if (unmapped)
7046	unmapped(event, vma->vm_mm);
7047
7048	/*
7049	* The AUX buffer is strictly a sub-buffer, serialize using aux_mutex
7050	* to avoid complications.
7051	*/
7052	if (rb_has_aux(rb) && vma_start_pgoff(vma) == rb->aux_pgoff &&
7053	refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {
7054	/*
7055	* Stop all AUX events that are writing to this buffer,
7056	* so that we can free its AUX pages and corresponding PMU
7057	* data. Note that after rb::aux_mmap_count dropped to zero,
7058	* they won't start any more (see perf_aux_output_begin()).
7059	*/
7060	perf_pmu_output_stop(event);
7061
7062	/* now it's safe to free the pages */
7063	atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->perf_mlock);
7064	atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);
7065
7066	/* this has to be the last one */
7067	rb_free_aux(rb);
7068	WARN_ON_ONCE(refcount_read(&rb->aux_refcount));
7069
7070	mutex_unlock(&rb->aux_mutex);
7071	}
7072
7073	/*
7074	* Drop references in reverse order of perf_mmap() to prevent
7075	* rb revival after rb->mmap_count reaches zero.
7076	*/
7077	if (refcount_dec_and_mutex_lock(&event->mmap_count,
7078	&event->mmap_mutex)) {
7079	ring_buffer_attach(event, NULL);
7080	mutex_unlock(&event->mmap_mutex);
7081	}
7082
7083	/* If there's still other mmap()s of this buffer, we're done. */
7084	if (!refcount_dec_and_test(&rb->mmap_count))
7085	goto out_put;
7086
7087	/*
7088	* No other mmap()s, detach from all other events that might redirect
7089	* into the now unreachable buffer. Somewhat complicated by the
7090	* fact that rb::event_lock otherwise nests inside mmap_mutex.
7091	*/
7092	again:
7093	rcu_read_lock();
7094	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
7095	if (!atomic_long_inc_not_zero(&event->refcount)) {
7096	/*
7097	* This event is en-route to free_event() which will
7098	* detach it and remove it from the list.
7099	*/
7100	continue;
7101	}
7102	rcu_read_unlock();
7103
7104	mutex_lock(&event->mmap_mutex);
7105	/*
7106	* Check we didn't race with perf_event_set_output() which can
7107	* swizzle the rb from under us while we were waiting to
7108	* acquire mmap_mutex.
7109	*
7110	* If we find a different rb; ignore this event, a next
7111	* iteration will no longer find it on the list. We have to
7112	* still restart the iteration to make sure we're not now
7113	* iterating the wrong list.
7114	*/
7115	if (event->rb == rb)
7116	ring_buffer_attach(event, NULL);
7117
7118	mutex_unlock(&event->mmap_mutex);
7119	put_event(event);
7120
7121	/*
7122	* Restart the iteration; either we're on the wrong list or
7123	* destroyed its integrity by doing a deletion.
7124	*/
7125	goto again;
7126	}
7127	rcu_read_unlock();
7128
7129	/*
7130	* It could be there's still a few 0-ref events on the list; they'll
7131	* get cleaned up by free_event() -- they'll also still have their
7132	* ref on the rb and will free it whenever they are done with it.
7133	*
7134	* Aside from that, this buffer is 'fully' detached and unmapped,
7135	* undo the VM accounting.
7136	*/
7137	perf_mmap_unaccount(vma, rb);
7138
7139	out_put:
7140	ring_buffer_put(rb); /* could be last */
7141	}
7142
7143	static vm_fault_t perf_mmap_pfn_mkwrite(struct vm_fault *vmf)
7144	{
7145	/* The first page is the user control page, others are read-only. */
7146	return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS;
7147	}
7148
7149	static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr)
7150	{
7151	/*
7152	* Forbid splitting perf mappings to prevent refcount leaks due to
7153	* the resulting non-matching offsets and sizes. See open()/close().
7154	*/
7155	return -EINVAL;
7156	}
7157
7158	static const struct vm_operations_struct perf_mmap_vmops = {
7159	.open = perf_mmap_open,
7160	.close = perf_mmap_close, /* non mergeable */
7161	.pfn_mkwrite = perf_mmap_pfn_mkwrite,
7162	.may_split = perf_mmap_may_split,
7163	};
7164
7165	static int map_range(struct perf_buffer rb, struct vm_area_struct vma)
7166	{
7167	unsigned long nr_pages = vma_pages(vma);
7168	int err = 0;
7169	unsigned long pagenum;
7170
7171	guard(mutex)(&rb->aux_mutex);
7172
7173	/*
7174	* We map this as a VM_PFNMAP VMA.
7175	*
7176	* This is not ideal as this is designed broadly for mappings of PFNs
7177	* referencing memory-mapped I/O ranges or non-system RAM i.e. for which
7178	* !pfn_valid(pfn).
7179	*
7180	* We are mapping kernel-allocated memory (memory we manage ourselves)
7181	* which would more ideally be mapped using vm_insert_page() or a
7182	* similar mechanism, that is as a VM_MIXEDMAP mapping.
7183	*
7184	* However this won't work here, because:
7185	*
7186	* 1. It uses vma->vm_page_prot, but this field has not been completely
7187	* setup at the point of the f_op->mmp() hook, so we are unable to
7188	* indicate that this should be mapped CoW in order that the
7189	* mkwrite() hook can be invoked to make the first page R/W and the
7190	* rest R/O as desired.
7191	*
7192	* 2. Anything other than a VM_PFNMAP of valid PFNs will result in
7193	* vm_normal_page() returning a struct page * pointer, which means
7194	* vm_ops->page_mkwrite() will be invoked rather than
7195	* vm_ops->pfn_mkwrite(), and this means we have to set page->mapping
7196	* to work around retry logic in the fault handler, however this
7197	* field is no longer allowed to be used within struct page.
7198	*
7199	* 3. Having a struct page * made available in the fault logic also
7200	* means that the page gets put on the rmap and becomes
7201	* inappropriately accessible and subject to map and ref counting.
7202	*
7203	* Ideally we would have a mechanism that could explicitly express our
7204	* desires, but this is not currently the case, so we instead use
7205	* VM_PFNMAP.
7206	*
7207	* We manage the lifetime of these mappings with internal refcounts (see
7208	* perf_mmap_open() and perf_mmap_close()) so we ensure the lifetime of
7209	* this mapping is maintained correctly.
7210	*/
7211	for (pagenum = 0; pagenum < nr_pages; pagenum++) {
7212	unsigned long va = vma->vm_start + PAGE_SIZE * pagenum;
7213	struct page *page = perf_mmap_to_page(rb,
7214	vma_start_pgoff(vma) + pagenum);
7215
7216	if (page == NULL) {
7217	err = -EINVAL;
7218	break;
7219	}
7220
7221	/* Map readonly, perf_mmap_pfn_mkwrite() called on write fault. */
7222	err = remap_pfn_range(vma, va, page_to_pfn(page), PAGE_SIZE,
7223	vm_get_page_prot(vma->vm_flags & ~VM_SHARED));
7224	if (err)
7225	break;
7226	}
7227
7228	#ifdef CONFIG_MMU
7229	/* Clear any partial mappings on error. */
7230	if (err)
7231	zap_vma_range(vma, vma->vm_start, nr_pages * PAGE_SIZE);
7232	#endif
7233
7234	return err;
7235	}
7236
7237	static bool perf_mmap_calc_limits(struct vm_area_struct vma, long user_extra, long *extra)
7238	{
7239	unsigned long user_locked, user_lock_limit, locked, lock_limit;
7240	struct user_struct *user = current_user();
7241
7242	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
7243	/* Increase the limit linearly with more CPUs */
7244	user_lock_limit *= num_online_cpus();
7245
7246	user_locked = atomic_long_read(&user->perf_mlock);
7247
7248	/*
7249	* sysctl_perf_event_mlock may have changed, so that
7250	* user->perf_mlock > user_lock_limit
7251	*/
7252	if (user_locked > user_lock_limit)
7253	user_locked = user_lock_limit;
7254	user_locked += *user_extra;
7255
7256	if (user_locked > user_lock_limit) {
7257	/*
7258	* charge perf_mlock until it hits user_lock_limit;
7259	* charge the rest from pinned_vm
7260	*/
7261	*extra = user_locked - user_lock_limit;
7262	user_extra -= extra;
7263	}
7264
7265	lock_limit = rlimit(RLIMIT_MEMLOCK);
7266	lock_limit >>= PAGE_SHIFT;
7267	locked = atomic64_read(&vma->vm_mm->pinned_vm) + *extra;
7268
7269	return locked <= lock_limit \|\| !perf_is_paranoid() \|\| capable(CAP_IPC_LOCK);
7270	}
7271
7272	static void perf_mmap_account(struct vm_area_struct *vma, long user_extra, long extra)
7273	{
7274	struct user_struct *user = current_user();
7275
7276	atomic_long_add(user_extra, &user->perf_mlock);
7277	atomic64_add(extra, &vma->vm_mm->pinned_vm);
7278	}
7279
7280	static void perf_mmap_unaccount(struct vm_area_struct vma, struct perf_buffer rb)
7281	{
7282	struct user_struct *user = rb->mmap_user;
7283
7284	atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked,
7285	&user->perf_mlock);
7286	atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm);
7287	}
7288
7289	static int perf_mmap_rb(struct vm_area_struct vma, struct perf_event event,
7290	unsigned long nr_pages)
7291	{
7292	long extra = 0, user_extra = nr_pages;
7293	struct perf_buffer *rb;
7294	int rb_flags = 0;
7295
7296	nr_pages -= 1;
7297
7298	/*
7299	* If we have rb pages ensure they're a power-of-two number, so we
7300	* can do bitmasks instead of modulo.
7301	*/
7302	if (nr_pages != 0 && !is_power_of_2(nr_pages))
7303	return -EINVAL;
7304
7305	WARN_ON_ONCE(event->ctx->parent_ctx);
7306
7307	if (event->rb) {
7308	if (data_page_nr(event->rb) != nr_pages)
7309	return -EINVAL;
7310
7311	/*
7312	* If this event doesn't have mmap_count, we're attempting to
7313	* create an alias of another event's mmap(); this would mean
7314	* both events will end up scribbling the same user_page;
7315	* which makes no sense.
7316	*/
7317	if (!refcount_read(&event->mmap_count))
7318	return -EBUSY;
7319
7320	if (refcount_inc_not_zero(&event->rb->mmap_count)) {
7321	/*
7322	* Success -- managed to mmap() the same buffer
7323	* multiple times.
7324	*/
7325	perf_mmap_account(vma, user_extra, extra);
7326	refcount_inc(&event->mmap_count);
7327	return 0;
7328	}
7329
7330	/*
7331	* Raced against perf_mmap_close()'s
7332	* refcount_dec_and_mutex_lock() remove the
7333	* event and continue as if !event->rb
7334	*/
7335	ring_buffer_attach(event, NULL);
7336	}
7337
7338	if (!perf_mmap_calc_limits(vma, &user_extra, &extra))
7339	return -EPERM;
7340
7341	if (vma->vm_flags & VM_WRITE)
7342	rb_flags \|= RING_BUFFER_WRITABLE;
7343
7344	rb = rb_alloc(nr_pages,
7345	event->attr.watermark ? event->attr.wakeup_watermark : 0,
7346	event->cpu, rb_flags);
7347
7348	if (!rb)
7349	return -ENOMEM;
7350
7351	rb->mmap_locked = extra;
7352
7353	ring_buffer_attach(event, rb);
7354
7355	perf_event_update_time(event);
7356	perf_event_init_userpage(event);
7357	perf_event_update_userpage(event);
7358
7359	perf_mmap_account(vma, user_extra, extra);
7360	refcount_set(&event->mmap_count, 1);
7361
7362	return 0;
7363	}
7364
7365	static int perf_mmap_aux(struct vm_area_struct vma, struct perf_event event,
7366	unsigned long nr_pages)
7367	{
7368	const pgoff_t pgoff_start = vma_start_pgoff(vma);
7369	long extra = 0, user_extra = nr_pages;
7370	u64 aux_offset, aux_size;
7371	struct perf_buffer *rb;
7372	int ret, rb_flags = 0;
7373
7374	rb = event->rb;
7375	if (!rb)
7376	return -EINVAL;
7377
7378	guard(mutex)(&rb->aux_mutex);
7379
7380	/*
7381	* AUX area mapping: if rb->aux_nr_pages != 0, it's already
7382	* mapped, all subsequent mappings should have the same size
7383	* and offset. Must be above the normal perf buffer.
7384	*/
7385	aux_offset = READ_ONCE(rb->user_page->aux_offset);
7386	aux_size = READ_ONCE(rb->user_page->aux_size);
7387
7388	if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
7389	return -EINVAL;
7390
7391	if (aux_offset != pgoff_start << PAGE_SHIFT)
7392	return -EINVAL;
7393
7394	/* already mapped with a different offset */
7395	if (rb_has_aux(rb) && rb->aux_pgoff != pgoff_start)
7396	return -EINVAL;
7397
7398	if (aux_size != nr_pages * PAGE_SIZE)
7399	return -EINVAL;
7400
7401	/* already mapped with a different size */
7402	if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
7403	return -EINVAL;
7404
7405	if (!is_power_of_2(nr_pages))
7406	return -EINVAL;
7407
7408	if (!refcount_inc_not_zero(&rb->mmap_count))
7409	return -EINVAL;
7410
7411	if (rb_has_aux(rb)) {
7412	refcount_inc(&rb->aux_mmap_count);
7413
7414	} else {
7415	if (!perf_mmap_calc_limits(vma, &user_extra, &extra)) {
7416	refcount_dec(&rb->mmap_count);
7417	return -EPERM;
7418	}
7419
7420	WARN_ON(!rb && event->rb);
7421
7422	if (vma->vm_flags & VM_WRITE)
7423	rb_flags \|= RING_BUFFER_WRITABLE;
7424
7425	ret = rb_alloc_aux(rb, event, pgoff_start, nr_pages,
7426	event->attr.aux_watermark, rb_flags);
7427	if (ret) {
7428	refcount_dec(&rb->mmap_count);
7429	return ret;
7430	}
7431
7432	refcount_set(&rb->aux_mmap_count, 1);
7433	rb->aux_mmap_locked = extra;
7434	}
7435
7436	perf_mmap_account(vma, user_extra, extra);
7437	refcount_inc(&event->mmap_count);
7438
7439	return 0;
7440	}
7441
7442	static int perf_mmap(struct file file, struct vm_area_struct vma)
7443	{
7444	struct perf_event *event = file->private_data;
7445	unsigned long vma_size, nr_pages;
7446	mapped_f mapped;
7447	int ret;
7448
7449	/*
7450	* Don't allow mmap() of inherited per-task counters. This would
7451	* create a performance issue due to all children writing to the
7452	* same rb.
7453	*/
7454	if (event->cpu == -1 && event->attr.inherit)
7455	return -EINVAL;
7456
7457	if (!(vma->vm_flags & VM_SHARED))
7458	return -EINVAL;
7459
7460	ret = security_perf_event_read(event);
7461	if (ret)
7462	return ret;
7463
7464	vma_size = vma->vm_end - vma->vm_start;
7465	nr_pages = vma_size / PAGE_SIZE;
7466
7467	if (nr_pages > INT_MAX)
7468	return -ENOMEM;
7469
7470	if (vma_size != PAGE_SIZE * nr_pages)
7471	return -EINVAL;
7472
7473	scoped_guard (mutex, &event->mmap_mutex) {
7474	/*
7475	* This relies on __pmu_detach_event() taking mmap_mutex after marking
7476	* the event REVOKED. Either we observe the state, or __pmu_detach_event()
7477	* will detach the rb created here.
7478	*/
7479	if (event->state <= PERF_EVENT_STATE_REVOKED)
7480	return -ENODEV;
7481
7482	if (!vma_start_pgoff(vma))
7483	ret = perf_mmap_rb(vma, event, nr_pages);
7484	else
7485	ret = perf_mmap_aux(vma, event, nr_pages);
7486	if (ret)
7487	return ret;
7488
7489	/*
7490	* Since pinned accounting is per vm we cannot allow fork() to copy our
7491	* vma.
7492	*/
7493	vm_flags_set(vma, VM_DONTCOPY \| VM_DONTEXPAND \| VM_DONTDUMP);
7494	vma->vm_ops = &perf_mmap_vmops;
7495
7496	mapped = get_mapped(event, event_mapped);
7497	if (mapped)
7498	mapped(event, vma->vm_mm);
7499
7500	/*
7501	* Try to map it into the page table. On fail undo the above,
7502	* as the callsite expects full cleanup in this case and
7503	* therefore does not invoke vmops::close().
7504	*/
7505	ret = map_range(event->rb, vma);
7506	if (likely(!ret))
7507	return 0;
7508
7509	/* Error path */
7510
7511	/*
7512	* If this is the first mmap(), then event->mmap_count should
7513	* be stable at 1. It is only modified by:
7514	* perf_mmap_{open,close}() and perf_mmap().
7515	*
7516	* The former are not possible because this mmap() hasn't been
7517	* successful yet, and the latter is serialized by
7518	* event->mmap_mutex which we still hold (note that mmap_lock
7519	* is not strictly sufficient here, because the event fd can
7520	* be passed to another process through trivial means like
7521	* fork(), leading to concurrent mmap() from different mm).
7522	*
7523	* Make sure to remove event->rb before releasing
7524	* event->mmap_mutex, such that any concurrent mmap() will not
7525	* attempt use this failed buffer.
7526	*/
7527	if (refcount_read(&event->mmap_count) == 1) {
7528	/*
7529	* Minimal perf_mmap_close(); there can't be AUX or
7530	* other events on account of this being the first.
7531	*/
7532	mapped = get_mapped(event, event_unmapped);
7533	if (mapped)
7534	mapped(event, vma->vm_mm);
7535	perf_mmap_unaccount(vma, event->rb);
7536	ring_buffer_attach(event, NULL); /* drops last rb->refcount */
7537	refcount_set(&event->mmap_count, 0);
7538	return ret;
7539	}
7540
7541	/*
7542	* Otherwise this is an already existing buffer, and there is
7543	* no race vs first exposure, so fall-through and call
7544	* perf_mmap_close().
7545	*/
7546	}
7547
7548	perf_mmap_close(vma);
7549	return ret;
7550	}
7551
7552	static int perf_fasync(int fd, struct file *filp, int on)
7553	{
7554	struct inode *inode = file_inode(filp);
7555	struct perf_event *event = filp->private_data;
7556	int retval;
7557
7558	if (event->state <= PERF_EVENT_STATE_REVOKED)
7559	return -ENODEV;
7560
7561	inode_lock(inode);
7562	retval = fasync_helper(fd, filp, on, &event->fasync);
7563	inode_unlock(inode);
7564
7565	if (retval < 0)
7566	return retval;
7567
7568	return 0;
7569	}
7570
7571	static void perf_show_fdinfo(struct seq_file m, struct file f)
7572	{
7573	struct perf_event *event = f->private_data;
7574	struct perf_event_context *ctx;
7575	struct mutex *child_mutex;
7576
7577	ctx = perf_event_ctx_lock(event);
7578	child_mutex = event->parent ? &event->parent->child_mutex : &event->child_mutex;
7579	mutex_lock(child_mutex);
7580
7581	seq_printf(m, "perf_event_attr.type:\t%u\n", event->orig_type);
7582	if (event->pmu)
7583	seq_printf(m, "pmu_type:\t%u\n", event->pmu->type);
7584	seq_printf(m, "perf_event_attr.config:\t0x%llx\n", (unsigned long long)event->attr.config);
7585	seq_printf(m, "perf_event_attr.config1:\t0x%llx\n",
7586	(unsigned long long)event->attr.config1);
7587	seq_printf(m, "perf_event_attr.config2:\t0x%llx\n",
7588	(unsigned long long)event->attr.config2);
7589	seq_printf(m, "perf_event_attr.config3:\t0x%llx\n",
7590	(unsigned long long)event->attr.config3);
7591	seq_printf(m, "perf_event_attr.config4:\t0x%llx\n",
7592	(unsigned long long)event->attr.config4);
7593
7594	mutex_unlock(child_mutex);
7595	perf_event_ctx_unlock(event, ctx);
7596	}
7597
7598	static const struct file_operations perf_fops = {
7599	.release = perf_release,
7600	.read = perf_read,
7601	.poll = perf_poll,
7602	.unlocked_ioctl = perf_ioctl,
7603	.compat_ioctl = perf_compat_ioctl,
7604	.mmap = perf_mmap,
7605	.fasync = perf_fasync,
7606	.show_fdinfo = perf_show_fdinfo,
7607	};
7608
7609	/*
7610	* Perf event wakeup
7611	*
7612	* If there's data, ensure we set the poll() state and publish everything
7613	* to user-space before waking everybody up.
7614	*/
7615
7616	void perf_event_wakeup(struct perf_event *event)
7617	{
7618	ring_buffer_wakeup(event);
7619
7620	if (event->pending_kill) {
7621	kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
7622	event->pending_kill = 0;
7623	}
7624	}
7625
7626	static void perf_sigtrap(struct perf_event *event)
7627	{
7628	/*
7629	* Both perf_pending_task() and perf_pending_irq() can race with the
7630	* task exiting or exec-ing. We can determine if such a race has
7631	* occurred by checking if perf_event_exit_task(), which will set
7632	* ctx->task to TASK_TOMBSTONE, has already been called.
7633	*/
7634	if (event->ctx->task == TASK_TOMBSTONE)
7635	return;
7636
7637	/*
7638	* We'd expect this to only occur if the irq_work is delayed and either
7639	* ctx->task or current has changed in the meantime. This can be the
7640	* case on architectures that do not implement arch_irq_work_raise().
7641	*/
7642	if (WARN_ON_ONCE(event->ctx->task != current))
7643	return;
7644
7645	send_sig_perf((void __user *)event->pending_addr,
7646	event->orig_type, event->attr.sig_data);
7647	}
7648
7649	/*
7650	* Deliver the pending work in-event-context or follow the context.
7651	*/
7652	static void __perf_pending_disable(struct perf_event *event)
7653	{
7654	int cpu = READ_ONCE(event->oncpu);
7655
7656	/*
7657	* If the event isn't running; we done. event_sched_out() will have
7658	* taken care of things.
7659	*/
7660	if (cpu < 0)
7661	return;
7662
7663	/*
7664	* Yay, we hit home and are in the context of the event.
7665	*/
7666	if (cpu == smp_processor_id()) {
7667	if (event->pending_disable) {
7668	event->pending_disable = 0;
7669	perf_event_disable_local(event);
7670	}
7671	return;
7672	}
7673
7674	/*
7675	* CPU-A CPU-B
7676	*
7677	* perf_event_disable_inatomic()
7678	* @pending_disable = 1;
7679	* irq_work_queue();
7680	*
7681	* sched-out
7682	* @pending_disable = 0;
7683	*
7684	* sched-in
7685	* perf_event_disable_inatomic()
7686	* @pending_disable = 1;
7687	* irq_work_queue(); // FAILS
7688	*
7689	* irq_work_run()
7690	* perf_pending_disable()
7691	*
7692	* But the event runs on CPU-B and wants disabling there.
7693	*/
7694	irq_work_queue_on(&event->pending_disable_irq, cpu);
7695	}
7696
7697	static void perf_pending_disable(struct irq_work *entry)
7698	{
7699	struct perf_event *event = container_of(entry, struct perf_event, pending_disable_irq);
7700	int rctx;
7701
7702	/*
7703	* If we 'fail' here, that's OK, it means recursion is already disabled
7704	* and we won't recurse 'further'.
7705	*/
7706	rctx = perf_swevent_get_recursion_context();
7707	__perf_pending_disable(event);
7708	if (rctx >= 0)
7709	perf_swevent_put_recursion_context(rctx);
7710	}
7711
7712	static void perf_pending_irq(struct irq_work *entry)
7713	{
7714	struct perf_event *event = container_of(entry, struct perf_event, pending_irq);
7715	int rctx;
7716
7717	/*
7718	* If we 'fail' here, that's OK, it means recursion is already disabled
7719	* and we won't recurse 'further'.
7720	*/
7721	rctx = perf_swevent_get_recursion_context();
7722
7723	/*
7724	* The wakeup isn't bound to the context of the event -- it can happen
7725	* irrespective of where the event is.
7726	*/
7727	if (event->pending_wakeup) {
7728	event->pending_wakeup = 0;
7729	perf_event_wakeup(event);
7730	}
7731
7732	if (rctx >= 0)
7733	perf_swevent_put_recursion_context(rctx);
7734	}
7735
7736	static void perf_pending_task(struct callback_head *head)
7737	{
7738	struct perf_event *event = container_of(head, struct perf_event, pending_task);
7739	int rctx;
7740
7741	/*
7742	* If we 'fail' here, that's OK, it means recursion is already disabled
7743	* and we won't recurse 'further'.
7744	*/
7745	rctx = perf_swevent_get_recursion_context();
7746
7747	if (event->pending_work) {
7748	event->pending_work = 0;
7749	perf_sigtrap(event);
7750	local_dec(&event->ctx->nr_no_switch_fast);
7751	}
7752	put_event(event);
7753
7754	if (rctx >= 0)
7755	perf_swevent_put_recursion_context(rctx);
7756	}
7757
7758	#ifdef CONFIG_GUEST_PERF_EVENTS
7759	struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
7760
7761	DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
7762	DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
7763	DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
7764	DEFINE_STATIC_CALL_RET0(__perf_guest_handle_mediated_pmi, *perf_guest_cbs->handle_mediated_pmi);
7765
7766	void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
7767	{
7768	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs)))
7769	return;
7770
7771	rcu_assign_pointer(perf_guest_cbs, cbs);
7772	static_call_update(__perf_guest_state, cbs->state);
7773	static_call_update(__perf_guest_get_ip, cbs->get_ip);
7774
7775	/* Implementing ->handle_intel_pt_intr is optional. */
7776	if (cbs->handle_intel_pt_intr)
7777	static_call_update(__perf_guest_handle_intel_pt_intr,
7778	cbs->handle_intel_pt_intr);
7779
7780	if (cbs->handle_mediated_pmi)
7781	static_call_update(__perf_guest_handle_mediated_pmi,
7782	cbs->handle_mediated_pmi);
7783	}
7784	EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
7785
7786	void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
7787	{
7788	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs))
7789	return;
7790
7791	rcu_assign_pointer(perf_guest_cbs, NULL);
7792	static_call_update(__perf_guest_state, (void *)&__static_call_return0);
7793	static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0);
7794	static_call_update(__perf_guest_handle_intel_pt_intr, (void *)&__static_call_return0);
7795	static_call_update(__perf_guest_handle_mediated_pmi, (void *)&__static_call_return0);
7796	synchronize_rcu();
7797	}
7798	EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
7799	#endif
7800
7801	static bool should_sample_guest(struct perf_event *event)
7802	{
7803	return !event->attr.exclude_guest && perf_guest_state();
7804	}
7805
7806	unsigned long perf_misc_flags(struct perf_event *event,
7807	struct pt_regs *regs)
7808	{
7809	if (should_sample_guest(event))
7810	return perf_arch_guest_misc_flags(regs);
7811
7812	return perf_arch_misc_flags(regs);
7813	}
7814
7815	unsigned long perf_instruction_pointer(struct perf_event *event,
7816	struct pt_regs *regs)
7817	{
7818	/*
7819	* Hardware skid can lead to a scenario where a PMI is
7820	* delivered after the CPU has already entered kernel mode.
7821	* In that case, user-space sampling must not expose kernel
7822	* register state.
7823	*/
7824	if (should_sample_guest(event)) {
7825	return event->attr.exclude_kernel &&
7826	!(perf_guest_state() & PERF_GUEST_USER) ?
7827	0 : perf_guest_get_ip();
7828	}
7829
7830	return event->attr.exclude_kernel && !user_mode(regs) ?
7831	0 : perf_arch_instruction_pointer(regs);
7832	}
7833
7834	u64 __weak perf_reg_value(struct pt_regs *regs, int idx)
7835	{
7836	return 0;
7837	}
7838
7839	int __weak perf_reg_validate(u64 mask, bool simd_enabled)
7840	{
7841	return mask ? -ENOSYS : 0;
7842	}
7843
7844	u64 __weak perf_reg_abi(struct task_struct *task)
7845	{
7846	return PERF_SAMPLE_REGS_ABI_NONE;
7847	}
7848
7849	void __weak perf_get_regs_user(struct perf_regs *regs_user,
7850	struct pt_regs *regs)
7851	{
7852	regs_user->regs = task_pt_regs(current);
7853	regs_user->abi = perf_reg_abi(current);
7854	}
7855
7856	#define word_for_each_set_bit(bit, val) \
7857	for (unsigned long long __v = (val); \
7858	__v && ((bit = __builtin_ctzll(__v)), 1); \
7859	__v &= __v - 1)
7860
7861	static void
7862	perf_output_sample_regs(struct perf_output_handle *handle,
7863	struct pt_regs *regs, u64 mask)
7864	{
7865	int bit;
7866
7867	word_for_each_set_bit(bit, mask) {
7868	u64 val = perf_reg_value(regs, bit);
7869	perf_output_put(handle, val);
7870	}
7871	}
7872
7873	static void
7874	perf_output_sample_simd_regs(struct perf_output_handle *handle,
7875	struct perf_event *event,
7876	struct pt_regs *regs,
7877	u64 mask, u32 pred_mask)
7878	{
7879	u64 pred_qwords = event->attr.sample_simd_pred_reg_qwords;
7880	u64 vec_qwords = event->attr.sample_simd_vec_reg_qwords;
7881	u64 nr_vectors = hweight64(mask);
7882	u64 nr_pred = hweight32(pred_mask);
7883	int bit;
7884
7885	perf_output_put(handle, nr_vectors);
7886	perf_output_put(handle, vec_qwords);
7887	perf_output_put(handle, nr_pred);
7888	perf_output_put(handle, pred_qwords);
7889
7890	if (nr_vectors) {
7891	word_for_each_set_bit(bit, mask) {
7892	for (int i = 0; i < vec_qwords; i++) {
7893	u64 val = perf_simd_reg_value(regs, bit,
7894	i, false);
7895	perf_output_put(handle, val);
7896	}
7897	}
7898	}
7899	if (nr_pred) {
7900	word_for_each_set_bit(bit, pred_mask) {
7901	for (int i = 0; i < pred_qwords; i++) {
7902	u64 val = perf_simd_reg_value(regs, bit,
7903	i, true);
7904	perf_output_put(handle, val);
7905	}
7906	}
7907	}
7908	}
7909
7910	static void perf_sample_regs_user(struct perf_regs *regs_user,
7911	struct pt_regs *regs)
7912	{
7913	if (user_mode(regs)) {
7914	regs_user->abi = perf_reg_abi(current);
7915	regs_user->regs = regs;
7916	} else if (is_user_task(current)) {
7917	perf_get_regs_user(regs_user, regs);
7918	} else {
7919	regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
7920	regs_user->regs = NULL;
7921	}
7922	}
7923
7924	static void perf_sample_regs_intr(struct perf_regs *regs_intr,
7925	struct pt_regs *regs,
7926	bool exclude_kernel)
7927	{
7928	/*
7929	* Hardware skid can lead to a scenario where a PMI is
7930	* delivered after the CPU has already entered kernel mode.
7931	* In that case, user-space sampling must not expose kernel
7932	* register state.
7933	*/
7934	if (exclude_kernel && !user_mode(regs)) {
7935	regs_intr->abi = PERF_SAMPLE_REGS_ABI_NONE;
7936	regs_intr->regs = NULL;
7937	} else {
7938	regs_intr->regs = regs;
7939	regs_intr->abi = perf_reg_abi(current);
7940	}
7941	}
7942
7943	int __weak perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask,
7944	u16 pred_qwords, u32 pred_mask)
7945	{
7946	return -EINVAL;
7947	}
7948
7949	u64 __weak perf_simd_reg_value(struct pt_regs *regs, int idx,
7950	u16 qwords_idx, bool pred)
7951	{
7952	return 0;
7953	}
7954
7955	/*
7956	* Get remaining task size from user stack pointer.
7957	*
7958	* It'd be better to take stack vma map and limit this more
7959	* precisely, but there's no way to get it safely under interrupt,
7960	* so using TASK_SIZE as limit.
7961	*/
7962	static u64 perf_ustack_task_size(struct pt_regs *regs)
7963	{
7964	unsigned long addr = perf_user_stack_pointer(regs);
7965
7966	if (!addr \|\| addr >= TASK_SIZE)
7967	return 0;
7968
7969	return TASK_SIZE - addr;
7970	}
7971
7972	static u16
7973	perf_sample_ustack_size(u16 stack_size, u16 header_size,
7974	struct pt_regs *regs)
7975	{
7976	u64 task_size;
7977
7978	/* No regs, no stack pointer, no dump. */
7979	if (!regs)
7980	return 0;
7981
7982	/* No mm, no stack, no dump. */
7983	if (!current->mm)
7984	return 0;
7985
7986	/*
7987	* Check if we fit in with the requested stack size into the:
7988	* - TASK_SIZE
7989	* If we don't, we limit the size to the TASK_SIZE.
7990	*
7991	* - remaining sample size
7992	* If we don't, we customize the stack size to
7993	* fit in to the remaining sample size.
7994	*/
7995
7996	task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
7997	stack_size = min(stack_size, (u16) task_size);
7998
7999	/* Current header size plus static size and dynamic size. */
8000	header_size += 2 * sizeof(u64);
8001
8002	/* Do we fit in with the current stack dump size? */
8003	if ((u16) (header_size + stack_size) < header_size) {
8004	/*
8005	* If we overflow the maximum size for the sample,
8006	* we customize the stack dump size to fit in.
8007	*/
8008	stack_size = USHRT_MAX - header_size - sizeof(u64);
8009	stack_size = round_up(stack_size, sizeof(u64));
8010	}
8011
8012	return stack_size;
8013	}
8014
8015	static void
8016	perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
8017	struct pt_regs *regs)
8018	{
8019	/* Case of a kernel thread, nothing to dump */
8020	if (!regs) {
8021	u64 size = 0;
8022	perf_output_put(handle, size);
8023	} else {
8024	unsigned long sp;
8025	unsigned int rem;
8026	u64 dyn_size;
8027
8028	/*
8029	* We dump:
8030	* static size
8031	* - the size requested by user or the best one we can fit
8032	* in to the sample max size
8033	* data
8034	* - user stack dump data
8035	* dynamic size
8036	* - the actual dumped size
8037	*/
8038
8039	/* Static size. */
8040	perf_output_put(handle, dump_size);
8041
8042	/* Data. */
8043	sp = perf_user_stack_pointer(regs);
8044	rem = __output_copy_user(handle, (void *) sp, dump_size);
8045	dyn_size = dump_size - rem;
8046
8047	perf_output_skip(handle, rem);
8048
8049	/* Dynamic size. */
8050	perf_output_put(handle, dyn_size);
8051	}
8052	}
8053
8054	static unsigned long perf_prepare_sample_aux(struct perf_event *event,
8055	struct perf_sample_data *data,
8056	size_t size)
8057	{
8058	struct perf_event *sampler = event->aux_event;
8059	struct perf_buffer *rb;
8060
8061	data->aux_size = 0;
8062
8063	if (!sampler)
8064	goto out;
8065
8066	if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))
8067	goto out;
8068
8069	if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))
8070	goto out;
8071
8072	rb = ring_buffer_get(sampler);
8073	if (!rb)
8074	goto out;
8075
8076	/*
8077	* If this is an NMI hit inside sampling code, don't take
8078	* the sample. See also perf_aux_sample_output().
8079	*/
8080	if (READ_ONCE(rb->aux_in_sampling)) {
8081	data->aux_size = 0;
8082	} else {
8083	size = min_t(size_t, size, perf_aux_size(rb));
8084	data->aux_size = ALIGN(size, sizeof(u64));
8085	}
8086	ring_buffer_put(rb);
8087
8088	out:
8089	return data->aux_size;
8090	}
8091
8092	static long perf_pmu_snapshot_aux(struct perf_buffer *rb,
8093	struct perf_event *event,
8094	struct perf_output_handle *handle,
8095	unsigned long size)
8096	{
8097	unsigned long flags;
8098	long ret;
8099
8100	/*
8101	* Normal ->start()/->stop() callbacks run in IRQ mode in scheduler
8102	* paths. If we start calling them in NMI context, they may race with
8103	* the IRQ ones, that is, for example, re-starting an event that's just
8104	* been stopped, which is why we're using a separate callback that
8105	* doesn't change the event state.
8106	*
8107	* IRQs need to be disabled to prevent IPIs from racing with us.
8108	*/
8109	local_irq_save(flags);
8110	/*
8111	* Guard against NMI hits inside the critical section;
8112	* see also perf_prepare_sample_aux().
8113	*/
8114	WRITE_ONCE(rb->aux_in_sampling, 1);
8115	barrier();
8116
8117	ret = event->pmu->snapshot_aux(event, handle, size);
8118
8119	barrier();
8120	WRITE_ONCE(rb->aux_in_sampling, 0);
8121	local_irq_restore(flags);
8122
8123	return ret;
8124	}
8125
8126	static void perf_aux_sample_output(struct perf_event *event,
8127	struct perf_output_handle *handle,
8128	struct perf_sample_data *data)
8129	{
8130	struct perf_event *sampler = event->aux_event;
8131	struct perf_buffer *rb;
8132	unsigned long pad;
8133	long size;
8134
8135	if (WARN_ON_ONCE(!sampler \|\| !data->aux_size))
8136	return;
8137
8138	rb = ring_buffer_get(sampler);
8139	if (!rb)
8140	return;
8141
8142	size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);
8143
8144	/*
8145	* An error here means that perf_output_copy() failed (returned a
8146	* non-zero surplus that it didn't copy), which in its current
8147	* enlightened implementation is not possible. If that changes, we'd
8148	* like to know.
8149	*/
8150	if (WARN_ON_ONCE(size < 0))
8151	goto out_put;
8152
8153	/*
8154	* The pad comes from ALIGN()ing data->aux_size up to u64 in
8155	* perf_prepare_sample_aux(), so should not be more than that.
8156	*/
8157	pad = data->aux_size - size;
8158	if (WARN_ON_ONCE(pad >= sizeof(u64)))
8159	pad = 8;
8160
8161	if (pad) {
8162	u64 zero = 0;
8163	perf_output_copy(handle, &zero, pad);
8164	}
8165
8166	out_put:
8167	ring_buffer_put(rb);
8168	}
8169
8170	/*
8171	* A set of common sample data types saved even for non-sample records
8172	* when event->attr.sample_id_all is set.
8173	*/
8174	#define PERF_SAMPLE_ID_ALL (PERF_SAMPLE_TID \| PERF_SAMPLE_TIME \| \
8175	PERF_SAMPLE_ID \| PERF_SAMPLE_STREAM_ID \| \
8176	PERF_SAMPLE_CPU \| PERF_SAMPLE_IDENTIFIER)
8177
8178	static void __perf_event_header__init_id(struct perf_sample_data *data,
8179	struct perf_event *event,
8180	u64 sample_type)
8181	{
8182	data->type = event->attr.sample_type;
8183	data->sample_flags \|= data->type & PERF_SAMPLE_ID_ALL;
8184
8185	if (sample_type & PERF_SAMPLE_TID) {
8186	/* namespace issues */
8187	data->tid_entry.pid = perf_event_pid(event, current);
8188	data->tid_entry.tid = perf_event_tid(event, current);
8189	}
8190
8191	if (sample_type & PERF_SAMPLE_TIME)
8192	data->time = perf_event_clock(event);
8193
8194	if (sample_type & (PERF_SAMPLE_ID \| PERF_SAMPLE_IDENTIFIER))
8195	data->id = primary_event_id(event);
8196
8197	if (sample_type & PERF_SAMPLE_STREAM_ID)
8198	data->stream_id = event->id;
8199
8200	if (sample_type & PERF_SAMPLE_CPU) {
8201	data->cpu_entry.cpu = raw_smp_processor_id();
8202	data->cpu_entry.reserved = 0;
8203	}
8204	}
8205
8206	void perf_event_header__init(struct perf_event_header *header,
8207	struct perf_sample_data *data,
8208	u32 type, u16 misc, u16 size,
8209	struct perf_event *event)
8210	{
8211	header->type = type;
8212	header->misc = misc;
8213	header->size = size;
8214
8215	if (event->attr.sample_id_all) {
8216	header->size += event->id_header_size;
8217	__perf_event_header__init_id(data, event, event->attr.sample_type);
8218	}
8219	}
8220
8221	static void __perf_event__output_id_sample(struct perf_output_handle *handle,
8222	struct perf_sample_data *data)
8223	{
8224	u64 sample_type = data->type;
8225
8226	if (sample_type & PERF_SAMPLE_TID)
8227	perf_output_put(handle, data->tid_entry);
8228
8229	if (sample_type & PERF_SAMPLE_TIME)
8230	perf_output_put(handle, data->time);
8231
8232	if (sample_type & PERF_SAMPLE_ID)
8233	perf_output_put(handle, data->id);
8234
8235	if (sample_type & PERF_SAMPLE_STREAM_ID)
8236	perf_output_put(handle, data->stream_id);
8237
8238	if (sample_type & PERF_SAMPLE_CPU)
8239	perf_output_put(handle, data->cpu_entry);
8240
8241	if (sample_type & PERF_SAMPLE_IDENTIFIER)
8242	perf_output_put(handle, data->id);
8243	}
8244
8245	void perf_event__output_id_sample(struct perf_event *event,
8246	struct perf_output_handle *handle,
8247	struct perf_sample_data *sample)
8248	{
8249	if (event->attr.sample_id_all)
8250	__perf_event__output_id_sample(handle, sample);
8251	}
8252
8253	static void perf_output_read_one(struct perf_output_handle *handle,
8254	struct perf_event *event,
8255	u64 enabled, u64 running)
8256	{
8257	u64 read_format = event->attr.read_format;
8258	u64 values[5];
8259	int n = 0;
8260
8261	values[n++] = perf_event_count(event, has_inherit_and_sample_read(&event->attr));
8262	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
8263	values[n++] = enabled +
8264	atomic64_read(&event->child_total_time_enabled);
8265	}
8266	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
8267	values[n++] = running +
8268	atomic64_read(&event->child_total_time_running);
8269	}
8270	if (read_format & PERF_FORMAT_ID)
8271	values[n++] = primary_event_id(event);
8272	if (read_format & PERF_FORMAT_LOST)
8273	values[n++] = atomic64_read(&event->lost_samples);
8274
8275	__output_copy(handle, values, n * sizeof(u64));
8276	}
8277
8278	static void perf_output_read_group(struct perf_output_handle *handle,
8279	struct perf_event *event,
8280	u64 enabled, u64 running)
8281	{
8282	struct perf_event leader = event->group_leader, sub;
8283	u64 read_format = event->attr.read_format;
8284	unsigned long flags;
8285	u64 values[6];
8286	int n = 0;
8287	bool self = has_inherit_and_sample_read(&event->attr);
8288
8289	/*
8290	* Disabling interrupts avoids all counter scheduling
8291	* (context switches, timer based rotation and IPIs).
8292	*/
8293	local_irq_save(flags);
8294
8295	values[n++] = 1 + leader->nr_siblings;
8296
8297	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
8298	values[n++] = enabled;
8299
8300	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
8301	values[n++] = running;
8302
8303	if ((leader != event) && !handle->skip_read)
8304	perf_pmu_read(leader);
8305
8306	values[n++] = perf_event_count(leader, self);
8307	if (read_format & PERF_FORMAT_ID)
8308	values[n++] = primary_event_id(leader);
8309	if (read_format & PERF_FORMAT_LOST)
8310	values[n++] = atomic64_read(&leader->lost_samples);
8311
8312	__output_copy(handle, values, n * sizeof(u64));
8313
8314	for_each_sibling_event(sub, leader) {
8315	n = 0;
8316
8317	if ((sub != event) && !handle->skip_read)
8318	perf_pmu_read(sub);
8319
8320	values[n++] = perf_event_count(sub, self);
8321	if (read_format & PERF_FORMAT_ID)
8322	values[n++] = primary_event_id(sub);
8323	if (read_format & PERF_FORMAT_LOST)
8324	values[n++] = atomic64_read(&sub->lost_samples);
8325
8326	__output_copy(handle, values, n * sizeof(u64));
8327	}
8328
8329	local_irq_restore(flags);
8330	}
8331
8332	#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED\|\
8333	PERF_FORMAT_TOTAL_TIME_RUNNING)
8334
8335	/*
8336	* XXX PERF_SAMPLE_READ vs inherited events seems difficult.
8337	*
8338	* The problem is that its both hard and excessively expensive to iterate the
8339	* child list, not to mention that its impossible to IPI the children running
8340	* on another CPU, from interrupt/NMI context.
8341	*
8342	* Instead the combination of PERF_SAMPLE_READ and inherit will track per-thread
8343	* counts rather than attempting to accumulate some value across all children on
8344	* all cores.
8345	*/
8346	static void perf_output_read(struct perf_output_handle *handle,
8347	struct perf_event *event)
8348	{
8349	u64 enabled = 0, running = 0, now;
8350	u64 read_format = event->attr.read_format;
8351
8352	/*
8353	* Compute total_time_enabled, total_time_running based on snapshot
8354	* values taken when the event was last scheduled in.
8355	*
8356	* We cannot simply call update_context_time() because doing so would
8357	* lead to deadlock when called from NMI context.
8358	*/
8359	if (read_format & PERF_FORMAT_TOTAL_TIMES)
8360	calc_timer_values(event, &now, &enabled, &running);
8361
8362	if (event->attr.read_format & PERF_FORMAT_GROUP)
8363	perf_output_read_group(handle, event, enabled, running);
8364	else
8365	perf_output_read_one(handle, event, enabled, running);
8366	}
8367
8368	void perf_output_sample(struct perf_output_handle *handle,
8369	struct perf_event_header *header,
8370	struct perf_sample_data *data,
8371	struct perf_event *event)
8372	{
8373	u64 sample_type = data->type;
8374
8375	if (data->sample_flags & PERF_SAMPLE_READ)
8376	handle->skip_read = 1;
8377
8378	perf_output_put(handle, *header);
8379
8380	if (sample_type & PERF_SAMPLE_IDENTIFIER)
8381	perf_output_put(handle, data->id);
8382
8383	if (sample_type & PERF_SAMPLE_IP)
8384	perf_output_put(handle, data->ip);
8385
8386	if (sample_type & PERF_SAMPLE_TID)
8387	perf_output_put(handle, data->tid_entry);
8388
8389	if (sample_type & PERF_SAMPLE_TIME)
8390	perf_output_put(handle, data->time);
8391
8392	if (sample_type & PERF_SAMPLE_ADDR)
8393	perf_output_put(handle, data->addr);
8394
8395	if (sample_type & PERF_SAMPLE_ID)
8396	perf_output_put(handle, data->id);
8397
8398	if (sample_type & PERF_SAMPLE_STREAM_ID)
8399	perf_output_put(handle, data->stream_id);
8400
8401	if (sample_type & PERF_SAMPLE_CPU)
8402	perf_output_put(handle, data->cpu_entry);
8403
8404	if (sample_type & PERF_SAMPLE_PERIOD)
8405	perf_output_put(handle, data->period);
8406
8407	if (sample_type & PERF_SAMPLE_READ)
8408	perf_output_read(handle, event);
8409
8410	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
8411	int size = 1;
8412
8413	size += data->callchain->nr;
8414	size *= sizeof(u64);
8415	__output_copy(handle, data->callchain, size);
8416	}
8417
8418	if (sample_type & PERF_SAMPLE_RAW) {
8419	struct perf_raw_record *raw = data->raw;
8420
8421	if (raw) {
8422	struct perf_raw_frag *frag = &raw->frag;
8423
8424	perf_output_put(handle, raw->size);
8425	do {
8426	if (frag->copy) {
8427	__output_custom(handle, frag->copy,
8428	frag->data, frag->size);
8429	} else {
8430	__output_copy(handle, frag->data,
8431	frag->size);
8432	}
8433	if (perf_raw_frag_last(frag))
8434	break;
8435	frag = frag->next;
8436	} while (1);
8437	if (frag->pad)
8438	__output_skip(handle, NULL, frag->pad);
8439	} else {
8440	struct {
8441	u32 size;
8442	u32 data;
8443	} raw = {
8444	.size = sizeof(u32),
8445	.data = 0,
8446	};
8447	perf_output_put(handle, raw);
8448	}
8449	}
8450
8451	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
8452	if (data->br_stack) {
8453	size_t size;
8454
8455	size = data->br_stack->nr
8456	* sizeof(struct perf_branch_entry);
8457
8458	perf_output_put(handle, data->br_stack->nr);
8459	if (branch_sample_hw_index(event))
8460	perf_output_put(handle, data->br_stack->hw_idx);
8461	perf_output_copy(handle, data->br_stack->entries, size);
8462	/*
8463	* Add the extension space which is appended
8464	* right after the struct perf_branch_stack.
8465	*/
8466	if (data->br_stack_cntr) {
8467	size = data->br_stack->nr * sizeof(u64);
8468	perf_output_copy(handle, data->br_stack_cntr, size);
8469	}
8470	} else {
8471	/*
8472	* we always store at least the value of nr
8473	*/
8474	u64 nr = 0;
8475	perf_output_put(handle, nr);
8476	}
8477	}
8478
8479	if (sample_type & PERF_SAMPLE_REGS_USER) {
8480	u64 abi = data->regs_user.abi;
8481
8482	/*
8483	* If there are no regs to dump, notice it through
8484	* first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
8485	*/
8486	perf_output_put(handle, abi);
8487
8488	if (abi) {
8489	struct perf_event_attr *attr = &event->attr;
8490	u64 mask = attr->sample_regs_user;
8491	perf_output_sample_regs(handle,
8492	data->regs_user.regs,
8493	mask);
8494	if (abi & PERF_SAMPLE_REGS_ABI_SIMD) {
8495	perf_output_sample_simd_regs(handle, event,
8496	data->regs_user.regs,
8497	attr->sample_simd_vec_reg_user,
8498	attr->sample_simd_pred_reg_user);
8499	}
8500	}
8501	}
8502
8503	if (sample_type & PERF_SAMPLE_STACK_USER) {
8504	perf_output_sample_ustack(handle,
8505	data->stack_user_size,
8506	data->regs_user.regs);
8507	}
8508
8509	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
8510	perf_output_put(handle, data->weight.full);
8511
8512	if (sample_type & PERF_SAMPLE_DATA_SRC)
8513	perf_output_put(handle, data->data_src.val);
8514
8515	if (sample_type & PERF_SAMPLE_TRANSACTION)
8516	perf_output_put(handle, data->txn);
8517
8518	if (sample_type & PERF_SAMPLE_REGS_INTR) {
8519	u64 abi = data->regs_intr.abi;
8520	/*
8521	* If there are no regs to dump, notice it through
8522	* first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
8523	*/
8524	perf_output_put(handle, abi);
8525
8526	if (abi) {
8527	struct perf_event_attr *attr = &event->attr;
8528	u64 mask = attr->sample_regs_intr;
8529
8530	perf_output_sample_regs(handle,
8531	data->regs_intr.regs,
8532	mask);
8533	if (abi & PERF_SAMPLE_REGS_ABI_SIMD) {
8534	perf_output_sample_simd_regs(handle, event,
8535	data->regs_intr.regs,
8536	attr->sample_simd_vec_reg_intr,
8537	attr->sample_simd_pred_reg_intr);
8538	}
8539	}
8540	}
8541
8542	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
8543	perf_output_put(handle, data->phys_addr);
8544
8545	if (sample_type & PERF_SAMPLE_CGROUP)
8546	perf_output_put(handle, data->cgroup);
8547
8548	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
8549	perf_output_put(handle, data->data_page_size);
8550
8551	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
8552	perf_output_put(handle, data->code_page_size);
8553
8554	if (sample_type & PERF_SAMPLE_AUX) {
8555	perf_output_put(handle, data->aux_size);
8556
8557	if (data->aux_size)
8558	perf_aux_sample_output(event, handle, data);
8559	}
8560
8561	if (!event->attr.watermark) {
8562	int wakeup_events = event->attr.wakeup_events;
8563
8564	if (wakeup_events) {
8565	struct perf_buffer *rb = handle->rb;
8566	int events = local_inc_return(&rb->events);
8567
8568	if (events >= wakeup_events) {
8569	local_sub(wakeup_events, &rb->events);
8570	local_inc(&rb->wakeup);
8571	}
8572	}
8573	}
8574	}
8575
8576	static u64 perf_virt_to_phys(u64 virt)
8577	{
8578	u64 phys_addr = 0;
8579
8580	if (!virt)
8581	return 0;
8582
8583	if (virt >= TASK_SIZE) {
8584	/* If it's vmalloc()d memory, leave phys_addr as 0 */
8585	if (virt_addr_valid((void *)(uintptr_t)virt) &&
8586	!(virt >= VMALLOC_START && virt < VMALLOC_END))
8587	phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
8588	} else {
8589	/*
8590	* Walking the pages tables for user address.
8591	* Interrupts are disabled, so it prevents any tear down
8592	* of the page tables.
8593	* Try IRQ-safe get_user_page_fast_only first.
8594	* If failed, leave phys_addr as 0.
8595	*/
8596	if (is_user_task(current)) {
8597	struct page *p;
8598
8599	pagefault_disable();
8600	if (get_user_page_fast_only(virt, 0, &p)) {
8601	phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
8602	put_page(p);
8603	}
8604	pagefault_enable();
8605	}
8606	}
8607
8608	return phys_addr;
8609	}
8610
8611	/*
8612	* Return the pagetable size of a given virtual address.
8613	*/
8614	static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr)
8615	{
8616	u64 size = 0;
8617
8618	#ifdef CONFIG_HAVE_GUP_FAST
8619	pgd_t *pgdp, pgd;
8620	p4d_t *p4dp, p4d;
8621	pud_t *pudp, pud;
8622	pmd_t *pmdp, pmd;
8623	pte_t *ptep, pte;
8624
8625	pgdp = pgd_offset(mm, addr);
8626	pgd = pgdp_get(pgdp);
8627	if (pgd_none(pgd))
8628	return 0;
8629
8630	if (pgd_leaf(pgd))
8631	return pgd_leaf_size(pgd);
8632
8633	p4dp = p4d_offset_lockless(pgdp, pgd, addr);
8634	p4d = p4dp_get(p4dp);
8635	if (!p4d_present(p4d))
8636	return 0;
8637
8638	if (p4d_leaf(p4d))
8639	return p4d_leaf_size(p4d);
8640
8641	pudp = pud_offset_lockless(p4dp, p4d, addr);
8642	pud = pudp_get(pudp);
8643	if (!pud_present(pud))
8644	return 0;
8645
8646	if (pud_leaf(pud))
8647	return pud_leaf_size(pud);
8648
8649	pmdp = pmd_offset_lockless(pudp, pud, addr);
8650	again:
8651	pmd = pmdp_get_lockless(pmdp);
8652	if (!pmd_present(pmd))
8653	return 0;
8654
8655	if (pmd_leaf(pmd))
8656	return pmd_leaf_size(pmd);
8657
8658	ptep = pte_offset_map(&pmd, addr);
8659	if (!ptep)
8660	goto again;
8661
8662	pte = ptep_get_lockless(ptep);
8663	if (pte_present(pte))
8664	size = __pte_leaf_size(pmd, pte);
8665	pte_unmap(ptep);
8666	#endif /* CONFIG_HAVE_GUP_FAST */
8667
8668	return size;
8669	}
8670
8671	static u64 perf_get_page_size(unsigned long addr)
8672	{
8673	struct mm_struct *mm;
8674	unsigned long flags;
8675	u64 size;
8676
8677	if (!addr)
8678	return 0;
8679
8680	/*
8681	* Software page-table walkers must disable IRQs,
8682	* which prevents any tear down of the page tables.
8683	*/
8684	local_irq_save(flags);
8685
8686	mm = current->mm;
8687	if (!mm) {
8688	/*
8689	* For kernel threads and the like, use init_mm so that
8690	* we can find kernel memory.
8691	*/
8692	mm = &init_mm;
8693	}
8694
8695	size = perf_get_pgtable_size(mm, addr);
8696
8697	local_irq_restore(flags);
8698
8699	return size;
8700	}
8701
8702	static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
8703
8704	static struct unwind_work perf_unwind_work;
8705
8706	struct perf_callchain_entry *
8707	perf_callchain(struct perf_event event, struct pt_regs regs)
8708	{
8709	bool kernel = !event->attr.exclude_callchain_kernel;
8710	bool user = !event->attr.exclude_callchain_user &&
8711	is_user_task(current);
8712	/* Disallow cross-task user callchains. */
8713	bool crosstask = event->ctx->task && event->ctx->task != current;
8714	bool defer_user = IS_ENABLED(CONFIG_UNWIND_USER) && user &&
8715	event->attr.defer_callchain;
8716	const u32 max_stack = event->attr.sample_max_stack;
8717	struct perf_callchain_entry *callchain;
8718	u64 defer_cookie;
8719
8720	if (!current->mm)
8721	user = false;
8722
8723	if (!kernel && !user)
8724	return &__empty_callchain;
8725
8726	if (!(user && defer_user && !crosstask &&
8727	unwind_deferred_request(&perf_unwind_work, &defer_cookie) >= 0))
8728	defer_cookie = 0;
8729
8730	callchain = get_perf_callchain(regs, kernel, user, max_stack,
8731	crosstask, true, defer_cookie);
8732
8733	return callchain ?: &__empty_callchain;
8734	}
8735
8736	static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d)
8737	{
8738	return d * !!(flags & s);
8739	}
8740
8741	u64 perf_update_xregs_size(struct perf_event *event, bool intr)
8742	{
8743	u16 pred_qwords = event->attr.sample_simd_pred_reg_qwords;
8744	u16 vec_qwords = event->attr.sample_simd_vec_reg_qwords;
8745	u64 pred_mask;
8746	u64 mask;
8747	int size;
8748
8749	if (intr) {
8750	mask = event->attr.sample_simd_vec_reg_intr;
8751	pred_mask = event->attr.sample_simd_pred_reg_intr;
8752	} else {
8753	mask = event->attr.sample_simd_vec_reg_user;
8754	pred_mask = event->attr.sample_simd_pred_reg_user;
8755	}
8756
8757	size = sizeof(u64) * 4;
8758	size += (hweight64(mask) * vec_qwords +
8759	hweight64(pred_mask) * pred_qwords) * sizeof(u64);
8760
8761	return size;
8762	}
8763
8764	void perf_prepare_sample(struct perf_sample_data *data,
8765	struct perf_event *event,
8766	struct pt_regs *regs)
8767	{
8768	u64 sample_type = event->attr.sample_type;
8769	u64 filtered_sample_type;
8770
8771	/*
8772	* Add the sample flags that are dependent to others. And clear the
8773	* sample flags that have already been done by the PMU driver.
8774	*/
8775	filtered_sample_type = sample_type;
8776	filtered_sample_type \|= __cond_set(sample_type, PERF_SAMPLE_CODE_PAGE_SIZE,
8777	PERF_SAMPLE_IP);
8778	filtered_sample_type \|= __cond_set(sample_type, PERF_SAMPLE_DATA_PAGE_SIZE \|
8779	PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR);
8780	filtered_sample_type \|= __cond_set(sample_type, PERF_SAMPLE_STACK_USER,
8781	PERF_SAMPLE_REGS_USER);
8782	filtered_sample_type &= ~data->sample_flags;
8783
8784	if (filtered_sample_type == 0) {
8785	/* Make sure it has the correct data->type for output */
8786	data->type = event->attr.sample_type;
8787	return;
8788	}
8789
8790	__perf_event_header__init_id(data, event, filtered_sample_type);
8791
8792	if (filtered_sample_type & PERF_SAMPLE_IP) {
8793	data->ip = perf_instruction_pointer(event, regs);
8794	data->sample_flags \|= PERF_SAMPLE_IP;
8795	}
8796
8797	if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN)
8798	perf_sample_save_callchain(data, event, regs);
8799
8800	if (filtered_sample_type & PERF_SAMPLE_RAW) {
8801	data->raw = NULL;
8802	data->dyn_size += sizeof(u64);
8803	data->sample_flags \|= PERF_SAMPLE_RAW;
8804	}
8805
8806	if (filtered_sample_type & PERF_SAMPLE_BRANCH_STACK) {
8807	data->br_stack = NULL;
8808	data->dyn_size += sizeof(u64);
8809	data->sample_flags \|= PERF_SAMPLE_BRANCH_STACK;
8810	}
8811
8812	if (filtered_sample_type & PERF_SAMPLE_REGS_USER)
8813	perf_sample_regs_user(&data->regs_user, regs);
8814
8815	/*
8816	* It cannot use the filtered_sample_type here as REGS_USER can be set
8817	* by STACK_USER (using __cond_set() above) and we don't want to update
8818	* the dyn_size if it's not requested by users.
8819	*/
8820	if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) {
8821	/* regs dump ABI info */
8822	int size = sizeof(u64);
8823
8824	if (data->regs_user.regs) {
8825	u64 mask = event->attr.sample_regs_user;
8826
8827	size += hweight64(mask) * sizeof(u64);
8828	if (event_has_simd_regs(event)) {
8829	size += perf_update_xregs_size(event, false);
8830	data->regs_user.abi \|= PERF_SAMPLE_REGS_ABI_SIMD;
8831	}
8832	}
8833
8834	data->dyn_size += size;
8835	data->sample_flags \|= PERF_SAMPLE_REGS_USER;
8836	}
8837
8838	if (filtered_sample_type & PERF_SAMPLE_STACK_USER) {
8839	/*
8840	* Either we need PERF_SAMPLE_STACK_USER bit to be always
8841	* processed as the last one or have additional check added
8842	* in case new sample type is added, because we could eat
8843	* up the rest of the sample size.
8844	*/
8845	u16 stack_size = event->attr.sample_stack_user;
8846	u16 header_size = perf_sample_data_size(data, event);
8847	u16 size = sizeof(u64);
8848
8849	stack_size = perf_sample_ustack_size(stack_size, header_size,
8850	data->regs_user.regs);
8851
8852	/*
8853	* If there is something to dump, add space for the dump
8854	* itself and for the field that tells the dynamic size,
8855	* which is how many have been actually dumped.
8856	*/
8857	if (stack_size)
8858	size += sizeof(u64) + stack_size;
8859
8860	data->stack_user_size = stack_size;
8861	data->dyn_size += size;
8862	data->sample_flags \|= PERF_SAMPLE_STACK_USER;
8863	}
8864
8865	if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
8866	data->weight.full = 0;
8867	data->sample_flags \|= PERF_SAMPLE_WEIGHT_TYPE;
8868	}
8869
8870	if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) {
8871	data->data_src.val = PERF_MEM_NA;
8872	data->sample_flags \|= PERF_SAMPLE_DATA_SRC;
8873	}
8874
8875	if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) {
8876	data->txn = 0;
8877	data->sample_flags \|= PERF_SAMPLE_TRANSACTION;
8878	}
8879
8880	if (filtered_sample_type & PERF_SAMPLE_ADDR) {
8881	data->addr = 0;
8882	data->sample_flags \|= PERF_SAMPLE_ADDR;
8883	}
8884
8885	if (filtered_sample_type & PERF_SAMPLE_REGS_INTR) {
8886	/* regs dump ABI info */
8887	int size = sizeof(u64);
8888
8889	perf_sample_regs_intr(&data->regs_intr, regs,
8890	event->attr.exclude_kernel);
8891
8892	if (data->regs_intr.regs) {
8893	u64 mask = event->attr.sample_regs_intr;
8894
8895	size += hweight64(mask) * sizeof(u64);
8896	if (event_has_simd_regs(event)) {
8897	size += perf_update_xregs_size(event, true);
8898	data->regs_intr.abi \|= PERF_SAMPLE_REGS_ABI_SIMD;
8899	}
8900	}
8901
8902	data->dyn_size += size;
8903	data->sample_flags \|= PERF_SAMPLE_REGS_INTR;
8904	}
8905
8906	if (filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) {
8907	data->phys_addr = perf_virt_to_phys(data->addr);
8908	data->sample_flags \|= PERF_SAMPLE_PHYS_ADDR;
8909	}
8910
8911	#ifdef CONFIG_CGROUP_PERF
8912	if (filtered_sample_type & PERF_SAMPLE_CGROUP) {
8913	struct cgroup *cgrp;
8914
8915	/* protected by RCU */
8916	cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;
8917	data->cgroup = cgroup_id(cgrp);
8918	data->sample_flags \|= PERF_SAMPLE_CGROUP;
8919	}
8920	#endif
8921
8922	/*
8923	* PERF_DATA_PAGE_SIZE requires PERF_SAMPLE_ADDR. If the user doesn't
8924	* require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,
8925	* but the value will not dump to the userspace.
8926	*/
8927	if (filtered_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) {
8928	data->data_page_size = perf_get_page_size(data->addr);
8929	data->sample_flags \|= PERF_SAMPLE_DATA_PAGE_SIZE;
8930	}
8931
8932	if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) {
8933	data->code_page_size = perf_get_page_size(data->ip);
8934	data->sample_flags \|= PERF_SAMPLE_CODE_PAGE_SIZE;
8935	}
8936
8937	if (filtered_sample_type & PERF_SAMPLE_AUX) {
8938	u64 size;
8939	u16 header_size = perf_sample_data_size(data, event);
8940
8941	header_size += sizeof(u64); /* size */
8942
8943	/*
8944	* Given the 16bit nature of header::size, an AUX sample can
8945	* easily overflow it, what with all the preceding sample bits.
8946	* Make sure this doesn't happen by using up to U16_MAX bytes
8947	* per sample in total (rounded down to 8 byte boundary).
8948	*/
8949	size = min_t(size_t, U16_MAX - header_size,
8950	event->attr.aux_sample_size);
8951	size = rounddown(size, 8);
8952	size = perf_prepare_sample_aux(event, data, size);
8953
8954	WARN_ON_ONCE(size + header_size > U16_MAX);
8955	data->dyn_size += size + sizeof(u64); /* size above */
8956	data->sample_flags \|= PERF_SAMPLE_AUX;
8957	}
8958	}
8959
8960	void perf_prepare_header(struct perf_event_header *header,
8961	struct perf_sample_data *data,
8962	struct perf_event *event,
8963	struct pt_regs *regs)
8964	{
8965	header->type = PERF_RECORD_SAMPLE;
8966	header->size = perf_sample_data_size(data, event);
8967	header->misc = perf_misc_flags(event, regs);
8968
8969	/*
8970	* If you're adding more sample types here, you likely need to do
8971	* something about the overflowing header::size, like repurpose the
8972	* lowest 3 bits of size, which should be always zero at the moment.
8973	* This raises a more important question, do we really need 512k sized
8974	* samples and why, so good argumentation is in order for whatever you
8975	* do here next.
8976	*/
8977	WARN_ON_ONCE(header->size & 7);
8978	}
8979
8980	static void __perf_event_aux_pause(struct perf_event *event, bool pause)
8981	{
8982	if (pause) {
8983	if (!event->hw.aux_paused) {
8984	event->hw.aux_paused = 1;
8985	event->pmu->stop(event, PERF_EF_PAUSE);
8986	}
8987	} else {
8988	if (event->hw.aux_paused) {
8989	event->hw.aux_paused = 0;
8990	event->pmu->start(event, PERF_EF_RESUME);
8991	}
8992	}
8993	}
8994
8995	static void perf_event_aux_pause(struct perf_event *event, bool pause)
8996	{
8997	struct perf_buffer *rb;
8998
8999	if (WARN_ON_ONCE(!event))
9000	return;
9001
9002	rb = ring_buffer_get(event);
9003	if (!rb)
9004	return;
9005
9006	scoped_guard (irqsave) {
9007	/*
9008	* Guard against self-recursion here. Another event could trip
9009	* this same from NMI context.
9010	*/
9011	if (READ_ONCE(rb->aux_in_pause_resume))
9012	break;
9013
9014	WRITE_ONCE(rb->aux_in_pause_resume, 1);
9015	barrier();
9016	__perf_event_aux_pause(event, pause);
9017	barrier();
9018	WRITE_ONCE(rb->aux_in_pause_resume, 0);
9019	}
9020	ring_buffer_put(rb);
9021	}
9022
9023	static __always_inline int
9024	__perf_event_output(struct perf_event *event,
9025	struct perf_sample_data *data,
9026	struct pt_regs *regs,
9027	int (output_begin)(struct perf_output_handle ,
9028	struct perf_sample_data *,
9029	struct perf_event *,
9030	unsigned int))
9031	{
9032	struct perf_output_handle handle;
9033	struct perf_event_header header;
9034	int err;
9035
9036	/* protect the callchain buffers */
9037	rcu_read_lock();
9038
9039	perf_prepare_sample(data, event, regs);
9040	perf_prepare_header(&header, data, event, regs);
9041
9042	err = output_begin(&handle, data, event, header.size);
9043	if (err)
9044	goto exit;
9045
9046	perf_output_sample(&handle, &header, data, event);
9047
9048	perf_output_end(&handle);
9049
9050	exit:
9051	rcu_read_unlock();
9052	return err;
9053	}
9054
9055	void
9056	perf_event_output_forward(struct perf_event *event,
9057	struct perf_sample_data *data,
9058	struct pt_regs *regs)
9059	{
9060	__perf_event_output(event, data, regs, perf_output_begin_forward);
9061	}
9062
9063	void
9064	perf_event_output_backward(struct perf_event *event,
9065	struct perf_sample_data *data,
9066	struct pt_regs *regs)
9067	{
9068	__perf_event_output(event, data, regs, perf_output_begin_backward);
9069	}
9070
9071	int
9072	perf_event_output(struct perf_event *event,
9073	struct perf_sample_data *data,
9074	struct pt_regs *regs)
9075	{
9076	return __perf_event_output(event, data, regs, perf_output_begin);
9077	}
9078
9079	/*
9080	* read event_id
9081	*/
9082
9083	struct perf_read_event {
9084	struct perf_event_header header;
9085
9086	u32 pid;
9087	u32 tid;
9088	};
9089
9090	static void
9091	perf_event_read_event(struct perf_event *event,
9092	struct task_struct *task)
9093	{
9094	struct perf_output_handle handle;
9095	struct perf_sample_data sample;
9096	struct perf_read_event read_event = {
9097	.pid = perf_event_pid(event, task),
9098	.tid = perf_event_tid(event, task),
9099	};
9100	int ret;
9101
9102	perf_event_header__init(&read_event.header, &sample,
9103	PERF_RECORD_READ,
9104	/* misc= */ 0,
9105	sizeof(read_event) + event->read_size,
9106	event);
9107	ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
9108	if (ret)
9109	return;
9110
9111	perf_output_put(&handle, read_event);
9112	perf_output_read(&handle, event);
9113	perf_event__output_id_sample(event, &handle, &sample);
9114
9115	perf_output_end(&handle);
9116	}
9117
9118	typedef void (perf_iterate_f)(struct perf_event event, void data);
9119
9120	static void
9121	perf_iterate_ctx(struct perf_event_context *ctx,
9122	perf_iterate_f output,
9123	void *data, bool all)
9124	{
9125	struct perf_event *event;
9126
9127	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
9128	if (!all) {
9129	if (event->state < PERF_EVENT_STATE_INACTIVE)
9130	continue;
9131	if (!event_filter_match(event))
9132	continue;
9133	}
9134
9135	output(event, data);
9136	}
9137	}
9138
9139	static void perf_iterate_sb_cpu(perf_iterate_f output, void *data)
9140	{
9141	struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events);
9142	struct perf_event *event;
9143
9144	list_for_each_entry_rcu(event, &pel->list, sb_list) {
9145	/*
9146	* Skip events that are not fully formed yet; ensure that
9147	* if we observe event->ctx, both event and ctx will be
9148	* complete enough. See perf_install_in_context().
9149	*/
9150	if (!smp_load_acquire(&event->ctx))
9151	continue;
9152
9153	if (event->state < PERF_EVENT_STATE_INACTIVE)
9154	continue;
9155	if (!event_filter_match(event))
9156	continue;
9157	output(event, data);
9158	}
9159	}
9160
9161	/*
9162	* Iterate all events that need to receive side-band events.
9163	*
9164	* For new callers; ensure that account_pmu_sb_event() includes
9165	* your event, otherwise it might not get delivered.
9166	*/
9167	static void
9168	perf_iterate_sb(perf_iterate_f output, void *data,
9169	struct perf_event_context *task_ctx)
9170	{
9171	struct perf_event_context *ctx;
9172
9173	rcu_read_lock();
9174	preempt_disable();
9175
9176	/*
9177	* If we have task_ctx != NULL we only notify the task context itself.
9178	* The task_ctx is set only for EXIT events before releasing task
9179	* context.
9180	*/
9181	if (task_ctx) {
9182	perf_iterate_ctx(task_ctx, output, data, false);
9183	goto done;
9184	}
9185
9186	perf_iterate_sb_cpu(output, data);
9187
9188	ctx = rcu_dereference(current->perf_event_ctxp);
9189	if (ctx)
9190	perf_iterate_ctx(ctx, output, data, false);
9191	done:
9192	preempt_enable();
9193	rcu_read_unlock();
9194	}
9195
9196	/*
9197	* Clear all file-based filters at exec, they'll have to be
9198	* re-instated when/if these objects are mmapped again.
9199	*/
9200	static void perf_event_addr_filters_exec(struct perf_event event, void data)
9201	{
9202	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
9203	struct perf_addr_filter *filter;
9204	unsigned int restart = 0, count = 0;
9205	unsigned long flags;
9206
9207	if (!has_addr_filter(event))
9208	return;
9209
9210	raw_spin_lock_irqsave(&ifh->lock, flags);
9211	list_for_each_entry(filter, &ifh->list, entry) {
9212	if (filter->path.dentry) {
9213	event->addr_filter_ranges[count].start = 0;
9214	event->addr_filter_ranges[count].size = 0;
9215	restart++;
9216	}
9217
9218	count++;
9219	}
9220
9221	if (restart)
9222	event->addr_filters_gen++;
9223	raw_spin_unlock_irqrestore(&ifh->lock, flags);
9224
9225	if (restart)
9226	perf_event_stop(event, 1);
9227	}
9228
9229	void perf_event_exec(void)
9230	{
9231	struct perf_event_context *ctx;
9232
9233	ctx = perf_pin_task_context(current);
9234	if (!ctx)
9235	return;
9236
9237	perf_event_enable_on_exec(ctx);
9238	perf_event_remove_on_exec(ctx);
9239	scoped_guard(rcu)
9240	perf_iterate_ctx(ctx, perf_event_addr_filters_exec, NULL, true);
9241
9242	perf_unpin_context(ctx);
9243	put_ctx(ctx);
9244	}
9245
9246	struct remote_output {
9247	struct perf_buffer *rb;
9248	int err;
9249	};
9250
9251	static void __perf_event_output_stop(struct perf_event event, void data)
9252	{
9253	struct perf_event *parent = event->parent;
9254	struct remote_output *ro = data;
9255	struct perf_buffer *rb = ro->rb;
9256	struct stop_event_data sd = {
9257	.event = event,
9258	};
9259
9260	if (!has_aux(event))
9261	return;
9262
9263	if (!parent)
9264	parent = event;
9265
9266	/*
9267	* In case of inheritance, it will be the parent that links to the
9268	* ring-buffer, but it will be the child that's actually using it.
9269	*
9270	* We are using event::rb to determine if the event should be stopped,
9271	* however this may race with ring_buffer_attach() (through set_output),
9272	* which will make us skip the event that actually needs to be stopped.
9273	* So ring_buffer_attach() has to stop an aux event before re-assigning
9274	* its rb pointer.
9275	*/
9276	if (rcu_dereference(parent->rb) == rb)
9277	ro->err = __perf_event_stop(&sd);
9278	}
9279
9280	static int __perf_pmu_output_stop(void *info)
9281	{
9282	struct perf_event *event = info;
9283	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
9284	struct remote_output ro = {
9285	.rb = event->rb,
9286	};
9287
9288	rcu_read_lock();
9289	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);
9290	if (cpuctx->task_ctx)
9291	perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,
9292	&ro, false);
9293	rcu_read_unlock();
9294
9295	return ro.err;
9296	}
9297
9298	static void perf_pmu_output_stop(struct perf_event *event)
9299	{
9300	struct perf_event *iter;
9301	int err, cpu;
9302
9303	restart:
9304	rcu_read_lock();
9305	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {
9306	/*
9307	* For per-CPU events, we need to make sure that neither they
9308	* nor their children are running; for cpu==-1 events it's
9309	* sufficient to stop the event itself if it's active, since
9310	* it can't have children.
9311	*/
9312	cpu = iter->cpu;
9313	if (cpu == -1)
9314	cpu = READ_ONCE(iter->oncpu);
9315
9316	if (cpu == -1)
9317	continue;
9318
9319	err = cpu_function_call(cpu, __perf_pmu_output_stop, event);
9320	if (err == -EAGAIN) {
9321	rcu_read_unlock();
9322	goto restart;
9323	}
9324	}
9325	rcu_read_unlock();
9326	}
9327
9328	/*
9329	* task tracking -- fork/exit
9330	*
9331	* enabled by: attr.comm \| attr.mmap \| attr.mmap2 \| attr.mmap_data \| attr.task
9332	*/
9333
9334	struct perf_task_event {
9335	struct task_struct *task;
9336	struct perf_event_context *task_ctx;
9337
9338	struct {
9339	struct perf_event_header header;
9340
9341	u32 pid;
9342	u32 ppid;
9343	u32 tid;
9344	u32 ptid;
9345	u64 time;
9346	} event_id;
9347	int new;
9348	};
9349
9350	static int perf_event_task_match(struct perf_event *event)
9351	{
9352	return event->attr.comm \|\| event->attr.mmap \|\|
9353	event->attr.mmap2 \|\| event->attr.mmap_data \|\|
9354	event->attr.task;
9355	}
9356
9357	static void perf_event_task_output(struct perf_event *event,
9358	void *data)
9359	{
9360	struct perf_task_event *task_event = data;
9361	struct perf_output_handle handle;
9362	struct perf_sample_data sample;
9363	struct task_struct *task = task_event->task;
9364	int ret;
9365
9366	if (!perf_event_task_match(event))
9367	return;
9368
9369	perf_event_header__init(&task_event->event_id.header, &sample,
9370	task_event->new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
9371	/* misc= */ 0,
9372	sizeof(task_event->event_id),
9373	event);
9374
9375	ret = perf_output_begin(&handle, &sample, event,
9376	task_event->event_id.header.size);
9377	if (ret)
9378	return;
9379
9380	task_event->event_id.pid = perf_event_pid(event, task);
9381	task_event->event_id.tid = perf_event_tid(event, task);
9382
9383	if (task_event->event_id.header.type == PERF_RECORD_EXIT) {
9384	task_event->event_id.ppid = perf_event_pid(event,
9385	task->real_parent);
9386	task_event->event_id.ptid = perf_event_pid(event,
9387	task->real_parent);
9388	} else { /* PERF_RECORD_FORK */
9389	task_event->event_id.ppid = perf_event_pid(event, current);
9390	task_event->event_id.ptid = perf_event_tid(event, current);
9391	}
9392
9393	task_event->event_id.time = perf_event_clock(event);
9394
9395	perf_output_put(&handle, task_event->event_id);
9396
9397	perf_event__output_id_sample(event, &handle, &sample);
9398
9399	perf_output_end(&handle);
9400	}
9401
9402	static void perf_event_task(struct task_struct *task,
9403	struct perf_event_context *task_ctx,
9404	int new)
9405	{
9406	struct perf_task_event task_event;
9407
9408	if (!atomic_read(&nr_comm_events) &&
9409	!atomic_read(&nr_mmap_events) &&
9410	!atomic_read(&nr_task_events))
9411	return;
9412
9413	task_event = (struct perf_task_event){
9414	.task = task,
9415	.task_ctx = task_ctx,
9416	.new = new,
9417	};
9418
9419	perf_iterate_sb(perf_event_task_output,
9420	&task_event,
9421	task_ctx);
9422	}
9423
9424	/*
9425	* Allocate data for a new task when profiling system-wide
9426	* events which require PMU specific data
9427	*/
9428	static void
9429	perf_event_alloc_task_data(struct task_struct *child,
9430	struct task_struct *parent)
9431	{
9432	struct kmem_cache *ctx_cache = NULL;
9433	struct perf_ctx_data *cd;
9434
9435	if (!refcount_read(&global_ctx_data_ref))
9436	return;
9437
9438	scoped_guard (rcu) {
9439	cd = rcu_dereference(parent->perf_ctx_data);
9440	if (cd)
9441	ctx_cache = cd->ctx_cache;
9442	}
9443
9444	if (!ctx_cache)
9445	return;
9446
9447	guard(percpu_read)(&global_ctx_data_rwsem);
9448	scoped_guard (rcu) {
9449	cd = rcu_dereference(child->perf_ctx_data);
9450	if (!cd) {
9451	/*
9452	* A system-wide event may be unaccount,
9453	* when attaching the perf_ctx_data.
9454	*/
9455	if (!refcount_read(&global_ctx_data_ref))
9456	return;
9457	goto attach;
9458	}
9459
9460	if (!cd->global) {
9461	cd->global = 1;
9462	refcount_inc(&cd->refcount);
9463	}
9464	}
9465
9466	return;
9467	attach:
9468	attach_task_ctx_data(child, ctx_cache, true, GFP_KERNEL);
9469	}
9470
9471	void perf_event_fork(struct task_struct *task)
9472	{
9473	perf_event_task(task, NULL, 1);
9474	perf_event_namespaces(task);
9475	perf_event_alloc_task_data(task, current);
9476	}
9477
9478	/*
9479	* comm tracking
9480	*/
9481
9482	struct perf_comm_event {
9483	struct task_struct *task;
9484	char *comm;
9485	int comm_size;
9486
9487	struct {
9488	struct perf_event_header header;
9489
9490	u32 pid;
9491	u32 tid;
9492	} event_id;
9493	bool exec;
9494	};
9495
9496	static int perf_event_comm_match(struct perf_event *event)
9497	{
9498	return event->attr.comm;
9499	}
9500
9501	static void perf_event_comm_output(struct perf_event *event,
9502	void *data)
9503	{
9504	struct perf_comm_event *comm_event = data;
9505	struct perf_output_handle handle;
9506	struct perf_sample_data sample;
9507	int ret;
9508
9509	if (!perf_event_comm_match(event))
9510	return;
9511
9512	perf_event_header__init(&comm_event->event_id.header, &sample,
9513	PERF_RECORD_COMM,
9514	comm_event->exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
9515	sizeof(comm_event->event_id) + comm_event->comm_size,
9516	event);
9517	ret = perf_output_begin(&handle, &sample, event,
9518	comm_event->event_id.header.size);
9519
9520	if (ret)
9521	return;
9522
9523	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
9524	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
9525
9526	perf_output_put(&handle, comm_event->event_id);
9527	__output_copy(&handle, comm_event->comm,
9528	comm_event->comm_size);
9529
9530	perf_event__output_id_sample(event, &handle, &sample);
9531
9532	perf_output_end(&handle);
9533	}
9534
9535	static void perf_event_comm_event(struct perf_comm_event *comm_event)
9536	{
9537	char comm[TASK_COMM_LEN];
9538	unsigned int size;
9539
9540	memset(comm, 0, sizeof(comm));
9541	strscpy(comm, comm_event->task->comm);
9542	size = ALIGN(strlen(comm)+1, sizeof(u64));
9543
9544	comm_event->comm = comm;
9545	comm_event->comm_size = size;
9546
9547	perf_iterate_sb(perf_event_comm_output,
9548	comm_event,
9549	NULL);
9550	}
9551
9552	void perf_event_comm(struct task_struct *task, bool exec)
9553	{
9554	struct perf_comm_event comm_event;
9555
9556	if (!atomic_read(&nr_comm_events))
9557	return;
9558
9559	comm_event = (struct perf_comm_event){
9560	.task = task,
9561	/* .comm */
9562	/* .comm_size */
9563	/* .event_id */
9564	.exec = exec,
9565	};
9566
9567	perf_event_comm_event(&comm_event);
9568	}
9569
9570	/*
9571	* namespaces tracking
9572	*/
9573
9574	struct perf_namespaces_event {
9575	struct task_struct *task;
9576
9577	struct {
9578	struct perf_event_header header;
9579
9580	u32 pid;
9581	u32 tid;
9582	u64 nr_namespaces;
9583	struct perf_ns_link_info link_info[NR_NAMESPACES];
9584	} event_id;
9585	};
9586
9587	static int perf_event_namespaces_match(struct perf_event *event)
9588	{
9589	return event->attr.namespaces;
9590	}
9591
9592	static void perf_event_namespaces_output(struct perf_event *event,
9593	void *data)
9594	{
9595	struct perf_namespaces_event *namespaces_event = data;
9596	struct perf_output_handle handle;
9597	struct perf_sample_data sample;
9598	int ret;
9599
9600	if (!perf_event_namespaces_match(event))
9601	return;
9602
9603	perf_event_header__init(&namespaces_event->event_id.header, &sample,
9604	PERF_RECORD_NAMESPACES,
9605	/* misc= */ 0,
9606	sizeof(namespaces_event->event_id),
9607	event);
9608	ret = perf_output_begin(&handle, &sample, event,
9609	namespaces_event->event_id.header.size);
9610	if (ret)
9611	return;
9612
9613	namespaces_event->event_id.pid = perf_event_pid(event,
9614	namespaces_event->task);
9615	namespaces_event->event_id.tid = perf_event_tid(event,
9616	namespaces_event->task);
9617
9618	perf_output_put(&handle, namespaces_event->event_id);
9619
9620	perf_event__output_id_sample(event, &handle, &sample);
9621
9622	perf_output_end(&handle);
9623	}
9624
9625	static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info,
9626	struct task_struct *task,
9627	const struct proc_ns_operations *ns_ops)
9628	{
9629	struct path ns_path;
9630	struct inode *ns_inode;
9631	int error;
9632
9633	error = ns_get_path(&ns_path, task, ns_ops);
9634	if (!error) {
9635	ns_inode = ns_path.dentry->d_inode;
9636	ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev);
9637	ns_link_info->ino = ns_inode->i_ino;
9638	path_put(&ns_path);
9639	}
9640	}
9641
9642	void perf_event_namespaces(struct task_struct *task)
9643	{
9644	struct perf_namespaces_event namespaces_event;
9645	struct perf_ns_link_info *ns_link_info;
9646
9647	if (!atomic_read(&nr_namespaces_events))
9648	return;
9649
9650	namespaces_event = (struct perf_namespaces_event){
9651	.task = task,
9652	.event_id = {
9653	/* .header */
9654	/* .pid */
9655	/* .tid */
9656	.nr_namespaces = NR_NAMESPACES,
9657	/* .link_info[NR_NAMESPACES] */
9658	},
9659	};
9660
9661	ns_link_info = namespaces_event.event_id.link_info;
9662
9663	perf_fill_ns_link_info(&ns_link_info[MNT_NS_INDEX],
9664	task, &mntns_operations);
9665
9666	#ifdef CONFIG_USER_NS
9667	perf_fill_ns_link_info(&ns_link_info[USER_NS_INDEX],
9668	task, &userns_operations);
9669	#endif
9670	#ifdef CONFIG_NET_NS
9671	perf_fill_ns_link_info(&ns_link_info[NET_NS_INDEX],
9672	task, &netns_operations);
9673	#endif
9674	#ifdef CONFIG_UTS_NS
9675	perf_fill_ns_link_info(&ns_link_info[UTS_NS_INDEX],
9676	task, &utsns_operations);
9677	#endif
9678	#ifdef CONFIG_IPC_NS
9679	perf_fill_ns_link_info(&ns_link_info[IPC_NS_INDEX],
9680	task, &ipcns_operations);
9681	#endif
9682	#ifdef CONFIG_PID_NS
9683	perf_fill_ns_link_info(&ns_link_info[PID_NS_INDEX],
9684	task, &pidns_operations);
9685	#endif
9686	#ifdef CONFIG_CGROUPS
9687	perf_fill_ns_link_info(&ns_link_info[CGROUP_NS_INDEX],
9688	task, &cgroupns_operations);
9689	#endif
9690
9691	perf_iterate_sb(perf_event_namespaces_output,
9692	&namespaces_event,
9693	NULL);
9694	}
9695
9696	/*
9697	* cgroup tracking
9698	*/
9699	#ifdef CONFIG_CGROUP_PERF
9700
9701	struct perf_cgroup_event {
9702	char *path;
9703	int path_size;
9704	struct {
9705	struct perf_event_header header;
9706	u64 id;
9707	char path[];
9708	} event_id;
9709	};
9710
9711	static int perf_event_cgroup_match(struct perf_event *event)
9712	{
9713	return event->attr.cgroup;
9714	}
9715
9716	static void perf_event_cgroup_output(struct perf_event event, void data)
9717	{
9718	struct perf_cgroup_event *cgroup_event = data;
9719	struct perf_output_handle handle;
9720	struct perf_sample_data sample;
9721	int ret;
9722	u16 size = sizeof(cgroup_event->event_id) + cgroup_event->path_size;
9723
9724	if (!perf_event_cgroup_match(event))
9725	return;
9726
9727	perf_event_header__init(&cgroup_event->event_id.header, &sample,
9728	PERF_RECORD_CGROUP, /* misc= */ 0, size,
9729	event);
9730	ret = perf_output_begin(&handle, &sample, event,
9731	cgroup_event->event_id.header.size);
9732	if (ret)
9733	return;
9734
9735	perf_output_put(&handle, cgroup_event->event_id);
9736	__output_copy(&handle, cgroup_event->path, cgroup_event->path_size);
9737
9738	perf_event__output_id_sample(event, &handle, &sample);
9739
9740	perf_output_end(&handle);
9741	}
9742
9743	static void perf_event_cgroup(struct cgroup *cgrp)
9744	{
9745	struct perf_cgroup_event cgroup_event;
9746	char path_enomem[16] = "//enomem";
9747	char *pathname;
9748	size_t size;
9749
9750	if (!atomic_read(&nr_cgroup_events))
9751	return;
9752
9753	cgroup_event = (struct perf_cgroup_event){
9754	.event_id = {
9755	.id = cgroup_id(cgrp),
9756	},
9757	};
9758
9759	pathname = kmalloc(PATH_MAX, GFP_KERNEL);
9760	if (pathname == NULL) {
9761	cgroup_event.path = path_enomem;
9762	} else {
9763	/* just to be sure to have enough space for alignment */
9764	cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64));
9765	cgroup_event.path = pathname;
9766	}
9767
9768	/*
9769	* Since our buffer works in 8 byte units we need to align our string
9770	* size to a multiple of 8. However, we must guarantee the tail end is
9771	* zero'd out to avoid leaking random bits to userspace.
9772	*/
9773	size = strlen(cgroup_event.path) + 1;
9774	while (!IS_ALIGNED(size, sizeof(u64)))
9775	cgroup_event.path[size++] = '\0';
9776
9777	cgroup_event.path_size = size;
9778
9779	perf_iterate_sb(perf_event_cgroup_output,
9780	&cgroup_event,
9781	NULL);
9782
9783	kfree(pathname);
9784	}
9785
9786	#endif
9787
9788	/*
9789	* mmap tracking
9790	*/
9791
9792	struct perf_mmap_event {
9793	struct vm_area_struct *vma;
9794
9795	const char *file_name;
9796	int file_size;
9797	int maj, min;
9798	u64 ino;
9799	u64 ino_generation;
9800	u32 prot, flags;
9801	u8 build_id[BUILD_ID_SIZE_MAX];
9802	u32 build_id_size;
9803
9804	struct {
9805	struct perf_event_header header;
9806
9807	u32 pid;
9808	u32 tid;
9809	u64 start;
9810	u64 len;
9811	u64 pgoff;
9812	} event_id;
9813	};
9814
9815	static int perf_event_mmap_match(struct perf_event *event,
9816	void *data)
9817	{
9818	struct perf_mmap_event *mmap_event = data;
9819	struct vm_area_struct *vma = mmap_event->vma;
9820	int executable = vma->vm_flags & VM_EXEC;
9821
9822	return (!executable && event->attr.mmap_data) \|\|
9823	(executable && (event->attr.mmap \|\| event->attr.mmap2));
9824	}
9825
9826	static void perf_event_mmap_output(struct perf_event *event,
9827	void *data)
9828	{
9829	struct perf_mmap_event *mmap_event = data;
9830	struct perf_output_handle handle;
9831	struct perf_sample_data sample;
9832	int size = sizeof(mmap_event->event_id) + mmap_event->file_size;
9833	u32 type = PERF_RECORD_MMAP;
9834	u16 misc = PERF_RECORD_MISC_USER;
9835	bool use_build_id = false;
9836	int ret;
9837
9838	if (!perf_event_mmap_match(event, data))
9839	return;
9840
9841	if (event->attr.mmap2) {
9842	type = PERF_RECORD_MMAP2;
9843	size += sizeof(mmap_event->maj);
9844	size += sizeof(mmap_event->min);
9845	size += sizeof(mmap_event->ino);
9846	size += sizeof(mmap_event->ino_generation);
9847	size += sizeof(mmap_event->prot);
9848	size += sizeof(mmap_event->flags);
9849	use_build_id = event->attr.build_id && mmap_event->build_id_size;
9850	if (use_build_id)
9851	misc \|= PERF_RECORD_MISC_MMAP_BUILD_ID;
9852	}
9853	if (!(mmap_event->vma->vm_flags & VM_EXEC))
9854	misc \|= PERF_RECORD_MISC_MMAP_DATA;
9855
9856	perf_event_header__init(&mmap_event->event_id.header, &sample,
9857	type, misc, size, event);
9858	ret = perf_output_begin(&handle, &sample, event,
9859	mmap_event->event_id.header.size);
9860	if (ret)
9861	return;
9862
9863	mmap_event->event_id.pid = perf_event_pid(event, current);
9864	mmap_event->event_id.tid = perf_event_tid(event, current);
9865
9866	perf_output_put(&handle, mmap_event->event_id);
9867
9868	if (event->attr.mmap2) {
9869	if (use_build_id) {
9870	u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 };
9871
9872	__output_copy(&handle, size, 4);
9873	__output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX);
9874	} else {
9875	perf_output_put(&handle, mmap_event->maj);
9876	perf_output_put(&handle, mmap_event->min);
9877	perf_output_put(&handle, mmap_event->ino);
9878	perf_output_put(&handle, mmap_event->ino_generation);
9879	}
9880	perf_output_put(&handle, mmap_event->prot);
9881	perf_output_put(&handle, mmap_event->flags);
9882	}
9883
9884	__output_copy(&handle, mmap_event->file_name,
9885	mmap_event->file_size);
9886
9887	perf_event__output_id_sample(event, &handle, &sample);
9888
9889	perf_output_end(&handle);
9890	}
9891
9892	static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
9893	{
9894	struct vm_area_struct *vma = mmap_event->vma;
9895	struct file *file = vma->vm_file;
9896	int maj = 0, min = 0;
9897	u64 ino = 0, gen = 0;
9898	u32 prot = 0, flags = 0;
9899	unsigned int size;
9900	char tmp[16];
9901	char *buf = NULL;
9902	char *name = NULL;
9903
9904	if (vma->vm_flags & VM_READ)
9905	prot \|= PROT_READ;
9906	if (vma->vm_flags & VM_WRITE)
9907	prot \|= PROT_WRITE;
9908	if (vma->vm_flags & VM_EXEC)
9909	prot \|= PROT_EXEC;
9910
9911	if (vma->vm_flags & VM_MAYSHARE)
9912	flags = MAP_SHARED;
9913	else
9914	flags = MAP_PRIVATE;
9915
9916	if (vma->vm_flags & VM_LOCKED)
9917	flags \|= MAP_LOCKED;
9918	if (is_vm_hugetlb_page(vma))
9919	flags \|= MAP_HUGETLB;
9920
9921	if (file) {
9922	const struct inode *inode;
9923	dev_t dev;
9924
9925	buf = kmalloc(PATH_MAX, GFP_KERNEL);
9926	if (!buf) {
9927	name = "//enomem";
9928	goto cpy_name;
9929	}
9930	/*
9931	* d_path() works from the end of the rb backwards, so we
9932	* need to add enough zero bytes after the string to handle
9933	* the 64bit alignment we do later.
9934	*/
9935	name = d_path(file_user_path(file), buf, PATH_MAX - sizeof(u64));
9936	if (IS_ERR(name)) {
9937	name = "//toolong";
9938	goto cpy_name;
9939	}
9940	inode = file_user_inode(vma->vm_file);
9941	dev = inode->i_sb->s_dev;
9942	ino = inode->i_ino;
9943	gen = inode->i_generation;
9944	maj = MAJOR(dev);
9945	min = MINOR(dev);
9946
9947	goto got_name;
9948	} else {
9949	if (vma->vm_ops && vma->vm_ops->name)
9950	name = (char *) vma->vm_ops->name(vma);
9951	if (!name)
9952	name = (char *)arch_vma_name(vma);
9953	if (!name) {
9954	if (vma_is_initial_heap(vma))
9955	name = "[heap]";
9956	else if (vma_is_initial_stack(vma))
9957	name = "[stack]";
9958	else
9959	name = "//anon";
9960	}
9961	}
9962
9963	cpy_name:
9964	strscpy(tmp, name);
9965	name = tmp;
9966	got_name:
9967	/*
9968	* Since our buffer works in 8 byte units we need to align our string
9969	* size to a multiple of 8. However, we must guarantee the tail end is
9970	* zero'd out to avoid leaking random bits to userspace.
9971	*/
9972	size = strlen(name)+1;
9973	while (!IS_ALIGNED(size, sizeof(u64)))
9974	name[size++] = '\0';
9975
9976	mmap_event->file_name = name;
9977	mmap_event->file_size = size;
9978	mmap_event->maj = maj;
9979	mmap_event->min = min;
9980	mmap_event->ino = ino;
9981	mmap_event->ino_generation = gen;
9982	mmap_event->prot = prot;
9983	mmap_event->flags = flags;
9984
9985	if (atomic_read(&nr_build_id_events))
9986	build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size);
9987
9988	perf_iterate_sb(perf_event_mmap_output,
9989	mmap_event,
9990	NULL);
9991
9992	kfree(buf);
9993	}
9994
9995	/*
9996	* Check whether inode and address range match filter criteria.
9997	*/
9998	static bool perf_addr_filter_match(struct perf_addr_filter *filter,
9999	struct file *file, unsigned long offset,
10000	unsigned long size)
10001	{
10002	/* d_inode(NULL) won't be equal to any mapped user-space file */
10003	if (!filter->path.dentry)
10004	return false;
10005
10006	if (d_inode(filter->path.dentry) != file_user_inode(file))
10007	return false;
10008
10009	if (filter->offset > offset + size)
10010	return false;
10011
10012	if (filter->offset + filter->size < offset)
10013	return false;
10014
10015	return true;
10016	}
10017
10018	static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter,
10019	struct vm_area_struct *vma,
10020	struct perf_addr_filter_range *fr)
10021	{
10022	unsigned long vma_size = vma->vm_end - vma->vm_start;
10023	unsigned long off = vma_start_pgoff(vma) << PAGE_SHIFT;
10024	struct file *file = vma->vm_file;
10025
10026	if (!perf_addr_filter_match(filter, file, off, vma_size))
10027	return false;
10028
10029	if (filter->offset < off) {
10030	fr->start = vma->vm_start;
10031	fr->size = min(vma_size, filter->size - (off - filter->offset));
10032	} else {
10033	fr->start = vma->vm_start + filter->offset - off;
10034	fr->size = min(vma->vm_end - fr->start, filter->size);
10035	}
10036
10037	return true;
10038	}
10039
10040	static void __perf_addr_filters_adjust(struct perf_event event, void data)
10041	{
10042	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
10043	struct vm_area_struct *vma = data;
10044	struct perf_addr_filter *filter;
10045	unsigned int restart = 0, count = 0;
10046	unsigned long flags;
10047
10048	if (!has_addr_filter(event))
10049	return;
10050
10051	if (!vma->vm_file)
10052	return;
10053
10054	raw_spin_lock_irqsave(&ifh->lock, flags);
10055	list_for_each_entry(filter, &ifh->list, entry) {
10056	if (perf_addr_filter_vma_adjust(filter, vma,
10057	&event->addr_filter_ranges[count]))
10058	restart++;
10059
10060	count++;
10061	}
10062
10063	if (restart)
10064	event->addr_filters_gen++;
10065	raw_spin_unlock_irqrestore(&ifh->lock, flags);
10066
10067	if (restart)
10068	perf_event_stop(event, 1);
10069	}
10070
10071	/*
10072	* Adjust all task's events' filters to the new vma
10073	*/
10074	static void perf_addr_filters_adjust(struct vm_area_struct *vma)
10075	{
10076	struct perf_event_context *ctx;
10077
10078	/*
10079	* Data tracing isn't supported yet and as such there is no need
10080	* to keep track of anything that isn't related to executable code:
10081	*/
10082	if (!(vma->vm_flags & VM_EXEC))
10083	return;
10084
10085	rcu_read_lock();
10086	ctx = rcu_dereference(current->perf_event_ctxp);
10087	if (ctx)
10088	perf_iterate_ctx(ctx, __perf_addr_filters_adjust, vma, true);
10089	rcu_read_unlock();
10090	}
10091
10092	void perf_event_mmap(struct vm_area_struct *vma)
10093	{
10094	struct perf_mmap_event mmap_event;
10095
10096	if (!atomic_read(&nr_mmap_events))
10097	return;
10098
10099	mmap_event = (struct perf_mmap_event){
10100	.vma = vma,
10101	/* .file_name */
10102	/* .file_size */
10103	.event_id = {
10104	/* .header */
10105	/* .pid */
10106	/* .tid */
10107	.start = vma->vm_start,
10108	.len = vma->vm_end - vma->vm_start,
10109	.pgoff = (u64)vma_start_pgoff(vma) << PAGE_SHIFT,
10110	},
10111	/* .maj (attr_mmap2 only) */
10112	/* .min (attr_mmap2 only) */
10113	/* .ino (attr_mmap2 only) */
10114	/* .ino_generation (attr_mmap2 only) */
10115	/* .prot (attr_mmap2 only) */
10116	/* .flags (attr_mmap2 only) */
10117	};
10118
10119	perf_addr_filters_adjust(vma);
10120	perf_event_mmap_event(&mmap_event);
10121	}
10122
10123	void perf_event_aux_event(struct perf_event *event, unsigned long head,
10124	unsigned long size, u64 flags)
10125	{
10126	struct perf_output_handle handle;
10127	struct perf_sample_data sample;
10128	struct perf_aux_event {
10129	struct perf_event_header header;
10130	u64 offset;
10131	u64 size;
10132	u64 flags;
10133	} rec = {
10134	.offset = head,
10135	.size = size,
10136	.flags = flags,
10137	};
10138	int ret;
10139
10140	perf_event_header__init(&rec.header, &sample,
10141	PERF_RECORD_AUX, /* misc= */ 0, sizeof(rec),
10142	event);
10143	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
10144
10145	if (ret)
10146	return;
10147
10148	perf_output_put(&handle, rec);
10149	perf_event__output_id_sample(event, &handle, &sample);
10150
10151	perf_output_end(&handle);
10152	}
10153
10154	/*
10155	* Lost/dropped samples logging
10156	*/
10157	void perf_log_lost_samples(struct perf_event *event, u64 lost)
10158	{
10159	struct perf_output_handle handle;
10160	struct perf_sample_data sample;
10161	int ret;
10162
10163	struct {
10164	struct perf_event_header header;
10165	u64 lost;
10166	} lost_samples_event = {
10167	.lost = lost,
10168	};
10169
10170	perf_event_header__init(&lost_samples_event.header, &sample,
10171	PERF_RECORD_LOST_SAMPLES,
10172	/* misc= */ 0,
10173	sizeof(lost_samples_event),
10174	event);
10175
10176	ret = perf_output_begin(&handle, &sample, event,
10177	lost_samples_event.header.size);
10178	if (ret)
10179	return;
10180
10181	perf_output_put(&handle, lost_samples_event);
10182	perf_event__output_id_sample(event, &handle, &sample);
10183	perf_output_end(&handle);
10184	}
10185
10186	/*
10187	* context_switch tracking
10188	*/
10189
10190	struct perf_switch_event {
10191	struct task_struct *task;
10192	struct task_struct *next_prev;
10193
10194	struct {
10195	struct perf_event_header header;
10196	u32 next_prev_pid;
10197	u32 next_prev_tid;
10198	} event_id;
10199	bool sched_in;
10200	bool preempt;
10201	};
10202
10203	static int perf_event_switch_match(struct perf_event *event)
10204	{
10205	return event->attr.context_switch;
10206	}
10207
10208	static void perf_event_switch_output(struct perf_event event, void data)
10209	{
10210	struct perf_switch_event *se = data;
10211	struct perf_output_handle handle;
10212	struct perf_sample_data sample;
10213	u32 type;
10214	u16 misc;
10215	u16 size;
10216	int ret;
10217
10218	if (!perf_event_switch_match(event))
10219	return;
10220
10221	/* Only CPU-wide events are allowed to see next/prev pid/tid */
10222	if (event->ctx->task) {
10223	type = PERF_RECORD_SWITCH;
10224	size = sizeof(se->event_id.header);
10225	} else {
10226	type = PERF_RECORD_SWITCH_CPU_WIDE;
10227	size = sizeof(se->event_id);
10228	se->event_id.next_prev_pid =
10229	perf_event_pid(event, se->next_prev);
10230	se->event_id.next_prev_tid =
10231	perf_event_tid(event, se->next_prev);
10232	}
10233	misc = se->sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT;
10234	if (se->preempt)
10235	misc \|= PERF_RECORD_MISC_SWITCH_OUT_PREEMPT;
10236
10237	perf_event_header__init(&se->event_id.header, &sample,
10238	type, misc, size, event);
10239
10240	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
10241	if (ret)
10242	return;
10243
10244	if (event->ctx->task)
10245	perf_output_put(&handle, se->event_id.header);
10246	else
10247	perf_output_put(&handle, se->event_id);
10248
10249	perf_event__output_id_sample(event, &handle, &sample);
10250
10251	perf_output_end(&handle);
10252	}
10253
10254	static void perf_event_switch(struct task_struct *task,
10255	struct task_struct *next_prev, bool sched_in)
10256	{
10257	struct perf_switch_event switch_event;
10258
10259	/* N.B. caller checks nr_switch_events != 0 */
10260
10261	switch_event = (struct perf_switch_event){
10262	.task = task,
10263	.next_prev = next_prev,
10264	/* .event_id */
10265	.sched_in = sched_in,
10266	.preempt = !sched_in && task_is_runnable(task),
10267	};
10268
10269	perf_iterate_sb(perf_event_switch_output, &switch_event, NULL);
10270	}
10271
10272	/*
10273	* IRQ throttle logging
10274	*/
10275
10276	static void perf_log_throttle(struct perf_event *event, int enable)
10277	{
10278	struct perf_output_handle handle;
10279	struct perf_sample_data sample;
10280	int ret;
10281
10282	struct {
10283	struct perf_event_header header;
10284	u64 time;
10285	u64 id;
10286	u64 stream_id;
10287	} throttle_event = {
10288	.time = perf_event_clock(event),
10289	.id = primary_event_id(event),
10290	.stream_id = event->id,
10291	};
10292
10293	perf_event_header__init(&throttle_event.header, &sample,
10294	enable ? PERF_RECORD_UNTHROTTLE
10295	: PERF_RECORD_THROTTLE,
10296	/* misc= */ 0,
10297	sizeof(throttle_event),
10298	event);
10299
10300	ret = perf_output_begin(&handle, &sample, event,
10301	throttle_event.header.size);
10302	if (ret)
10303	return;
10304
10305	perf_output_put(&handle, throttle_event);
10306	perf_event__output_id_sample(event, &handle, &sample);
10307	perf_output_end(&handle);
10308	}
10309
10310	/*
10311	* ksymbol register/unregister tracking
10312	*/
10313
10314	struct perf_ksymbol_event {
10315	const char *name;
10316	int name_len;
10317	struct {
10318	struct perf_event_header header;
10319	u64 addr;
10320	u32 len;
10321	u16 ksym_type;
10322	u16 flags;
10323	} event_id;
10324	};
10325
10326	static int perf_event_ksymbol_match(struct perf_event *event)
10327	{
10328	return event->attr.ksymbol;
10329	}
10330
10331	static void perf_event_ksymbol_output(struct perf_event event, void data)
10332	{
10333	struct perf_ksymbol_event *ksymbol_event = data;
10334	struct perf_output_handle handle;
10335	struct perf_sample_data sample;
10336	int ret;
10337	u16 size = sizeof(ksymbol_event->event_id) + ksymbol_event->name_len;
10338
10339	if (!perf_event_ksymbol_match(event))
10340	return;
10341
10342	perf_event_header__init(&ksymbol_event->event_id.header, &sample,
10343	PERF_RECORD_KSYMBOL, /* misc= */ 0, size,
10344	event);
10345	ret = perf_output_begin(&handle, &sample, event,
10346	ksymbol_event->event_id.header.size);
10347	if (ret)
10348	return;
10349
10350	perf_output_put(&handle, ksymbol_event->event_id);
10351	__output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len);
10352	perf_event__output_id_sample(event, &handle, &sample);
10353
10354	perf_output_end(&handle);
10355	}
10356
10357	void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister,
10358	const char *sym)
10359	{
10360	struct perf_ksymbol_event ksymbol_event;
10361	char name[KSYM_NAME_LEN];
10362	u16 flags = 0;
10363	int name_len;
10364
10365	if (!atomic_read(&nr_ksymbol_events))
10366	return;
10367
10368	if (ksym_type >= PERF_RECORD_KSYMBOL_TYPE_MAX \|\|
10369	ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN)
10370	goto err;
10371
10372	strscpy(name, sym);
10373	name_len = strlen(name) + 1;
10374	while (!IS_ALIGNED(name_len, sizeof(u64)))
10375	name[name_len++] = '\0';
10376	BUILD_BUG_ON(KSYM_NAME_LEN % sizeof(u64));
10377
10378	if (unregister)
10379	flags \|= PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER;
10380
10381	ksymbol_event = (struct perf_ksymbol_event){
10382	.name = name,
10383	.name_len = name_len,
10384	.event_id = {
10385	.addr = addr,
10386	.len = len,
10387	.ksym_type = ksym_type,
10388	.flags = flags,
10389	},
10390	};
10391
10392	perf_iterate_sb(perf_event_ksymbol_output, &ksymbol_event, NULL);
10393	return;
10394	err:
10395	WARN_ONCE(1, "%s: Invalid KSYMBOL type 0x%x\n", __func__, ksym_type);
10396	}
10397
10398	/*
10399	* bpf program load/unload tracking
10400	*/
10401
10402	struct perf_bpf_event {
10403	struct bpf_prog *prog;
10404	struct {
10405	struct perf_event_header header;
10406	u16 type;
10407	u16 flags;
10408	u32 id;
10409	u8 tag[BPF_TAG_SIZE];
10410	} event_id;
10411	};
10412
10413	static int perf_event_bpf_match(struct perf_event *event)
10414	{
10415	return event->attr.bpf_event;
10416	}
10417
10418	static void perf_event_bpf_output(struct perf_event event, void data)
10419	{
10420	struct perf_bpf_event *bpf_event = data;
10421	struct perf_output_handle handle;
10422	struct perf_sample_data sample;
10423	int ret;
10424
10425	if (!perf_event_bpf_match(event))
10426	return;
10427
10428	perf_event_header__init(&bpf_event->event_id.header, &sample,
10429	PERF_RECORD_BPF_EVENT,
10430	/* misc= */ 0,
10431	sizeof(bpf_event->event_id),
10432	event);
10433	ret = perf_output_begin(&handle, &sample, event,
10434	bpf_event->event_id.header.size);
10435	if (ret)
10436	return;
10437
10438	perf_output_put(&handle, bpf_event->event_id);
10439	perf_event__output_id_sample(event, &handle, &sample);
10440
10441	perf_output_end(&handle);
10442	}
10443
10444	static void perf_event_bpf_emit_ksymbols(struct bpf_prog *prog,
10445	enum perf_bpf_event_type type)
10446	{
10447	bool unregister = type == PERF_BPF_EVENT_PROG_UNLOAD;
10448	int i;
10449
10450	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF,
10451	(u64)(unsigned long)prog->bpf_func,
10452	prog->jited_len, unregister,
10453	prog->aux->ksym.name);
10454
10455	for (i = 1; i < prog->aux->func_cnt; i++) {
10456	struct bpf_prog *subprog = prog->aux->func[i];
10457
10458	perf_event_ksymbol(
10459	PERF_RECORD_KSYMBOL_TYPE_BPF,
10460	(u64)(unsigned long)subprog->bpf_func,
10461	subprog->jited_len, unregister,
10462	subprog->aux->ksym.name);
10463	}
10464	}
10465
10466	void perf_event_bpf_event(struct bpf_prog *prog,
10467	enum perf_bpf_event_type type,
10468	u16 flags)
10469	{
10470	struct perf_bpf_event bpf_event;
10471
10472	switch (type) {
10473	case PERF_BPF_EVENT_PROG_LOAD:
10474	case PERF_BPF_EVENT_PROG_UNLOAD:
10475	if (atomic_read(&nr_ksymbol_events))
10476	perf_event_bpf_emit_ksymbols(prog, type);
10477	break;
10478	default:
10479	return;
10480	}
10481
10482	if (!atomic_read(&nr_bpf_events))
10483	return;
10484
10485	bpf_event = (struct perf_bpf_event){
10486	.prog = prog,
10487	.event_id = {
10488	.type = type,
10489	.flags = flags,
10490	.id = prog->aux->id,
10491	},
10492	};
10493
10494	BUILD_BUG_ON(BPF_TAG_SIZE % sizeof(u64));
10495
10496	memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);
10497	perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL);
10498	}
10499
10500	struct perf_callchain_deferred_event {
10501	struct unwind_stacktrace *trace;
10502	struct {
10503	struct perf_event_header header;
10504	u64 cookie;
10505	u64 nr;
10506	u64 ips[];
10507	} event;
10508	};
10509
10510	static void perf_callchain_deferred_output(struct perf_event event, void data)
10511	{
10512	struct perf_callchain_deferred_event *deferred_event = data;
10513	struct perf_output_handle handle;
10514	struct perf_sample_data sample;
10515	int ret;
10516	u16 size = sizeof(deferred_event->event) + (deferred_event->trace->nr * sizeof(u64));
10517
10518	if (!event->attr.defer_output)
10519	return;
10520
10521	/* XXX do we really need sample_id_all for this ??? */
10522	perf_event_header__init(&deferred_event->event.header, &sample,
10523	PERF_RECORD_CALLCHAIN_DEFERRED,
10524	PERF_RECORD_MISC_USER,
10525	size,
10526	event);
10527
10528	ret = perf_output_begin(&handle, &sample, event,
10529	deferred_event->event.header.size);
10530	if (ret)
10531	return;
10532
10533	perf_output_put(&handle, deferred_event->event);
10534	for (int i = 0; i < deferred_event->trace->nr; i++) {
10535	u64 entry = deferred_event->trace->entries[i];
10536	perf_output_put(&handle, entry);
10537	}
10538	perf_event__output_id_sample(event, &handle, &sample);
10539
10540	perf_output_end(&handle);
10541	}
10542
10543	static void perf_unwind_deferred_callback(struct unwind_work *work,
10544	struct unwind_stacktrace *trace, u64 cookie)
10545	{
10546	struct perf_callchain_deferred_event deferred_event = {
10547	.trace = trace,
10548	.event = {
10549	.cookie = cookie,
10550	.nr = trace->nr,
10551	},
10552	};
10553
10554	perf_iterate_sb(perf_callchain_deferred_output, &deferred_event, NULL);
10555	}
10556
10557	struct perf_text_poke_event {
10558	const void *old_bytes;
10559	const void *new_bytes;
10560	u16 tot;
10561	u16 pad;
10562	u16 old_len;
10563	u16 new_len;
10564
10565	struct {
10566	struct perf_event_header header;
10567
10568	u64 addr;
10569	} event_id;
10570	};
10571
10572	static int perf_event_text_poke_match(struct perf_event *event)
10573	{
10574	return event->attr.text_poke;
10575	}
10576
10577	static void perf_event_text_poke_output(struct perf_event event, void data)
10578	{
10579	struct perf_text_poke_event *text_poke_event = data;
10580	struct perf_output_handle handle;
10581	struct perf_sample_data sample;
10582	u16 size = sizeof(text_poke_event->event_id) + text_poke_event->tot + text_poke_event->pad;
10583	u64 padding = 0;
10584	int ret;
10585
10586	if (!perf_event_text_poke_match(event))
10587	return;
10588
10589	perf_event_header__init(&text_poke_event->event_id.header, &sample,
10590	PERF_RECORD_TEXT_POKE,
10591	PERF_RECORD_MISC_KERNEL,
10592	size,
10593	event);
10594
10595	ret = perf_output_begin(&handle, &sample, event,
10596	text_poke_event->event_id.header.size);
10597	if (ret)
10598	return;
10599
10600	perf_output_put(&handle, text_poke_event->event_id);
10601	perf_output_put(&handle, text_poke_event->old_len);
10602	perf_output_put(&handle, text_poke_event->new_len);
10603
10604	__output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);
10605	__output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);
10606
10607	if (text_poke_event->pad)
10608	__output_copy(&handle, &padding, text_poke_event->pad);
10609
10610	perf_event__output_id_sample(event, &handle, &sample);
10611
10612	perf_output_end(&handle);
10613	}
10614
10615	void perf_event_text_poke(const void addr, const void old_bytes,
10616	size_t old_len, const void *new_bytes, size_t new_len)
10617	{
10618	struct perf_text_poke_event text_poke_event;
10619	size_t tot, pad;
10620
10621	if (!atomic_read(&nr_text_poke_events))
10622	return;
10623
10624	tot = sizeof(text_poke_event.old_len) + old_len;
10625	tot += sizeof(text_poke_event.new_len) + new_len;
10626	pad = ALIGN(tot, sizeof(u64)) - tot;
10627
10628	text_poke_event = (struct perf_text_poke_event){
10629	.old_bytes = old_bytes,
10630	.new_bytes = new_bytes,
10631	.tot = tot,
10632	.pad = pad,
10633	.old_len = old_len,
10634	.new_len = new_len,
10635	.event_id = {
10636	.addr = (unsigned long)addr,
10637	},
10638	};
10639
10640	perf_iterate_sb(perf_event_text_poke_output, &text_poke_event, NULL);
10641	}
10642
10643	void perf_event_itrace_started(struct perf_event *event)
10644	{
10645	WRITE_ONCE(event->attach_state, event->attach_state \| PERF_ATTACH_ITRACE);
10646	}
10647	EXPORT_SYMBOL_GPL(perf_event_itrace_started);
10648
10649	static void perf_log_itrace_start(struct perf_event *event)
10650	{
10651	struct perf_output_handle handle;
10652	struct perf_sample_data sample;
10653	struct perf_aux_event {
10654	struct perf_event_header header;
10655	u32 pid;
10656	u32 tid;
10657	} rec;
10658	int ret;
10659
10660	if (event->parent)
10661	event = event->parent;
10662
10663	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) \|\|
10664	event->attach_state & PERF_ATTACH_ITRACE)
10665	return;
10666
10667	rec.pid = perf_event_pid(event, current);
10668	rec.tid = perf_event_tid(event, current);
10669
10670	perf_event_header__init(&rec.header, &sample,
10671	PERF_RECORD_ITRACE_START, /* misc= */ 0, sizeof(rec),
10672	event);
10673	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
10674
10675	if (ret)
10676	return;
10677
10678	perf_output_put(&handle, rec);
10679	perf_event__output_id_sample(event, &handle, &sample);
10680
10681	perf_output_end(&handle);
10682	}
10683
10684	void perf_report_aux_output_id(struct perf_event *event, u64 hw_id)
10685	{
10686	struct perf_output_handle handle;
10687	struct perf_sample_data sample;
10688	struct perf_aux_event {
10689	struct perf_event_header header;
10690	u64 hw_id;
10691	} rec;
10692	int ret;
10693
10694	if (event->parent)
10695	event = event->parent;
10696
10697	rec.hw_id = hw_id;
10698	perf_event_header__init(&rec.header, &sample,
10699	PERF_RECORD_AUX_OUTPUT_HW_ID, /* misc= */ 0,
10700	sizeof(rec), event);
10701	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
10702
10703	if (ret)
10704	return;
10705
10706	perf_output_put(&handle, rec);
10707	perf_event__output_id_sample(event, &handle, &sample);
10708
10709	perf_output_end(&handle);
10710	}
10711	EXPORT_SYMBOL_GPL(perf_report_aux_output_id);
10712
10713	static int
10714	__perf_event_account_interrupt(struct perf_event *event, int throttle)
10715	{
10716	struct hw_perf_event *hwc = &event->hw;
10717	int ret = 0;
10718	u64 seq;
10719
10720	seq = __this_cpu_read(perf_throttled_seq);
10721	if (seq != hwc->interrupts_seq) {
10722	hwc->interrupts_seq = seq;
10723	hwc->interrupts = 1;
10724	} else {
10725	hwc->interrupts++;
10726	}
10727
10728	if (unlikely(throttle && hwc->interrupts >= max_samples_per_tick)) {
10729	__this_cpu_inc(perf_throttled_count);
10730	tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
10731	perf_event_throttle_group(event);
10732	ret = 1;
10733	}
10734
10735	if (event->attr.freq) {
10736	u64 now = perf_clock();
10737	s64 delta = now - hwc->freq_time_stamp;
10738
10739	hwc->freq_time_stamp = now;
10740
10741	if (delta > 0 && delta < 2*TICK_NSEC)
10742	perf_adjust_period(event, delta, hwc->last_period, true);
10743	}
10744
10745	return ret;
10746	}
10747
10748	int perf_event_account_interrupt(struct perf_event *event)
10749	{
10750	return __perf_event_account_interrupt(event, 1);
10751	}
10752
10753	static inline bool sample_is_allowed(struct perf_event event, struct pt_regs regs)
10754	{
10755	/*
10756	* Due to interrupt latency (AKA "skid"), we may enter the
10757	* kernel before taking an overflow, even if the PMU is only
10758	* counting user events.
10759	*/
10760	if (event->attr.exclude_kernel && !user_mode(regs))
10761	return false;
10762
10763	return true;
10764	}
10765
10766	#ifdef CONFIG_BPF_SYSCALL
10767	static int bpf_overflow_handler(struct perf_event *event,
10768	struct perf_sample_data *data,
10769	struct pt_regs *regs)
10770	{
10771	struct bpf_perf_event_data_kern ctx = {
10772	.data = data,
10773	.event = event,
10774	};
10775	struct bpf_prog *prog;
10776	int ret = 0;
10777
10778	ctx.regs = perf_arch_bpf_user_pt_regs(regs);
10779	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
10780	goto out;
10781	rcu_read_lock();
10782	prog = READ_ONCE(event->prog);
10783	if (prog) {
10784	perf_prepare_sample(data, event, regs);
10785	ret = bpf_prog_run(prog, &ctx);
10786	}
10787	rcu_read_unlock();
10788	out:
10789	__this_cpu_dec(bpf_prog_active);
10790
10791	return ret;
10792	}
10793
10794	static inline int perf_event_set_bpf_handler(struct perf_event *event,
10795	struct bpf_prog *prog,
10796	u64 bpf_cookie)
10797	{
10798	if (event->overflow_handler_context)
10799	/* hw breakpoint or kernel counter */
10800	return -EINVAL;
10801
10802	if (event->prog)
10803	return -EEXIST;
10804
10805	if (prog->type != BPF_PROG_TYPE_PERF_EVENT)
10806	return -EINVAL;
10807
10808	if (event->attr.precise_ip &&
10809	prog->call_get_stack &&
10810	(!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) \|\|
10811	event->attr.exclude_callchain_kernel \|\|
10812	event->attr.exclude_callchain_user)) {
10813	/*
10814	* On perf_event with precise_ip, calling bpf_get_stack()
10815	* may trigger unwinder warnings and occasional crashes.
10816	* bpf_get_[stack\|stackid] works around this issue by using
10817	* callchain attached to perf_sample_data. If the
10818	* perf_event does not full (kernel and user) callchain
10819	* attached to perf_sample_data, do not allow attaching BPF
10820	* program that calls bpf_get_[stack\|stackid].
10821	*/
10822	return -EPROTO;
10823	}
10824
10825	event->prog = prog;
10826	event->bpf_cookie = bpf_cookie;
10827	return 0;
10828	}
10829
10830	static inline void perf_event_free_bpf_handler(struct perf_event *event)
10831	{
10832	struct bpf_prog *prog = event->prog;
10833
10834	if (!prog)
10835	return;
10836
10837	event->prog = NULL;
10838	bpf_prog_put(prog);
10839	}
10840	#else
10841	static inline int bpf_overflow_handler(struct perf_event *event,
10842	struct perf_sample_data *data,
10843	struct pt_regs *regs)
10844	{
10845	return 1;
10846	}
10847
10848	static inline int perf_event_set_bpf_handler(struct perf_event *event,
10849	struct bpf_prog *prog,
10850	u64 bpf_cookie)
10851	{
10852	return -EOPNOTSUPP;
10853	}
10854
10855	static inline void perf_event_free_bpf_handler(struct perf_event *event)
10856	{
10857	}
10858	#endif
10859
10860	/*
10861	* Generic event overflow handling, sampling.
10862	*/
10863
10864	static int __perf_event_overflow(struct perf_event *event,
10865	int throttle, struct perf_sample_data *data,
10866	struct pt_regs *regs)
10867	{
10868	int events = atomic_read(&event->event_limit);
10869	int ret = 0;
10870
10871	/*
10872	* Non-sampling counters might still use the PMI to fold short
10873	* hardware counters, ignore those.
10874	*/
10875	if (unlikely(!is_sampling_event(event)))
10876	return 0;
10877
10878	ret = __perf_event_account_interrupt(event, throttle);
10879
10880	if (event->attr.aux_pause)
10881	perf_event_aux_pause(event->aux_event, true);
10882
10883	if (event->prog && event->prog->type == BPF_PROG_TYPE_PERF_EVENT &&
10884	!bpf_overflow_handler(event, data, regs))
10885	goto out;
10886
10887	/*
10888	* XXX event_limit might not quite work as expected on inherited
10889	* events
10890	*/
10891
10892	event->pending_kill = POLL_IN;
10893	if (events && atomic_dec_and_test(&event->event_limit)) {
10894	ret = 1;
10895	event->pending_kill = POLL_HUP;
10896	perf_event_disable_inatomic(event);
10897	event->pmu->stop(event, 0);
10898	}
10899
10900	if (event->attr.sigtrap) {
10901	/*
10902	* The desired behaviour of sigtrap vs invalid samples is a bit
10903	* tricky; on the one hand, one should not loose the SIGTRAP if
10904	* it is the first event, on the other hand, we should also not
10905	* trigger the WARN or override the data address.
10906	*/
10907	bool valid_sample = sample_is_allowed(event, regs);
10908	unsigned int pending_id = 1;
10909	enum task_work_notify_mode notify_mode;
10910
10911	if (regs)
10912	pending_id = hash32_ptr((void *)instruction_pointer(regs)) ?: 1;
10913
10914	notify_mode = in_nmi() ? TWA_NMI_CURRENT : TWA_RESUME;
10915
10916	if (!event->pending_work &&
10917	!task_work_add(current, &event->pending_task, notify_mode)) {
10918	event->pending_work = pending_id;
10919	local_inc(&event->ctx->nr_no_switch_fast);
10920	WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
10921
10922	event->pending_addr = 0;
10923	if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))
10924	event->pending_addr = data->addr;
10925
10926	} else if (event->attr.exclude_kernel && valid_sample) {
10927	/*
10928	* Should not be able to return to user space without
10929	* consuming pending_work; with exceptions:
10930	*
10931	* 1. Where !exclude_kernel, events can overflow again
10932	* in the kernel without returning to user space.
10933	*
10934	* 2. Events that can overflow again before the IRQ-
10935	* work without user space progress (e.g. hrtimer).
10936	* To approximate progress (with false negatives),
10937	* check 32-bit hash of the current IP.
10938	*/
10939	WARN_ON_ONCE(event->pending_work != pending_id);
10940	}
10941	}
10942
10943	READ_ONCE(event->overflow_handler)(event, data, regs);
10944
10945	if (*perf_event_fasync(event) && event->pending_kill) {
10946	event->pending_wakeup = 1;
10947	irq_work_queue(&event->pending_irq);
10948	}
10949	out:
10950	if (event->attr.aux_resume)
10951	perf_event_aux_pause(event->aux_event, false);
10952
10953	return ret;
10954	}
10955
10956	int perf_event_overflow(struct perf_event *event,
10957	struct perf_sample_data *data,
10958	struct pt_regs *regs)
10959	{
10960	/*
10961	* Entry point from hardware PMI, interrupts should be disabled here.
10962	* This serializes us against perf_event_remove_from_context() in
10963	* things like perf_event_release_kernel().
10964	*/
10965	lockdep_assert_irqs_disabled();
10966
10967	return __perf_event_overflow(event, 1, data, regs);
10968	}
10969
10970	/*
10971	* Generic software event infrastructure
10972	*/
10973
10974	struct swevent_htable {
10975	struct swevent_hlist *swevent_hlist;
10976	struct mutex hlist_mutex;
10977	int hlist_refcount;
10978	};
10979	static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
10980
10981	/*
10982	* We directly increment event->count and keep a second value in
10983	* event->hw.period_left to count intervals. This period event
10984	* is kept in the range [-sample_period, 0] so that we can use the
10985	* sign as trigger.
10986	*/
10987
10988	u64 perf_swevent_set_period(struct perf_event *event)
10989	{
10990	struct hw_perf_event *hwc = &event->hw;
10991	u64 period = hwc->last_period;
10992	u64 nr, offset;
10993	s64 old, val;
10994
10995	hwc->last_period = hwc->sample_period;
10996
10997	old = local64_read(&hwc->period_left);
10998	do {
10999	val = old;
11000	if (val < 0)
11001	return 0;
11002
11003	nr = div64_u64(period + val, period);
11004	offset = nr * period;
11005	val -= offset;
11006	} while (!local64_try_cmpxchg(&hwc->period_left, &old, val));
11007
11008	return nr;
11009	}
11010
11011	static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
11012	struct perf_sample_data *data,
11013	struct pt_regs *regs)
11014	{
11015	struct hw_perf_event *hwc = &event->hw;
11016	int throttle = 0;
11017
11018	if (!overflow)
11019	overflow = perf_swevent_set_period(event);
11020
11021	if (hwc->interrupts == MAX_INTERRUPTS)
11022	return;
11023
11024	for (; overflow; overflow--) {
11025	if (__perf_event_overflow(event, throttle,
11026	data, regs)) {
11027	/*
11028	* We inhibit the overflow from happening when
11029	* hwc->interrupts == MAX_INTERRUPTS.
11030	*/
11031	break;
11032	}
11033	throttle = 1;
11034	}
11035	}
11036
11037	static void perf_swevent_event(struct perf_event *event, u64 nr,
11038	struct perf_sample_data *data,
11039	struct pt_regs *regs)
11040	{
11041	struct hw_perf_event *hwc = &event->hw;
11042
11043	/*
11044	* This is:
11045	* - software preempt
11046	* - tracepoint preempt
11047	* - tp_target_task irq (ctx->lock)
11048	* - uprobes preempt/irq
11049	* - kprobes preempt/irq
11050	* - hw_breakpoint irq
11051	*
11052	* Any of these are sufficient to hold off RCU and thus ensure @event
11053	* exists.
11054	*/
11055	lockdep_assert_preemption_disabled();
11056	local64_add(nr, &event->count);
11057
11058	if (!regs)
11059	return;
11060
11061	if (!is_sampling_event(event))
11062	return;
11063
11064	/*
11065	* Serialize against event_function_call() IPIs like normal overflow
11066	* event handling. Specifically, must not allow
11067	* perf_event_release_kernel() -> perf_remove_from_context() to make
11068	* progress and 'release' the event from under us.
11069	*/
11070	guard(irqsave)();
11071	if (event->state != PERF_EVENT_STATE_ACTIVE)
11072	return;
11073
11074	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
11075	data->period = nr;
11076	return perf_swevent_overflow(event, 1, data, regs);
11077	} else
11078	data->period = event->hw.last_period;
11079
11080	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
11081	return perf_swevent_overflow(event, 1, data, regs);
11082
11083	if (local64_add_negative(nr, &hwc->period_left))
11084	return;
11085
11086	perf_swevent_overflow(event, 0, data, regs);
11087	}
11088
11089	int perf_exclude_event(struct perf_event event, struct pt_regs regs)
11090	{
11091	if (event->hw.state & PERF_HES_STOPPED)
11092	return 1;
11093
11094	if (regs) {
11095	if (event->attr.exclude_user && user_mode(regs))
11096	return 1;
11097
11098	if (event->attr.exclude_kernel && !user_mode(regs))
11099	return 1;
11100	}
11101
11102	return 0;
11103	}
11104
11105	static int perf_swevent_match(struct perf_event *event,
11106	enum perf_type_id type,
11107	u32 event_id,
11108	struct perf_sample_data *data,
11109	struct pt_regs *regs)
11110	{
11111	if (event->attr.type != type)
11112	return 0;
11113
11114	if (event->attr.config != event_id)
11115	return 0;
11116
11117	if (perf_exclude_event(event, regs))
11118	return 0;
11119
11120	return 1;
11121	}
11122
11123	static inline u64 swevent_hash(u64 type, u32 event_id)
11124	{
11125	u64 val = event_id \| (type << 32);
11126
11127	return hash_64(val, SWEVENT_HLIST_BITS);
11128	}
11129
11130	static inline struct hlist_head *
11131	__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
11132	{
11133	u64 hash = swevent_hash(type, event_id);
11134
11135	return &hlist->heads[hash];
11136	}
11137
11138	/* For the read side: events when they trigger */
11139	static inline struct hlist_head *
11140	find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
11141	{
11142	struct swevent_hlist *hlist;
11143
11144	hlist = rcu_dereference(swhash->swevent_hlist);
11145	if (!hlist)
11146	return NULL;
11147
11148	return __find_swevent_head(hlist, type, event_id);
11149	}
11150
11151	/* For the event head insertion and removal in the hlist */
11152	static inline struct hlist_head *
11153	find_swevent_head(struct swevent_htable swhash, struct perf_event event)
11154	{
11155	struct swevent_hlist *hlist;
11156	u32 event_id = event->attr.config;
11157	u64 type = event->attr.type;
11158
11159	/*
11160	* Event scheduling is always serialized against hlist allocation
11161	* and release. Which makes the protected version suitable here.
11162	* The context lock guarantees that.
11163	*/
11164	hlist = rcu_dereference_protected(swhash->swevent_hlist,
11165	lockdep_is_held(&event->ctx->lock));
11166	if (!hlist)
11167	return NULL;
11168
11169	return __find_swevent_head(hlist, type, event_id);
11170	}
11171
11172	static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
11173	u64 nr,
11174	struct perf_sample_data *data,
11175	struct pt_regs *regs)
11176	{
11177	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
11178	struct perf_event *event;
11179	struct hlist_head *head;
11180
11181	rcu_read_lock();
11182	head = find_swevent_head_rcu(swhash, type, event_id);
11183	if (!head)
11184	goto end;
11185
11186	hlist_for_each_entry_rcu(event, head, hlist_entry) {
11187	if (perf_swevent_match(event, type, event_id, data, regs))
11188	perf_swevent_event(event, nr, data, regs);
11189	}
11190	end:
11191	rcu_read_unlock();
11192	}
11193
11194	DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]);
11195
11196	int perf_swevent_get_recursion_context(void)
11197	{
11198	return get_recursion_context(current->perf_recursion);
11199	}
11200	EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
11201
11202	void perf_swevent_put_recursion_context(int rctx)
11203	{
11204	put_recursion_context(current->perf_recursion, rctx);
11205	}
11206
11207	void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
11208	{
11209	struct perf_sample_data data;
11210
11211	if (WARN_ON_ONCE(!regs))
11212	return;
11213
11214	perf_sample_data_init(&data, addr, 0);
11215	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
11216	}
11217
11218	void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
11219	{
11220	int rctx;
11221
11222	preempt_disable_notrace();
11223	rctx = perf_swevent_get_recursion_context();
11224	if (unlikely(rctx < 0))
11225	goto fail;
11226
11227	___perf_sw_event(event_id, nr, regs, addr);
11228
11229	perf_swevent_put_recursion_context(rctx);
11230	fail:
11231	preempt_enable_notrace();
11232	}
11233
11234	static void perf_swevent_read(struct perf_event *event)
11235	{
11236	}
11237
11238	static int perf_swevent_add(struct perf_event *event, int flags)
11239	{
11240	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
11241	struct hw_perf_event *hwc = &event->hw;
11242	struct hlist_head *head;
11243
11244	if (is_sampling_event(event)) {
11245	hwc->last_period = hwc->sample_period;
11246	perf_swevent_set_period(event);
11247	}
11248
11249	hwc->state = !(flags & PERF_EF_START);
11250
11251	head = find_swevent_head(swhash, event);
11252	if (WARN_ON_ONCE(!head))
11253	return -EINVAL;
11254
11255	hlist_add_head_rcu(&event->hlist_entry, head);
11256	perf_event_update_userpage(event);
11257
11258	return 0;
11259	}
11260
11261	static void perf_swevent_del(struct perf_event *event, int flags)
11262	{
11263	hlist_del_rcu(&event->hlist_entry);
11264	}
11265
11266	static void perf_swevent_start(struct perf_event *event, int flags)
11267	{
11268	event->hw.state = 0;
11269	}
11270
11271	static void perf_swevent_stop(struct perf_event *event, int flags)
11272	{
11273	event->hw.state = PERF_HES_STOPPED;
11274	}
11275
11276	/* Deref the hlist from the update side */
11277	static inline struct swevent_hlist *
11278	swevent_hlist_deref(struct swevent_htable *swhash)
11279	{
11280	return rcu_dereference_protected(swhash->swevent_hlist,
11281	lockdep_is_held(&swhash->hlist_mutex));
11282	}
11283
11284	static void swevent_hlist_release(struct swevent_htable *swhash)
11285	{
11286	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
11287
11288	if (!hlist)
11289	return;
11290
11291	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
11292	kfree_rcu(hlist, rcu_head);
11293	}
11294
11295	static void swevent_hlist_put_cpu(int cpu)
11296	{
11297	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
11298
11299	mutex_lock(&swhash->hlist_mutex);
11300
11301	if (!--swhash->hlist_refcount)
11302	swevent_hlist_release(swhash);
11303
11304	mutex_unlock(&swhash->hlist_mutex);
11305	}
11306
11307	static void swevent_hlist_put(void)
11308	{
11309	int cpu;
11310
11311	for_each_possible_cpu(cpu)
11312	swevent_hlist_put_cpu(cpu);
11313	}
11314
11315	static int swevent_hlist_get_cpu(int cpu)
11316	{
11317	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
11318	int err = 0;
11319
11320	mutex_lock(&swhash->hlist_mutex);
11321	if (!swevent_hlist_deref(swhash) &&
11322	cpumask_test_cpu(cpu, perf_online_mask)) {
11323	struct swevent_hlist *hlist;
11324
11325	hlist = kzalloc_obj(*hlist);
11326	if (!hlist) {
11327	err = -ENOMEM;
11328	goto exit;
11329	}
11330	rcu_assign_pointer(swhash->swevent_hlist, hlist);
11331	}
11332	swhash->hlist_refcount++;
11333	exit:
11334	mutex_unlock(&swhash->hlist_mutex);
11335
11336	return err;
11337	}
11338
11339	static int swevent_hlist_get(void)
11340	{
11341	int err, cpu, failed_cpu;
11342
11343	mutex_lock(&pmus_lock);
11344	for_each_possible_cpu(cpu) {
11345	err = swevent_hlist_get_cpu(cpu);
11346	if (err) {
11347	failed_cpu = cpu;
11348	goto fail;
11349	}
11350	}
11351	mutex_unlock(&pmus_lock);
11352	return 0;
11353	fail:
11354	for_each_possible_cpu(cpu) {
11355	if (cpu == failed_cpu)
11356	break;
11357	swevent_hlist_put_cpu(cpu);
11358	}
11359	mutex_unlock(&pmus_lock);
11360	return err;
11361	}
11362
11363	struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
11364
11365	static void sw_perf_event_destroy(struct perf_event *event)
11366	{
11367	u64 event_id = event->attr.config;
11368
11369	WARN_ON(event->parent);
11370
11371	static_key_slow_dec(&perf_swevent_enabled[event_id]);
11372	swevent_hlist_put();
11373	}
11374
11375	static struct pmu perf_cpu_clock; /* fwd declaration */
11376	static struct pmu perf_task_clock;
11377
11378	static int perf_swevent_init(struct perf_event *event)
11379	{
11380	u64 event_id = event->attr.config;
11381
11382	if (event->attr.type != PERF_TYPE_SOFTWARE)
11383	return -ENOENT;
11384
11385	/*
11386	* no branch sampling for software events
11387	*/
11388	if (has_branch_stack(event))
11389	return -EOPNOTSUPP;
11390
11391	switch (event_id) {
11392	case PERF_COUNT_SW_CPU_CLOCK:
11393	event->attr.type = perf_cpu_clock.type;
11394	return -ENOENT;
11395	case PERF_COUNT_SW_TASK_CLOCK:
11396	event->attr.type = perf_task_clock.type;
11397	return -ENOENT;
11398
11399	default:
11400	break;
11401	}
11402
11403	if (event_id >= PERF_COUNT_SW_MAX)
11404	return -ENOENT;
11405
11406	if (!event->parent) {
11407	int err;
11408
11409	err = swevent_hlist_get();
11410	if (err)
11411	return err;
11412
11413	static_key_slow_inc(&perf_swevent_enabled[event_id]);
11414	event->destroy = sw_perf_event_destroy;
11415	}
11416
11417	return 0;
11418	}
11419
11420	static struct pmu perf_swevent = {
11421	.task_ctx_nr = perf_sw_context,
11422
11423	.capabilities = PERF_PMU_CAP_NO_NMI,
11424
11425	.event_init = perf_swevent_init,
11426	.add = perf_swevent_add,
11427	.del = perf_swevent_del,
11428	.start = perf_swevent_start,
11429	.stop = perf_swevent_stop,
11430	.read = perf_swevent_read,
11431	};
11432
11433	#ifdef CONFIG_EVENT_TRACING
11434
11435	static void tp_perf_event_destroy(struct perf_event *event)
11436	{
11437	perf_trace_destroy(event);
11438	}
11439
11440	static int perf_tp_event_init(struct perf_event *event)
11441	{
11442	int err;
11443
11444	if (event->attr.type != PERF_TYPE_TRACEPOINT)
11445	return -ENOENT;
11446
11447	/*
11448	* no branch sampling for tracepoint events
11449	*/
11450	if (has_branch_stack(event))
11451	return -EOPNOTSUPP;
11452
11453	err = perf_trace_init(event);
11454	if (err)
11455	return err;
11456
11457	event->destroy = tp_perf_event_destroy;
11458
11459	return 0;
11460	}
11461
11462	static struct pmu perf_tracepoint = {
11463	.task_ctx_nr = perf_sw_context,
11464
11465	.event_init = perf_tp_event_init,
11466	.add = perf_trace_add,
11467	.del = perf_trace_del,
11468	.start = perf_swevent_start,
11469	.stop = perf_swevent_stop,
11470	.read = perf_swevent_read,
11471	};
11472
11473	static int perf_tp_filter_match(struct perf_event *event,
11474	struct perf_raw_record *raw)
11475	{
11476	void *record = raw->frag.data;
11477
11478	/* only top level events have filters set */
11479	if (event->parent)
11480	event = event->parent;
11481
11482	if (likely(!event->filter) \|\| filter_match_preds(event->filter, record))
11483	return 1;
11484	return 0;
11485	}
11486
11487	static int perf_tp_event_match(struct perf_event *event,
11488	struct perf_raw_record *raw,
11489	struct pt_regs *regs)
11490	{
11491	if (event->hw.state & PERF_HES_STOPPED)
11492	return 0;
11493	/*
11494	* If exclude_kernel, only trace user-space tracepoints (uprobes)
11495	*/
11496	if (event->attr.exclude_kernel && !user_mode(regs))
11497	return 0;
11498
11499	if (!perf_tp_filter_match(event, raw))
11500	return 0;
11501
11502	return 1;
11503	}
11504
11505	void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
11506	struct trace_event_call *call, u64 count,
11507	struct pt_regs regs, struct hlist_head head,
11508	struct task_struct *task)
11509	{
11510	if (bpf_prog_array_valid(call)) {
11511	(struct pt_regs *)raw_data = regs;
11512	if (!trace_call_bpf(call, raw_data) \|\| hlist_empty(head)) {
11513	perf_swevent_put_recursion_context(rctx);
11514	return;
11515	}
11516	}
11517	perf_tp_event(call->event.type, count, raw_data, size, regs, head,
11518	rctx, task);
11519	}
11520	EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit);
11521
11522	static void __perf_tp_event_target_task(u64 count, void *record,
11523	struct pt_regs *regs,
11524	struct perf_sample_data *data,
11525	struct perf_raw_record *raw,
11526	struct perf_event *event)
11527	{
11528	struct trace_entry *entry = record;
11529
11530	if (event->attr.config != entry->type)
11531	return;
11532	/* Cannot deliver synchronous signal to other task. */
11533	if (event->attr.sigtrap)
11534	return;
11535	if (perf_tp_event_match(event, raw, regs)) {
11536	perf_sample_data_init(data, 0, 0);
11537	perf_sample_save_raw_data(data, event, raw);
11538	perf_swevent_event(event, count, data, regs);
11539	}
11540	}
11541
11542	static void perf_tp_event_target_task(u64 count, void *record,
11543	struct pt_regs *regs,
11544	struct perf_sample_data *data,
11545	struct perf_raw_record *raw,
11546	struct perf_event_context *ctx)
11547	{
11548	unsigned int cpu = smp_processor_id();
11549	struct pmu *pmu = &perf_tracepoint;
11550	struct perf_event event, sibling;
11551
11552	perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {
11553	__perf_tp_event_target_task(count, record, regs, data, raw, event);
11554	for_each_sibling_event(sibling, event)
11555	__perf_tp_event_target_task(count, record, regs, data, raw, sibling);
11556	}
11557
11558	perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {
11559	__perf_tp_event_target_task(count, record, regs, data, raw, event);
11560	for_each_sibling_event(sibling, event)
11561	__perf_tp_event_target_task(count, record, regs, data, raw, sibling);
11562	}
11563	}
11564
11565	void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
11566	struct pt_regs regs, struct hlist_head head, int rctx,
11567	struct task_struct *task)
11568	{
11569	struct perf_sample_data data;
11570	struct perf_event *event;
11571
11572	/*
11573	* Per being a tracepoint, this runs with preemption disabled.
11574	*/
11575	lockdep_assert_preemption_disabled();
11576
11577	struct perf_raw_record raw = {
11578	.frag = {
11579	.size = entry_size,
11580	.data = record,
11581	},
11582	};
11583
11584	perf_trace_buf_update(record, event_type);
11585
11586	hlist_for_each_entry_rcu(event, head, hlist_entry) {
11587	if (perf_tp_event_match(event, &raw, regs)) {
11588	/*
11589	* Here use the same on-stack perf_sample_data,
11590	* some members in data are event-specific and
11591	* need to be re-computed for different sweveents.
11592	* Re-initialize data->sample_flags safely to avoid
11593	* the problem that next event skips preparing data
11594	* because data->sample_flags is set.
11595	*/
11596	perf_sample_data_init(&data, 0, 0);
11597	perf_sample_save_raw_data(&data, event, &raw);
11598	perf_swevent_event(event, count, &data, regs);
11599	}
11600	}
11601
11602	/*
11603	* If we got specified a target task, also iterate its context and
11604	* deliver this event there too.
11605	*/
11606	if (task && task != current) {
11607	struct perf_event_context *ctx;
11608
11609	rcu_read_lock();
11610	ctx = rcu_dereference(task->perf_event_ctxp);
11611	if (!ctx)
11612	goto unlock;
11613
11614	raw_spin_lock(&ctx->lock);
11615	perf_tp_event_target_task(count, record, regs, &data, &raw, ctx);
11616	raw_spin_unlock(&ctx->lock);
11617	unlock:
11618	rcu_read_unlock();
11619	}
11620
11621	perf_swevent_put_recursion_context(rctx);
11622	}
11623	EXPORT_SYMBOL_GPL(perf_tp_event);
11624
11625	#if defined(CONFIG_KPROBE_EVENTS) \|\| defined(CONFIG_UPROBE_EVENTS)
11626	/*
11627	* Flags in config, used by dynamic PMU kprobe and uprobe
11628	* The flags should match following PMU_FORMAT_ATTR().
11629	*
11630	* PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe
11631	* if not set, create kprobe/uprobe
11632	*
11633	* The following values specify a reference counter (or semaphore in the
11634	* terminology of tools like dtrace, systemtap, etc.) Userspace Statically
11635	* Defined Tracepoints (USDT). Currently, we use 40 bit for the offset.
11636	*
11637	* PERF_UPROBE_REF_CTR_OFFSET_BITS # of bits in config as th offset
11638	* PERF_UPROBE_REF_CTR_OFFSET_SHIFT # of bits to shift left
11639	*/
11640	enum perf_probe_config {
11641	PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */
11642	PERF_UPROBE_REF_CTR_OFFSET_BITS = 32,
11643	PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
11644	};
11645
11646	PMU_FORMAT_ATTR(retprobe, "config:0");
11647	#endif
11648
11649	#ifdef CONFIG_KPROBE_EVENTS
11650	static struct attribute *kprobe_attrs[] = {
11651	&format_attr_retprobe.attr,
11652	NULL,
11653	};
11654
11655	static struct attribute_group kprobe_format_group = {
11656	.name = "format",
11657	.attrs = kprobe_attrs,
11658	};
11659
11660	static const struct attribute_group *kprobe_attr_groups[] = {
11661	&kprobe_format_group,
11662	NULL,
11663	};
11664
11665	static int perf_kprobe_event_init(struct perf_event *event);
11666	static struct pmu perf_kprobe = {
11667	.task_ctx_nr = perf_sw_context,
11668	.event_init = perf_kprobe_event_init,
11669	.add = perf_trace_add,
11670	.del = perf_trace_del,
11671	.start = perf_swevent_start,
11672	.stop = perf_swevent_stop,
11673	.read = perf_swevent_read,
11674	.attr_groups = kprobe_attr_groups,
11675	};
11676
11677	static int perf_kprobe_event_init(struct perf_event *event)
11678	{
11679	int err;
11680	bool is_retprobe;
11681
11682	if (event->attr.type != perf_kprobe.type)
11683	return -ENOENT;
11684
11685	if (!perfmon_capable())
11686	return -EACCES;
11687
11688	/*
11689	* no branch sampling for probe events
11690	*/
11691	if (has_branch_stack(event))
11692	return -EOPNOTSUPP;
11693
11694	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
11695	err = perf_kprobe_init(event, is_retprobe);
11696	if (err)
11697	return err;
11698
11699	event->destroy = perf_kprobe_destroy;
11700
11701	return 0;
11702	}
11703	#endif /* CONFIG_KPROBE_EVENTS */
11704
11705	#ifdef CONFIG_UPROBE_EVENTS
11706	PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
11707
11708	static struct attribute *uprobe_attrs[] = {
11709	&format_attr_retprobe.attr,
11710	&format_attr_ref_ctr_offset.attr,
11711	NULL,
11712	};
11713
11714	static struct attribute_group uprobe_format_group = {
11715	.name = "format",
11716	.attrs = uprobe_attrs,
11717	};
11718
11719	static const struct attribute_group *uprobe_attr_groups[] = {
11720	&uprobe_format_group,
11721	NULL,
11722	};
11723
11724	static int perf_uprobe_event_init(struct perf_event *event);
11725	static struct pmu perf_uprobe = {
11726	.task_ctx_nr = perf_sw_context,
11727	.event_init = perf_uprobe_event_init,
11728	.add = perf_trace_add,
11729	.del = perf_trace_del,
11730	.start = perf_swevent_start,
11731	.stop = perf_swevent_stop,
11732	.read = perf_swevent_read,
11733	.attr_groups = uprobe_attr_groups,
11734	};
11735
11736	static int perf_uprobe_event_init(struct perf_event *event)
11737	{
11738	int err;
11739	unsigned long ref_ctr_offset;
11740	bool is_retprobe;
11741
11742	if (event->attr.type != perf_uprobe.type)
11743	return -ENOENT;
11744
11745	if (!capable(CAP_SYS_ADMIN))
11746	return -EACCES;
11747
11748	/*
11749	* no branch sampling for probe events
11750	*/
11751	if (has_branch_stack(event))
11752	return -EOPNOTSUPP;
11753
11754	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
11755	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
11756	err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);
11757	if (err)
11758	return err;
11759
11760	event->destroy = perf_uprobe_destroy;
11761
11762	return 0;
11763	}
11764	#endif /* CONFIG_UPROBE_EVENTS */
11765
11766	static inline void perf_tp_register(void)
11767	{
11768	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
11769	#ifdef CONFIG_KPROBE_EVENTS
11770	perf_pmu_register(&perf_kprobe, "kprobe", -1);
11771	#endif
11772	#ifdef CONFIG_UPROBE_EVENTS
11773	perf_pmu_register(&perf_uprobe, "uprobe", -1);
11774	#endif
11775	}
11776
11777	static void perf_event_free_filter(struct perf_event *event)
11778	{
11779	ftrace_profile_free_filter(event);
11780	}
11781
11782	/*
11783	* returns true if the event is a tracepoint, or a kprobe/upprobe created
11784	* with perf_event_open()
11785	*/
11786	static inline bool perf_event_is_tracing(struct perf_event *event)
11787	{
11788	if (event->pmu == &perf_tracepoint)
11789	return true;
11790	#ifdef CONFIG_KPROBE_EVENTS
11791	if (event->pmu == &perf_kprobe)
11792	return true;
11793	#endif
11794	#ifdef CONFIG_UPROBE_EVENTS
11795	if (event->pmu == &perf_uprobe)
11796	return true;
11797	#endif
11798	return false;
11799	}
11800
11801	static int __perf_event_set_bpf_prog(struct perf_event *event,
11802	struct bpf_prog *prog,
11803	u64 bpf_cookie)
11804	{
11805	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;
11806
11807	if (event->state <= PERF_EVENT_STATE_REVOKED)
11808	return -ENODEV;
11809
11810	if (!perf_event_is_tracing(event))
11811	return perf_event_set_bpf_handler(event, prog, bpf_cookie);
11812
11813	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;
11814	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;
11815	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
11816	is_syscall_tp = is_syscall_trace_event(event->tp_event);
11817	if (!is_kprobe && !is_uprobe && !is_tracepoint && !is_syscall_tp)
11818	/* bpf programs can only be attached to u/kprobe or tracepoint */
11819	return -EINVAL;
11820
11821	if (((is_kprobe \|\| is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) \|\|
11822	(is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) \|\|
11823	(is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))
11824	return -EINVAL;
11825
11826	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->sleepable && !is_uprobe)
11827	/* only uprobe programs are allowed to be sleepable */
11828	return -EINVAL;
11829
11830	if (prog->type == BPF_PROG_TYPE_TRACEPOINT && prog->sleepable) {
11831	/*
11832	* Sleepable tracepoint programs can only attach to faultable
11833	* tracepoints. Currently only syscall tracepoints are faultable.
11834	*/
11835	if (!is_syscall_tp)
11836	return -EINVAL;
11837	}
11838
11839	/* Kprobe override only works for kprobes, not uprobes. */
11840	if (prog->kprobe_override && !is_kprobe)
11841	return -EINVAL;
11842
11843	/* Writing to context allowed only for uprobes. */
11844	if (prog->aux->kprobe_write_ctx && !is_uprobe)
11845	return -EINVAL;
11846
11847	if (is_tracepoint \|\| is_syscall_tp) {
11848	int off = trace_event_get_offsets(event->tp_event);
11849
11850	if (prog->aux->max_ctx_offset > off)
11851	return -EACCES;
11852	}
11853
11854	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
11855	}
11856
11857	int perf_event_set_bpf_prog(struct perf_event *event,
11858	struct bpf_prog *prog,
11859	u64 bpf_cookie)
11860	{
11861	struct perf_event_context *ctx;
11862	int ret;
11863
11864	ctx = perf_event_ctx_lock(event);
11865	ret = __perf_event_set_bpf_prog(event, prog, bpf_cookie);
11866	perf_event_ctx_unlock(event, ctx);
11867
11868	return ret;
11869	}
11870
11871	void perf_event_free_bpf_prog(struct perf_event *event)
11872	{
11873	if (!event->prog)
11874	return;
11875
11876	if (!perf_event_is_tracing(event)) {
11877	perf_event_free_bpf_handler(event);
11878	return;
11879	}
11880	perf_event_detach_bpf_prog(event);
11881	}
11882
11883	#else
11884
11885	static inline void perf_tp_register(void)
11886	{
11887	}
11888
11889	static void perf_event_free_filter(struct perf_event *event)
11890	{
11891	}
11892
11893	static int __perf_event_set_bpf_prog(struct perf_event *event,
11894	struct bpf_prog *prog,
11895	u64 bpf_cookie)
11896	{
11897	return -ENOENT;
11898	}
11899
11900	int perf_event_set_bpf_prog(struct perf_event *event,
11901	struct bpf_prog *prog,
11902	u64 bpf_cookie)
11903	{
11904	return -ENOENT;
11905	}
11906
11907	void perf_event_free_bpf_prog(struct perf_event *event)
11908	{
11909	}
11910	#endif /* CONFIG_EVENT_TRACING */
11911
11912	#ifdef CONFIG_HAVE_HW_BREAKPOINT
11913	void perf_bp_event(struct perf_event bp, void data)
11914	{
11915	struct perf_sample_data sample;
11916	struct pt_regs *regs = data;
11917
11918	/*
11919	* Exception context, will have interrupts disabled.
11920	*/
11921	lockdep_assert_irqs_disabled();
11922
11923	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
11924
11925	if (!bp->hw.state && !perf_exclude_event(bp, regs))
11926	perf_swevent_event(bp, 1, &sample, regs);
11927	}
11928	#endif
11929
11930	/*
11931	* Allocate a new address filter
11932	*/
11933	static struct perf_addr_filter *
11934	perf_addr_filter_new(struct perf_event event, struct list_head filters)
11935	{
11936	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);
11937	struct perf_addr_filter *filter;
11938
11939	filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node);
11940	if (!filter)
11941	return NULL;
11942
11943	INIT_LIST_HEAD(&filter->entry);
11944	list_add_tail(&filter->entry, filters);
11945
11946	return filter;
11947	}
11948
11949	static void free_filters_list(struct list_head *filters)
11950	{
11951	struct perf_addr_filter filter, iter;
11952
11953	list_for_each_entry_safe(filter, iter, filters, entry) {
11954	path_put(&filter->path);
11955	list_del(&filter->entry);
11956	kfree(filter);
11957	}
11958	}
11959
11960	/*
11961	* Free existing address filters and optionally install new ones
11962	*/
11963	static void perf_addr_filters_splice(struct perf_event *event,
11964	struct list_head *head)
11965	{
11966	unsigned long flags;
11967	LIST_HEAD(list);
11968
11969	if (!has_addr_filter(event))
11970	return;
11971
11972	/* don't bother with children, they don't have their own filters */
11973	if (event->parent)
11974	return;
11975
11976	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);
11977
11978	list_splice_init(&event->addr_filters.list, &list);
11979	if (head)
11980	list_splice(head, &event->addr_filters.list);
11981
11982	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);
11983
11984	free_filters_list(&list);
11985	}
11986
11987	static void perf_free_addr_filters(struct perf_event *event)
11988	{
11989	/*
11990	* Used during free paths, there is no concurrency.
11991	*/
11992	if (list_empty(&event->addr_filters.list))
11993	return;
11994
11995	perf_addr_filters_splice(event, NULL);
11996	}
11997
11998	/*
11999	* Scan through mm's vmas and see if one of them matches the
12000	* @filter; if so, adjust filter's address range.
12001	* Called with mm::mmap_lock down for reading.
12002	*/
12003	static void perf_addr_filter_apply(struct perf_addr_filter *filter,
12004	struct mm_struct *mm,
12005	struct perf_addr_filter_range *fr)
12006	{
12007	struct vm_area_struct *vma;
12008	VMA_ITERATOR(vmi, mm, 0);
12009
12010	for_each_vma(vmi, vma) {
12011	if (!vma->vm_file)
12012	continue;
12013
12014	if (perf_addr_filter_vma_adjust(filter, vma, fr))
12015	return;
12016	}
12017	}
12018
12019	/*
12020	* Update event's address range filters based on the
12021	* task's existing mappings, if any.
12022	*/
12023	static void perf_event_addr_filters_apply(struct perf_event *event)
12024	{
12025	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
12026	struct task_struct *task = READ_ONCE(event->ctx->task);
12027	struct perf_addr_filter *filter;
12028	struct mm_struct *mm = NULL;
12029	unsigned int count = 0;
12030	unsigned long flags;
12031
12032	/*
12033	* We may observe TASK_TOMBSTONE, which means that the event tear-down
12034	* will stop on the parent's child_mutex that our caller is also holding
12035	*/
12036	if (task == TASK_TOMBSTONE)
12037	return;
12038
12039	if (ifh->nr_file_filters) {
12040	mm = get_task_mm(task);
12041	if (!mm)
12042	goto restart;
12043
12044	mmap_read_lock(mm);
12045	}
12046
12047	raw_spin_lock_irqsave(&ifh->lock, flags);
12048	list_for_each_entry(filter, &ifh->list, entry) {
12049	if (filter->path.dentry) {
12050	/*
12051	* Adjust base offset if the filter is associated to a
12052	* binary that needs to be mapped:
12053	*/
12054	event->addr_filter_ranges[count].start = 0;
12055	event->addr_filter_ranges[count].size = 0;
12056
12057	perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);
12058	} else {
12059	event->addr_filter_ranges[count].start = filter->offset;
12060	event->addr_filter_ranges[count].size = filter->size;
12061	}
12062
12063	count++;
12064	}
12065
12066	event->addr_filters_gen++;
12067	raw_spin_unlock_irqrestore(&ifh->lock, flags);
12068
12069	if (ifh->nr_file_filters) {
12070	mmap_read_unlock(mm);
12071
12072	mmput(mm);
12073	}
12074
12075	restart:
12076	perf_event_stop(event, 1);
12077	}
12078
12079	/*
12080	* Address range filtering: limiting the data to certain
12081	* instruction address ranges. Filters are ioctl()ed to us from
12082	* userspace as ascii strings.
12083	*
12084	* Filter string format:
12085	*
12086	* ACTION RANGE_SPEC
12087	* where ACTION is one of the
12088	* * "filter": limit the trace to this region
12089	* * "start": start tracing from this address
12090	* * "stop": stop tracing at this address/region;
12091	* RANGE_SPEC is
12092	* * for kernel addresses: <start address>[/<size>]
12093	* * for object files: <start address>[/<size>]@</path/to/object/file>
12094	*
12095	* if <size> is not specified or is zero, the range is treated as a single
12096	* address; not valid for ACTION=="filter".
12097	*/
12098	enum {
12099	IF_ACT_NONE = -1,
12100	IF_ACT_FILTER,
12101	IF_ACT_START,
12102	IF_ACT_STOP,
12103	IF_SRC_FILE,
12104	IF_SRC_KERNEL,
12105	IF_SRC_FILEADDR,
12106	IF_SRC_KERNELADDR,
12107	};
12108
12109	enum {
12110	IF_STATE_ACTION = 0,
12111	IF_STATE_SOURCE,
12112	IF_STATE_END,
12113	};
12114
12115	static const match_table_t if_tokens = {
12116	{ IF_ACT_FILTER, "filter" },
12117	{ IF_ACT_START, "start" },
12118	{ IF_ACT_STOP, "stop" },
12119	{ IF_SRC_FILE, "%u/%u@%s" },
12120	{ IF_SRC_KERNEL, "%u/%u" },
12121	{ IF_SRC_FILEADDR, "%u@%s" },
12122	{ IF_SRC_KERNELADDR, "%u" },
12123	{ IF_ACT_NONE, NULL },
12124	};
12125
12126	/*
12127	* Address filter string parser
12128	*/
12129	static int
12130	perf_event_parse_addr_filter(struct perf_event event, char fstr,
12131	struct list_head *filters)
12132	{
12133	struct perf_addr_filter *filter = NULL;
12134	char start, orig, *filename = NULL;
12135	substring_t args[MAX_OPT_ARGS];
12136	int state = IF_STATE_ACTION, token;
12137	unsigned int kernel = 0;
12138	int ret = -EINVAL;
12139
12140	orig = fstr = kstrdup(fstr, GFP_KERNEL);
12141	if (!fstr)
12142	return -ENOMEM;
12143
12144	while ((start = strsep(&fstr, " ,\n")) != NULL) {
12145	static const enum perf_addr_filter_action_t actions[] = {
12146	[IF_ACT_FILTER] = PERF_ADDR_FILTER_ACTION_FILTER,
12147	[IF_ACT_START] = PERF_ADDR_FILTER_ACTION_START,
12148	[IF_ACT_STOP] = PERF_ADDR_FILTER_ACTION_STOP,
12149	};
12150	ret = -EINVAL;
12151
12152	if (!*start)
12153	continue;
12154
12155	/* filter definition begins */
12156	if (state == IF_STATE_ACTION) {
12157	filter = perf_addr_filter_new(event, filters);
12158	if (!filter)
12159	goto fail;
12160	}
12161
12162	token = match_token(start, if_tokens, args);
12163	switch (token) {
12164	case IF_ACT_FILTER:
12165	case IF_ACT_START:
12166	case IF_ACT_STOP:
12167	if (state != IF_STATE_ACTION)
12168	goto fail;
12169
12170	filter->action = actions[token];
12171	state = IF_STATE_SOURCE;
12172	break;
12173
12174	case IF_SRC_KERNELADDR:
12175	case IF_SRC_KERNEL:
12176	kernel = 1;
12177	fallthrough;
12178
12179	case IF_SRC_FILEADDR:
12180	case IF_SRC_FILE:
12181	if (state != IF_STATE_SOURCE)
12182	goto fail;
12183
12184	*args[0].to = 0;
12185	ret = kstrtoul(args[0].from, 0, &filter->offset);
12186	if (ret)
12187	goto fail;
12188
12189	if (token == IF_SRC_KERNEL \|\| token == IF_SRC_FILE) {
12190	*args[1].to = 0;
12191	ret = kstrtoul(args[1].from, 0, &filter->size);
12192	if (ret)
12193	goto fail;
12194	}
12195
12196	if (token == IF_SRC_FILE \|\| token == IF_SRC_FILEADDR) {
12197	int fpos = token == IF_SRC_FILE ? 2 : 1;
12198
12199	kfree(filename);
12200	filename = match_strdup(&args[fpos]);
12201	if (!filename) {
12202	ret = -ENOMEM;
12203	goto fail;
12204	}
12205	}
12206
12207	state = IF_STATE_END;
12208	break;
12209
12210	default:
12211	goto fail;
12212	}
12213
12214	/*
12215	* Filter definition is fully parsed, validate and install it.
12216	* Make sure that it doesn't contradict itself or the event's
12217	* attribute.
12218	*/
12219	if (state == IF_STATE_END) {
12220	ret = -EINVAL;
12221
12222	/*
12223	* ACTION "filter" must have a non-zero length region
12224	* specified.
12225	*/
12226	if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&
12227	!filter->size)
12228	goto fail;
12229
12230	if (!kernel) {
12231	if (!filename)
12232	goto fail;
12233
12234	/*
12235	* For now, we only support file-based filters
12236	* in per-task events; doing so for CPU-wide
12237	* events requires additional context switching
12238	* trickery, since same object code will be
12239	* mapped at different virtual addresses in
12240	* different processes.
12241	*/
12242	ret = -EOPNOTSUPP;
12243	if (!event->ctx->task)
12244	goto fail;
12245
12246	/* look up the path and grab its inode */
12247	ret = kern_path(filename, LOOKUP_FOLLOW,
12248	&filter->path);
12249	if (ret)
12250	goto fail;
12251
12252	ret = -EINVAL;
12253	if (!filter->path.dentry \|\|
12254	!S_ISREG(d_inode(filter->path.dentry)
12255	->i_mode))
12256	goto fail;
12257
12258	event->addr_filters.nr_file_filters++;
12259	}
12260
12261	/* ready to consume more filters */
12262	kfree(filename);
12263	filename = NULL;
12264	state = IF_STATE_ACTION;
12265	filter = NULL;
12266	kernel = 0;
12267	}
12268	}
12269
12270	if (state != IF_STATE_ACTION)
12271	goto fail;
12272
12273	kfree(filename);
12274	kfree(orig);
12275
12276	return 0;
12277
12278	fail:
12279	kfree(filename);
12280	free_filters_list(filters);
12281	kfree(orig);
12282
12283	return ret;
12284	}
12285
12286	static int
12287	perf_event_set_addr_filter(struct perf_event event, char filter_str)
12288	{
12289	LIST_HEAD(filters);
12290	int ret;
12291
12292	/*
12293	* Since this is called in perf_ioctl() path, we're already holding
12294	* ctx::mutex.
12295	*/
12296	lockdep_assert_held(&event->ctx->mutex);
12297
12298	if (WARN_ON_ONCE(event->parent))
12299	return -EINVAL;
12300
12301	ret = perf_event_parse_addr_filter(event, filter_str, &filters);
12302	if (ret)
12303	goto fail_clear_files;
12304
12305	ret = event->pmu->addr_filters_validate(&filters);
12306	if (ret)
12307	goto fail_free_filters;
12308
12309	/* remove existing filters, if any */
12310	perf_addr_filters_splice(event, &filters);
12311
12312	/* install new filters */
12313	perf_event_for_each_child(event, perf_event_addr_filters_apply);
12314
12315	return ret;
12316
12317	fail_free_filters:
12318	free_filters_list(&filters);
12319
12320	fail_clear_files:
12321	event->addr_filters.nr_file_filters = 0;
12322
12323	return ret;
12324	}
12325
12326	static int perf_event_set_filter(struct perf_event event, void __user arg)
12327	{
12328	int ret = -EINVAL;
12329	char *filter_str;
12330
12331	filter_str = strndup_user(arg, PAGE_SIZE);
12332	if (IS_ERR(filter_str))
12333	return PTR_ERR(filter_str);
12334
12335	#ifdef CONFIG_EVENT_TRACING
12336	if (perf_event_is_tracing(event)) {
12337	struct perf_event_context *ctx = event->ctx;
12338
12339	/*
12340	* Beware, here be dragons!!
12341	*
12342	* the tracepoint muck will deadlock against ctx->mutex, but
12343	* the tracepoint stuff does not actually need it. So
12344	* temporarily drop ctx->mutex. As per perf_event_ctx_lock() we
12345	* already have a reference on ctx.
12346	*
12347	* This can result in event getting moved to a different ctx,
12348	* but that does not affect the tracepoint state.
12349	*/
12350	mutex_unlock(&ctx->mutex);
12351	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
12352	mutex_lock(&ctx->mutex);
12353	} else
12354	#endif
12355	if (has_addr_filter(event))
12356	ret = perf_event_set_addr_filter(event, filter_str);
12357
12358	kfree(filter_str);
12359	return ret;
12360	}
12361
12362	/*
12363	* hrtimer based swevent callback
12364	*/
12365
12366	static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
12367	{
12368	enum hrtimer_restart ret = HRTIMER_RESTART;
12369	struct perf_sample_data data;
12370	struct pt_regs *regs;
12371	struct perf_event *event;
12372	u64 period;
12373
12374	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
12375
12376	if (event->state != PERF_EVENT_STATE_ACTIVE \|\|
12377	event->hw.state & PERF_HES_STOPPED)
12378	return HRTIMER_NORESTART;
12379
12380	event->pmu->read(event);
12381
12382	perf_sample_data_init(&data, 0, event->hw.last_period);
12383	regs = get_irq_regs();
12384
12385	if (regs && !perf_exclude_event(event, regs)) {
12386	if (!(event->attr.exclude_idle && is_idle_task(current)))
12387	if (perf_event_overflow(event, &data, regs))
12388	ret = HRTIMER_NORESTART;
12389	}
12390
12391	period = max_t(u64, 10000, event->hw.sample_period);
12392	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
12393
12394	return ret;
12395	}
12396
12397	static void perf_swevent_start_hrtimer(struct perf_event *event)
12398	{
12399	struct hw_perf_event *hwc = &event->hw;
12400	s64 period;
12401
12402	if (!is_sampling_event(event))
12403	return;
12404
12405	period = local64_read(&hwc->period_left);
12406	if (period) {
12407	if (period < 0)
12408	period = 10000;
12409
12410	local64_set(&hwc->period_left, 0);
12411	} else {
12412	period = max_t(u64, 10000, hwc->sample_period);
12413	}
12414	hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),
12415	HRTIMER_MODE_REL_PINNED_HARD);
12416	}
12417
12418	static void perf_swevent_cancel_hrtimer(struct perf_event *event)
12419	{
12420	struct hw_perf_event *hwc = &event->hw;
12421
12422	/*
12423	* Careful: this function can be triggered in the hrtimer handler,
12424	* for cpu-clock events, so hrtimer_cancel() would cause a
12425	* deadlock.
12426	*
12427	* So use hrtimer_try_to_cancel() to try to stop the hrtimer,
12428	* and the cpu-clock handler also sets the PERF_HES_STOPPED flag,
12429	* which guarantees that perf_swevent_hrtimer() will stop the
12430	* hrtimer once it sees the PERF_HES_STOPPED flag.
12431	*/
12432	if (is_sampling_event(event) && (hwc->interrupts != MAX_INTERRUPTS)) {
12433	ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
12434	local64_set(&hwc->period_left, ktime_to_ns(remaining));
12435
12436	hrtimer_try_to_cancel(&hwc->hrtimer);
12437	}
12438	}
12439
12440	static void perf_swevent_destroy_hrtimer(struct perf_event *event)
12441	{
12442	hrtimer_cancel(&event->hw.hrtimer);
12443	}
12444
12445	static void perf_swevent_init_hrtimer(struct perf_event *event)
12446	{
12447	struct hw_perf_event *hwc = &event->hw;
12448
12449	if (!is_sampling_event(event))
12450	return;
12451
12452	hrtimer_setup(&hwc->hrtimer, perf_swevent_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
12453	event->destroy = perf_swevent_destroy_hrtimer;
12454
12455	/*
12456	* Since hrtimers have a fixed rate, we can do a static freq->period
12457	* mapping and avoid the whole period adjust feedback stuff.
12458	*/
12459	if (event->attr.freq) {
12460	long freq = event->attr.sample_freq;
12461
12462	event->attr.sample_period = NSEC_PER_SEC / freq;
12463	hwc->sample_period = event->attr.sample_period;
12464	local64_set(&hwc->period_left, hwc->sample_period);
12465	hwc->last_period = hwc->sample_period;
12466	event->attr.freq = 0;
12467	}
12468	}
12469
12470	/*
12471	* Software event: cpu wall time clock
12472	*/
12473
12474	static void cpu_clock_event_update(struct perf_event *event)
12475	{
12476	s64 prev;
12477	u64 now;
12478
12479	now = local_clock();
12480	prev = local64_xchg(&event->hw.prev_count, now);
12481	local64_add(now - prev, &event->count);
12482	}
12483
12484	static void cpu_clock_event_start(struct perf_event *event, int flags)
12485	{
12486	event->hw.state = 0;
12487	local64_set(&event->hw.prev_count, local_clock());
12488	perf_swevent_start_hrtimer(event);
12489	}
12490
12491	static void cpu_clock_event_stop(struct perf_event *event, int flags)
12492	{
12493	event->hw.state = PERF_HES_STOPPED;
12494	perf_swevent_cancel_hrtimer(event);
12495	if (flags & PERF_EF_UPDATE)
12496	cpu_clock_event_update(event);
12497	}
12498
12499	static int cpu_clock_event_add(struct perf_event *event, int flags)
12500	{
12501	if (flags & PERF_EF_START)
12502	cpu_clock_event_start(event, flags);
12503	perf_event_update_userpage(event);
12504
12505	return 0;
12506	}
12507
12508	static void cpu_clock_event_del(struct perf_event *event, int flags)
12509	{
12510	cpu_clock_event_stop(event, PERF_EF_UPDATE);
12511	}
12512
12513	static void cpu_clock_event_read(struct perf_event *event)
12514	{
12515	cpu_clock_event_update(event);
12516	}
12517
12518	static int cpu_clock_event_init(struct perf_event *event)
12519	{
12520	if (event->attr.type != perf_cpu_clock.type)
12521	return -ENOENT;
12522
12523	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
12524	return -ENOENT;
12525
12526	/*
12527	* no branch sampling for software events
12528	*/
12529	if (has_branch_stack(event))
12530	return -EOPNOTSUPP;
12531
12532	perf_swevent_init_hrtimer(event);
12533
12534	return 0;
12535	}
12536
12537	static struct pmu perf_cpu_clock = {
12538	.task_ctx_nr = perf_sw_context,
12539
12540	.capabilities = PERF_PMU_CAP_NO_NMI,
12541	.dev = PMU_NULL_DEV,
12542
12543	.event_init = cpu_clock_event_init,
12544	.add = cpu_clock_event_add,
12545	.del = cpu_clock_event_del,
12546	.start = cpu_clock_event_start,
12547	.stop = cpu_clock_event_stop,
12548	.read = cpu_clock_event_read,
12549	};
12550
12551	/*
12552	* Software event: task time clock
12553	*/
12554
12555	static void task_clock_event_update(struct perf_event *event, u64 now)
12556	{
12557	u64 prev;
12558	s64 delta;
12559
12560	prev = local64_xchg(&event->hw.prev_count, now);
12561	delta = now - prev;
12562	local64_add(delta, &event->count);
12563	}
12564
12565	static void task_clock_event_start(struct perf_event *event, int flags)
12566	{
12567	event->hw.state = 0;
12568	local64_set(&event->hw.prev_count, event->ctx->time.time);
12569	perf_swevent_start_hrtimer(event);
12570	}
12571
12572	static void task_clock_event_stop(struct perf_event *event, int flags)
12573	{
12574	event->hw.state = PERF_HES_STOPPED;
12575	perf_swevent_cancel_hrtimer(event);
12576	if (flags & PERF_EF_UPDATE)
12577	task_clock_event_update(event, event->ctx->time.time);
12578	}
12579
12580	static int task_clock_event_add(struct perf_event *event, int flags)
12581	{
12582	if (flags & PERF_EF_START)
12583	task_clock_event_start(event, flags);
12584	perf_event_update_userpage(event);
12585
12586	return 0;
12587	}
12588
12589	static void task_clock_event_del(struct perf_event *event, int flags)
12590	{
12591	task_clock_event_stop(event, PERF_EF_UPDATE);
12592	}
12593
12594	static void task_clock_event_read(struct perf_event *event)
12595	{
12596	u64 now = perf_clock();
12597	u64 delta = now - event->ctx->time.stamp;
12598	u64 time = event->ctx->time.time + delta;
12599
12600	task_clock_event_update(event, time);
12601	}
12602
12603	static int task_clock_event_init(struct perf_event *event)
12604	{
12605	if (event->attr.type != perf_task_clock.type)
12606	return -ENOENT;
12607
12608	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
12609	return -ENOENT;
12610
12611	/*
12612	* no branch sampling for software events
12613	*/
12614	if (has_branch_stack(event))
12615	return -EOPNOTSUPP;
12616
12617	perf_swevent_init_hrtimer(event);
12618
12619	return 0;
12620	}
12621
12622	static struct pmu perf_task_clock = {
12623	.task_ctx_nr = perf_sw_context,
12624
12625	.capabilities = PERF_PMU_CAP_NO_NMI,
12626	.dev = PMU_NULL_DEV,
12627
12628	.event_init = task_clock_event_init,
12629	.add = task_clock_event_add,
12630	.del = task_clock_event_del,
12631	.start = task_clock_event_start,
12632	.stop = task_clock_event_stop,
12633	.read = task_clock_event_read,
12634	};
12635
12636	static void perf_pmu_nop_void(struct pmu *pmu)
12637	{
12638	}
12639
12640	static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags)
12641	{
12642	}
12643
12644	static int perf_pmu_nop_int(struct pmu *pmu)
12645	{
12646	return 0;
12647	}
12648
12649	static int perf_event_nop_int(struct perf_event *event, u64 value)
12650	{
12651	return 0;
12652	}
12653
12654	static DEFINE_PER_CPU(unsigned int, nop_txn_flags);
12655
12656	static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags)
12657	{
12658	__this_cpu_write(nop_txn_flags, flags);
12659
12660	if (flags & ~PERF_PMU_TXN_ADD)
12661	return;
12662
12663	perf_pmu_disable(pmu);
12664	}
12665
12666	static int perf_pmu_commit_txn(struct pmu *pmu)
12667	{
12668	unsigned int flags = __this_cpu_read(nop_txn_flags);
12669
12670	__this_cpu_write(nop_txn_flags, 0);
12671
12672	if (flags & ~PERF_PMU_TXN_ADD)
12673	return 0;
12674
12675	perf_pmu_enable(pmu);
12676	return 0;
12677	}
12678
12679	static void perf_pmu_cancel_txn(struct pmu *pmu)
12680	{
12681	unsigned int flags = __this_cpu_read(nop_txn_flags);
12682
12683	__this_cpu_write(nop_txn_flags, 0);
12684
12685	if (flags & ~PERF_PMU_TXN_ADD)
12686	return;
12687
12688	perf_pmu_enable(pmu);
12689	}
12690
12691	static int perf_event_idx_default(struct perf_event *event)
12692	{
12693	return 0;
12694	}
12695
12696	/*
12697	* Let userspace know that this PMU supports address range filtering:
12698	*/
12699	static ssize_t nr_addr_filters_show(struct device *dev,
12700	struct device_attribute *attr,
12701	char *page)
12702	{
12703	struct pmu *pmu = dev_get_drvdata(dev);
12704
12705	return sysfs_emit(page, "%d\n", pmu->nr_addr_filters);
12706	}
12707	DEVICE_ATTR_RO(nr_addr_filters);
12708
12709	static struct idr pmu_idr;
12710
12711	static ssize_t
12712	type_show(struct device dev, struct device_attribute attr, char *page)
12713	{
12714	struct pmu *pmu = dev_get_drvdata(dev);
12715
12716	return sysfs_emit(page, "%d\n", pmu->type);
12717	}
12718	static DEVICE_ATTR_RO(type);
12719
12720	static ssize_t
12721	perf_event_mux_interval_ms_show(struct device *dev,
12722	struct device_attribute *attr,
12723	char *page)
12724	{
12725	struct pmu *pmu = dev_get_drvdata(dev);
12726
12727	return sysfs_emit(page, "%d\n", pmu->hrtimer_interval_ms);
12728	}
12729
12730	static DEFINE_MUTEX(mux_interval_mutex);
12731
12732	static ssize_t
12733	perf_event_mux_interval_ms_store(struct device *dev,
12734	struct device_attribute *attr,
12735	const char *buf, size_t count)
12736	{
12737	struct pmu *pmu = dev_get_drvdata(dev);
12738	int timer, cpu, ret;
12739
12740	ret = kstrtoint(buf, 0, &timer);
12741	if (ret)
12742	return ret;
12743
12744	if (timer < 1)
12745	return -EINVAL;
12746
12747	/* same value, noting to do */
12748	if (timer == pmu->hrtimer_interval_ms)
12749	return count;
12750
12751	mutex_lock(&mux_interval_mutex);
12752	pmu->hrtimer_interval_ms = timer;
12753
12754	/* update all cpuctx for this PMU */
12755	cpus_read_lock();
12756	for_each_online_cpu(cpu) {
12757	struct perf_cpu_pmu_context *cpc;
12758	cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu);
12759	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
12760
12761	cpu_function_call(cpu, perf_mux_hrtimer_restart_ipi, cpc);
12762	}
12763	cpus_read_unlock();
12764	mutex_unlock(&mux_interval_mutex);
12765
12766	return count;
12767	}
12768	static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
12769
12770	static inline const struct cpumask *perf_scope_cpu_topology_cpumask(unsigned int scope, int cpu)
12771	{
12772	switch (scope) {
12773	case PERF_PMU_SCOPE_CORE:
12774	return topology_sibling_cpumask(cpu);
12775	case PERF_PMU_SCOPE_DIE:
12776	return topology_die_cpumask(cpu);
12777	case PERF_PMU_SCOPE_CLUSTER:
12778	return topology_cluster_cpumask(cpu);
12779	case PERF_PMU_SCOPE_PKG:
12780	return topology_core_cpumask(cpu);
12781	case PERF_PMU_SCOPE_SYS_WIDE:
12782	return cpu_online_mask;
12783	}
12784
12785	return NULL;
12786	}
12787
12788	static inline struct cpumask *perf_scope_cpumask(unsigned int scope)
12789	{
12790	switch (scope) {
12791	case PERF_PMU_SCOPE_CORE:
12792	return perf_online_core_mask;
12793	case PERF_PMU_SCOPE_DIE:
12794	return perf_online_die_mask;
12795	case PERF_PMU_SCOPE_CLUSTER:
12796	return perf_online_cluster_mask;
12797	case PERF_PMU_SCOPE_PKG:
12798	return perf_online_pkg_mask;
12799	case PERF_PMU_SCOPE_SYS_WIDE:
12800	return perf_online_sys_mask;
12801	}
12802
12803	return NULL;
12804	}
12805
12806	static ssize_t cpumask_show(struct device dev, struct device_attribute attr,
12807	char *buf)
12808	{
12809	struct pmu *pmu = dev_get_drvdata(dev);
12810	struct cpumask *mask = perf_scope_cpumask(pmu->scope);
12811
12812	if (mask)
12813	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(mask));
12814	return 0;
12815	}
12816
12817	static DEVICE_ATTR_RO(cpumask);
12818
12819	static struct attribute *pmu_dev_attrs[] = {
12820	&dev_attr_type.attr,
12821	&dev_attr_perf_event_mux_interval_ms.attr,
12822	&dev_attr_nr_addr_filters.attr,
12823	&dev_attr_cpumask.attr,
12824	NULL,
12825	};
12826
12827	static umode_t pmu_dev_is_visible(struct kobject kobj, struct attribute a, int n)
12828	{
12829	struct device *dev = kobj_to_dev(kobj);
12830	struct pmu *pmu = dev_get_drvdata(dev);
12831
12832	if (n == 2 && !pmu->nr_addr_filters)
12833	return 0;
12834
12835	/* cpumask */
12836	if (n == 3 && pmu->scope == PERF_PMU_SCOPE_NONE)
12837	return 0;
12838
12839	return a->mode;
12840	}
12841
12842	static struct attribute_group pmu_dev_attr_group = {
12843	.is_visible = pmu_dev_is_visible,
12844	.attrs = pmu_dev_attrs,
12845	};
12846
12847	static const struct attribute_group *pmu_dev_groups[] = {
12848	&pmu_dev_attr_group,
12849	NULL,
12850	};
12851
12852	static int pmu_bus_running;
12853	static const struct bus_type pmu_bus = {
12854	.name = "event_source",
12855	.dev_groups = pmu_dev_groups,
12856	};
12857
12858	static void pmu_dev_release(struct device *dev)
12859	{
12860	kfree(dev);
12861	}
12862
12863	static int pmu_dev_alloc(struct pmu *pmu)
12864	{
12865	int ret = -ENOMEM;
12866
12867	pmu->dev = kzalloc_obj(struct device);
12868	if (!pmu->dev)
12869	goto out;
12870
12871	pmu->dev->groups = pmu->attr_groups;
12872	device_initialize(pmu->dev);
12873
12874	dev_set_drvdata(pmu->dev, pmu);
12875	pmu->dev->bus = &pmu_bus;
12876	pmu->dev->parent = pmu->parent;
12877	pmu->dev->release = pmu_dev_release;
12878
12879	ret = dev_set_name(pmu->dev, "%s", pmu->name);
12880	if (ret)
12881	goto free_dev;
12882
12883	ret = device_add(pmu->dev);
12884	if (ret)
12885	goto free_dev;
12886
12887	if (pmu->attr_update) {
12888	ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);
12889	if (ret)
12890	goto del_dev;
12891	}
12892
12893	out:
12894	return ret;
12895
12896	del_dev:
12897	device_del(pmu->dev);
12898
12899	free_dev:
12900	put_device(pmu->dev);
12901	pmu->dev = NULL;
12902	goto out;
12903	}
12904
12905	static struct lock_class_key cpuctx_mutex;
12906	static struct lock_class_key cpuctx_lock;
12907
12908	static bool idr_cmpxchg(struct idr idr, unsigned long id, void old, void *new)
12909	{
12910	void tmp, val = idr_find(idr, id);
12911
12912	if (val != old)
12913	return false;
12914
12915	tmp = idr_replace(idr, new, id);
12916	if (IS_ERR(tmp))
12917	return false;
12918
12919	WARN_ON_ONCE(tmp != val);
12920	return true;
12921	}
12922
12923	static void perf_pmu_free(struct pmu *pmu)
12924	{
12925	if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) {
12926	if (pmu->nr_addr_filters)
12927	device_remove_file(pmu->dev, &dev_attr_nr_addr_filters);
12928	device_del(pmu->dev);
12929	put_device(pmu->dev);
12930	}
12931
12932	if (pmu->cpu_pmu_context) {
12933	int cpu;
12934
12935	for_each_possible_cpu(cpu) {
12936	struct perf_cpu_pmu_context *cpc;
12937
12938	cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu);
12939	if (!cpc)
12940	continue;
12941	if (cpc->epc.embedded) {
12942	/* refcount managed */
12943	put_pmu_ctx(&cpc->epc);
12944	continue;
12945	}
12946	kfree(cpc);
12947	}
12948	free_percpu(pmu->cpu_pmu_context);
12949	}
12950	}
12951
12952	DEFINE_FREE(pmu_unregister, struct pmu *, if (_T) perf_pmu_free(_T))
12953
12954	int perf_pmu_register(struct pmu _pmu, const char name, int type)
12955	{
12956	int cpu, max = PERF_TYPE_MAX;
12957
12958	struct pmu *pmu __free(pmu_unregister) = _pmu;
12959	guard(mutex)(&pmus_lock);
12960
12961	if (WARN_ONCE(!name, "Can not register anonymous pmu.\n"))
12962	return -EINVAL;
12963
12964	if (WARN_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE,
12965	"Can not register a pmu with an invalid scope.\n"))
12966	return -EINVAL;
12967
12968	pmu->name = name;
12969
12970	if (type >= 0)
12971	max = type;
12972
12973	CLASS(idr_alloc, pmu_type)(&pmu_idr, NULL, max, 0, GFP_KERNEL);
12974	if (pmu_type.id < 0)
12975	return pmu_type.id;
12976
12977	WARN_ON(type >= 0 && pmu_type.id != type);
12978
12979	pmu->type = pmu_type.id;
12980	atomic_set(&pmu->exclusive_cnt, 0);
12981
12982	if (pmu_bus_running && !pmu->dev) {
12983	int ret = pmu_dev_alloc(pmu);
12984	if (ret)
12985	return ret;
12986	}
12987
12988	pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context *);
12989	if (!pmu->cpu_pmu_context)
12990	return -ENOMEM;
12991
12992	for_each_possible_cpu(cpu) {
12993	struct perf_cpu_pmu_context *cpc =
12994	kmalloc_node(sizeof(struct perf_cpu_pmu_context),
12995	GFP_KERNEL \| __GFP_ZERO,
12996	cpu_to_node(cpu));
12997
12998	if (!cpc)
12999	return -ENOMEM;
13000
13001	*per_cpu_ptr(pmu->cpu_pmu_context, cpu) = cpc;
13002	__perf_init_event_pmu_context(&cpc->epc, pmu);
13003	__perf_mux_hrtimer_init(cpc, cpu);
13004	}
13005
13006	if (!pmu->start_txn) {
13007	if (pmu->pmu_enable) {
13008	/*
13009	* If we have pmu_enable/pmu_disable calls, install
13010	* transaction stubs that use that to try and batch
13011	* hardware accesses.
13012	*/
13013	pmu->start_txn = perf_pmu_start_txn;
13014	pmu->commit_txn = perf_pmu_commit_txn;
13015	pmu->cancel_txn = perf_pmu_cancel_txn;
13016	} else {
13017	pmu->start_txn = perf_pmu_nop_txn;
13018	pmu->commit_txn = perf_pmu_nop_int;
13019	pmu->cancel_txn = perf_pmu_nop_void;
13020	}
13021	}
13022
13023	if (!pmu->pmu_enable) {
13024	pmu->pmu_enable = perf_pmu_nop_void;
13025	pmu->pmu_disable = perf_pmu_nop_void;
13026	}
13027
13028	if (!pmu->check_period)
13029	pmu->check_period = perf_event_nop_int;
13030
13031	if (!pmu->event_idx)
13032	pmu->event_idx = perf_event_idx_default;
13033
13034	INIT_LIST_HEAD(&pmu->events);
13035	spin_lock_init(&pmu->events_lock);
13036
13037	/*
13038	* Now that the PMU is complete, make it visible to perf_try_init_event().
13039	*/
13040	if (!idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu))
13041	return -EINVAL;
13042	list_add_rcu(&pmu->entry, &pmus);
13043
13044	take_idr_id(pmu_type);
13045	_pmu = no_free_ptr(pmu); // let it rip
13046	return 0;
13047	}
13048	EXPORT_SYMBOL_GPL(perf_pmu_register);
13049
13050	static void __pmu_detach_event(struct pmu pmu, struct perf_event event,
13051	struct perf_event_context *ctx)
13052	{
13053	/*
13054	* De-schedule the event and mark it REVOKED.
13055	*/
13056	perf_event_exit_event(event, ctx, ctx->task, DETACH_REVOKE);
13057
13058	/*
13059	* All _free_event() bits that rely on event->pmu:
13060	*
13061	* Notably, perf_mmap() relies on the ordering here.
13062	*/
13063	scoped_guard (mutex, &event->mmap_mutex) {
13064	WARN_ON_ONCE(pmu->event_unmapped);
13065	/*
13066	* Mostly an empty lock sequence, such that perf_mmap(), which
13067	* relies on mmap_mutex, is sure to observe the state change.
13068	*/
13069	}
13070
13071	perf_event_free_bpf_prog(event);
13072	perf_free_addr_filters(event);
13073
13074	if (event->destroy) {
13075	event->destroy(event);
13076	event->destroy = NULL;
13077	}
13078
13079	if (event->pmu_ctx) {
13080	put_pmu_ctx(event->pmu_ctx);
13081	event->pmu_ctx = NULL;
13082	}
13083
13084	exclusive_event_destroy(event);
13085	module_put(pmu->module);
13086
13087	mediated_pmu_unaccount_event(event);
13088	event->pmu = NULL; /* force fault instead of UAF */
13089	}
13090
13091	static void pmu_detach_event(struct pmu pmu, struct perf_event event)
13092	{
13093	struct perf_event_context *ctx;
13094
13095	ctx = perf_event_ctx_lock(event);
13096	__pmu_detach_event(pmu, event, ctx);
13097	perf_event_ctx_unlock(event, ctx);
13098
13099	scoped_guard (spinlock, &pmu->events_lock)
13100	list_del(&event->pmu_list);
13101	}
13102
13103	static struct perf_event pmu_get_event(struct pmu pmu)
13104	{
13105	struct perf_event *event;
13106
13107	guard(spinlock)(&pmu->events_lock);
13108	list_for_each_entry(event, &pmu->events, pmu_list) {
13109	if (atomic_long_inc_not_zero(&event->refcount))
13110	return event;
13111	}
13112
13113	return NULL;
13114	}
13115
13116	static bool pmu_empty(struct pmu *pmu)
13117	{
13118	guard(spinlock)(&pmu->events_lock);
13119	return list_empty(&pmu->events);
13120	}
13121
13122	static void pmu_detach_events(struct pmu *pmu)
13123	{
13124	struct perf_event *event;
13125
13126	for (;;) {
13127	event = pmu_get_event(pmu);
13128	if (!event)
13129	break;
13130
13131	pmu_detach_event(pmu, event);
13132	put_event(event);
13133	}
13134
13135	/*
13136	* wait for pending _free_event()s
13137	*/
13138	wait_var_event(pmu, pmu_empty(pmu));
13139	}
13140
13141	int perf_pmu_unregister(struct pmu *pmu)
13142	{
13143	scoped_guard (mutex, &pmus_lock) {
13144	if (!idr_cmpxchg(&pmu_idr, pmu->type, pmu, NULL))
13145	return -EINVAL;
13146
13147	list_del_rcu(&pmu->entry);
13148	}
13149
13150	/*
13151	* We dereference the pmu list under both SRCU and regular RCU, so
13152	* synchronize against both of those.
13153	*
13154	* Notably, the entirety of event creation, from perf_init_event()
13155	* (which will now fail, because of the above) until
13156	* perf_install_in_context() should be under SRCU such that
13157	* this synchronizes against event creation. This avoids trying to
13158	* detach events that are not fully formed.
13159	*/
13160	synchronize_srcu(&pmus_srcu);
13161	synchronize_rcu();
13162
13163	if (pmu->event_unmapped && !pmu_empty(pmu)) {
13164	/*
13165	* Can't force remove events when pmu::event_unmapped()
13166	* is used in perf_mmap_close().
13167	*/
13168	guard(mutex)(&pmus_lock);
13169	idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu);
13170	list_add_rcu(&pmu->entry, &pmus);
13171	return -EBUSY;
13172	}
13173
13174	scoped_guard (mutex, &pmus_lock)
13175	idr_remove(&pmu_idr, pmu->type);
13176
13177	/*
13178	* PMU is removed from the pmus list, so no new events will
13179	* be created, now take care of the existing ones.
13180	*/
13181	pmu_detach_events(pmu);
13182
13183	/*
13184	* PMU is unused, make it go away.
13185	*/
13186	perf_pmu_free(pmu);
13187	return 0;
13188	}
13189	EXPORT_SYMBOL_GPL(perf_pmu_unregister);
13190
13191	static int perf_try_init_event(struct pmu pmu, struct perf_event event)
13192	{
13193	struct perf_event_context *ctx = NULL;
13194	int ret;
13195
13196	if (!try_module_get(pmu->module))
13197	return -ENODEV;
13198
13199	/*
13200	* A number of pmu->event_init() methods iterate the sibling_list to,
13201	* for example, validate if the group fits on the PMU. Therefore,
13202	* if this is a sibling event, acquire the ctx->mutex to protect
13203	* the sibling_list.
13204	*/
13205	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {
13206	/*
13207	* This ctx->mutex can nest when we're called through
13208	* inheritance. See the perf_event_ctx_lock_nested() comment.
13209	*/
13210	ctx = perf_event_ctx_lock_nested(event->group_leader,
13211	SINGLE_DEPTH_NESTING);
13212	BUG_ON(!ctx);
13213	}
13214
13215	event->pmu = pmu;
13216	ret = pmu->event_init(event);
13217
13218	if (ctx)
13219	perf_event_ctx_unlock(event->group_leader, ctx);
13220
13221	if (ret)
13222	goto err_pmu;
13223
13224	if (!(pmu->capabilities & PERF_PMU_CAP_SIMD_REGS) &&
13225	event_has_simd_regs(event)) {
13226	ret = -EOPNOTSUPP;
13227	goto err_destroy;
13228	}
13229
13230	if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&
13231	event_has_extended_regs(event)) {
13232	ret = -EOPNOTSUPP;
13233	goto err_destroy;
13234	}
13235
13236	if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&
13237	event_has_any_exclude_flag(event)) {
13238	ret = -EINVAL;
13239	goto err_destroy;
13240	}
13241
13242	if (pmu->scope != PERF_PMU_SCOPE_NONE && event->cpu >= 0) {
13243	const struct cpumask *cpumask;
13244	struct cpumask *pmu_cpumask;
13245	int cpu;
13246
13247	cpumask = perf_scope_cpu_topology_cpumask(pmu->scope, event->cpu);
13248	pmu_cpumask = perf_scope_cpumask(pmu->scope);
13249
13250	ret = -ENODEV;
13251	if (!pmu_cpumask \|\| !cpumask)
13252	goto err_destroy;
13253
13254	cpu = cpumask_any_and(pmu_cpumask, cpumask);
13255	if (cpu >= nr_cpu_ids)
13256	goto err_destroy;
13257
13258	event->event_caps \|= PERF_EV_CAP_READ_SCOPE;
13259	}
13260
13261	return 0;
13262
13263	err_destroy:
13264	if (event->destroy) {
13265	event->destroy(event);
13266	event->destroy = NULL;
13267	}
13268
13269	err_pmu:
13270	event->pmu = NULL;
13271	module_put(pmu->module);
13272	return ret;
13273	}
13274
13275	static struct pmu perf_init_event(struct perf_event event)
13276	{
13277	bool extended_type = false;
13278	struct pmu *pmu;
13279	int type, ret;
13280
13281	guard(srcu)(&pmus_srcu); /* pmu idr/list access */
13282
13283	/*
13284	* Save original type before calling pmu->event_init() since certain
13285	* pmus overwrites event->attr.type to forward event to another pmu.
13286	*/
13287	event->orig_type = event->attr.type;
13288
13289	/* Try parent's PMU first: */
13290	if (event->parent && event->parent->pmu) {
13291	pmu = event->parent->pmu;
13292	ret = perf_try_init_event(pmu, event);
13293	if (!ret)
13294	return pmu;
13295	}
13296
13297	/*
13298	* PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
13299	* are often aliases for PERF_TYPE_RAW.
13300	*/
13301	type = event->attr.type;
13302	if (type == PERF_TYPE_HARDWARE \|\| type == PERF_TYPE_HW_CACHE) {
13303	type = event->attr.config >> PERF_PMU_TYPE_SHIFT;
13304	if (!type) {
13305	type = PERF_TYPE_RAW;
13306	} else {
13307	extended_type = true;
13308	event->attr.config &= PERF_HW_EVENT_MASK;
13309	}
13310	}
13311
13312	again:
13313	scoped_guard (rcu)
13314	pmu = idr_find(&pmu_idr, type);
13315	if (pmu) {
13316	if (event->attr.type != type && type != PERF_TYPE_RAW &&
13317	!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))
13318	return ERR_PTR(-ENOENT);
13319
13320	ret = perf_try_init_event(pmu, event);
13321	if (ret == -ENOENT && event->attr.type != type && !extended_type) {
13322	type = event->attr.type;
13323	goto again;
13324	}
13325
13326	if (ret)
13327	return ERR_PTR(ret);
13328
13329	return pmu;
13330	}
13331
13332	list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) {
13333	ret = perf_try_init_event(pmu, event);
13334	if (!ret)
13335	return pmu;
13336
13337	if (ret != -ENOENT)
13338	return ERR_PTR(ret);
13339	}
13340
13341	return ERR_PTR(-ENOENT);
13342	}
13343
13344	static void attach_sb_event(struct perf_event *event)
13345	{
13346	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
13347
13348	raw_spin_lock(&pel->lock);
13349	list_add_rcu(&event->sb_list, &pel->list);
13350	raw_spin_unlock(&pel->lock);
13351	}
13352
13353	/*
13354	* We keep a list of all !task (and therefore per-cpu) events
13355	* that need to receive side-band records.
13356	*
13357	* This avoids having to scan all the various PMU per-cpu contexts
13358	* looking for them.
13359	*/
13360	static void account_pmu_sb_event(struct perf_event *event)
13361	{
13362	if (is_sb_event(event))
13363	attach_sb_event(event);
13364	}
13365
13366	/* Freq events need the tick to stay alive (see perf_event_task_tick). */
13367	static void account_freq_event_nohz(void)
13368	{
13369	#ifdef CONFIG_NO_HZ_FULL
13370	/* Lock so we don't race with concurrent unaccount */
13371	spin_lock(&nr_freq_lock);
13372	if (atomic_inc_return(&nr_freq_events) == 1)
13373	tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS);
13374	spin_unlock(&nr_freq_lock);
13375	#endif
13376	}
13377
13378	static void account_freq_event(void)
13379	{
13380	if (tick_nohz_full_enabled())
13381	account_freq_event_nohz();
13382	else
13383	atomic_inc(&nr_freq_events);
13384	}
13385
13386
13387	static void account_event(struct perf_event *event)
13388	{
13389	bool inc = false;
13390
13391	if (event->parent)
13392	return;
13393
13394	if (event->attach_state & (PERF_ATTACH_TASK \| PERF_ATTACH_SCHED_CB))
13395	inc = true;
13396	if (event->attr.mmap \|\| event->attr.mmap_data)
13397	atomic_inc(&nr_mmap_events);
13398	if (event->attr.build_id)
13399	atomic_inc(&nr_build_id_events);
13400	if (event->attr.comm)
13401	atomic_inc(&nr_comm_events);
13402	if (event->attr.namespaces)
13403	atomic_inc(&nr_namespaces_events);
13404	if (event->attr.cgroup)
13405	atomic_inc(&nr_cgroup_events);
13406	if (event->attr.task)
13407	atomic_inc(&nr_task_events);
13408	if (event->attr.freq)
13409	account_freq_event();
13410	if (event->attr.context_switch) {
13411	atomic_inc(&nr_switch_events);
13412	inc = true;
13413	}
13414	if (has_branch_stack(event))
13415	inc = true;
13416	if (is_cgroup_event(event))
13417	inc = true;
13418	if (event->attr.ksymbol)
13419	atomic_inc(&nr_ksymbol_events);
13420	if (event->attr.bpf_event)
13421	atomic_inc(&nr_bpf_events);
13422	if (event->attr.text_poke)
13423	atomic_inc(&nr_text_poke_events);
13424
13425	if (inc) {
13426	/*
13427	* We need the mutex here because static_branch_enable()
13428	* must complete before the perf_sched_count increment
13429	* becomes visible.
13430	*/
13431	if (atomic_inc_not_zero(&perf_sched_count))
13432	goto enabled;
13433
13434	mutex_lock(&perf_sched_mutex);
13435	if (!atomic_read(&perf_sched_count)) {
13436	static_branch_enable(&perf_sched_events);
13437	/*
13438	* Guarantee that all CPUs observe they key change and
13439	* call the perf scheduling hooks before proceeding to
13440	* install events that need them.
13441	*/
13442	synchronize_rcu();
13443	}
13444	/*
13445	* Now that we have waited for the sync_sched(), allow further
13446	* increments to by-pass the mutex.
13447	*/
13448	atomic_inc(&perf_sched_count);
13449	mutex_unlock(&perf_sched_mutex);
13450	}
13451	enabled:
13452
13453	account_pmu_sb_event(event);
13454	}
13455
13456	/*
13457	* Allocate and initialize an event structure
13458	*/
13459	static struct perf_event *
13460	perf_event_alloc(struct perf_event_attr *attr, int cpu,
13461	struct task_struct *task,
13462	struct perf_event *group_leader,
13463	struct perf_event *parent_event,
13464	perf_overflow_handler_t overflow_handler,
13465	void *context, int cgroup_fd)
13466	{
13467	struct pmu *pmu;
13468	struct hw_perf_event *hwc;
13469	long err = -EINVAL;
13470	int node;
13471
13472	if ((unsigned)cpu >= nr_cpu_ids) {
13473	if (!task \|\| cpu != -1)
13474	return ERR_PTR(-EINVAL);
13475	}
13476	if (attr->sigtrap && !task) {
13477	/* Requires a task: avoid signalling random tasks. */
13478	return ERR_PTR(-EINVAL);
13479	}
13480
13481	node = (cpu >= 0) ? cpu_to_node(cpu) : -1;
13482	struct perf_event *event __free(__free_event) =
13483	kmem_cache_alloc_node(perf_event_cache, GFP_KERNEL \| __GFP_ZERO, node);
13484	if (!event)
13485	return ERR_PTR(-ENOMEM);
13486
13487	/*
13488	* Single events are their own group leaders, with an
13489	* empty sibling list:
13490	*/
13491	if (!group_leader)
13492	group_leader = event;
13493
13494	mutex_init(&event->child_mutex);
13495	INIT_LIST_HEAD(&event->child_list);
13496
13497	INIT_LIST_HEAD(&event->event_entry);
13498	INIT_LIST_HEAD(&event->sibling_list);
13499	INIT_LIST_HEAD(&event->active_list);
13500	init_event_group(event);
13501	INIT_LIST_HEAD(&event->rb_entry);
13502	INIT_LIST_HEAD(&event->active_entry);
13503	INIT_LIST_HEAD(&event->addr_filters.list);
13504	INIT_HLIST_NODE(&event->hlist_entry);
13505	INIT_LIST_HEAD(&event->pmu_list);
13506
13507
13508	init_waitqueue_head(&event->waitq);
13509	init_irq_work(&event->pending_irq, perf_pending_irq);
13510	event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable);
13511	init_task_work(&event->pending_task, perf_pending_task);
13512
13513	mutex_init(&event->mmap_mutex);
13514	raw_spin_lock_init(&event->addr_filters.lock);
13515
13516	atomic_long_set(&event->refcount, 1);
13517	event->cpu = cpu;
13518	event->attr = *attr;
13519	event->group_leader = group_leader;
13520	event->pmu = NULL;
13521	event->oncpu = -1;
13522
13523	event->parent = parent_event;
13524
13525	event->ns = get_pid_ns(task_active_pid_ns(current));
13526	event->id = atomic64_inc_return(&perf_event_id);
13527
13528	event->state = PERF_EVENT_STATE_INACTIVE;
13529
13530	if (parent_event)
13531	event->event_caps = parent_event->event_caps;
13532
13533	if (task) {
13534	event->attach_state = PERF_ATTACH_TASK;
13535	/*
13536	* XXX pmu::event_init needs to know what task to account to
13537	* and we cannot use the ctx information because we need the
13538	* pmu before we get a ctx.
13539	*/
13540	event->hw.target = get_task_struct(task);
13541	}
13542
13543	event->clock = &local_clock;
13544	if (parent_event)
13545	event->clock = parent_event->clock;
13546
13547	if (!overflow_handler && parent_event) {
13548	overflow_handler = parent_event->overflow_handler;
13549	context = parent_event->overflow_handler_context;
13550	#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
13551	if (parent_event->prog) {
13552	struct bpf_prog *prog = parent_event->prog;
13553
13554	bpf_prog_inc(prog);
13555	event->prog = prog;
13556	}
13557	#endif
13558	}
13559
13560	if (overflow_handler) {
13561	event->overflow_handler = overflow_handler;
13562	event->overflow_handler_context = context;
13563	} else if (is_write_backward(event)){
13564	event->overflow_handler = perf_event_output_backward;
13565	event->overflow_handler_context = NULL;
13566	} else {
13567	event->overflow_handler = perf_event_output_forward;
13568	event->overflow_handler_context = NULL;
13569	}
13570
13571	perf_event__state_init(event);
13572
13573	pmu = NULL;
13574
13575	hwc = &event->hw;
13576	hwc->sample_period = attr->sample_period;
13577	if (is_event_in_freq_mode(event))
13578	hwc->sample_period = 1;
13579	hwc->last_period = hwc->sample_period;
13580
13581	local64_set(&hwc->period_left, hwc->sample_period);
13582
13583	/*
13584	* We do not support PERF_SAMPLE_READ on inherited events unless
13585	* PERF_SAMPLE_TID is also selected, which allows inherited events to
13586	* collect per-thread samples.
13587	* See perf_output_read().
13588	*/
13589	if (has_inherit_and_sample_read(attr) && !(attr->sample_type & PERF_SAMPLE_TID))
13590	return ERR_PTR(-EINVAL);
13591
13592	if (!has_branch_stack(event))
13593	event->attr.branch_sample_type = 0;
13594
13595	pmu = perf_init_event(event);
13596	if (IS_ERR(pmu))
13597	return (void*)pmu;
13598
13599	/*
13600	* The PERF_ATTACH_TASK_DATA is set in the event_init()->hw_config().
13601	* The attach should be right after the perf_init_event().
13602	* Otherwise, the __free_event() would mistakenly detach the non-exist
13603	* perf_ctx_data because of the other errors between them.
13604	*/
13605	if (event->attach_state & PERF_ATTACH_TASK_DATA) {
13606	err = attach_perf_ctx_data(event);
13607	if (err)
13608	return ERR_PTR(err);
13609	}
13610
13611	/*
13612	* Disallow uncore-task events. Similarly, disallow uncore-cgroup
13613	* events (they don't make sense as the cgroup will be different
13614	* on other CPUs in the uncore mask).
13615	*/
13616	if (pmu->task_ctx_nr == perf_invalid_context && (task \|\| cgroup_fd != -1))
13617	return ERR_PTR(-EINVAL);
13618
13619	if (event->attr.aux_output &&
13620	(!(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT) \|\|
13621	event->attr.aux_pause \|\| event->attr.aux_resume))
13622	return ERR_PTR(-EOPNOTSUPP);
13623
13624	if (event->attr.aux_pause && event->attr.aux_resume)
13625	return ERR_PTR(-EINVAL);
13626
13627	if (event->attr.aux_start_paused) {
13628	if (!(pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE))
13629	return ERR_PTR(-EOPNOTSUPP);
13630	event->hw.aux_paused = 1;
13631	}
13632
13633	if (cgroup_fd != -1) {
13634	err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);
13635	if (err)
13636	return ERR_PTR(err);
13637	}
13638
13639	err = exclusive_event_init(event);
13640	if (err)
13641	return ERR_PTR(err);
13642
13643	if (has_addr_filter(event)) {
13644	event->addr_filter_ranges = kzalloc_objs(struct perf_addr_filter_range,
13645	pmu->nr_addr_filters);
13646	if (!event->addr_filter_ranges)
13647	return ERR_PTR(-ENOMEM);
13648
13649	/*
13650	* Clone the parent's vma offsets: they are valid until exec()
13651	* even if the mm is not shared with the parent.
13652	*/
13653	if (event->parent) {
13654	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
13655
13656	raw_spin_lock_irq(&ifh->lock);
13657	memcpy(event->addr_filter_ranges,
13658	event->parent->addr_filter_ranges,
13659	pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));
13660	raw_spin_unlock_irq(&ifh->lock);
13661	}
13662
13663	/* force hw sync on the address filters */
13664	event->addr_filters_gen = 1;
13665	}
13666
13667	if (!event->parent) {
13668	if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
13669	err = get_callchain_buffers(attr->sample_max_stack);
13670	if (err)
13671	return ERR_PTR(err);
13672	event->attach_state \|= PERF_ATTACH_CALLCHAIN;
13673	}
13674	}
13675
13676	err = security_perf_event_alloc(event);
13677	if (err)
13678	return ERR_PTR(err);
13679
13680	err = mediated_pmu_account_event(event);
13681	if (err)
13682	return ERR_PTR(err);
13683
13684	/* symmetric to unaccount_event() in _free_event() */
13685	account_event(event);
13686
13687	/*
13688	* Event creation should be under SRCU, see perf_pmu_unregister().
13689	*/
13690	lockdep_assert_held(&pmus_srcu);
13691	scoped_guard (spinlock, &pmu->events_lock)
13692	list_add(&event->pmu_list, &pmu->events);
13693
13694	return_ptr(event);
13695	}
13696
13697	static int perf_copy_attr(struct perf_event_attr __user *uattr,
13698	struct perf_event_attr *attr)
13699	{
13700	u32 size;
13701	int ret;
13702
13703	/* Zero the full structure, so that a short copy will be nice. */
13704	memset(attr, 0, sizeof(*attr));
13705
13706	ret = get_user(size, &uattr->size);
13707	if (ret)
13708	return ret;
13709
13710	/* ABI compatibility quirk: */
13711	if (!size)
13712	size = PERF_ATTR_SIZE_VER0;
13713	if (size < PERF_ATTR_SIZE_VER0 \|\| size > PAGE_SIZE)
13714	goto err_size;
13715
13716	ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size);
13717	if (ret) {
13718	if (ret == -E2BIG)
13719	goto err_size;
13720	return ret;
13721	}
13722
13723	attr->size = size;
13724
13725	if (attr->__reserved_1 \|\| attr->__reserved_2 \|\|
13726	attr->__reserved_3 \|\| attr->__reserved_4)
13727	return -EINVAL;
13728
13729	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
13730	return -EINVAL;
13731
13732	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
13733	return -EINVAL;
13734
13735	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
13736	u64 mask = attr->branch_sample_type;
13737
13738	/* only using defined bits */
13739	if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
13740	return -EINVAL;
13741
13742	/* at least one branch bit must be set */
13743	if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
13744	return -EINVAL;
13745
13746	/* propagate priv level, when not set for branch */
13747	if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
13748
13749	/* exclude_kernel checked on syscall entry */
13750	if (!attr->exclude_kernel)
13751	mask \|= PERF_SAMPLE_BRANCH_KERNEL;
13752
13753	if (!attr->exclude_user)
13754	mask \|= PERF_SAMPLE_BRANCH_USER;
13755
13756	if (!attr->exclude_hv)
13757	mask \|= PERF_SAMPLE_BRANCH_HV;
13758	/*
13759	* adjust user setting (for HW filter setup)
13760	*/
13761	attr->branch_sample_type = mask;
13762	}
13763	/* privileged levels capture (kernel, hv): check permissions */
13764	if (mask & PERF_SAMPLE_BRANCH_PERM_PLM) {
13765	ret = perf_allow_kernel();
13766	if (ret)
13767	return ret;
13768	}
13769	}
13770
13771	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
13772	ret = perf_reg_validate(attr->sample_regs_user,
13773	attr->sample_simd_regs_enabled);
13774	if (ret)
13775	return ret;
13776	if (attr->sample_simd_regs_enabled) {
13777	ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords,
13778	attr->sample_simd_vec_reg_user,
13779	attr->sample_simd_pred_reg_qwords,
13780	attr->sample_simd_pred_reg_user);
13781	if (ret)
13782	return ret;
13783	}
13784	}
13785
13786	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
13787	if (!arch_perf_have_user_stack_dump())
13788	return -ENOSYS;
13789
13790	/*
13791	* We have __u32 type for the size, but so far
13792	* we can only use __u16 as maximum due to the
13793	* __u16 sample size limit.
13794	*/
13795	if (attr->sample_stack_user >= USHRT_MAX)
13796	return -EINVAL;
13797	else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
13798	return -EINVAL;
13799	}
13800
13801	if (!attr->sample_max_stack)
13802	attr->sample_max_stack = sysctl_perf_event_max_stack;
13803
13804	if (attr->sample_type & PERF_SAMPLE_REGS_INTR) {
13805	ret = perf_reg_validate(attr->sample_regs_intr,
13806	attr->sample_simd_regs_enabled);
13807	if (ret)
13808	return ret;
13809	if (attr->sample_simd_regs_enabled) {
13810	ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords,
13811	attr->sample_simd_vec_reg_intr,
13812	attr->sample_simd_pred_reg_qwords,
13813	attr->sample_simd_pred_reg_intr);
13814	if (ret)
13815	return ret;
13816	}
13817	}
13818
13819	#ifndef CONFIG_CGROUP_PERF
13820	if (attr->sample_type & PERF_SAMPLE_CGROUP)
13821	return -EINVAL;
13822	#endif
13823	if ((attr->sample_type & PERF_SAMPLE_WEIGHT) &&
13824	(attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT))
13825	return -EINVAL;
13826
13827	if (!attr->inherit && attr->inherit_thread)
13828	return -EINVAL;
13829
13830	if (attr->remove_on_exec && attr->enable_on_exec)
13831	return -EINVAL;
13832
13833	if (attr->sigtrap && !attr->remove_on_exec)
13834	return -EINVAL;
13835
13836	out:
13837	return ret;
13838
13839	err_size:
13840	put_user(sizeof(*attr), &uattr->size);
13841	ret = -E2BIG;
13842	goto out;
13843	}
13844
13845	static void mutex_lock_double(struct mutex a, struct mutex b)
13846	{
13847	if (b < a)
13848	swap(a, b);
13849
13850	mutex_lock(a);
13851	mutex_lock_nested(b, SINGLE_DEPTH_NESTING);
13852	}
13853
13854	static int
13855	perf_event_set_output(struct perf_event event, struct perf_event output_event)
13856	{
13857	struct perf_buffer *rb = NULL;
13858	int ret = -EINVAL;
13859
13860	if (!output_event) {
13861	mutex_lock(&event->mmap_mutex);
13862	goto set;
13863	}
13864
13865	/* don't allow circular references */
13866	if (event == output_event)
13867	goto out;
13868
13869	/*
13870	* Don't allow cross-cpu buffers
13871	*/
13872	if (output_event->cpu != event->cpu)
13873	goto out;
13874
13875	/*
13876	* If its not a per-cpu rb, it must be the same task.
13877	*/
13878	if (output_event->cpu == -1 && output_event->hw.target != event->hw.target)
13879	goto out;
13880
13881	/*
13882	* Mixing clocks in the same buffer is trouble you don't need.
13883	*/
13884	if (output_event->clock != event->clock)
13885	goto out;
13886
13887	/*
13888	* Either writing ring buffer from beginning or from end.
13889	* Mixing is not allowed.
13890	*/
13891	if (is_write_backward(output_event) != is_write_backward(event))
13892	goto out;
13893
13894	/*
13895	* If both events generate aux data, they must be on the same PMU
13896	*/
13897	if (has_aux(event) && has_aux(output_event) &&
13898	event->pmu != output_event->pmu)
13899	goto out;
13900
13901	/*
13902	* Hold both mmap_mutex to serialize against perf_mmap_close(). Since
13903	* output_event is already on rb->event_list, and the list iteration
13904	* restarts after every removal, it is guaranteed this new event is
13905	* observed OR if output_event is already removed, it's guaranteed we
13906	* observe !rb->mmap_count.
13907	*/
13908	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);
13909	set:
13910	/* Can't redirect output if we've got an active mmap() */
13911	if (refcount_read(&event->mmap_count))
13912	goto unlock;
13913
13914	if (output_event) {
13915	if (output_event->state <= PERF_EVENT_STATE_REVOKED)
13916	goto unlock;
13917
13918	/* get the rb we want to redirect to */
13919	rb = ring_buffer_get(output_event);
13920	if (!rb)
13921	goto unlock;
13922
13923	/* did we race against perf_mmap_close() */
13924	if (!refcount_read(&rb->mmap_count)) {
13925	ring_buffer_put(rb);
13926	goto unlock;
13927	}
13928	}
13929
13930	ring_buffer_attach(event, rb);
13931
13932	ret = 0;
13933	unlock:
13934	mutex_unlock(&event->mmap_mutex);
13935	if (output_event)
13936	mutex_unlock(&output_event->mmap_mutex);
13937
13938	out:
13939	return ret;
13940	}
13941
13942	static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)
13943	{
13944	bool nmi_safe = false;
13945
13946	switch (clk_id) {
13947	case CLOCK_MONOTONIC:
13948	event->clock = &ktime_get_mono_fast_ns;
13949	nmi_safe = true;
13950	break;
13951
13952	case CLOCK_MONOTONIC_RAW:
13953	event->clock = &ktime_get_raw_fast_ns;
13954	nmi_safe = true;
13955	break;
13956
13957	case CLOCK_REALTIME:
13958	event->clock = &ktime_get_real_ns;
13959	break;
13960
13961	case CLOCK_BOOTTIME:
13962	event->clock = &ktime_get_boottime_ns;
13963	break;
13964
13965	case CLOCK_TAI:
13966	event->clock = &ktime_get_clocktai_ns;
13967	break;
13968
13969	default:
13970	return -EINVAL;
13971	}
13972
13973	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))
13974	return -EINVAL;
13975
13976	return 0;
13977	}
13978
13979	static bool
13980	perf_check_permission(struct perf_event_attr attr, struct task_struct task)
13981	{
13982	unsigned int ptrace_mode = PTRACE_MODE_READ_REALCREDS;
13983	bool is_capable = perfmon_capable();
13984
13985	if (attr->sigtrap) {
13986	/*
13987	* perf_event_attr::sigtrap sends signals to the other task.
13988	* Require the current task to also have CAP_KILL.
13989	*/
13990	rcu_read_lock();
13991	is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL);
13992	rcu_read_unlock();
13993
13994	/*
13995	* If the required capabilities aren't available, checks for
13996	* ptrace permissions: upgrade to ATTACH, since sending signals
13997	* can effectively change the target task.
13998	*/
13999	ptrace_mode = PTRACE_MODE_ATTACH_REALCREDS;
14000	}
14001
14002	/*
14003	* Preserve ptrace permission check for backwards compatibility. The
14004	* ptrace check also includes checks that the current task and other
14005	* task have matching uids, and is therefore not done here explicitly.
14006	*/
14007	return is_capable \|\| ptrace_may_access(task, ptrace_mode);
14008	}
14009
14010	/**
14011	* sys_perf_event_open - open a performance event, associate it to a task/cpu
14012	*
14013	* @attr_uptr: event_id type attributes for monitoring/sampling
14014	* @pid: target pid
14015	* @cpu: target cpu
14016	* @group_fd: group leader event fd
14017	* @flags: perf event open flags
14018	*/
14019	SYSCALL_DEFINE5(perf_event_open,
14020	struct perf_event_attr __user *, attr_uptr,
14021	pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
14022	{
14023	struct perf_event group_leader = NULL, output_event = NULL;
14024	struct perf_event_pmu_context *pmu_ctx;
14025	struct perf_event event, sibling;
14026	struct perf_event_attr attr;
14027	struct perf_event_context *ctx;
14028	struct file *event_file = NULL;
14029	struct task_struct *task = NULL;
14030	struct pmu *pmu;
14031	int event_fd;
14032	int move_group = 0;
14033	int err;
14034	int f_flags = O_RDWR;
14035	int cgroup_fd = -1;
14036
14037	/* for future expandability... */
14038	if (flags & ~PERF_FLAG_ALL)
14039	return -EINVAL;
14040
14041	err = perf_copy_attr(attr_uptr, &attr);
14042	if (err)
14043	return err;
14044
14045	/* Do we allow access to perf_event_open(2) ? */
14046	err = security_perf_event_open(PERF_SECURITY_OPEN);
14047	if (err)
14048	return err;
14049
14050	if (!attr.exclude_kernel \|\| attr.text_poke \|\|
14051	((attr.sample_type & PERF_SAMPLE_CALLCHAIN) &&
14052	!attr.exclude_callchain_kernel)) {
14053	err = perf_allow_kernel();
14054	if (err)
14055	return err;
14056	}
14057
14058	if (attr.namespaces) {
14059	if (!perfmon_capable())
14060	return -EACCES;
14061	}
14062
14063	if (attr.freq) {
14064	if (attr.sample_freq > sysctl_perf_event_sample_rate)
14065	return -EINVAL;
14066	} else {
14067	if (attr.sample_period & (1ULL << 63))
14068	return -EINVAL;
14069	}
14070
14071	/* Only privileged users can get physical addresses */
14072	if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR)) {
14073	err = perf_allow_kernel();
14074	if (err)
14075	return err;
14076	}
14077
14078	/* REGS_INTR can leak data, lockdown must prevent this */
14079	if (attr.sample_type & PERF_SAMPLE_REGS_INTR) {
14080	err = security_locked_down(LOCKDOWN_PERF);
14081	if (err)
14082	return err;
14083	}
14084
14085	/*
14086	* In cgroup mode, the pid argument is used to pass the fd
14087	* opened to the cgroup directory in cgroupfs. The cpu argument
14088	* designates the cpu on which to monitor threads from that
14089	* cgroup.
14090	*/
14091	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 \|\| cpu == -1))
14092	return -EINVAL;
14093
14094	if (flags & PERF_FLAG_FD_CLOEXEC)
14095	f_flags \|= O_CLOEXEC;
14096
14097	event_fd = get_unused_fd_flags(f_flags);
14098	if (event_fd < 0)
14099	return event_fd;
14100
14101	/*
14102	* Event creation should be under SRCU, see perf_pmu_unregister().
14103	*/
14104	guard(srcu)(&pmus_srcu);
14105
14106	CLASS(fd, group)(group_fd); // group_fd == -1 => empty
14107	if (group_fd != -1) {
14108	if (!is_perf_file(group)) {
14109	err = -EBADF;
14110	goto err_fd;
14111	}
14112	group_leader = fd_file(group)->private_data;
14113	if (group_leader->state <= PERF_EVENT_STATE_EXIT) {
14114	err = -ENODEV;
14115	goto err_fd;
14116	}
14117	if (flags & PERF_FLAG_FD_OUTPUT)
14118	output_event = group_leader;
14119	if (flags & PERF_FLAG_FD_NO_GROUP)
14120	group_leader = NULL;
14121	}
14122
14123	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
14124	task = find_lively_task_by_vpid(pid);
14125	if (IS_ERR(task)) {
14126	err = PTR_ERR(task);
14127	goto err_fd;
14128	}
14129	}
14130
14131	if (task && group_leader &&
14132	group_leader->attr.inherit != attr.inherit) {
14133	err = -EINVAL;
14134	goto err_task;
14135	}
14136
14137	if (flags & PERF_FLAG_PID_CGROUP)
14138	cgroup_fd = pid;
14139
14140	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
14141	NULL, NULL, cgroup_fd);
14142	if (IS_ERR(event)) {
14143	err = PTR_ERR(event);
14144	goto err_task;
14145	}
14146
14147	if (is_sampling_event(event)) {
14148	if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
14149	err = -EOPNOTSUPP;
14150	goto err_alloc;
14151	}
14152	}
14153
14154	/*
14155	* Special case software events and allow them to be part of
14156	* any hardware group.
14157	*/
14158	pmu = event->pmu;
14159
14160	if (attr.use_clockid) {
14161	err = perf_event_set_clock(event, attr.clockid);
14162	if (err)
14163	goto err_alloc;
14164	}
14165
14166	if (pmu->task_ctx_nr == perf_sw_context)
14167	event->event_caps \|= PERF_EV_CAP_SOFTWARE;
14168
14169	if (task) {
14170	err = down_read_interruptible(&task->signal->exec_update_lock);
14171	if (err)
14172	goto err_alloc;
14173
14174	/*
14175	* We must hold exec_update_lock across this and any potential
14176	* perf_install_in_context() call for this new event to
14177	* serialize against exec() altering our credentials (and the
14178	* perf_event_exit_task() that could imply).
14179	*/
14180	err = -EACCES;
14181	if (!perf_check_permission(&attr, task))
14182	goto err_cred;
14183	}
14184
14185	/*
14186	* Get the target context (task or percpu):
14187	*/
14188	ctx = find_get_context(task, event);
14189	if (IS_ERR(ctx)) {
14190	err = PTR_ERR(ctx);
14191	goto err_cred;
14192	}
14193
14194	mutex_lock(&ctx->mutex);
14195
14196	if (ctx->task == TASK_TOMBSTONE) {
14197	err = -ESRCH;
14198	goto err_locked;
14199	}
14200
14201	if (!task) {
14202	/*
14203	* Check if the @cpu we're creating an event for is online.
14204	*
14205	* We use the perf_cpu_context::ctx::mutex to serialize against
14206	* the hotplug notifiers. See perf_event_{init,exit}_cpu().
14207	*/
14208	struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
14209
14210	if (!cpuctx->online) {
14211	err = -ENODEV;
14212	goto err_locked;
14213	}
14214	}
14215
14216	if (group_leader) {
14217	err = -EINVAL;
14218
14219	/*
14220	* Do not allow a recursive hierarchy (this new sibling
14221	* becoming part of another group-sibling):
14222	*/
14223	if (group_leader->group_leader != group_leader)
14224	goto err_locked;
14225
14226	/* All events in a group should have the same clock */
14227	if (group_leader->clock != event->clock)
14228	goto err_locked;
14229
14230	/*
14231	* Make sure we're both events for the same CPU;
14232	* grouping events for different CPUs is broken; since
14233	* you can never concurrently schedule them anyhow.
14234	*/
14235	if (group_leader->cpu != event->cpu)
14236	goto err_locked;
14237
14238	/*
14239	* Make sure we're both on the same context; either task or cpu.
14240	*/
14241	if (group_leader->ctx != ctx)
14242	goto err_locked;
14243
14244	/* Recheck under ctx::mutex to serialize against remove-on-exec. */
14245	if (group_leader->state <= PERF_EVENT_STATE_EXIT) {
14246	err = -ENODEV;
14247	goto err_locked;
14248	}
14249
14250	/*
14251	* Only a group leader can be exclusive or pinned
14252	*/
14253	if (attr.exclusive \|\| attr.pinned)
14254	goto err_locked;
14255
14256	if (is_software_event(event) &&
14257	!in_software_context(group_leader)) {
14258	/*
14259	* If the event is a sw event, but the group_leader
14260	* is on hw context.
14261	*
14262	* Allow the addition of software events to hw
14263	* groups, this is safe because software events
14264	* never fail to schedule.
14265	*
14266	* Note the comment that goes with struct
14267	* perf_event_pmu_context.
14268	*/
14269	pmu = group_leader->pmu_ctx->pmu;
14270	} else if (!is_software_event(event)) {
14271	if (is_software_event(group_leader) &&
14272	(group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {
14273	/*
14274	* In case the group is a pure software group, and we
14275	* try to add a hardware event, move the whole group to
14276	* the hardware context.
14277	*/
14278	move_group = 1;
14279	}
14280
14281	/* Don't allow group of multiple hw events from different pmus */
14282	if (!in_software_context(group_leader) &&
14283	group_leader->pmu_ctx->pmu != pmu)
14284	goto err_locked;
14285	}
14286	}
14287
14288	/*
14289	* Now that we're certain of the pmu; find the pmu_ctx.
14290	*/
14291	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
14292	if (IS_ERR(pmu_ctx)) {
14293	err = PTR_ERR(pmu_ctx);
14294	goto err_locked;
14295	}
14296	event->pmu_ctx = pmu_ctx;
14297
14298	if (output_event) {
14299	err = perf_event_set_output(event, output_event);
14300	if (err)
14301	goto err_context;
14302	}
14303
14304	if (!perf_event_validate_size(event)) {
14305	err = -E2BIG;
14306	goto err_context;
14307	}
14308
14309	if (perf_need_aux_event(event) && !perf_get_aux_event(event, group_leader)) {
14310	err = -EINVAL;
14311	goto err_context;
14312	}
14313
14314	/*
14315	* Must be under the same ctx::mutex as perf_install_in_context(),
14316	* because we need to serialize with concurrent event creation.
14317	*/
14318	if (!exclusive_event_installable(event, ctx)) {
14319	err = -EBUSY;
14320	goto err_context;
14321	}
14322
14323	WARN_ON_ONCE(ctx->parent_ctx);
14324
14325	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, f_flags);
14326	if (IS_ERR(event_file)) {
14327	err = PTR_ERR(event_file);
14328	event_file = NULL;
14329	goto err_context;
14330	}
14331
14332	/*
14333	* This is the point on no return; we cannot fail hereafter. This is
14334	* where we start modifying current state.
14335	*/
14336
14337	if (move_group) {
14338	perf_remove_from_context(group_leader, 0);
14339	put_pmu_ctx(group_leader->pmu_ctx);
14340
14341	for_each_sibling_event(sibling, group_leader) {
14342	perf_remove_from_context(sibling, 0);
14343	put_pmu_ctx(sibling->pmu_ctx);
14344	}
14345
14346	/*
14347	* Install the group siblings before the group leader.
14348	*
14349	* Because a group leader will try and install the entire group
14350	* (through the sibling list, which is still in-tact), we can
14351	* end up with siblings installed in the wrong context.
14352	*
14353	* By installing siblings first we NO-OP because they're not
14354	* reachable through the group lists.
14355	*/
14356	for_each_sibling_event(sibling, group_leader) {
14357	sibling->pmu_ctx = pmu_ctx;
14358	get_pmu_ctx(pmu_ctx);
14359	perf_event__state_init(sibling);
14360	perf_install_in_context(ctx, sibling, sibling->cpu);
14361	}
14362
14363	/*
14364	* Removing from the context ends up with disabled
14365	* event. What we want here is event in the initial
14366	* startup state, ready to be add into new context.
14367	*/
14368	group_leader->pmu_ctx = pmu_ctx;
14369	get_pmu_ctx(pmu_ctx);
14370	perf_event__state_init(group_leader);
14371	perf_install_in_context(ctx, group_leader, group_leader->cpu);
14372	}
14373
14374	/*
14375	* Precalculate sample_data sizes; do while holding ctx::mutex such
14376	* that we're serialized against further additions and before
14377	* perf_install_in_context() which is the point the event is active and
14378	* can use these values.
14379	*/
14380	perf_event__header_size(event);
14381	perf_event__id_header_size(event);
14382
14383	event->owner = current;
14384
14385	perf_install_in_context(ctx, event, event->cpu);
14386	perf_unpin_context(ctx);
14387
14388	mutex_unlock(&ctx->mutex);
14389
14390	if (task) {
14391	up_read(&task->signal->exec_update_lock);
14392	put_task_struct(task);
14393	}
14394
14395	mutex_lock(&current->perf_event_mutex);
14396	list_add_tail(&event->owner_entry, &current->perf_event_list);
14397	mutex_unlock(&current->perf_event_mutex);
14398
14399	/*
14400	* File reference in group guarantees that group_leader has been
14401	* kept alive until we place the new event on the sibling_list.
14402	* This ensures destruction of the group leader will find
14403	* the pointer to itself in perf_group_detach().
14404	*/
14405	fd_install(event_fd, event_file);
14406	return event_fd;
14407
14408	err_context:
14409	put_pmu_ctx(event->pmu_ctx);
14410	event->pmu_ctx = NULL; /* _free_event() */
14411	err_locked:
14412	mutex_unlock(&ctx->mutex);
14413	perf_unpin_context(ctx);
14414	put_ctx(ctx);
14415	err_cred:
14416	if (task)
14417	up_read(&task->signal->exec_update_lock);
14418	err_alloc:
14419	put_event(event);
14420	err_task:
14421	if (task)
14422	put_task_struct(task);
14423	err_fd:
14424	put_unused_fd(event_fd);
14425	return err;
14426	}
14427
14428	/**
14429	* perf_event_create_kernel_counter
14430	*
14431	* @attr: attributes of the counter to create
14432	* @cpu: cpu in which the counter is bound
14433	* @task: task to profile (NULL for percpu)
14434	* @overflow_handler: callback to trigger when we hit the event
14435	* @context: context data could be used in overflow_handler callback
14436	*/
14437	struct perf_event *
14438	perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
14439	struct task_struct *task,
14440	perf_overflow_handler_t overflow_handler,
14441	void *context)
14442	{
14443	struct perf_event_pmu_context *pmu_ctx;
14444	struct perf_event_context *ctx;
14445	struct perf_event *event;
14446	struct pmu *pmu;
14447	int err;
14448
14449	/*
14450	* Grouping is not supported for kernel events, neither is 'AUX',
14451	* make sure the caller's intentions are adjusted.
14452	*/
14453	if (attr->aux_output \|\| attr->aux_action)
14454	return ERR_PTR(-EINVAL);
14455
14456	/*
14457	* Event creation should be under SRCU, see perf_pmu_unregister().
14458	*/
14459	guard(srcu)(&pmus_srcu);
14460
14461	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
14462	overflow_handler, context, -1);
14463	if (IS_ERR(event)) {
14464	err = PTR_ERR(event);
14465	goto err;
14466	}
14467
14468	/* Mark owner so we could distinguish it from user events. */
14469	event->owner = TASK_TOMBSTONE;
14470	pmu = event->pmu;
14471
14472	if (pmu->task_ctx_nr == perf_sw_context)
14473	event->event_caps \|= PERF_EV_CAP_SOFTWARE;
14474
14475	/*
14476	* Get the target context (task or percpu):
14477	*/
14478	ctx = find_get_context(task, event);
14479	if (IS_ERR(ctx)) {
14480	err = PTR_ERR(ctx);
14481	goto err_alloc;
14482	}
14483
14484	WARN_ON_ONCE(ctx->parent_ctx);
14485	mutex_lock(&ctx->mutex);
14486	if (ctx->task == TASK_TOMBSTONE) {
14487	err = -ESRCH;
14488	goto err_unlock;
14489	}
14490
14491	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
14492	if (IS_ERR(pmu_ctx)) {
14493	err = PTR_ERR(pmu_ctx);
14494	goto err_unlock;
14495	}
14496	event->pmu_ctx = pmu_ctx;
14497
14498	if (!task) {
14499	/*
14500	* Check if the @cpu we're creating an event for is online.
14501	*
14502	* We use the perf_cpu_context::ctx::mutex to serialize against
14503	* the hotplug notifiers. See perf_event_{init,exit}_cpu().
14504	*/
14505	struct perf_cpu_context *cpuctx =
14506	container_of(ctx, struct perf_cpu_context, ctx);
14507	if (!cpuctx->online) {
14508	err = -ENODEV;
14509	goto err_pmu_ctx;
14510	}
14511	}
14512
14513	if (!exclusive_event_installable(event, ctx)) {
14514	err = -EBUSY;
14515	goto err_pmu_ctx;
14516	}
14517
14518	perf_install_in_context(ctx, event, event->cpu);
14519	perf_unpin_context(ctx);
14520	mutex_unlock(&ctx->mutex);
14521
14522	return event;
14523
14524	err_pmu_ctx:
14525	put_pmu_ctx(pmu_ctx);
14526	event->pmu_ctx = NULL; /* _free_event() */
14527	err_unlock:
14528	mutex_unlock(&ctx->mutex);
14529	perf_unpin_context(ctx);
14530	put_ctx(ctx);
14531	err_alloc:
14532	put_event(event);
14533	err:
14534	return ERR_PTR(err);
14535	}
14536	EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
14537
14538	static void __perf_pmu_remove(struct perf_event_context ctx, struct pmu pmu,
14539	struct list_head *events)
14540	{
14541	struct perf_event event, tmp;
14542
14543	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) {
14544	if (event->pmu_ctx->pmu != pmu)
14545	continue;
14546
14547	perf_remove_from_context(event, 0);
14548	put_pmu_ctx(event->pmu_ctx);
14549	list_add(&event->migrate_entry, events);
14550	}
14551	}
14552
14553	static void __perf_pmu_install_event(struct pmu *pmu,
14554	struct perf_event_context *ctx,
14555	int cpu, struct perf_event *event)
14556	{
14557	struct perf_event_pmu_context *epc;
14558	struct perf_event_context *old_ctx = event->ctx;
14559
14560	get_ctx(ctx); /* normally find_get_context() */
14561
14562	event->cpu = cpu;
14563	epc = find_get_pmu_context(pmu, ctx, event);
14564	event->pmu_ctx = epc;
14565
14566	if (event->state >= PERF_EVENT_STATE_OFF)
14567	event->state = PERF_EVENT_STATE_INACTIVE;
14568	perf_install_in_context(ctx, event, cpu);
14569
14570	/*
14571	* Now that event->ctx is updated and visible, put the old ctx.
14572	*/
14573	put_ctx(old_ctx);
14574	}
14575
14576	static void __perf_pmu_install(struct perf_event_context *ctx,
14577	int cpu, struct pmu pmu, struct list_head events)
14578	{
14579	struct perf_event event, tmp;
14580
14581	/*
14582	* Re-instate events in 2 passes.
14583	*
14584	* Skip over group leaders and only install siblings on this first
14585	* pass, siblings will not get enabled without a leader, however a
14586	* leader will enable its siblings, even if those are still on the old
14587	* context.
14588	*/
14589	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
14590	if (event->group_leader == event)
14591	continue;
14592
14593	list_del(&event->migrate_entry);
14594	__perf_pmu_install_event(pmu, ctx, cpu, event);
14595	}
14596
14597	/*
14598	* Once all the siblings are setup properly, install the group leaders
14599	* to make it go.
14600	*/
14601	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
14602	list_del(&event->migrate_entry);
14603	__perf_pmu_install_event(pmu, ctx, cpu, event);
14604	}
14605	}
14606
14607	void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
14608	{
14609	struct perf_event_context src_ctx, dst_ctx;
14610	LIST_HEAD(events);
14611
14612	/*
14613	* Since per-cpu context is persistent, no need to grab an extra
14614	* reference.
14615	*/
14616	src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;
14617	dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;
14618
14619	/*
14620	* See perf_event_ctx_lock() for comments on the details
14621	* of swizzling perf_event::ctx.
14622	*/
14623	mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);
14624
14625	__perf_pmu_remove(src_ctx, pmu, &events);
14626
14627	if (!list_empty(&events)) {
14628	/*
14629	* Wait for the events to quiesce before re-instating them.
14630	*/
14631	synchronize_rcu();
14632
14633	__perf_pmu_install(dst_ctx, dst_cpu, pmu, &events);
14634	}
14635
14636	mutex_unlock(&dst_ctx->mutex);
14637	mutex_unlock(&src_ctx->mutex);
14638	}
14639	EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
14640
14641	static void sync_child_event(struct perf_event *child_event,
14642	struct task_struct *task)
14643	{
14644	struct perf_event *parent_event = child_event->parent;
14645	u64 child_val;
14646
14647	if (child_event->attr.inherit_stat) {
14648	if (task && task != TASK_TOMBSTONE)
14649	perf_event_read_event(child_event, task);
14650	}
14651
14652	child_val = perf_event_count(child_event, false);
14653
14654	/*
14655	* Add back the child's count to the parent's count:
14656	*/
14657	atomic64_add(child_val, &parent_event->child_count);
14658	atomic64_add(child_event->total_time_enabled,
14659	&parent_event->child_total_time_enabled);
14660	atomic64_add(child_event->total_time_running,
14661	&parent_event->child_total_time_running);
14662	}
14663
14664	static void
14665	perf_event_exit_event(struct perf_event *event,
14666	struct perf_event_context *ctx,
14667	struct task_struct *task,
14668	unsigned long detach_flags)
14669	{
14670	struct perf_event *parent_event = event->parent;
14671	unsigned int attach_state;
14672
14673	detach_flags \|= DETACH_EXIT;
14674
14675	if (parent_event) {
14676	/*
14677	* Do not destroy the 'original' grouping; because of the
14678	* context switch optimization the original events could've
14679	* ended up in a random child task.
14680	*
14681	* If we were to destroy the original group, all group related
14682	* operations would cease to function properly after this
14683	* random child dies.
14684	*
14685	* Do destroy all inherited groups, we don't care about those
14686	* and being thorough is better.
14687	*/
14688	detach_flags \|= DETACH_GROUP \| DETACH_CHILD;
14689	mutex_lock(&parent_event->child_mutex);
14690	/* PERF_ATTACH_ITRACE might be set concurrently */
14691	attach_state = READ_ONCE(event->attach_state);
14692
14693	if (attach_state & PERF_ATTACH_CHILD)
14694	sync_child_event(event, task);
14695	}
14696
14697	if (detach_flags & DETACH_REVOKE)
14698	detach_flags \|= DETACH_GROUP;
14699
14700	perf_remove_from_context(event, detach_flags);
14701	/*
14702	* Child events can be freed.
14703	*/
14704	if (parent_event) {
14705	mutex_unlock(&parent_event->child_mutex);
14706
14707	/*
14708	* Match the refcount initialization. Make sure it doesn't happen
14709	* twice if pmu_detach_event() calls it on an already exited task.
14710	*/
14711	if (attach_state & PERF_ATTACH_CHILD) {
14712	/*
14713	* Kick perf_poll() for is_event_hup();
14714	*/
14715	perf_event_wakeup(parent_event);
14716	/*
14717	* pmu_detach_event() will have an extra refcount.
14718	* perf_pending_task() might have one too.
14719	*/
14720	put_event(event);
14721	}
14722
14723	return;
14724	}
14725
14726	/*
14727	* Parent events are governed by their filedesc, retain them.
14728	*/
14729	perf_event_wakeup(event);
14730	}
14731
14732	static void perf_event_exit_task_context(struct task_struct *task, bool exit)
14733	{
14734	struct perf_event_context ctx, clone_ctx = NULL;
14735	struct perf_event child_event, next;
14736
14737	ctx = perf_pin_task_context(task);
14738	if (!ctx)
14739	return;
14740
14741	/*
14742	* In order to reduce the amount of tricky in ctx tear-down, we hold
14743	* ctx::mutex over the entire thing. This serializes against almost
14744	* everything that wants to access the ctx.
14745	*
14746	* The exception is sys_perf_event_open() /
14747	* perf_event_create_kernel_count() which does find_get_context()
14748	* without ctx::mutex (it cannot because of the move_group double mutex
14749	* lock thing). See the comments in perf_install_in_context().
14750	*/
14751	mutex_lock(&ctx->mutex);
14752
14753	/*
14754	* In a single ctx::lock section, de-schedule the events and detach the
14755	* context from the task such that we cannot ever get it scheduled back
14756	* in.
14757	*/
14758	raw_spin_lock_irq(&ctx->lock);
14759	if (exit)
14760	task_ctx_sched_out(ctx, NULL, EVENT_ALL);
14761
14762	/*
14763	* Now that the context is inactive, destroy the task <-> ctx relation
14764	* and mark the context dead.
14765	*/
14766	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);
14767	put_ctx(ctx); /* cannot be last */
14768	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
14769	put_task_struct(task); /* cannot be last */
14770
14771	clone_ctx = unclone_ctx(ctx);
14772	raw_spin_unlock_irq(&ctx->lock);
14773
14774	if (clone_ctx)
14775	put_ctx(clone_ctx);
14776
14777	/*
14778	* Report the task dead after unscheduling the events so that we
14779	* won't get any samples after PERF_RECORD_EXIT. We can however still
14780	* get a few PERF_RECORD_READ events.
14781	*/
14782	if (exit)
14783	perf_event_task(task, ctx, 0);
14784
14785	list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry)
14786	perf_event_exit_event(child_event, ctx, exit ? task : NULL, 0);
14787
14788	mutex_unlock(&ctx->mutex);
14789
14790	if (!exit) {
14791	/*
14792	* perf_event_release_kernel() could still have a reference on
14793	* this context. In that case we must wait for these events to
14794	* have been freed (in particular all their references to this
14795	* task must've been dropped).
14796	*
14797	* Without this copy_process() will unconditionally free this
14798	* task (irrespective of its reference count) and
14799	* _free_event()'s put_task_struct(event->hw.target) will be a
14800	* use-after-free.
14801	*
14802	* Wait for all events to drop their context reference.
14803	*/
14804	wait_var_event(&ctx->refcount,
14805	refcount_read(&ctx->refcount) == 1);
14806	}
14807	put_ctx(ctx);
14808	}
14809
14810	/*
14811	* When a task exits, feed back event values to parent events.
14812	*
14813	* Can be called with exec_update_lock held when called from
14814	* setup_new_exec().
14815	*/
14816	void perf_event_exit_task(struct task_struct *task)
14817	{
14818	struct perf_event event, tmp;
14819
14820	WARN_ON_ONCE(task != current);
14821
14822	mutex_lock(&task->perf_event_mutex);
14823	list_for_each_entry_safe(event, tmp, &task->perf_event_list,
14824	owner_entry) {
14825	list_del_init(&event->owner_entry);
14826
14827	/*
14828	* Ensure the list deletion is visible before we clear
14829	* the owner, closes a race against perf_release() where
14830	* we need to serialize on the owner->perf_event_mutex.
14831	*/
14832	smp_store_release(&event->owner, NULL);
14833	}
14834	mutex_unlock(&task->perf_event_mutex);
14835
14836	perf_event_exit_task_context(task, true);
14837
14838	/*
14839	* The perf_event_exit_task_context calls perf_event_task
14840	* with task's task_ctx, which generates EXIT events for
14841	* task contexts and sets task->perf_event_ctxp[] to NULL.
14842	* At this point we need to send EXIT events to cpu contexts.
14843	*/
14844	perf_event_task(task, NULL, 0);
14845
14846	/*
14847	* Detach the perf_ctx_data for the system-wide event.
14848	*
14849	* Done without holding global_ctx_data_rwsem; typically
14850	* attach_global_ctx_data() will skip over this task, but otherwise
14851	* attach_task_ctx_data() will observe PF_EXITING.
14852	*/
14853	detach_task_ctx_data(task);
14854	}
14855
14856	/*
14857	* Free a context as created by inheritance by perf_event_init_task() below,
14858	* used by fork() in case of fail.
14859	*
14860	* Even though the task has never lived, the context and events have been
14861	* exposed through the child_list, so we must take care tearing it all down.
14862	*/
14863	void perf_event_free_task(struct task_struct *task)
14864	{
14865	perf_event_exit_task_context(task, false);
14866	}
14867
14868	void perf_event_delayed_put(struct task_struct *task)
14869	{
14870	WARN_ON_ONCE(task->perf_event_ctxp);
14871	}
14872
14873	struct file *perf_event_get(unsigned int fd)
14874	{
14875	struct file *file = fget(fd);
14876	if (!file)
14877	return ERR_PTR(-EBADF);
14878
14879	if (file->f_op != &perf_fops) {
14880	fput(file);
14881	return ERR_PTR(-EBADF);
14882	}
14883
14884	return file;
14885	}
14886
14887	const struct perf_event perf_get_event(struct file file)
14888	{
14889	if (file->f_op != &perf_fops)
14890	return ERR_PTR(-EINVAL);
14891
14892	return file->private_data;
14893	}
14894
14895	const struct perf_event_attr perf_event_attrs(struct perf_event event)
14896	{
14897	if (!event)
14898	return ERR_PTR(-EINVAL);
14899
14900	return &event->attr;
14901	}
14902
14903	int perf_allow_kernel(void)
14904	{
14905	if (sysctl_perf_event_paranoid > 1 && !perfmon_capable())
14906	return -EACCES;
14907
14908	return security_perf_event_open(PERF_SECURITY_KERNEL);
14909	}
14910	EXPORT_SYMBOL_GPL(perf_allow_kernel);
14911
14912	int perf_allow_cpu(void)
14913	{
14914	if (sysctl_perf_event_paranoid > 0 && !perfmon_capable())
14915	return -EACCES;
14916
14917	return security_perf_event_open(PERF_SECURITY_CPU);
14918	}
14919	EXPORT_SYMBOL_GPL(perf_allow_cpu);
14920
14921	int perf_allow_tracepoint(void)
14922	{
14923	if (sysctl_perf_event_paranoid > -1 && !perfmon_capable())
14924	return -EPERM;
14925
14926	return security_perf_event_open(PERF_SECURITY_TRACEPOINT);
14927	}
14928	EXPORT_SYMBOL_GPL(perf_allow_tracepoint);
14929
14930	/*
14931	* Inherit an event from parent task to child task.
14932	*
14933	* Returns:
14934	* - valid pointer on success
14935	* - NULL for orphaned events
14936	* - IS_ERR() on error
14937	*/
14938	static struct perf_event *
14939	inherit_event(struct perf_event *parent_event,
14940	struct task_struct *parent,
14941	struct perf_event_context *parent_ctx,
14942	struct task_struct *child,
14943	struct perf_event *group_leader,
14944	struct perf_event_context *child_ctx)
14945	{
14946	enum perf_event_state parent_state = parent_event->state;
14947	struct perf_event_pmu_context *pmu_ctx;
14948	struct perf_event *child_event;
14949	unsigned long flags;
14950
14951	/*
14952	* Instead of creating recursive hierarchies of events,
14953	* we link inherited events back to the original parent,
14954	* which has a filp for sure, which we use as the reference
14955	* count:
14956	*/
14957	if (parent_event->parent)
14958	parent_event = parent_event->parent;
14959
14960	if (parent_event->state <= PERF_EVENT_STATE_REVOKED)
14961	return NULL;
14962
14963	/*
14964	* Event creation should be under SRCU, see perf_pmu_unregister().
14965	*/
14966	guard(srcu)(&pmus_srcu);
14967
14968	child_event = perf_event_alloc(&parent_event->attr,
14969	parent_event->cpu,
14970	child,
14971	group_leader, parent_event,
14972	NULL, NULL, -1);
14973	if (IS_ERR(child_event))
14974	return child_event;
14975
14976	get_ctx(child_ctx);
14977	child_event->ctx = child_ctx;
14978
14979	pmu_ctx = find_get_pmu_context(parent_event->pmu_ctx->pmu, child_ctx, child_event);
14980	if (IS_ERR(pmu_ctx)) {
14981	free_event(child_event);
14982	return ERR_CAST(pmu_ctx);
14983	}
14984	child_event->pmu_ctx = pmu_ctx;
14985
14986	/*
14987	* is_orphaned_event() and list_add_tail(&parent_event->child_list)
14988	* must be under the same lock in order to serialize against
14989	* perf_event_release_kernel(), such that either we must observe
14990	* is_orphaned_event() or they will observe us on the child_list.
14991	*/
14992	mutex_lock(&parent_event->child_mutex);
14993	if (is_orphaned_event(parent_event) \|\|
14994	!atomic_long_inc_not_zero(&parent_event->refcount)) {
14995	mutex_unlock(&parent_event->child_mutex);
14996	free_event(child_event);
14997	return NULL;
14998	}
14999
15000	/*
15001	* Make the child state follow the state of the parent event,
15002	* not its attr.disabled bit. We hold the parent's mutex,
15003	* so we won't race with perf_event_{en, dis}able_family.
15004	*/
15005	if (parent_state >= PERF_EVENT_STATE_INACTIVE)
15006	child_event->state = PERF_EVENT_STATE_INACTIVE;
15007	else
15008	child_event->state = PERF_EVENT_STATE_OFF;
15009
15010	if (parent_event->attr.freq) {
15011	u64 sample_period = parent_event->hw.sample_period;
15012	struct hw_perf_event *hwc = &child_event->hw;
15013
15014	hwc->sample_period = sample_period;
15015	hwc->last_period = sample_period;
15016
15017	local64_set(&hwc->period_left, sample_period);
15018	}
15019
15020	child_event->overflow_handler = parent_event->overflow_handler;
15021	child_event->overflow_handler_context
15022	= parent_event->overflow_handler_context;
15023
15024	/*
15025	* Precalculate sample_data sizes
15026	*/
15027	perf_event__header_size(child_event);
15028	perf_event__id_header_size(child_event);
15029
15030	/*
15031	* Link it up in the child's context:
15032	*/
15033	raw_spin_lock_irqsave(&child_ctx->lock, flags);
15034	add_event_to_ctx(child_event, child_ctx);
15035	child_event->attach_state \|= PERF_ATTACH_CHILD;
15036	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
15037
15038	/*
15039	* Link this into the parent event's child list
15040	*/
15041	list_add_tail(&child_event->child_list, &parent_event->child_list);
15042	mutex_unlock(&parent_event->child_mutex);
15043
15044	return child_event;
15045	}
15046
15047	/*
15048	* Inherits an event group.
15049	*
15050	* This will quietly suppress orphaned events; !inherit_event() is not an error.
15051	* This matches with perf_event_release_kernel() removing all child events.
15052	*
15053	* Returns:
15054	* - 0 on success
15055	* - <0 on error
15056	*/
15057	static int inherit_group(struct perf_event *parent_event,
15058	struct task_struct *parent,
15059	struct perf_event_context *parent_ctx,
15060	struct task_struct *child,
15061	struct perf_event_context *child_ctx)
15062	{
15063	struct perf_event *leader;
15064	struct perf_event *sub;
15065	struct perf_event *child_ctr;
15066
15067	leader = inherit_event(parent_event, parent, parent_ctx,
15068	child, NULL, child_ctx);
15069	if (IS_ERR(leader))
15070	return PTR_ERR(leader);
15071	/*
15072	* @leader can be NULL here because of is_orphaned_event(). In this
15073	* case inherit_event() will create individual events, similar to what
15074	* perf_group_detach() would do anyway.
15075	*/
15076	for_each_sibling_event(sub, parent_event) {
15077	child_ctr = inherit_event(sub, parent, parent_ctx,
15078	child, leader, child_ctx);
15079	if (IS_ERR(child_ctr))
15080	return PTR_ERR(child_ctr);
15081
15082	if (sub->aux_event == parent_event && child_ctr &&
15083	!perf_get_aux_event(child_ctr, leader))
15084	return -EINVAL;
15085	}
15086	if (leader)
15087	leader->group_generation = parent_event->group_generation;
15088	return 0;
15089	}
15090
15091	/*
15092	* Creates the child task context and tries to inherit the event-group.
15093	*
15094	* Clears @inherited_all on !attr.inherited or error. Note that we'll leave
15095	* inherited_all set when we 'fail' to inherit an orphaned event; this is
15096	* consistent with perf_event_release_kernel() removing all child events.
15097	*
15098	* Returns:
15099	* - 0 on success
15100	* - <0 on error
15101	*/
15102	static int
15103	inherit_task_group(struct perf_event event, struct task_struct parent,
15104	struct perf_event_context *parent_ctx,
15105	struct task_struct *child,
15106	u64 clone_flags, int *inherited_all)
15107	{
15108	struct perf_event_context *child_ctx;
15109	int ret;
15110
15111	if (!event->attr.inherit \|\|
15112	(event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) \|\|
15113	/* Do not inherit if sigtrap and signal handlers were cleared. */
15114	(event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {
15115	*inherited_all = 0;
15116	return 0;
15117	}
15118
15119	child_ctx = child->perf_event_ctxp;
15120	if (!child_ctx) {
15121	/*
15122	* This is executed from the parent task context, so
15123	* inherit events that have been marked for cloning.
15124	* First allocate and initialize a context for the
15125	* child.
15126	*/
15127	child_ctx = alloc_perf_context(child);
15128	if (!child_ctx)
15129	return -ENOMEM;
15130
15131	child->perf_event_ctxp = child_ctx;
15132	}
15133
15134	ret = inherit_group(event, parent, parent_ctx, child, child_ctx);
15135	if (ret)
15136	*inherited_all = 0;
15137
15138	return ret;
15139	}
15140
15141	/*
15142	* Initialize the perf_event context in task_struct
15143	*/
15144	static int perf_event_init_context(struct task_struct *child, u64 clone_flags)
15145	{
15146	struct perf_event_context child_ctx, parent_ctx;
15147	struct perf_event_context *cloned_ctx;
15148	struct perf_event *event;
15149	struct task_struct *parent = current;
15150	int inherited_all = 1;
15151	unsigned long flags;
15152	int ret = 0;
15153
15154	if (likely(!parent->perf_event_ctxp))
15155	return 0;
15156
15157	/*
15158	* If the parent's context is a clone, pin it so it won't get
15159	* swapped under us.
15160	*/
15161	parent_ctx = perf_pin_task_context(parent);
15162	if (!parent_ctx)
15163	return 0;
15164
15165	/*
15166	* No need to check if parent_ctx != NULL here; since we saw
15167	* it non-NULL earlier, the only reason for it to become NULL
15168	* is if we exit, and since we're currently in the middle of
15169	* a fork we can't be exiting at the same time.
15170	*/
15171
15172	/*
15173	* Lock the parent list. No need to lock the child - not PID
15174	* hashed yet and not running, so nobody can access it.
15175	*/
15176	mutex_lock(&parent_ctx->mutex);
15177
15178	/*
15179	* We dont have to disable NMIs - we are only looking at
15180	* the list, not manipulating it:
15181	*/
15182	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {
15183	ret = inherit_task_group(event, parent, parent_ctx,
15184	child, clone_flags, &inherited_all);
15185	if (ret)
15186	goto out_unlock;
15187	}
15188
15189	/*
15190	* We can't hold ctx->lock when iterating the ->flexible_group list due
15191	* to allocations, but we need to prevent rotation because
15192	* rotate_ctx() will change the list from interrupt context.
15193	*/
15194	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
15195	parent_ctx->rotate_disable = 1;
15196	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
15197
15198	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {
15199	ret = inherit_task_group(event, parent, parent_ctx,
15200	child, clone_flags, &inherited_all);
15201	if (ret)
15202	goto out_unlock;
15203	}
15204
15205	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
15206	parent_ctx->rotate_disable = 0;
15207
15208	child_ctx = child->perf_event_ctxp;
15209
15210	if (child_ctx && inherited_all) {
15211	/*
15212	* Mark the child context as a clone of the parent
15213	* context, or of whatever the parent is a clone of.
15214	*
15215	* Note that if the parent is a clone, the holding of
15216	* parent_ctx->lock avoids it from being uncloned.
15217	*/
15218	cloned_ctx = parent_ctx->parent_ctx;
15219	if (cloned_ctx) {
15220	child_ctx->parent_ctx = cloned_ctx;
15221	child_ctx->parent_gen = parent_ctx->parent_gen;
15222	} else {
15223	child_ctx->parent_ctx = parent_ctx;
15224	child_ctx->parent_gen = parent_ctx->generation;
15225	}
15226	get_ctx(child_ctx->parent_ctx);
15227	}
15228
15229	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
15230	out_unlock:
15231	mutex_unlock(&parent_ctx->mutex);
15232
15233	perf_unpin_context(parent_ctx);
15234	put_ctx(parent_ctx);
15235
15236	return ret;
15237	}
15238
15239	/*
15240	* Initialize the perf_event context in task_struct
15241	*/
15242	int perf_event_init_task(struct task_struct *child, u64 clone_flags)
15243	{
15244	int ret;
15245
15246	memset(child->perf_recursion, 0, sizeof(child->perf_recursion));
15247	child->perf_event_ctxp = NULL;
15248	mutex_init(&child->perf_event_mutex);
15249	INIT_LIST_HEAD(&child->perf_event_list);
15250	child->perf_ctx_data = NULL;
15251
15252	ret = perf_event_init_context(child, clone_flags);
15253	if (ret) {
15254	perf_event_free_task(child);
15255	return ret;
15256	}
15257
15258	return 0;
15259	}
15260
15261	static void __init perf_event_init_all_cpus(void)
15262	{
15263	struct swevent_htable *swhash;
15264	struct perf_cpu_context *cpuctx;
15265	int cpu;
15266
15267	zalloc_cpumask_var(&perf_online_mask, GFP_KERNEL);
15268	zalloc_cpumask_var(&perf_online_core_mask, GFP_KERNEL);
15269	zalloc_cpumask_var(&perf_online_die_mask, GFP_KERNEL);
15270	zalloc_cpumask_var(&perf_online_cluster_mask, GFP_KERNEL);
15271	zalloc_cpumask_var(&perf_online_pkg_mask, GFP_KERNEL);
15272	zalloc_cpumask_var(&perf_online_sys_mask, GFP_KERNEL);
15273
15274
15275	for_each_possible_cpu(cpu) {
15276	swhash = &per_cpu(swevent_htable, cpu);
15277	mutex_init(&swhash->hlist_mutex);
15278
15279	INIT_LIST_HEAD(&per_cpu(pmu_sb_events.list, cpu));
15280	raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu));
15281
15282	INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));
15283
15284	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
15285	__perf_event_init_context(&cpuctx->ctx);
15286	lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
15287	lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
15288	cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);
15289	cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);
15290	cpuctx->heap = cpuctx->heap_default;
15291	}
15292	}
15293
15294	static void perf_swevent_init_cpu(unsigned int cpu)
15295	{
15296	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
15297
15298	mutex_lock(&swhash->hlist_mutex);
15299	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {
15300	struct swevent_hlist *hlist;
15301
15302	hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
15303	WARN_ON(!hlist);
15304	rcu_assign_pointer(swhash->swevent_hlist, hlist);
15305	}
15306	mutex_unlock(&swhash->hlist_mutex);
15307	}
15308
15309	#if defined CONFIG_HOTPLUG_CPU \|\| defined CONFIG_KEXEC_CORE
15310	static void __perf_event_exit_context(void *__info)
15311	{
15312	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
15313	struct perf_event_context *ctx = __info;
15314	struct perf_event *event;
15315
15316	raw_spin_lock(&ctx->lock);
15317	ctx_sched_out(ctx, NULL, EVENT_TIME);
15318	list_for_each_entry(event, &ctx->event_list, event_entry)
15319	__perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);
15320	raw_spin_unlock(&ctx->lock);
15321	}
15322
15323	static void perf_event_clear_cpumask(unsigned int cpu)
15324	{
15325	int target[PERF_PMU_MAX_SCOPE];
15326	unsigned int scope;
15327	struct pmu *pmu;
15328
15329	cpumask_clear_cpu(cpu, perf_online_mask);
15330
15331	for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) {
15332	const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(scope, cpu);
15333	struct cpumask *pmu_cpumask = perf_scope_cpumask(scope);
15334
15335	target[scope] = -1;
15336	if (WARN_ON_ONCE(!pmu_cpumask \|\| !cpumask))
15337	continue;
15338
15339	if (!cpumask_test_and_clear_cpu(cpu, pmu_cpumask))
15340	continue;
15341	target[scope] = cpumask_any_but(cpumask, cpu);
15342	if (target[scope] < nr_cpu_ids)
15343	cpumask_set_cpu(target[scope], pmu_cpumask);
15344	}
15345
15346	/* migrate */
15347	list_for_each_entry(pmu, &pmus, entry) {
15348	if (pmu->scope == PERF_PMU_SCOPE_NONE \|\|
15349	WARN_ON_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE))
15350	continue;
15351
15352	if (target[pmu->scope] >= 0 && target[pmu->scope] < nr_cpu_ids)
15353	perf_pmu_migrate_context(pmu, cpu, target[pmu->scope]);
15354	}
15355	}
15356
15357	static void perf_event_exit_cpu_context(int cpu)
15358	{
15359	struct perf_cpu_context *cpuctx;
15360	struct perf_event_context *ctx;
15361
15362	// XXX simplify cpuctx->online
15363	mutex_lock(&pmus_lock);
15364	/*
15365	* Clear the cpumasks, and migrate to other CPUs if possible.
15366	* Must be invoked before the __perf_event_exit_context.
15367	*/
15368	perf_event_clear_cpumask(cpu);
15369	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
15370	ctx = &cpuctx->ctx;
15371
15372	mutex_lock(&ctx->mutex);
15373	if (ctx->nr_events)
15374	smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
15375	cpuctx->online = 0;
15376	mutex_unlock(&ctx->mutex);
15377	mutex_unlock(&pmus_lock);
15378	}
15379	#else
15380
15381	static void perf_event_exit_cpu_context(int cpu) { }
15382
15383	#endif
15384
15385	static void perf_event_setup_cpumask(unsigned int cpu)
15386	{
15387	struct cpumask *pmu_cpumask;
15388	unsigned int scope;
15389
15390	/*
15391	* Early boot stage, the cpumask hasn't been set yet.
15392	* The perf_online_<domain>_masks includes the first CPU of each domain.
15393	* Always unconditionally set the boot CPU for the perf_online_<domain>_masks.
15394	*/
15395	if (cpumask_empty(perf_online_mask)) {
15396	for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) {
15397	pmu_cpumask = perf_scope_cpumask(scope);
15398	if (WARN_ON_ONCE(!pmu_cpumask))
15399	continue;
15400	cpumask_set_cpu(cpu, pmu_cpumask);
15401	}
15402	goto end;
15403	}
15404
15405	for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) {
15406	const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(scope, cpu);
15407
15408	pmu_cpumask = perf_scope_cpumask(scope);
15409
15410	if (WARN_ON_ONCE(!pmu_cpumask \|\| !cpumask))
15411	continue;
15412
15413	if (!cpumask_empty(cpumask) &&
15414	cpumask_any_and(pmu_cpumask, cpumask) >= nr_cpu_ids)
15415	cpumask_set_cpu(cpu, pmu_cpumask);
15416	}
15417	end:
15418	cpumask_set_cpu(cpu, perf_online_mask);
15419	}
15420
15421	int perf_event_init_cpu(unsigned int cpu)
15422	{
15423	struct perf_cpu_context *cpuctx;
15424	struct perf_event_context *ctx;
15425
15426	perf_swevent_init_cpu(cpu);
15427
15428	mutex_lock(&pmus_lock);
15429	perf_event_setup_cpumask(cpu);
15430	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
15431	ctx = &cpuctx->ctx;
15432
15433	mutex_lock(&ctx->mutex);
15434	cpuctx->online = 1;
15435	mutex_unlock(&ctx->mutex);
15436	mutex_unlock(&pmus_lock);
15437
15438	return 0;
15439	}
15440
15441	int perf_event_exit_cpu(unsigned int cpu)
15442	{
15443	perf_event_exit_cpu_context(cpu);
15444	return 0;
15445	}
15446
15447	static int
15448	perf_reboot(struct notifier_block notifier, unsigned long val, void v)
15449	{
15450	int cpu;
15451
15452	for_each_online_cpu(cpu)
15453	perf_event_exit_cpu(cpu);
15454
15455	return NOTIFY_OK;
15456	}
15457
15458	/*
15459	* Run the perf reboot notifier at the very last possible moment so that
15460	* the generic watchdog code runs as long as possible.
15461	*/
15462	static struct notifier_block perf_reboot_notifier = {
15463	.notifier_call = perf_reboot,
15464	.priority = INT_MIN,
15465	};
15466
15467	void __init perf_event_init(void)
15468	{
15469	int ret;
15470
15471	idr_init(&pmu_idr);
15472
15473	unwind_deferred_init(&perf_unwind_work,
15474	perf_unwind_deferred_callback);
15475
15476	perf_event_init_all_cpus();
15477	init_srcu_struct(&pmus_srcu);
15478	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
15479	perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1);
15480	perf_pmu_register(&perf_task_clock, "task_clock", -1);
15481	perf_tp_register();
15482	perf_event_init_cpu(smp_processor_id());
15483	register_reboot_notifier(&perf_reboot_notifier);
15484
15485	ret = init_hw_breakpoint();
15486	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
15487
15488	perf_event_cache = KMEM_CACHE(perf_event, SLAB_PANIC);
15489
15490	/*
15491	* Build time assertion that we keep the data_head at the intended
15492	* location. IOW, validation we got the __reserved[] size right.
15493	*/
15494	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
15495	!= 1024);
15496	}
15497
15498	ssize_t perf_event_sysfs_show(struct device dev, struct device_attribute attr,
15499	char *page)
15500	{
15501	struct perf_pmu_events_attr *pmu_attr =
15502	container_of(attr, struct perf_pmu_events_attr, attr);
15503
15504	if (pmu_attr->event_str)
15505	return sprintf(page, "%s\n", pmu_attr->event_str);
15506
15507	return 0;
15508	}
15509	EXPORT_SYMBOL_GPL(perf_event_sysfs_show);
15510
15511	static int __init perf_event_sysfs_init(void)
15512	{
15513	struct pmu *pmu;
15514	int ret;
15515
15516	mutex_lock(&pmus_lock);
15517
15518	ret = bus_register(&pmu_bus);
15519	if (ret)
15520	goto unlock;
15521
15522	list_for_each_entry(pmu, &pmus, entry) {
15523	if (pmu->dev)
15524	continue;
15525
15526	ret = pmu_dev_alloc(pmu);
15527	WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
15528	}
15529	pmu_bus_running = 1;
15530	ret = 0;
15531
15532	unlock:
15533	mutex_unlock(&pmus_lock);
15534
15535	return ret;
15536	}
15537	device_initcall(perf_event_sysfs_init);
15538
15539	#ifdef CONFIG_CGROUP_PERF
15540	static struct cgroup_subsys_state *
15541	perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
15542	{
15543	struct perf_cgroup *jc;
15544
15545	jc = kzalloc_obj(*jc);
15546	if (!jc)
15547	return ERR_PTR(-ENOMEM);
15548
15549	jc->info = alloc_percpu(struct perf_cgroup_info);
15550	if (!jc->info) {
15551	kfree(jc);
15552	return ERR_PTR(-ENOMEM);
15553	}
15554
15555	return &jc->css;
15556	}
15557
15558	static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
15559	{
15560	struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
15561
15562	free_percpu(jc->info);
15563	kfree(jc);
15564	}
15565
15566	static int perf_cgroup_css_online(struct cgroup_subsys_state *css)
15567	{
15568	perf_event_cgroup(css->cgroup);
15569	return 0;
15570	}
15571
15572	static int __perf_cgroup_move(void *info)
15573	{
15574	struct task_struct *task = info;
15575
15576	preempt_disable();
15577	perf_cgroup_switch(task);
15578	preempt_enable();
15579
15580	return 0;
15581	}
15582
15583	static void perf_cgroup_attach(struct cgroup_taskset *tset)
15584	{
15585	struct task_struct *task;
15586	struct cgroup_subsys_state *css;
15587
15588	cgroup_taskset_for_each(task, css, tset)
15589	task_function_call(task, __perf_cgroup_move, task);
15590	}
15591
15592	struct cgroup_subsys perf_event_cgrp_subsys = {
15593	.css_alloc = perf_cgroup_css_alloc,
15594	.css_free = perf_cgroup_css_free,
15595	.css_online = perf_cgroup_css_online,
15596	.attach = perf_cgroup_attach,
15597	/*
15598	* Implicitly enable on dfl hierarchy so that perf events can
15599	* always be filtered by cgroup2 path as long as perf_event
15600	* controller is not mounted on a legacy hierarchy.
15601	*/
15602	.implicit_on_dfl = true,
15603	.threaded = true,
15604	};
15605	#endif /* CONFIG_CGROUP_PERF */
15606
15607	DEFINE_STATIC_CALL_RET0(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);