linux - kernel/events/core.c

15467 lines · 376.7 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Performance events core code:
4	*
5	* Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <[email protected]>
6	* Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7	* Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8	* Copyright © 2009 Paul Mackerras, IBM Corp. <[email protected]>
9	*/
10
11	#include <linux/fs.h>
12	#include <linux/mm.h>
13	#include <linux/cpu.h>
14	#include <linux/smp.h>
15	#include <linux/idr.h>
16	#include <linux/file.h>
17	#include <linux/poll.h>
18	#include <linux/slab.h>
19	#include <linux/hash.h>
20	#include <linux/tick.h>
21	#include <linux/sysfs.h>
22	#include <linux/dcache.h>
23	#include <linux/percpu.h>
24	#include <linux/ptrace.h>
25	#include <linux/reboot.h>
26	#include <linux/vmstat.h>
27	#include <linux/device.h>
28	#include <linux/export.h>
29	#include <linux/vmalloc.h>
30	#include <linux/hardirq.h>
31	#include <linux/hugetlb.h>
32	#include <linux/rculist.h>
33	#include <linux/uaccess.h>
34	#include <linux/syscalls.h>
35	#include <linux/anon_inodes.h>
36	#include <linux/kernel_stat.h>
37	#include <linux/cgroup.h>
38	#include <linux/perf_event.h>
39	#include <linux/trace_events.h>
40	#include <linux/hw_breakpoint.h>
41	#include <linux/mm_types.h>
42	#include <linux/module.h>
43	#include <linux/mman.h>
44	#include <linux/compat.h>
45	#include <linux/bpf.h>
46	#include <linux/filter.h>
47	#include <linux/namei.h>
48	#include <linux/parser.h>
49	#include <linux/sched/clock.h>
50	#include <linux/sched/mm.h>
51	#include <linux/proc_ns.h>
52	#include <linux/mount.h>
53	#include <linux/min_heap.h>
54	#include <linux/highmem.h>
55	#include <linux/pgtable.h>
56	#include <linux/buildid.h>
57	#include <linux/task_work.h>
58	#include <linux/percpu-rwsem.h>
59	#include <linux/unwind_deferred.h>
60	#include <linux/kvm_types.h>
61	#include <linux/seq_file.h>
62
63	#include "internal.h"
64
65	#include <asm/irq_regs.h>
66
67	typedef int (remote_function_f)(void );
68
69	struct remote_function_call {
70	struct task_struct *p;
71	remote_function_f func;
72	void *info;
73	int ret;
74	};
75
76	static void remote_function(void *data)
77	{
78	struct remote_function_call *tfc = data;
79	struct task_struct *p = tfc->p;
80
81	if (p) {
82	/* -EAGAIN */
83	if (task_cpu(p) != smp_processor_id())
84	return;
85
86	/*
87	* Now that we're on right CPU with IRQs disabled, we can test
88	* if we hit the right task without races.
89	*/
90
91	tfc->ret = -ESRCH; /* No such (running) process */
92	if (p != current)
93	return;
94	}
95
96	tfc->ret = tfc->func(tfc->info);
97	}
98
99	/**
100	* task_function_call - call a function on the cpu on which a task runs
101	* @p: the task to evaluate
102	* @func: the function to be called
103	* @info: the function call argument
104	*
105	* Calls the function @func when the task is currently running. This might
106	* be on the current CPU, which just calls the function directly. This will
107	* retry due to any failures in smp_call_function_single(), such as if the
108	* task_cpu() goes offline concurrently.
109	*
110	* returns @func return value or -ESRCH or -ENXIO when the process isn't running
111	*/
112	static int
113	task_function_call(struct task_struct p, remote_function_f func, void info)
114	{
115	struct remote_function_call data = {
116	.p = p,
117	.func = func,
118	.info = info,
119	.ret = -EAGAIN,
120	};
121	int ret;
122
123	for (;;) {
124	ret = smp_call_function_single(task_cpu(p), remote_function,
125	&data, 1);
126	if (!ret)
127	ret = data.ret;
128
129	if (ret != -EAGAIN)
130	break;
131
132	cond_resched();
133	}
134
135	return ret;
136	}
137
138	/**
139	* cpu_function_call - call a function on the cpu
140	* @cpu: target cpu to queue this function
141	* @func: the function to be called
142	* @info: the function call argument
143	*
144	* Calls the function @func on the remote cpu.
145	*
146	* returns: @func return value or -ENXIO when the cpu is offline
147	*/
148	static int cpu_function_call(int cpu, remote_function_f func, void *info)
149	{
150	struct remote_function_call data = {
151	.p = NULL,
152	.func = func,
153	.info = info,
154	.ret = -ENXIO, /* No such CPU */
155	};
156
157	smp_call_function_single(cpu, remote_function, &data, 1);
158
159	return data.ret;
160	}
161
162	enum event_type_t {
163	EVENT_FLEXIBLE = 0x01,
164	EVENT_PINNED = 0x02,
165	EVENT_TIME = 0x04,
166	EVENT_FROZEN = 0x08,
167	/* see ctx_resched() for details */
168	EVENT_CPU = 0x10,
169	EVENT_CGROUP = 0x20,
170
171	/*
172	* EVENT_GUEST is set when scheduling in/out events between the host
173	* and a guest with a mediated vPMU. Among other things, EVENT_GUEST
174	* is used:
175	*
176	* - In for_each_epc() to skip PMUs that don't support events in a
177	* MEDIATED_VPMU guest, i.e. don't need to be context switched.
178	* - To indicate the start/end point of the events in a guest. Guest
179	* running time is deducted for host-only (exclude_guest) events.
180	*/
181	EVENT_GUEST = 0x40,
182	EVENT_FLAGS = EVENT_CGROUP \| EVENT_GUEST,
183	/* compound helpers */
184	EVENT_ALL = EVENT_FLEXIBLE \| EVENT_PINNED,
185	EVENT_TIME_FROZEN = EVENT_TIME \| EVENT_FROZEN,
186	};
187
188	static inline void __perf_ctx_lock(struct perf_event_context *ctx)
189	{
190	raw_spin_lock(&ctx->lock);
191	WARN_ON_ONCE(ctx->is_active & EVENT_FROZEN);
192	}
193
194	static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
195	struct perf_event_context *ctx)
196	{
197	__perf_ctx_lock(&cpuctx->ctx);
198	if (ctx)
199	__perf_ctx_lock(ctx);
200	}
201
202	static inline void __perf_ctx_unlock(struct perf_event_context *ctx)
203	{
204	/*
205	* If ctx_sched_in() didn't again set any ALL flags, clean up
206	* after ctx_sched_out() by clearing is_active.
207	*/
208	if (ctx->is_active & EVENT_FROZEN) {
209	if (!(ctx->is_active & EVENT_ALL))
210	ctx->is_active = 0;
211	else
212	ctx->is_active &= ~EVENT_FROZEN;
213	}
214	raw_spin_unlock(&ctx->lock);
215	}
216
217	static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
218	struct perf_event_context *ctx)
219	{
220	if (ctx)
221	__perf_ctx_unlock(ctx);
222	__perf_ctx_unlock(&cpuctx->ctx);
223	}
224
225	typedef struct {
226	struct perf_cpu_context *cpuctx;
227	struct perf_event_context *ctx;
228	} class_perf_ctx_lock_t;
229
230	static inline void class_perf_ctx_lock_destructor(class_perf_ctx_lock_t *_T)
231	{ perf_ctx_unlock(_T->cpuctx, _T->ctx); }
232
233	static inline class_perf_ctx_lock_t
234	class_perf_ctx_lock_constructor(struct perf_cpu_context *cpuctx,
235	struct perf_event_context *ctx)
236	{ perf_ctx_lock(cpuctx, ctx); return (class_perf_ctx_lock_t){ cpuctx, ctx }; }
237
238	#define TASK_TOMBSTONE ((void *)-1L)
239
240	static bool is_kernel_event(struct perf_event *event)
241	{
242	return READ_ONCE(event->owner) == TASK_TOMBSTONE;
243	}
244
245	static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
246
247	struct perf_event_context *perf_cpu_task_ctx(void)
248	{
249	lockdep_assert_irqs_disabled();
250	return this_cpu_ptr(&perf_cpu_context)->task_ctx;
251	}
252
253	/*
254	* On task ctx scheduling...
255	*
256	* When !ctx->nr_events a task context will not be scheduled. This means
257	* we can disable the scheduler hooks (for performance) without leaving
258	* pending task ctx state.
259	*
260	* This however results in two special cases:
261	*
262	* - removing the last event from a task ctx; this is relatively straight
263	* forward and is done in __perf_remove_from_context.
264	*
265	* - adding the first event to a task ctx; this is tricky because we cannot
266	* rely on ctx->is_active and therefore cannot use event_function_call().
267	* See perf_install_in_context().
268	*
269	* If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
270	*/
271
272	typedef void (event_f)(struct perf_event , struct perf_cpu_context *,
273	struct perf_event_context , void );
274
275	struct event_function_struct {
276	struct perf_event *event;
277	event_f func;
278	void *data;
279	};
280
281	static int event_function(void *info)
282	{
283	struct event_function_struct *efs = info;
284	struct perf_event *event = efs->event;
285	struct perf_event_context *ctx = event->ctx;
286	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
287	struct perf_event_context *task_ctx = cpuctx->task_ctx;
288	int ret = 0;
289
290	lockdep_assert_irqs_disabled();
291
292	perf_ctx_lock(cpuctx, task_ctx);
293	/*
294	* Since we do the IPI call without holding ctx->lock things can have
295	* changed, double check we hit the task we set out to hit.
296	*/
297	if (ctx->task) {
298	if (ctx->task != current) {
299	ret = -ESRCH;
300	goto unlock;
301	}
302
303	/*
304	* We only use event_function_call() on established contexts,
305	* and event_function() is only ever called when active (or
306	* rather, we'll have bailed in task_function_call() or the
307	* above ctx->task != current test), therefore we must have
308	* ctx->is_active here.
309	*/
310	WARN_ON_ONCE(!ctx->is_active);
311	/*
312	* And since we have ctx->is_active, cpuctx->task_ctx must
313	* match.
314	*/
315	WARN_ON_ONCE(task_ctx != ctx);
316	} else {
317	WARN_ON_ONCE(&cpuctx->ctx != ctx);
318	}
319
320	efs->func(event, cpuctx, ctx, efs->data);
321	unlock:
322	perf_ctx_unlock(cpuctx, task_ctx);
323
324	return ret;
325	}
326
327	static void event_function_call(struct perf_event event, event_f func, void data)
328	{
329	struct perf_event_context *ctx = event->ctx;
330	struct task_struct task = READ_ONCE(ctx->task); / verified in event_function */
331	struct perf_cpu_context *cpuctx;
332	struct event_function_struct efs = {
333	.event = event,
334	.func = func,
335	.data = data,
336	};
337
338	if (!event->parent) {
339	/*
340	* If this is a !child event, we must hold ctx::mutex to
341	* stabilize the event->ctx relation. See
342	* perf_event_ctx_lock().
343	*/
344	lockdep_assert_held(&ctx->mutex);
345	}
346
347	if (!task) {
348	cpu_function_call(event->cpu, event_function, &efs);
349	return;
350	}
351
352	if (task == TASK_TOMBSTONE)
353	return;
354
355	again:
356	if (!task_function_call(task, event_function, &efs))
357	return;
358
359	local_irq_disable();
360	cpuctx = this_cpu_ptr(&perf_cpu_context);
361	perf_ctx_lock(cpuctx, ctx);
362	/*
363	* Reload the task pointer, it might have been changed by
364	* a concurrent perf_event_context_sched_out().
365	*/
366	task = ctx->task;
367	if (task == TASK_TOMBSTONE)
368	goto unlock;
369	if (ctx->is_active) {
370	perf_ctx_unlock(cpuctx, ctx);
371	local_irq_enable();
372	goto again;
373	}
374	func(event, NULL, ctx, data);
375	unlock:
376	perf_ctx_unlock(cpuctx, ctx);
377	local_irq_enable();
378	}
379
380	/*
381	* Similar to event_function_call() + event_function(), but hard assumes IRQs
382	* are already disabled and we're on the right CPU.
383	*/
384	static void event_function_local(struct perf_event event, event_f func, void data)
385	{
386	struct perf_event_context *ctx = event->ctx;
387	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
388	struct task_struct *task = READ_ONCE(ctx->task);
389	struct perf_event_context *task_ctx = NULL;
390
391	lockdep_assert_irqs_disabled();
392
393	if (task) {
394	if (task == TASK_TOMBSTONE)
395	return;
396
397	task_ctx = ctx;
398	}
399
400	perf_ctx_lock(cpuctx, task_ctx);
401
402	task = ctx->task;
403	if (task == TASK_TOMBSTONE)
404	goto unlock;
405
406	if (task) {
407	/*
408	* We must be either inactive or active and the right task,
409	* otherwise we're screwed, since we cannot IPI to somewhere
410	* else.
411	*/
412	if (ctx->is_active) {
413	if (WARN_ON_ONCE(task != current))
414	goto unlock;
415
416	if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))
417	goto unlock;
418	}
419	} else {
420	WARN_ON_ONCE(&cpuctx->ctx != ctx);
421	}
422
423	func(event, cpuctx, ctx, data);
424	unlock:
425	perf_ctx_unlock(cpuctx, task_ctx);
426	}
427
428	#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP \|\
429	PERF_FLAG_FD_OUTPUT \|\
430	PERF_FLAG_PID_CGROUP \|\
431	PERF_FLAG_FD_CLOEXEC)
432
433	/*
434	* branch priv levels that need permission checks
435	*/
436	#define PERF_SAMPLE_BRANCH_PERM_PLM \
437	(PERF_SAMPLE_BRANCH_KERNEL \|\
438	PERF_SAMPLE_BRANCH_HV)
439
440	/*
441	* perf_sched_events : >0 events exist
442	*/
443
444	static void perf_sched_delayed(struct work_struct *work);
445	DEFINE_STATIC_KEY_FALSE(perf_sched_events);
446	static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
447	static DEFINE_MUTEX(perf_sched_mutex);
448	static atomic_t perf_sched_count;
449
450	static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
451
452	static atomic_t nr_mmap_events __read_mostly;
453	static atomic_t nr_comm_events __read_mostly;
454	static atomic_t nr_namespaces_events __read_mostly;
455	static atomic_t nr_task_events __read_mostly;
456	static atomic_t nr_freq_events __read_mostly;
457	static atomic_t nr_switch_events __read_mostly;
458	static atomic_t nr_ksymbol_events __read_mostly;
459	static atomic_t nr_bpf_events __read_mostly;
460	static atomic_t nr_cgroup_events __read_mostly;
461	static atomic_t nr_text_poke_events __read_mostly;
462	static atomic_t nr_build_id_events __read_mostly;
463
464	static LIST_HEAD(pmus);
465	static DEFINE_MUTEX(pmus_lock);
466	static struct srcu_struct pmus_srcu;
467	static cpumask_var_t perf_online_mask;
468	static cpumask_var_t perf_online_core_mask;
469	static cpumask_var_t perf_online_die_mask;
470	static cpumask_var_t perf_online_cluster_mask;
471	static cpumask_var_t perf_online_pkg_mask;
472	static cpumask_var_t perf_online_sys_mask;
473	static struct kmem_cache *perf_event_cache;
474
475	#ifdef CONFIG_PERF_GUEST_MEDIATED_PMU
476	static DEFINE_PER_CPU(bool, guest_ctx_loaded);
477
478	static __always_inline bool is_guest_mediated_pmu_loaded(void)
479	{
480	return __this_cpu_read(guest_ctx_loaded);
481	}
482	#else
483	static __always_inline bool is_guest_mediated_pmu_loaded(void)
484	{
485	return false;
486	}
487	#endif
488
489	/*
490	* perf event paranoia level:
491	* -1 - not paranoid at all
492	* 0 - disallow raw tracepoint access for unpriv
493	* 1 - disallow cpu events for unpriv
494	* 2 - disallow kernel profiling for unpriv
495	*/
496	int sysctl_perf_event_paranoid __read_mostly = 2;
497
498	/* Minimum for 512 kiB + 1 user control page. 'free' kiB per user. */
499	static int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024);
500
501	/*
502	* max perf event sample rate
503	*/
504	#define DEFAULT_MAX_SAMPLE_RATE 100000
505	#define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
506	#define DEFAULT_CPU_TIME_MAX_PERCENT 25
507
508	int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
509	static int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
510
511	static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
512	static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS;
513
514	static int perf_sample_allowed_ns __read_mostly =
515	DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
516
517	static void update_perf_cpu_limits(void)
518	{
519	u64 tmp = perf_sample_period_ns;
520
521	tmp *= sysctl_perf_cpu_time_max_percent;
522	tmp = div_u64(tmp, 100);
523	if (!tmp)
524	tmp = 1;
525
526	WRITE_ONCE(perf_sample_allowed_ns, tmp);
527	}
528
529	static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc);
530
531	static int perf_event_max_sample_rate_handler(const struct ctl_table *table, int write,
532	void buffer, size_t lenp, loff_t *ppos)
533	{
534	int ret;
535	int perf_cpu = sysctl_perf_cpu_time_max_percent;
536	/*
537	* If throttling is disabled don't allow the write:
538	*/
539	if (write && (perf_cpu == 100 \|\| perf_cpu == 0))
540	return -EINVAL;
541
542	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
543	if (ret \|\| !write)
544	return ret;
545
546	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
547	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
548	update_perf_cpu_limits();
549
550	return 0;
551	}
552
553	static int perf_cpu_time_max_percent_handler(const struct ctl_table *table, int write,
554	void buffer, size_t lenp, loff_t *ppos)
555	{
556	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
557
558	if (ret \|\| !write)
559	return ret;
560
561	if (sysctl_perf_cpu_time_max_percent == 100 \|\|
562	sysctl_perf_cpu_time_max_percent == 0) {
563	printk(KERN_WARNING
564	"perf: Dynamic interrupt throttling disabled, can hang your system!\n");
565	WRITE_ONCE(perf_sample_allowed_ns, 0);
566	} else {
567	update_perf_cpu_limits();
568	}
569
570	return 0;
571	}
572
573	static const struct ctl_table events_core_sysctl_table[] = {
574	/*
575	* User-space relies on this file as a feature check for
576	* perf_events being enabled. It's an ABI, do not remove!
577	*/
578	{
579	.procname = "perf_event_paranoid",
580	.data = &sysctl_perf_event_paranoid,
581	.maxlen = sizeof(sysctl_perf_event_paranoid),
582	.mode = 0644,
583	.proc_handler = proc_dointvec,
584	},
585	{
586	.procname = "perf_event_mlock_kb",
587	.data = &sysctl_perf_event_mlock,
588	.maxlen = sizeof(sysctl_perf_event_mlock),
589	.mode = 0644,
590	.proc_handler = proc_dointvec,
591	},
592	{
593	.procname = "perf_event_max_sample_rate",
594	.data = &sysctl_perf_event_sample_rate,
595	.maxlen = sizeof(sysctl_perf_event_sample_rate),
596	.mode = 0644,
597	.proc_handler = perf_event_max_sample_rate_handler,
598	.extra1 = SYSCTL_ONE,
599	},
600	{
601	.procname = "perf_cpu_time_max_percent",
602	.data = &sysctl_perf_cpu_time_max_percent,
603	.maxlen = sizeof(sysctl_perf_cpu_time_max_percent),
604	.mode = 0644,
605	.proc_handler = perf_cpu_time_max_percent_handler,
606	.extra1 = SYSCTL_ZERO,
607	.extra2 = SYSCTL_ONE_HUNDRED,
608	},
609	};
610
611	static int __init init_events_core_sysctls(void)
612	{
613	register_sysctl_init("kernel", events_core_sysctl_table);
614	return 0;
615	}
616	core_initcall(init_events_core_sysctls);
617
618
619	/*
620	* perf samples are done in some very critical code paths (NMIs).
621	* If they take too much CPU time, the system can lock up and not
622	* get any real work done. This will drop the sample rate when
623	* we detect that events are taking too long.
624	*/
625	#define NR_ACCUMULATED_SAMPLES 128
626	static DEFINE_PER_CPU(u64, running_sample_length);
627
628	static u64 __report_avg;
629	static u64 __report_allowed;
630
631	static void perf_duration_warn(struct irq_work *w)
632	{
633	printk_ratelimited(KERN_INFO
634	"perf: interrupt took too long (%lld > %lld), lowering "
635	"kernel.perf_event_max_sample_rate to %d\n",
636	__report_avg, __report_allowed,
637	sysctl_perf_event_sample_rate);
638	}
639
640	static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
641
642	void perf_sample_event_took(u64 sample_len_ns)
643	{
644	u64 max_len = READ_ONCE(perf_sample_allowed_ns);
645	u64 running_len;
646	u64 avg_len;
647	u32 max;
648
649	if (max_len == 0)
650	return;
651
652	/* Decay the counter by 1 average sample. */
653	running_len = __this_cpu_read(running_sample_length);
654	running_len -= running_len/NR_ACCUMULATED_SAMPLES;
655	running_len += sample_len_ns;
656	__this_cpu_write(running_sample_length, running_len);
657
658	/*
659	* Note: this will be biased artificially low until we have
660	* seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
661	* from having to maintain a count.
662	*/
663	avg_len = running_len/NR_ACCUMULATED_SAMPLES;
664	if (avg_len <= max_len)
665	return;
666
667	__report_avg = avg_len;
668	__report_allowed = max_len;
669
670	/*
671	* Compute a throttle threshold 25% below the current duration.
672	*/
673	avg_len += avg_len / 4;
674	max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent;
675	if (avg_len < max)
676	max /= (u32)avg_len;
677	else
678	max = 1;
679
680	WRITE_ONCE(perf_sample_allowed_ns, avg_len);
681	WRITE_ONCE(max_samples_per_tick, max);
682
683	sysctl_perf_event_sample_rate = max * HZ;
684	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
685
686	if (!irq_work_queue(&perf_duration_work)) {
687	early_printk("perf: interrupt took too long (%lld > %lld), lowering "
688	"kernel.perf_event_max_sample_rate to %d\n",
689	__report_avg, __report_allowed,
690	sysctl_perf_event_sample_rate);
691	}
692	}
693
694	static atomic64_t perf_event_id;
695
696	static void update_context_time(struct perf_event_context *ctx);
697	static u64 perf_event_time(struct perf_event *event);
698
699	void __weak perf_event_print_debug(void) { }
700
701	static inline u64 perf_clock(void)
702	{
703	return local_clock();
704	}
705
706	static inline u64 perf_event_clock(struct perf_event *event)
707	{
708	return event->clock();
709	}
710
711	/*
712	* State based event timekeeping...
713	*
714	* The basic idea is to use event->state to determine which (if any) time
715	* fields to increment with the current delta. This means we only need to
716	* update timestamps when we change state or when they are explicitly requested
717	* (read).
718	*
719	* Event groups make things a little more complicated, but not terribly so. The
720	* rules for a group are that if the group leader is OFF the entire group is
721	* OFF, irrespective of what the group member states are. This results in
722	* __perf_effective_state().
723	*
724	* A further ramification is that when a group leader flips between OFF and
725	* !OFF, we need to update all group member times.
726	*
727	*
728	* NOTE: perf_event_time() is based on the (cgroup) context time, and thus we
729	* need to make sure the relevant context time is updated before we try and
730	* update our timestamps.
731	*/
732
733	static __always_inline enum perf_event_state
734	__perf_effective_state(struct perf_event *event)
735	{
736	struct perf_event *leader = event->group_leader;
737
738	if (leader->state <= PERF_EVENT_STATE_OFF)
739	return leader->state;
740
741	return event->state;
742	}
743
744	static __always_inline void
745	__perf_update_times(struct perf_event event, u64 now, u64 enabled, u64 *running)
746	{
747	enum perf_event_state state = __perf_effective_state(event);
748	u64 delta = now - event->tstamp;
749
750	*enabled = event->total_time_enabled;
751	if (state >= PERF_EVENT_STATE_INACTIVE)
752	*enabled += delta;
753
754	*running = event->total_time_running;
755	if (state >= PERF_EVENT_STATE_ACTIVE)
756	*running += delta;
757	}
758
759	static void perf_event_update_time(struct perf_event *event)
760	{
761	u64 now = perf_event_time(event);
762
763	__perf_update_times(event, now, &event->total_time_enabled,
764	&event->total_time_running);
765	event->tstamp = now;
766	}
767
768	static void perf_event_update_sibling_time(struct perf_event *leader)
769	{
770	struct perf_event *sibling;
771
772	for_each_sibling_event(sibling, leader)
773	perf_event_update_time(sibling);
774	}
775
776	static void
777	perf_event_set_state(struct perf_event *event, enum perf_event_state state)
778	{
779	if (event->state == state)
780	return;
781
782	perf_event_update_time(event);
783	/*
784	* If a group leader gets enabled/disabled all its siblings
785	* are affected too.
786	*/
787	if ((event->state < 0) ^ (state < 0))
788	perf_event_update_sibling_time(event);
789
790	WRITE_ONCE(event->state, state);
791	}
792
793	/*
794	* UP store-release, load-acquire
795	*/
796
797	#define __store_release(ptr, val) \
798	do { \
799	barrier(); \
800	WRITE_ONCE(*(ptr), (val)); \
801	} while (0)
802
803	#define __load_acquire(ptr) \
804	({ \
805	__unqual_scalar_typeof((ptr)) ___p = READ_ONCE((ptr)); \
806	barrier(); \
807	___p; \
808	})
809
810	static bool perf_skip_pmu_ctx(struct perf_event_pmu_context *pmu_ctx,
811	enum event_type_t event_type)
812	{
813	if ((event_type & EVENT_CGROUP) && !pmu_ctx->nr_cgroups)
814	return true;
815	if ((event_type & EVENT_GUEST) &&
816	!(pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU))
817	return true;
818	return false;
819	}
820
821	#define for_each_epc(_epc, _ctx, _pmu, _event_type) \
822	list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \
823	if (perf_skip_pmu_ctx(_epc, _event_type)) \
824	continue; \
825	else if (_pmu && _epc->pmu != _pmu) \
826	continue; \
827	else
828
829	static void perf_ctx_disable(struct perf_event_context *ctx,
830	enum event_type_t event_type)
831	{
832	struct perf_event_pmu_context *pmu_ctx;
833
834	for_each_epc(pmu_ctx, ctx, NULL, event_type)
835	perf_pmu_disable(pmu_ctx->pmu);
836	}
837
838	static void perf_ctx_enable(struct perf_event_context *ctx,
839	enum event_type_t event_type)
840	{
841	struct perf_event_pmu_context *pmu_ctx;
842
843	for_each_epc(pmu_ctx, ctx, NULL, event_type)
844	perf_pmu_enable(pmu_ctx->pmu);
845	}
846
847	static void ctx_sched_out(struct perf_event_context ctx, struct pmu pmu, enum event_type_t event_type);
848	static void ctx_sched_in(struct perf_event_context ctx, struct pmu pmu, enum event_type_t event_type);
849
850	static inline void update_perf_time_ctx(struct perf_time_ctx *time, u64 now, bool adv)
851	{
852	if (adv)
853	time->time += now - time->stamp;
854	time->stamp = now;
855
856	/*
857	* The above: time' = time + (now - timestamp), can be re-arranged
858	* into: time` = now + (time - timestamp), which gives a single value
859	* offset to compute future time without locks on.
860	*
861	* See perf_event_time_now(), which can be used from NMI context where
862	* it's (obviously) not possible to acquire ctx->lock in order to read
863	* both the above values in a consistent manner.
864	*/
865	WRITE_ONCE(time->offset, time->time - time->stamp);
866	}
867
868	static_assert(offsetof(struct perf_event_context, timeguest) -
869	offsetof(struct perf_event_context, time) ==
870	sizeof(struct perf_time_ctx));
871
872	#define T_TOTAL 0
873	#define T_GUEST 1
874
875	static inline u64 __perf_event_time_ctx(struct perf_event *event,
876	struct perf_time_ctx *times)
877	{
878	u64 time = times[T_TOTAL].time;
879
880	if (event->attr.exclude_guest)
881	time -= times[T_GUEST].time;
882
883	return time;
884	}
885
886	static inline u64 __perf_event_time_ctx_now(struct perf_event *event,
887	struct perf_time_ctx *times,
888	u64 now)
889	{
890	if (is_guest_mediated_pmu_loaded() && event->attr.exclude_guest) {
891	/*
892	* (now + times[total].offset) - (now + times[guest].offset) :=
893	* times[total].offset - times[guest].offset
894	*/
895	return READ_ONCE(times[T_TOTAL].offset) - READ_ONCE(times[T_GUEST].offset);
896	}
897
898	return now + READ_ONCE(times[T_TOTAL].offset);
899	}
900
901	#ifdef CONFIG_CGROUP_PERF
902
903	static inline bool
904	perf_cgroup_match(struct perf_event *event)
905	{
906	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
907
908	/* @event doesn't care about cgroup */
909	if (!event->cgrp)
910	return true;
911
912	/* wants specific cgroup scope but @cpuctx isn't associated with any */
913	if (!cpuctx->cgrp)
914	return false;
915
916	/*
917	* Cgroup scoping is recursive. An event enabled for a cgroup is
918	* also enabled for all its descendant cgroups. If @cpuctx's
919	* cgroup is a descendant of @event's (the test covers identity
920	* case), it's a match.
921	*/
922	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
923	event->cgrp->css.cgroup);
924	}
925
926	static inline void perf_detach_cgroup(struct perf_event *event)
927	{
928	css_put(&event->cgrp->css);
929	event->cgrp = NULL;
930	}
931
932	static inline int is_cgroup_event(struct perf_event *event)
933	{
934	return event->cgrp != NULL;
935	}
936
937	static_assert(offsetof(struct perf_cgroup_info, timeguest) -
938	offsetof(struct perf_cgroup_info, time) ==
939	sizeof(struct perf_time_ctx));
940
941	static inline u64 perf_cgroup_event_time(struct perf_event *event)
942	{
943	struct perf_cgroup_info *t;
944
945	t = per_cpu_ptr(event->cgrp->info, event->cpu);
946	return __perf_event_time_ctx(event, &t->time);
947	}
948
949	static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
950	{
951	struct perf_cgroup_info *t;
952
953	t = per_cpu_ptr(event->cgrp->info, event->cpu);
954	if (!__load_acquire(&t->active))
955	return __perf_event_time_ctx(event, &t->time);
956
957	return __perf_event_time_ctx_now(event, &t->time, now);
958	}
959
960	static inline void __update_cgrp_guest_time(struct perf_cgroup_info *info, u64 now, bool adv)
961	{
962	update_perf_time_ctx(&info->timeguest, now, adv);
963	}
964
965	static inline void update_cgrp_time(struct perf_cgroup_info *info, u64 now)
966	{
967	update_perf_time_ctx(&info->time, now, true);
968	if (is_guest_mediated_pmu_loaded())
969	__update_cgrp_guest_time(info, now, true);
970	}
971
972	static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
973	{
974	struct perf_cgroup *cgrp = cpuctx->cgrp;
975	struct cgroup_subsys_state *css;
976	struct perf_cgroup_info *info;
977
978	if (cgrp) {
979	u64 now = perf_clock();
980
981	for (css = &cgrp->css; css; css = css->parent) {
982	cgrp = container_of(css, struct perf_cgroup, css);
983	info = this_cpu_ptr(cgrp->info);
984
985	update_cgrp_time(info, now);
986	if (final)
987	__store_release(&info->active, 0);
988	}
989	}
990	}
991
992	static inline void update_cgrp_time_from_event(struct perf_event *event)
993	{
994	struct perf_cgroup_info *info;
995
996	/*
997	* ensure we access cgroup data only when needed and
998	* when we know the cgroup is pinned (css_get)
999	*/
1000	if (!is_cgroup_event(event))
1001	return;
1002
1003	info = this_cpu_ptr(event->cgrp->info);
1004	/*
1005	* Do not update time when cgroup is not active
1006	*/
1007	if (info->active)
1008	update_cgrp_time(info, perf_clock());
1009	}
1010
1011	static inline void
1012	perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest)
1013	{
1014	struct perf_event_context *ctx = &cpuctx->ctx;
1015	struct perf_cgroup *cgrp = cpuctx->cgrp;
1016	struct perf_cgroup_info *info;
1017	struct cgroup_subsys_state *css;
1018
1019	/*
1020	* ctx->lock held by caller
1021	* ensure we do not access cgroup data
1022	* unless we have the cgroup pinned (css_get)
1023	*/
1024	if (!cgrp)
1025	return;
1026
1027	WARN_ON_ONCE(!ctx->nr_cgroups);
1028
1029	for (css = &cgrp->css; css; css = css->parent) {
1030	cgrp = container_of(css, struct perf_cgroup, css);
1031	info = this_cpu_ptr(cgrp->info);
1032	if (guest) {
1033	__update_cgrp_guest_time(info, ctx->time.stamp, false);
1034	} else {
1035	update_perf_time_ctx(&info->time, ctx->time.stamp, false);
1036	__store_release(&info->active, 1);
1037	}
1038	}
1039	}
1040
1041	/*
1042	* reschedule events based on the cgroup constraint of task.
1043	*/
1044	static void perf_cgroup_switch(struct task_struct *task)
1045	{
1046	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
1047	struct perf_cgroup *cgrp;
1048
1049	/*
1050	* cpuctx->cgrp is set when the first cgroup event enabled,
1051	* and is cleared when the last cgroup event disabled.
1052	*/
1053	if (READ_ONCE(cpuctx->cgrp) == NULL)
1054	return;
1055
1056	cgrp = perf_cgroup_from_task(task, NULL);
1057	if (READ_ONCE(cpuctx->cgrp) == cgrp)
1058	return;
1059
1060	guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx);
1061	/*
1062	* Re-check, could've raced vs perf_remove_from_context().
1063	*/
1064	if (READ_ONCE(cpuctx->cgrp) == NULL)
1065	return;
1066
1067	WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
1068	perf_ctx_disable(&cpuctx->ctx, EVENT_CGROUP);
1069
1070	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL\|EVENT_CGROUP);
1071	/*
1072	* must not be done before ctxswout due
1073	* to update_cgrp_time_from_cpuctx() in
1074	* ctx_sched_out()
1075	*/
1076	cpuctx->cgrp = cgrp;
1077	/*
1078	* set cgrp before ctxsw in to allow
1079	* perf_cgroup_set_timestamp() in ctx_sched_in()
1080	* to not have to pass task around
1081	*/
1082	ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL\|EVENT_CGROUP);
1083
1084	perf_ctx_enable(&cpuctx->ctx, EVENT_CGROUP);
1085	}
1086
1087	static int perf_cgroup_ensure_storage(struct perf_event *event,
1088	struct cgroup_subsys_state *css)
1089	{
1090	struct perf_cpu_context *cpuctx;
1091	struct perf_event **storage;
1092	int cpu, heap_size, ret = 0;
1093
1094	/*
1095	* Allow storage to have sufficient space for an iterator for each
1096	* possibly nested cgroup plus an iterator for events with no cgroup.
1097	*/
1098	for (heap_size = 1; css; css = css->parent)
1099	heap_size++;
1100
1101	for_each_possible_cpu(cpu) {
1102	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
1103	if (heap_size <= cpuctx->heap_size)
1104	continue;
1105
1106	storage = kmalloc_node(heap_size * sizeof(struct perf_event *),
1107	GFP_KERNEL, cpu_to_node(cpu));
1108	if (!storage) {
1109	ret = -ENOMEM;
1110	break;
1111	}
1112
1113	raw_spin_lock_irq(&cpuctx->ctx.lock);
1114	if (cpuctx->heap_size < heap_size) {
1115	swap(cpuctx->heap, storage);
1116	if (storage == cpuctx->heap_default)
1117	storage = NULL;
1118	cpuctx->heap_size = heap_size;
1119	}
1120	raw_spin_unlock_irq(&cpuctx->ctx.lock);
1121
1122	kfree(storage);
1123	}
1124
1125	return ret;
1126	}
1127
1128	static inline int perf_cgroup_connect(int fd, struct perf_event *event,
1129	struct perf_event_attr *attr,
1130	struct perf_event *group_leader)
1131	{
1132	struct perf_cgroup *cgrp;
1133	struct cgroup_subsys_state *css;
1134	CLASS(fd, f)(fd);
1135	int ret = 0;
1136
1137	if (fd_empty(f))
1138	return -EBADF;
1139
1140	css = css_tryget_online_from_dir(fd_file(f)->f_path.dentry,
1141	&perf_event_cgrp_subsys);
1142	if (IS_ERR(css))
1143	return PTR_ERR(css);
1144
1145	ret = perf_cgroup_ensure_storage(event, css);
1146	if (ret)
1147	return ret;
1148
1149	cgrp = container_of(css, struct perf_cgroup, css);
1150	event->cgrp = cgrp;
1151
1152	/*
1153	* all events in a group must monitor
1154	* the same cgroup because a task belongs
1155	* to only one perf cgroup at a time
1156	*/
1157	if (group_leader && group_leader->cgrp != cgrp) {
1158	perf_detach_cgroup(event);
1159	ret = -EINVAL;
1160	}
1161	return ret;
1162	}
1163
1164	static inline void
1165	perf_cgroup_event_enable(struct perf_event event, struct perf_event_context ctx)
1166	{
1167	struct perf_cpu_context *cpuctx;
1168
1169	if (!is_cgroup_event(event))
1170	return;
1171
1172	event->pmu_ctx->nr_cgroups++;
1173
1174	/*
1175	* Because cgroup events are always per-cpu events,
1176	* @ctx == &cpuctx->ctx.
1177	*/
1178	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1179
1180	if (ctx->nr_cgroups++)
1181	return;
1182
1183	cpuctx->cgrp = perf_cgroup_from_task(current, ctx);
1184	}
1185
1186	static inline void
1187	perf_cgroup_event_disable(struct perf_event event, struct perf_event_context ctx)
1188	{
1189	struct perf_cpu_context *cpuctx;
1190
1191	if (!is_cgroup_event(event))
1192	return;
1193
1194	event->pmu_ctx->nr_cgroups--;
1195
1196	/*
1197	* Because cgroup events are always per-cpu events,
1198	* @ctx == &cpuctx->ctx.
1199	*/
1200	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1201
1202	if (--ctx->nr_cgroups)
1203	return;
1204
1205	cpuctx->cgrp = NULL;
1206	}
1207
1208	#else /* !CONFIG_CGROUP_PERF */
1209
1210	static inline bool
1211	perf_cgroup_match(struct perf_event *event)
1212	{
1213	return true;
1214	}
1215
1216	static inline void perf_detach_cgroup(struct perf_event *event)
1217	{}
1218
1219	static inline int is_cgroup_event(struct perf_event *event)
1220	{
1221	return 0;
1222	}
1223
1224	static inline void update_cgrp_time_from_event(struct perf_event *event)
1225	{
1226	}
1227
1228	static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx,
1229	bool final)
1230	{
1231	}
1232
1233	static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
1234	struct perf_event_attr *attr,
1235	struct perf_event *group_leader)
1236	{
1237	return -EINVAL;
1238	}
1239
1240	static inline void
1241	perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest)
1242	{
1243	}
1244
1245	static inline u64 perf_cgroup_event_time(struct perf_event *event)
1246	{
1247	return 0;
1248	}
1249
1250	static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
1251	{
1252	return 0;
1253	}
1254
1255	static inline void
1256	perf_cgroup_event_enable(struct perf_event event, struct perf_event_context ctx)
1257	{
1258	}
1259
1260	static inline void
1261	perf_cgroup_event_disable(struct perf_event event, struct perf_event_context ctx)
1262	{
1263	}
1264
1265	static void perf_cgroup_switch(struct task_struct *task)
1266	{
1267	}
1268	#endif
1269
1270	/*
1271	* set default to be dependent on timer tick just
1272	* like original code
1273	*/
1274	#define PERF_CPU_HRTIMER (1000 / HZ)
1275	/*
1276	* function must be called with interrupts disabled
1277	*/
1278	static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
1279	{
1280	struct perf_cpu_pmu_context *cpc;
1281	bool rotations;
1282
1283	lockdep_assert_irqs_disabled();
1284
1285	cpc = container_of(hr, struct perf_cpu_pmu_context, hrtimer);
1286	rotations = perf_rotate_context(cpc);
1287
1288	raw_spin_lock(&cpc->hrtimer_lock);
1289	if (rotations)
1290	hrtimer_forward_now(hr, cpc->hrtimer_interval);
1291	else
1292	cpc->hrtimer_active = 0;
1293	raw_spin_unlock(&cpc->hrtimer_lock);
1294
1295	return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
1296	}
1297
1298	static void __perf_mux_hrtimer_init(struct perf_cpu_pmu_context *cpc, int cpu)
1299	{
1300	struct hrtimer *timer = &cpc->hrtimer;
1301	struct pmu *pmu = cpc->epc.pmu;
1302	u64 interval;
1303
1304	/*
1305	* check default is sane, if not set then force to
1306	* default interval (1/tick)
1307	*/
1308	interval = pmu->hrtimer_interval_ms;
1309	if (interval < 1)
1310	interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
1311
1312	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
1313
1314	raw_spin_lock_init(&cpc->hrtimer_lock);
1315	hrtimer_setup(timer, perf_mux_hrtimer_handler, CLOCK_MONOTONIC,
1316	HRTIMER_MODE_ABS_PINNED_HARD);
1317	}
1318
1319	static int perf_mux_hrtimer_restart(struct perf_cpu_pmu_context *cpc)
1320	{
1321	struct hrtimer *timer = &cpc->hrtimer;
1322	unsigned long flags;
1323
1324	raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags);
1325	if (!cpc->hrtimer_active) {
1326	cpc->hrtimer_active = 1;
1327	hrtimer_forward_now(timer, cpc->hrtimer_interval);
1328	hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
1329	}
1330	raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags);
1331
1332	return 0;
1333	}
1334
1335	static int perf_mux_hrtimer_restart_ipi(void *arg)
1336	{
1337	return perf_mux_hrtimer_restart(arg);
1338	}
1339
1340	static __always_inline struct perf_cpu_pmu_context this_cpc(struct pmu pmu)
1341	{
1342	return *this_cpu_ptr(pmu->cpu_pmu_context);
1343	}
1344
1345	void perf_pmu_disable(struct pmu *pmu)
1346	{
1347	int *count = &this_cpc(pmu)->pmu_disable_count;
1348	if (!(*count)++)
1349	pmu->pmu_disable(pmu);
1350	}
1351
1352	void perf_pmu_enable(struct pmu *pmu)
1353	{
1354	int *count = &this_cpc(pmu)->pmu_disable_count;
1355	if (!--(*count))
1356	pmu->pmu_enable(pmu);
1357	}
1358
1359	static void perf_assert_pmu_disabled(struct pmu *pmu)
1360	{
1361	int *count = &this_cpc(pmu)->pmu_disable_count;
1362	WARN_ON_ONCE(*count == 0);
1363	}
1364
1365	static inline void perf_pmu_read(struct perf_event *event)
1366	{
1367	if (event->state == PERF_EVENT_STATE_ACTIVE)
1368	event->pmu->read(event);
1369	}
1370
1371	static void get_ctx(struct perf_event_context *ctx)
1372	{
1373	refcount_inc(&ctx->refcount);
1374	}
1375
1376	static void free_ctx(struct rcu_head *head)
1377	{
1378	struct perf_event_context *ctx;
1379
1380	ctx = container_of(head, struct perf_event_context, rcu_head);
1381	kfree(ctx);
1382	}
1383
1384	static void put_ctx(struct perf_event_context *ctx)
1385	{
1386	if (refcount_dec_and_test(&ctx->refcount)) {
1387	if (ctx->parent_ctx)
1388	put_ctx(ctx->parent_ctx);
1389	if (ctx->task && ctx->task != TASK_TOMBSTONE)
1390	put_task_struct(ctx->task);
1391	call_rcu(&ctx->rcu_head, free_ctx);
1392	} else {
1393	smp_mb__after_atomic(); /* pairs with wait_var_event() */
1394	if (ctx->task == TASK_TOMBSTONE)
1395	wake_up_var(&ctx->refcount);
1396	}
1397	}
1398
1399	/*
1400	* Because of perf_event::ctx migration in sys_perf_event_open::move_group and
1401	* perf_pmu_migrate_context() we need some magic.
1402	*
1403	* Those places that change perf_event::ctx will hold both
1404	* perf_event_ctx::mutex of the 'old' and 'new' ctx value.
1405	*
1406	* Lock ordering is by mutex address. There are two other sites where
1407	* perf_event_context::mutex nests and those are:
1408	*
1409	* - perf_event_exit_task_context() [ child , 0 ]
1410	* perf_event_exit_event()
1411	* put_event() [ parent, 1 ]
1412	*
1413	* - perf_event_init_context() [ parent, 0 ]
1414	* inherit_task_group()
1415	* inherit_group()
1416	* inherit_event()
1417	* perf_event_alloc()
1418	* perf_init_event()
1419	* perf_try_init_event() [ child , 1 ]
1420	*
1421	* While it appears there is an obvious deadlock here -- the parent and child
1422	* nesting levels are inverted between the two. This is in fact safe because
1423	* life-time rules separate them. That is an exiting task cannot fork, and a
1424	* spawning task cannot (yet) exit.
1425	*
1426	* But remember that these are parent<->child context relations, and
1427	* migration does not affect children, therefore these two orderings should not
1428	* interact.
1429	*
1430	* The change in perf_event::ctx does not affect children (as claimed above)
1431	* because the sys_perf_event_open() case will install a new event and break
1432	* the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1433	* concerned with cpuctx and that doesn't have children.
1434	*
1435	* The places that change perf_event::ctx will issue:
1436	*
1437	* perf_remove_from_context();
1438	* synchronize_rcu();
1439	* perf_install_in_context();
1440	*
1441	* to affect the change. The remove_from_context() + synchronize_rcu() should
1442	* quiesce the event, after which we can install it in the new location. This
1443	* means that only external vectors (perf_fops, prctl) can perturb the event
1444	* while in transit. Therefore all such accessors should also acquire
1445	* perf_event_context::mutex to serialize against this.
1446	*
1447	* However; because event->ctx can change while we're waiting to acquire
1448	* ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1449	* function.
1450	*
1451	* Lock order:
1452	* exec_update_lock
1453	* task_struct::perf_event_mutex
1454	* perf_event_context::mutex
1455	* perf_event::child_mutex;
1456	* perf_event_context::lock
1457	* mmap_lock
1458	* perf_event::mmap_mutex
1459	* perf_buffer::aux_mutex
1460	* perf_addr_filters_head::lock
1461	*
1462	* cpu_hotplug_lock
1463	* pmus_lock
1464	* cpuctx->mutex / perf_event_context::mutex
1465	*/
1466	static struct perf_event_context *
1467	perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
1468	{
1469	struct perf_event_context *ctx;
1470
1471	again:
1472	rcu_read_lock();
1473	ctx = READ_ONCE(event->ctx);
1474	if (!refcount_inc_not_zero(&ctx->refcount)) {
1475	rcu_read_unlock();
1476	goto again;
1477	}
1478	rcu_read_unlock();
1479
1480	mutex_lock_nested(&ctx->mutex, nesting);
1481	if (event->ctx != ctx) {
1482	mutex_unlock(&ctx->mutex);
1483	put_ctx(ctx);
1484	goto again;
1485	}
1486
1487	return ctx;
1488	}
1489
1490	static inline struct perf_event_context *
1491	perf_event_ctx_lock(struct perf_event *event)
1492	{
1493	return perf_event_ctx_lock_nested(event, 0);
1494	}
1495
1496	static void perf_event_ctx_unlock(struct perf_event *event,
1497	struct perf_event_context *ctx)
1498	{
1499	mutex_unlock(&ctx->mutex);
1500	put_ctx(ctx);
1501	}
1502
1503	/*
1504	* This must be done under the ctx->lock, such as to serialize against
1505	* context_equiv(), therefore we cannot call put_ctx() since that might end up
1506	* calling scheduler related locks and ctx->lock nests inside those.
1507	*/
1508	static __must_check struct perf_event_context *
1509	unclone_ctx(struct perf_event_context *ctx)
1510	{
1511	struct perf_event_context *parent_ctx = ctx->parent_ctx;
1512
1513	lockdep_assert_held(&ctx->lock);
1514
1515	if (parent_ctx)
1516	ctx->parent_ctx = NULL;
1517	ctx->generation++;
1518
1519	return parent_ctx;
1520	}
1521
1522	static u32 perf_event_pid_type(struct perf_event event, struct task_struct p,
1523	enum pid_type type)
1524	{
1525	u32 nr;
1526	/*
1527	* only top level events have the pid namespace they were created in
1528	*/
1529	if (event->parent)
1530	event = event->parent;
1531
1532	nr = __task_pid_nr_ns(p, type, event->ns);
1533	/* avoid -1 if it is idle thread or runs in another ns */
1534	if (!nr && !pid_alive(p))
1535	nr = -1;
1536	return nr;
1537	}
1538
1539	static u32 perf_event_pid(struct perf_event event, struct task_struct p)
1540	{
1541	return perf_event_pid_type(event, p, PIDTYPE_TGID);
1542	}
1543
1544	static u32 perf_event_tid(struct perf_event event, struct task_struct p)
1545	{
1546	return perf_event_pid_type(event, p, PIDTYPE_PID);
1547	}
1548
1549	/*
1550	* If we inherit events we want to return the parent event id
1551	* to userspace.
1552	*/
1553	static u64 primary_event_id(struct perf_event *event)
1554	{
1555	u64 id = event->id;
1556
1557	if (event->parent)
1558	id = event->parent->id;
1559
1560	return id;
1561	}
1562
1563	/*
1564	* Get the perf_event_context for a task and lock it.
1565	*
1566	* This has to cope with the fact that until it is locked,
1567	* the context could get moved to another task.
1568	*/
1569	static struct perf_event_context *
1570	perf_lock_task_context(struct task_struct task, unsigned long flags)
1571	{
1572	struct perf_event_context *ctx;
1573
1574	retry:
1575	/*
1576	* One of the few rules of preemptible RCU is that one cannot do
1577	* rcu_read_unlock() while holding a scheduler (or nested) lock when
1578	* part of the read side critical section was irqs-enabled -- see
1579	* rcu_read_unlock_special().
1580	*
1581	* Since ctx->lock nests under rq->lock we must ensure the entire read
1582	* side critical section has interrupts disabled.
1583	*/
1584	local_irq_save(*flags);
1585	rcu_read_lock();
1586	ctx = rcu_dereference(task->perf_event_ctxp);
1587	if (ctx) {
1588	/*
1589	* If this context is a clone of another, it might
1590	* get swapped for another underneath us by
1591	* perf_event_task_sched_out, though the
1592	* rcu_read_lock() protects us from any context
1593	* getting freed. Lock the context and check if it
1594	* got swapped before we could get the lock, and retry
1595	* if so. If we locked the right context, then it
1596	* can't get swapped on us any more.
1597	*/
1598	raw_spin_lock(&ctx->lock);
1599	if (ctx != rcu_dereference(task->perf_event_ctxp)) {
1600	raw_spin_unlock(&ctx->lock);
1601	rcu_read_unlock();
1602	local_irq_restore(*flags);
1603	goto retry;
1604	}
1605
1606	if (ctx->task == TASK_TOMBSTONE \|\|
1607	!refcount_inc_not_zero(&ctx->refcount)) {
1608	raw_spin_unlock(&ctx->lock);
1609	ctx = NULL;
1610	} else {
1611	WARN_ON_ONCE(ctx->task != task);
1612	}
1613	}
1614	rcu_read_unlock();
1615	if (!ctx)
1616	local_irq_restore(*flags);
1617	return ctx;
1618	}
1619
1620	/*
1621	* Get the context for a task and increment its pin_count so it
1622	* can't get swapped to another task. This also increments its
1623	* reference count so that the context can't get freed.
1624	*/
1625	static struct perf_event_context *
1626	perf_pin_task_context(struct task_struct *task)
1627	{
1628	struct perf_event_context *ctx;
1629	unsigned long flags;
1630
1631	ctx = perf_lock_task_context(task, &flags);
1632	if (ctx) {
1633	++ctx->pin_count;
1634	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1635	}
1636	return ctx;
1637	}
1638
1639	static void perf_unpin_context(struct perf_event_context *ctx)
1640	{
1641	unsigned long flags;
1642
1643	raw_spin_lock_irqsave(&ctx->lock, flags);
1644	--ctx->pin_count;
1645	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1646	}
1647
1648	/*
1649	* Update the record of the current time in a context.
1650	*/
1651	static void __update_context_time(struct perf_event_context *ctx, bool adv)
1652	{
1653	lockdep_assert_held(&ctx->lock);
1654
1655	update_perf_time_ctx(&ctx->time, perf_clock(), adv);
1656	}
1657
1658	static void __update_context_guest_time(struct perf_event_context *ctx, bool adv)
1659	{
1660	lockdep_assert_held(&ctx->lock);
1661
1662	/* must be called after __update_context_time(); */
1663	update_perf_time_ctx(&ctx->timeguest, ctx->time.stamp, adv);
1664	}
1665
1666	static void update_context_time(struct perf_event_context *ctx)
1667	{
1668	__update_context_time(ctx, true);
1669	if (is_guest_mediated_pmu_loaded())
1670	__update_context_guest_time(ctx, true);
1671	}
1672
1673	static u64 perf_event_time(struct perf_event *event)
1674	{
1675	struct perf_event_context *ctx = event->ctx;
1676
1677	if (unlikely(!ctx))
1678	return 0;
1679
1680	if (is_cgroup_event(event))
1681	return perf_cgroup_event_time(event);
1682
1683	return __perf_event_time_ctx(event, &ctx->time);
1684	}
1685
1686	static u64 perf_event_time_now(struct perf_event *event, u64 now)
1687	{
1688	struct perf_event_context *ctx = event->ctx;
1689
1690	if (unlikely(!ctx))
1691	return 0;
1692
1693	if (is_cgroup_event(event))
1694	return perf_cgroup_event_time_now(event, now);
1695
1696	if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
1697	return __perf_event_time_ctx(event, &ctx->time);
1698
1699	return __perf_event_time_ctx_now(event, &ctx->time, now);
1700	}
1701
1702	static enum event_type_t get_event_type(struct perf_event *event)
1703	{
1704	struct perf_event_context *ctx = event->ctx;
1705	enum event_type_t event_type;
1706
1707	lockdep_assert_held(&ctx->lock);
1708
1709	/*
1710	* It's 'group type', really, because if our group leader is
1711	* pinned, so are we.
1712	*/
1713	if (event->group_leader != event)
1714	event = event->group_leader;
1715
1716	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;
1717	if (!ctx->task)
1718	event_type \|= EVENT_CPU;
1719
1720	return event_type;
1721	}
1722
1723	/*
1724	* Helper function to initialize event group nodes.
1725	*/
1726	static void init_event_group(struct perf_event *event)
1727	{
1728	RB_CLEAR_NODE(&event->group_node);
1729	event->group_index = 0;
1730	}
1731
1732	/*
1733	* Extract pinned or flexible groups from the context
1734	* based on event attrs bits.
1735	*/
1736	static struct perf_event_groups *
1737	get_event_groups(struct perf_event event, struct perf_event_context ctx)
1738	{
1739	if (event->attr.pinned)
1740	return &ctx->pinned_groups;
1741	else
1742	return &ctx->flexible_groups;
1743	}
1744
1745	/*
1746	* Helper function to initializes perf_event_group trees.
1747	*/
1748	static void perf_event_groups_init(struct perf_event_groups *groups)
1749	{
1750	groups->tree = RB_ROOT;
1751	groups->index = 0;
1752	}
1753
1754	static inline struct cgroup event_cgroup(const struct perf_event event)
1755	{
1756	struct cgroup *cgroup = NULL;
1757
1758	#ifdef CONFIG_CGROUP_PERF
1759	if (event->cgrp)
1760	cgroup = event->cgrp->css.cgroup;
1761	#endif
1762
1763	return cgroup;
1764	}
1765
1766	/*
1767	* Compare function for event groups;
1768	*
1769	* Implements complex key that first sorts by CPU and then by virtual index
1770	* which provides ordering when rotating groups for the same CPU.
1771	*/
1772	static __always_inline int
1773	perf_event_groups_cmp(const int left_cpu, const struct pmu *left_pmu,
1774	const struct cgroup *left_cgroup, const u64 left_group_index,
1775	const struct perf_event *right)
1776	{
1777	if (left_cpu < right->cpu)
1778	return -1;
1779	if (left_cpu > right->cpu)
1780	return 1;
1781
1782	if (left_pmu) {
1783	if (left_pmu < right->pmu_ctx->pmu)
1784	return -1;
1785	if (left_pmu > right->pmu_ctx->pmu)
1786	return 1;
1787	}
1788
1789	#ifdef CONFIG_CGROUP_PERF
1790	{
1791	const struct cgroup *right_cgroup = event_cgroup(right);
1792
1793	if (left_cgroup != right_cgroup) {
1794	if (!left_cgroup) {
1795	/*
1796	* Left has no cgroup but right does, no
1797	* cgroups come first.
1798	*/
1799	return -1;
1800	}
1801	if (!right_cgroup) {
1802	/*
1803	* Right has no cgroup but left does, no
1804	* cgroups come first.
1805	*/
1806	return 1;
1807	}
1808	/* Two dissimilar cgroups, order by id. */
1809	if (cgroup_id(left_cgroup) < cgroup_id(right_cgroup))
1810	return -1;
1811
1812	return 1;
1813	}
1814	}
1815	#endif
1816
1817	if (left_group_index < right->group_index)
1818	return -1;
1819	if (left_group_index > right->group_index)
1820	return 1;
1821
1822	return 0;
1823	}
1824
1825	#define __node_2_pe(node) \
1826	rb_entry((node), struct perf_event, group_node)
1827
1828	static inline bool __group_less(struct rb_node a, const struct rb_node b)
1829	{
1830	struct perf_event *e = __node_2_pe(a);
1831	return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e),
1832	e->group_index, __node_2_pe(b)) < 0;
1833	}
1834
1835	struct __group_key {
1836	int cpu;
1837	struct pmu *pmu;
1838	struct cgroup *cgroup;
1839	};
1840
1841	static inline int __group_cmp(const void key, const struct rb_node node)
1842	{
1843	const struct __group_key *a = key;
1844	const struct perf_event *b = __node_2_pe(node);
1845
1846	/* partial/subtree match: @cpu, @pmu, @cgroup; ignore: @group_index */
1847	return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b);
1848	}
1849
1850	static inline int
1851	__group_cmp_ignore_cgroup(const void key, const struct rb_node node)
1852	{
1853	const struct __group_key *a = key;
1854	const struct perf_event *b = __node_2_pe(node);
1855
1856	/* partial/subtree match: @cpu, @pmu, ignore: @cgroup, @group_index */
1857	return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b),
1858	b->group_index, b);
1859	}
1860
1861	/*
1862	* Insert @event into @groups' tree; using
1863	* {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
1864	* as key. This places it last inside the {cpu,pmu,cgroup} subtree.
1865	*/
1866	static void
1867	perf_event_groups_insert(struct perf_event_groups *groups,
1868	struct perf_event *event)
1869	{
1870	event->group_index = ++groups->index;
1871
1872	rb_add(&event->group_node, &groups->tree, __group_less);
1873	}
1874
1875	/*
1876	* Helper function to insert event into the pinned or flexible groups.
1877	*/
1878	static void
1879	add_event_to_groups(struct perf_event event, struct perf_event_context ctx)
1880	{
1881	struct perf_event_groups *groups;
1882
1883	groups = get_event_groups(event, ctx);
1884	perf_event_groups_insert(groups, event);
1885	}
1886
1887	/*
1888	* Delete a group from a tree.
1889	*/
1890	static void
1891	perf_event_groups_delete(struct perf_event_groups *groups,
1892	struct perf_event *event)
1893	{
1894	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) \|\|
1895	RB_EMPTY_ROOT(&groups->tree));
1896
1897	rb_erase(&event->group_node, &groups->tree);
1898	init_event_group(event);
1899	}
1900
1901	/*
1902	* Helper function to delete event from its groups.
1903	*/
1904	static void
1905	del_event_from_groups(struct perf_event event, struct perf_event_context ctx)
1906	{
1907	struct perf_event_groups *groups;
1908
1909	groups = get_event_groups(event, ctx);
1910	perf_event_groups_delete(groups, event);
1911	}
1912
1913	/*
1914	* Get the leftmost event in the {cpu,pmu,cgroup} subtree.
1915	*/
1916	static struct perf_event *
1917	perf_event_groups_first(struct perf_event_groups *groups, int cpu,
1918	struct pmu pmu, struct cgroup cgrp)
1919	{
1920	struct __group_key key = {
1921	.cpu = cpu,
1922	.pmu = pmu,
1923	.cgroup = cgrp,
1924	};
1925	struct rb_node *node;
1926
1927	node = rb_find_first(&key, &groups->tree, __group_cmp);
1928	if (node)
1929	return __node_2_pe(node);
1930
1931	return NULL;
1932	}
1933
1934	static struct perf_event *
1935	perf_event_groups_next(struct perf_event event, struct pmu pmu)
1936	{
1937	struct __group_key key = {
1938	.cpu = event->cpu,
1939	.pmu = pmu,
1940	.cgroup = event_cgroup(event),
1941	};
1942	struct rb_node *next;
1943
1944	next = rb_next_match(&key, &event->group_node, __group_cmp);
1945	if (next)
1946	return __node_2_pe(next);
1947
1948	return NULL;
1949	}
1950
1951	#define perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu) \
1952	for (event = perf_event_groups_first(groups, cpu, pmu, NULL); \
1953	event; event = perf_event_groups_next(event, pmu))
1954
1955	/*
1956	* Iterate through the whole groups tree.
1957	*/
1958	#define perf_event_groups_for_each(event, groups) \
1959	for (event = rb_entry_safe(rb_first(&((groups)->tree)), \
1960	typeof(*event), group_node); event; \
1961	event = rb_entry_safe(rb_next(&event->group_node), \
1962	typeof(*event), group_node))
1963
1964	/*
1965	* Does the event attribute request inherit with PERF_SAMPLE_READ
1966	*/
1967	static inline bool has_inherit_and_sample_read(struct perf_event_attr *attr)
1968	{
1969	return attr->inherit && (attr->sample_type & PERF_SAMPLE_READ);
1970	}
1971
1972	/*
1973	* Add an event from the lists for its context.
1974	* Must be called with ctx->mutex and ctx->lock held.
1975	*/
1976	static void
1977	list_add_event(struct perf_event event, struct perf_event_context ctx)
1978	{
1979	lockdep_assert_held(&ctx->lock);
1980
1981	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1982	event->attach_state \|= PERF_ATTACH_CONTEXT;
1983
1984	event->tstamp = perf_event_time(event);
1985
1986	/*
1987	* If we're a stand alone event or group leader, we go to the context
1988	* list, group events are kept attached to the group so that
1989	* perf_group_detach can, at all times, locate all siblings.
1990	*/
1991	if (event->group_leader == event) {
1992	event->group_caps = event->event_caps;
1993	add_event_to_groups(event, ctx);
1994	}
1995
1996	list_add_rcu(&event->event_entry, &ctx->event_list);
1997	ctx->nr_events++;
1998	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
1999	ctx->nr_user++;
2000	if (event->attr.inherit_stat)
2001	ctx->nr_stat++;
2002	if (has_inherit_and_sample_read(&event->attr))
2003	local_inc(&ctx->nr_no_switch_fast);
2004
2005	if (event->state > PERF_EVENT_STATE_OFF)
2006	perf_cgroup_event_enable(event, ctx);
2007
2008	ctx->generation++;
2009	event->pmu_ctx->nr_events++;
2010	}
2011
2012	/*
2013	* Initialize event state based on the perf_event_attr::disabled.
2014	*/
2015	static inline void perf_event__state_init(struct perf_event *event)
2016	{
2017	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
2018	PERF_EVENT_STATE_INACTIVE;
2019	}
2020
2021	static int __perf_event_read_size(u64 read_format, int nr_siblings)
2022	{
2023	int entry = sizeof(u64); /* value */
2024	int size = 0;
2025	int nr = 1;
2026
2027	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2028	size += sizeof(u64);
2029
2030	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2031	size += sizeof(u64);
2032
2033	if (read_format & PERF_FORMAT_ID)
2034	entry += sizeof(u64);
2035
2036	if (read_format & PERF_FORMAT_LOST)
2037	entry += sizeof(u64);
2038
2039	if (read_format & PERF_FORMAT_GROUP) {
2040	nr += nr_siblings;
2041	size += sizeof(u64);
2042	}
2043
2044	/*
2045	* Since perf_event_validate_size() limits this to 16k and inhibits
2046	* adding more siblings, this will never overflow.
2047	*/
2048	return size + nr * entry;
2049	}
2050
2051	static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
2052	{
2053	struct perf_sample_data *data;
2054	u16 size = 0;
2055
2056	if (sample_type & PERF_SAMPLE_IP)
2057	size += sizeof(data->ip);
2058
2059	if (sample_type & PERF_SAMPLE_ADDR)
2060	size += sizeof(data->addr);
2061
2062	if (sample_type & PERF_SAMPLE_PERIOD)
2063	size += sizeof(data->period);
2064
2065	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
2066	size += sizeof(data->weight.full);
2067
2068	if (sample_type & PERF_SAMPLE_READ)
2069	size += event->read_size;
2070
2071	if (sample_type & PERF_SAMPLE_DATA_SRC)
2072	size += sizeof(data->data_src.val);
2073
2074	if (sample_type & PERF_SAMPLE_TRANSACTION)
2075	size += sizeof(data->txn);
2076
2077	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
2078	size += sizeof(data->phys_addr);
2079
2080	if (sample_type & PERF_SAMPLE_CGROUP)
2081	size += sizeof(data->cgroup);
2082
2083	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
2084	size += sizeof(data->data_page_size);
2085
2086	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
2087	size += sizeof(data->code_page_size);
2088
2089	event->header_size = size;
2090	}
2091
2092	/*
2093	* Called at perf_event creation and when events are attached/detached from a
2094	* group.
2095	*/
2096	static void perf_event__header_size(struct perf_event *event)
2097	{
2098	event->read_size =
2099	__perf_event_read_size(event->attr.read_format,
2100	event->group_leader->nr_siblings);
2101	__perf_event_header_size(event, event->attr.sample_type);
2102	}
2103
2104	static void perf_event__id_header_size(struct perf_event *event)
2105	{
2106	struct perf_sample_data *data;
2107	u64 sample_type = event->attr.sample_type;
2108	u16 size = 0;
2109
2110	if (sample_type & PERF_SAMPLE_TID)
2111	size += sizeof(data->tid_entry);
2112
2113	if (sample_type & PERF_SAMPLE_TIME)
2114	size += sizeof(data->time);
2115
2116	if (sample_type & PERF_SAMPLE_IDENTIFIER)
2117	size += sizeof(data->id);
2118
2119	if (sample_type & PERF_SAMPLE_ID)
2120	size += sizeof(data->id);
2121
2122	if (sample_type & PERF_SAMPLE_STREAM_ID)
2123	size += sizeof(data->stream_id);
2124
2125	if (sample_type & PERF_SAMPLE_CPU)
2126	size += sizeof(data->cpu_entry);
2127
2128	event->id_header_size = size;
2129	}
2130
2131	/*
2132	* Check that adding an event to the group does not result in anybody
2133	* overflowing the 64k event limit imposed by the output buffer.
2134	*
2135	* Specifically, check that the read_size for the event does not exceed 16k,
2136	* read_size being the one term that grows with groups size. Since read_size
2137	* depends on per-event read_format, also (re)check the existing events.
2138	*
2139	* This leaves 48k for the constant size fields and things like callchains,
2140	* branch stacks and register sets.
2141	*/
2142	static bool perf_event_validate_size(struct perf_event *event)
2143	{
2144	struct perf_event sibling, group_leader = event->group_leader;
2145
2146	if (__perf_event_read_size(event->attr.read_format,
2147	group_leader->nr_siblings + 1) > 16*1024)
2148	return false;
2149
2150	if (__perf_event_read_size(group_leader->attr.read_format,
2151	group_leader->nr_siblings + 1) > 16*1024)
2152	return false;
2153
2154	/*
2155	* When creating a new group leader, group_leader->ctx is initialized
2156	* after the size has been validated, but we cannot safely use
2157	* for_each_sibling_event() until group_leader->ctx is set. A new group
2158	* leader cannot have any siblings yet, so we can safely skip checking
2159	* the non-existent siblings.
2160	*/
2161	if (event == group_leader)
2162	return true;
2163
2164	for_each_sibling_event(sibling, group_leader) {
2165	if (__perf_event_read_size(sibling->attr.read_format,
2166	group_leader->nr_siblings + 1) > 16*1024)
2167	return false;
2168	}
2169
2170	return true;
2171	}
2172
2173	static void perf_group_attach(struct perf_event *event)
2174	{
2175	struct perf_event group_leader = event->group_leader, pos;
2176
2177	lockdep_assert_held(&event->ctx->lock);
2178
2179	/*
2180	* We can have double attach due to group movement (move_group) in
2181	* perf_event_open().
2182	*/
2183	if (event->attach_state & PERF_ATTACH_GROUP)
2184	return;
2185
2186	event->attach_state \|= PERF_ATTACH_GROUP;
2187
2188	if (group_leader == event)
2189	return;
2190
2191	WARN_ON_ONCE(group_leader->ctx != event->ctx);
2192
2193	group_leader->group_caps &= event->event_caps;
2194
2195	list_add_tail(&event->sibling_list, &group_leader->sibling_list);
2196	group_leader->nr_siblings++;
2197	group_leader->group_generation++;
2198
2199	perf_event__header_size(group_leader);
2200
2201	for_each_sibling_event(pos, group_leader)
2202	perf_event__header_size(pos);
2203	}
2204
2205	/*
2206	* Remove an event from the lists for its context.
2207	* Must be called with ctx->mutex and ctx->lock held.
2208	*/
2209	static void
2210	list_del_event(struct perf_event event, struct perf_event_context ctx)
2211	{
2212	WARN_ON_ONCE(event->ctx != ctx);
2213	lockdep_assert_held(&ctx->lock);
2214
2215	/*
2216	* We can have double detach due to exit/hot-unplug + close.
2217	*/
2218	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
2219	return;
2220
2221	event->attach_state &= ~PERF_ATTACH_CONTEXT;
2222
2223	ctx->nr_events--;
2224	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
2225	ctx->nr_user--;
2226	if (event->attr.inherit_stat)
2227	ctx->nr_stat--;
2228	if (has_inherit_and_sample_read(&event->attr))
2229	local_dec(&ctx->nr_no_switch_fast);
2230
2231	list_del_rcu(&event->event_entry);
2232
2233	if (event->group_leader == event)
2234	del_event_from_groups(event, ctx);
2235
2236	ctx->generation++;
2237	event->pmu_ctx->nr_events--;
2238	}
2239
2240	static int
2241	perf_aux_output_match(struct perf_event event, struct perf_event aux_event)
2242	{
2243	if (!has_aux(aux_event))
2244	return 0;
2245
2246	if (!event->pmu->aux_output_match)
2247	return 0;
2248
2249	return event->pmu->aux_output_match(aux_event);
2250	}
2251
2252	static void put_event(struct perf_event *event);
2253	static void __event_disable(struct perf_event *event,
2254	struct perf_event_context *ctx,
2255	enum perf_event_state state);
2256
2257	static void perf_put_aux_event(struct perf_event *event)
2258	{
2259	struct perf_event_context *ctx = event->ctx;
2260	struct perf_event *iter;
2261
2262	/*
2263	* If event uses aux_event tear down the link
2264	*/
2265	if (event->aux_event) {
2266	iter = event->aux_event;
2267	event->aux_event = NULL;
2268	put_event(iter);
2269	return;
2270	}
2271
2272	/*
2273	* If the event is an aux_event, tear down all links to
2274	* it from other events.
2275	*/
2276	for_each_sibling_event(iter, event) {
2277	if (iter->aux_event != event)
2278	continue;
2279
2280	iter->aux_event = NULL;
2281	put_event(event);
2282
2283	/*
2284	* If it's ACTIVE, schedule it out and put it into ERROR
2285	* state so that we don't try to schedule it again. Note
2286	* that perf_event_enable() will clear the ERROR status.
2287	*/
2288	__event_disable(iter, ctx, PERF_EVENT_STATE_ERROR);
2289	}
2290	}
2291
2292	static bool perf_need_aux_event(struct perf_event *event)
2293	{
2294	return event->attr.aux_output \|\| has_aux_action(event);
2295	}
2296
2297	static int perf_get_aux_event(struct perf_event *event,
2298	struct perf_event *group_leader)
2299	{
2300	/*
2301	* Our group leader must be an aux event if we want to be
2302	* an aux_output. This way, the aux event will precede its
2303	* aux_output events in the group, and therefore will always
2304	* schedule first.
2305	*/
2306	if (!group_leader)
2307	return 0;
2308
2309	/*
2310	* aux_output and aux_sample_size are mutually exclusive.
2311	*/
2312	if (event->attr.aux_output && event->attr.aux_sample_size)
2313	return 0;
2314
2315	if (event->attr.aux_output &&
2316	!perf_aux_output_match(event, group_leader))
2317	return 0;
2318
2319	if ((event->attr.aux_pause \|\| event->attr.aux_resume) &&
2320	!(group_leader->pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE))
2321	return 0;
2322
2323	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)
2324	return 0;
2325
2326	if (!atomic_long_inc_not_zero(&group_leader->refcount))
2327	return 0;
2328
2329	/*
2330	* Link aux_outputs to their aux event; this is undone in
2331	* perf_group_detach() by perf_put_aux_event(). When the
2332	* group in torn down, the aux_output events loose their
2333	* link to the aux_event and can't schedule any more.
2334	*/
2335	event->aux_event = group_leader;
2336
2337	return 1;
2338	}
2339
2340	static inline struct list_head get_event_list(struct perf_event event)
2341	{
2342	return event->attr.pinned ? &event->pmu_ctx->pinned_active :
2343	&event->pmu_ctx->flexible_active;
2344	}
2345
2346	/* @sibling must already be unlinked from its old leader's sibling_list. */
2347	static void perf_promote_sibling_to_leader(struct perf_event *sibling,
2348	struct perf_event_context *ctx,
2349	int group_caps)
2350	{
2351	/*
2352	* Events that have PERF_EV_CAP_SIBLING require being part of
2353	* a group and cannot exist on their own, schedule them out
2354	* and move them into the ERROR state. Also see
2355	* _perf_event_enable(), it will not be able to recover this
2356	* ERROR state.
2357	*/
2358	if (sibling->event_caps & PERF_EV_CAP_SIBLING)
2359	__event_disable(sibling, ctx, PERF_EVENT_STATE_ERROR);
2360
2361	sibling->group_leader = sibling;
2362	sibling->group_caps = group_caps;
2363
2364	if (sibling->attach_state & PERF_ATTACH_CONTEXT) {
2365	add_event_to_groups(sibling, ctx);
2366
2367	if (sibling->state == PERF_EVENT_STATE_ACTIVE)
2368	list_add_tail(&sibling->active_list, get_event_list(sibling));
2369	}
2370
2371	perf_event__header_size(sibling);
2372	}
2373
2374	static void perf_group_detach(struct perf_event *event)
2375	{
2376	struct perf_event *leader = event->group_leader;
2377	struct perf_event sibling, tmp;
2378	struct perf_event_context *ctx = event->ctx;
2379
2380	lockdep_assert_held(&ctx->lock);
2381
2382	/*
2383	* We can have double detach due to exit/hot-unplug + close.
2384	*/
2385	if (!(event->attach_state & PERF_ATTACH_GROUP))
2386	return;
2387
2388	event->attach_state &= ~PERF_ATTACH_GROUP;
2389
2390	perf_put_aux_event(event);
2391
2392	/*
2393	* If this is a sibling, remove it from its group.
2394	*/
2395	if (leader != event) {
2396	list_del_init(&event->sibling_list);
2397	leader->nr_siblings--;
2398	leader->group_generation++;
2399	perf_promote_sibling_to_leader(event, ctx, event->event_caps);
2400	goto out;
2401	}
2402
2403	/*
2404	* If this was a group event with sibling events then
2405	* upgrade the siblings to singleton events by adding them
2406	* to whatever list we are on.
2407	*/
2408	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {
2409	list_del_init(&sibling->sibling_list);
2410
2411	/* Inherit group flags from the previous leader */
2412	perf_promote_sibling_to_leader(sibling, ctx, event->group_caps);
2413
2414	WARN_ON_ONCE(sibling->ctx != event->ctx);
2415	}
2416	event->nr_siblings = 0;
2417
2418	out:
2419	for_each_sibling_event(tmp, leader)
2420	perf_event__header_size(tmp);
2421
2422	perf_event__header_size(leader);
2423	}
2424
2425	static void perf_child_detach(struct perf_event *event)
2426	{
2427	struct perf_event *parent_event = event->parent;
2428
2429	if (!(event->attach_state & PERF_ATTACH_CHILD))
2430	return;
2431
2432	event->attach_state &= ~PERF_ATTACH_CHILD;
2433
2434	if (WARN_ON_ONCE(!parent_event))
2435	return;
2436
2437	/*
2438	* Can't check this from an IPI, the holder is likey another CPU.
2439	*
2440	lockdep_assert_held(&parent_event->child_mutex);
2441	*/
2442
2443	list_del_init(&event->child_list);
2444	}
2445
2446	static bool is_orphaned_event(struct perf_event *event)
2447	{
2448	return event->state == PERF_EVENT_STATE_DEAD;
2449	}
2450
2451	static inline int
2452	event_filter_match(struct perf_event *event)
2453	{
2454	return (event->cpu == -1 \|\| event->cpu == smp_processor_id()) &&
2455	perf_cgroup_match(event);
2456	}
2457
2458	static inline bool is_event_in_freq_mode(struct perf_event *event)
2459	{
2460	return event->attr.freq && event->attr.sample_freq;
2461	}
2462
2463	static void
2464	event_sched_out(struct perf_event event, struct perf_event_context ctx)
2465	{
2466	struct perf_event_pmu_context *epc = event->pmu_ctx;
2467	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);
2468	enum perf_event_state state = PERF_EVENT_STATE_INACTIVE;
2469
2470	// XXX cpc serialization, probably per-cpu IRQ disabled
2471
2472	WARN_ON_ONCE(event->ctx != ctx);
2473	lockdep_assert_held(&ctx->lock);
2474
2475	if (event->state != PERF_EVENT_STATE_ACTIVE)
2476	return;
2477
2478	/*
2479	* Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but
2480	* we can schedule events _OUT_ individually through things like
2481	* __perf_remove_from_context().
2482	*/
2483	list_del_init(&event->active_list);
2484
2485	perf_pmu_disable(event->pmu);
2486
2487	event->pmu->del(event, 0);
2488	event->oncpu = -1;
2489
2490	if (event->pending_disable) {
2491	event->pending_disable = 0;
2492	perf_cgroup_event_disable(event, ctx);
2493	state = PERF_EVENT_STATE_OFF;
2494	}
2495
2496	perf_event_set_state(event, state);
2497
2498	if (!is_software_event(event))
2499	cpc->active_oncpu--;
2500	if (is_event_in_freq_mode(event)) {
2501	ctx->nr_freq--;
2502	epc->nr_freq--;
2503	}
2504	if (event->attr.exclusive \|\| !cpc->active_oncpu)
2505	cpc->exclusive = 0;
2506
2507	perf_pmu_enable(event->pmu);
2508	}
2509
2510	static void
2511	group_sched_out(struct perf_event group_event, struct perf_event_context ctx)
2512	{
2513	struct perf_event *event;
2514
2515	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
2516	return;
2517
2518	perf_assert_pmu_disabled(group_event->pmu_ctx->pmu);
2519
2520	event_sched_out(group_event, ctx);
2521
2522	/*
2523	* Schedule out siblings (if any):
2524	*/
2525	for_each_sibling_event(event, group_event)
2526	event_sched_out(event, ctx);
2527	}
2528
2529	static inline void
2530	__ctx_time_update(struct perf_cpu_context cpuctx, struct perf_event_context ctx,
2531	bool final, enum event_type_t event_type)
2532	{
2533	if (ctx->is_active & EVENT_TIME) {
2534	if (ctx->is_active & EVENT_FROZEN)
2535	return;
2536
2537	update_context_time(ctx);
2538	/* vPMU should not stop time */
2539	update_cgrp_time_from_cpuctx(cpuctx, !(event_type & EVENT_GUEST) && final);
2540	}
2541	}
2542
2543	static inline void
2544	ctx_time_update(struct perf_cpu_context cpuctx, struct perf_event_context ctx)
2545	{
2546	__ctx_time_update(cpuctx, ctx, false, 0);
2547	}
2548
2549	/*
2550	* To be used inside perf_ctx_lock() / perf_ctx_unlock(). Lasts until perf_ctx_unlock().
2551	*/
2552	static inline void
2553	ctx_time_freeze(struct perf_cpu_context cpuctx, struct perf_event_context ctx)
2554	{
2555	ctx_time_update(cpuctx, ctx);
2556	if (ctx->is_active & EVENT_TIME)
2557	ctx->is_active \|= EVENT_FROZEN;
2558	}
2559
2560	static inline void
2561	ctx_time_update_event(struct perf_event_context ctx, struct perf_event event)
2562	{
2563	if (ctx->is_active & EVENT_TIME) {
2564	if (ctx->is_active & EVENT_FROZEN)
2565	return;
2566	update_context_time(ctx);
2567	update_cgrp_time_from_event(event);
2568	}
2569	}
2570
2571	#define DETACH_GROUP 0x01UL
2572	#define DETACH_CHILD 0x02UL
2573	#define DETACH_EXIT 0x04UL
2574	#define DETACH_REVOKE 0x08UL
2575	#define DETACH_DEAD 0x10UL
2576
2577	/*
2578	* Cross CPU call to remove a performance event
2579	*
2580	* We disable the event on the hardware level first. After that we
2581	* remove it from the context list.
2582	*/
2583	static void
2584	__perf_remove_from_context(struct perf_event *event,
2585	struct perf_cpu_context *cpuctx,
2586	struct perf_event_context *ctx,
2587	void *info)
2588	{
2589	struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;
2590	enum perf_event_state state = PERF_EVENT_STATE_OFF;
2591	unsigned long flags = (unsigned long)info;
2592
2593	ctx_time_update(cpuctx, ctx);
2594
2595	/*
2596	* Ensure event_sched_out() switches to OFF, at the very least
2597	* this avoids raising perf_pending_task() at this time.
2598	*/
2599	if (flags & DETACH_EXIT)
2600	state = PERF_EVENT_STATE_EXIT;
2601	if (flags & DETACH_REVOKE)
2602	state = PERF_EVENT_STATE_REVOKED;
2603	if (flags & DETACH_DEAD)
2604	state = PERF_EVENT_STATE_DEAD;
2605
2606	__event_disable(event, ctx, state);
2607
2608	if (flags & DETACH_GROUP)
2609	perf_group_detach(event);
2610	if (flags & DETACH_CHILD)
2611	perf_child_detach(event);
2612	list_del_event(event, ctx);
2613
2614	if (!pmu_ctx->nr_events) {
2615	pmu_ctx->rotate_necessary = 0;
2616
2617	if (ctx->task && ctx->is_active) {
2618	struct perf_cpu_pmu_context *cpc = this_cpc(pmu_ctx->pmu);
2619
2620	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
2621	cpc->task_epc = NULL;
2622	}
2623	}
2624
2625	if (!ctx->nr_events && ctx->is_active) {
2626	if (ctx == &cpuctx->ctx)
2627	update_cgrp_time_from_cpuctx(cpuctx, true);
2628
2629	ctx->is_active = 0;
2630	if (ctx->task) {
2631	WARN_ON_ONCE(cpuctx->task_ctx != ctx);
2632	cpuctx->task_ctx = NULL;
2633	}
2634	}
2635	}
2636
2637	/*
2638	* Remove the event from a task's (or a CPU's) list of events.
2639	*
2640	* If event->ctx is a cloned context, callers must make sure that
2641	* every task struct that event->ctx->task could possibly point to
2642	* remains valid. This is OK when called from perf_release since
2643	* that only calls us on the top-level context, which can't be a clone.
2644	* When called from perf_event_exit_task, it's OK because the
2645	* context has been detached from its task.
2646	*/
2647	static void perf_remove_from_context(struct perf_event *event, unsigned long flags)
2648	{
2649	struct perf_event_context *ctx = event->ctx;
2650
2651	lockdep_assert_held(&ctx->mutex);
2652
2653	/*
2654	* Because of perf_event_exit_task(), perf_remove_from_context() ought
2655	* to work in the face of TASK_TOMBSTONE, unlike every other
2656	* event_function_call() user.
2657	*/
2658	raw_spin_lock_irq(&ctx->lock);
2659	if (!ctx->is_active) {
2660	__perf_remove_from_context(event, this_cpu_ptr(&perf_cpu_context),
2661	ctx, (void *)flags);
2662	raw_spin_unlock_irq(&ctx->lock);
2663	return;
2664	}
2665	raw_spin_unlock_irq(&ctx->lock);
2666
2667	event_function_call(event, __perf_remove_from_context, (void *)flags);
2668	}
2669
2670	static void __event_disable(struct perf_event *event,
2671	struct perf_event_context *ctx,
2672	enum perf_event_state state)
2673	{
2674	event_sched_out(event, ctx);
2675	if (event->state > PERF_EVENT_STATE_OFF)
2676	perf_cgroup_event_disable(event, ctx);
2677	perf_event_set_state(event, min(event->state, state));
2678	}
2679
2680	/*
2681	* Cross CPU call to disable a performance event
2682	*/
2683	static void __perf_event_disable(struct perf_event *event,
2684	struct perf_cpu_context *cpuctx,
2685	struct perf_event_context *ctx,
2686	void *info)
2687	{
2688	if (event->state < PERF_EVENT_STATE_INACTIVE)
2689	return;
2690
2691	perf_pmu_disable(event->pmu_ctx->pmu);
2692	ctx_time_update_event(ctx, event);
2693
2694	/*
2695	* When disabling a group leader, the whole group becomes ineligible
2696	* to run, so schedule out the full group.
2697	*/
2698	if (event == event->group_leader)
2699	group_sched_out(event, ctx);
2700
2701	/*
2702	* But only mark the leader OFF; the siblings will remain
2703	* INACTIVE.
2704	*/
2705	__event_disable(event, ctx, PERF_EVENT_STATE_OFF);
2706
2707	perf_pmu_enable(event->pmu_ctx->pmu);
2708	}
2709
2710	/*
2711	* Disable an event.
2712	*
2713	* If event->ctx is a cloned context, callers must make sure that
2714	* every task struct that event->ctx->task could possibly point to
2715	* remains valid. This condition is satisfied when called through
2716	* perf_event_for_each_child or perf_event_for_each because they
2717	* hold the top-level event's child_mutex, so any descendant that
2718	* goes to exit will block in perf_event_exit_event().
2719	*
2720	* When called from perf_pending_disable it's OK because event->ctx
2721	* is the current context on this CPU and preemption is disabled,
2722	* hence we can't get into perf_event_task_sched_out for this context.
2723	*/
2724	static void _perf_event_disable(struct perf_event *event)
2725	{
2726	struct perf_event_context *ctx = event->ctx;
2727
2728	raw_spin_lock_irq(&ctx->lock);
2729	if (event->state <= PERF_EVENT_STATE_OFF) {
2730	raw_spin_unlock_irq(&ctx->lock);
2731	return;
2732	}
2733	raw_spin_unlock_irq(&ctx->lock);
2734
2735	event_function_call(event, __perf_event_disable, NULL);
2736	}
2737
2738	void perf_event_disable_local(struct perf_event *event)
2739	{
2740	event_function_local(event, __perf_event_disable, NULL);
2741	}
2742
2743	/*
2744	* Strictly speaking kernel users cannot create groups and therefore this
2745	* interface does not need the perf_event_ctx_lock() magic.
2746	*/
2747	void perf_event_disable(struct perf_event *event)
2748	{
2749	struct perf_event_context *ctx;
2750
2751	ctx = perf_event_ctx_lock(event);
2752	_perf_event_disable(event);
2753	perf_event_ctx_unlock(event, ctx);
2754	}
2755	EXPORT_SYMBOL_GPL(perf_event_disable);
2756
2757	void perf_event_disable_inatomic(struct perf_event *event)
2758	{
2759	event->pending_disable = 1;
2760	irq_work_queue(&event->pending_disable_irq);
2761	}
2762
2763	#define MAX_INTERRUPTS (~0ULL)
2764
2765	static void perf_log_throttle(struct perf_event *event, int enable);
2766	static void perf_log_itrace_start(struct perf_event *event);
2767
2768	static void perf_event_unthrottle(struct perf_event *event, bool start)
2769	{
2770	if (event->state != PERF_EVENT_STATE_ACTIVE)
2771	return;
2772
2773	event->hw.interrupts = 0;
2774	if (start)
2775	event->pmu->start(event, 0);
2776	if (event == event->group_leader)
2777	perf_log_throttle(event, 1);
2778	}
2779
2780	static void perf_event_throttle(struct perf_event *event)
2781	{
2782	if (event->state != PERF_EVENT_STATE_ACTIVE)
2783	return;
2784
2785	event->hw.interrupts = MAX_INTERRUPTS;
2786	event->pmu->stop(event, 0);
2787	if (event == event->group_leader)
2788	perf_log_throttle(event, 0);
2789	}
2790
2791	static void perf_event_unthrottle_group(struct perf_event *event, bool skip_start_event)
2792	{
2793	struct perf_event sibling, leader = event->group_leader;
2794
2795	perf_event_unthrottle(leader, skip_start_event ? leader != event : true);
2796	for_each_sibling_event(sibling, leader)
2797	perf_event_unthrottle(sibling, skip_start_event ? sibling != event : true);
2798	}
2799
2800	static void perf_event_throttle_group(struct perf_event *event)
2801	{
2802	struct perf_event sibling, leader = event->group_leader;
2803
2804	perf_event_throttle(leader);
2805	for_each_sibling_event(sibling, leader)
2806	perf_event_throttle(sibling);
2807	}
2808
2809	static int
2810	event_sched_in(struct perf_event event, struct perf_event_context ctx)
2811	{
2812	struct perf_event_pmu_context *epc = event->pmu_ctx;
2813	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);
2814	int ret = 0;
2815
2816	WARN_ON_ONCE(event->ctx != ctx);
2817
2818	lockdep_assert_held(&ctx->lock);
2819
2820	if (event->state <= PERF_EVENT_STATE_OFF)
2821	return 0;
2822
2823	WRITE_ONCE(event->oncpu, smp_processor_id());
2824	/*
2825	* Order event::oncpu write to happen before the ACTIVE state is
2826	* visible. This allows perf_event_{stop,read}() to observe the correct
2827	* ->oncpu if it sees ACTIVE.
2828	*/
2829	smp_wmb();
2830	perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE);
2831
2832	/*
2833	* Unthrottle events, since we scheduled we might have missed several
2834	* ticks already, also for a heavily scheduling task there is little
2835	* guarantee it'll get a tick in a timely manner.
2836	*/
2837	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS))
2838	perf_event_unthrottle(event, false);
2839
2840	perf_pmu_disable(event->pmu);
2841
2842	perf_log_itrace_start(event);
2843
2844	if (event->pmu->add(event, PERF_EF_START)) {
2845	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2846	event->oncpu = -1;
2847	ret = -EAGAIN;
2848	goto out;
2849	}
2850
2851	if (!is_software_event(event))
2852	cpc->active_oncpu++;
2853	if (is_event_in_freq_mode(event)) {
2854	ctx->nr_freq++;
2855	epc->nr_freq++;
2856	}
2857	if (event->attr.exclusive)
2858	cpc->exclusive = 1;
2859
2860	out:
2861	perf_pmu_enable(event->pmu);
2862
2863	return ret;
2864	}
2865
2866	static int
2867	group_sched_in(struct perf_event group_event, struct perf_event_context ctx)
2868	{
2869	struct perf_event event, partial_group = NULL;
2870	struct pmu *pmu = group_event->pmu_ctx->pmu;
2871
2872	if (group_event->state == PERF_EVENT_STATE_OFF)
2873	return 0;
2874
2875	pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
2876
2877	if (event_sched_in(group_event, ctx))
2878	goto error;
2879
2880	/*
2881	* Schedule in siblings as one group (if any):
2882	*/
2883	for_each_sibling_event(event, group_event) {
2884	if (event_sched_in(event, ctx)) {
2885	partial_group = event;
2886	goto group_error;
2887	}
2888	}
2889
2890	if (!pmu->commit_txn(pmu))
2891	return 0;
2892
2893	group_error:
2894	/*
2895	* Groups can be scheduled in as one unit only, so undo any
2896	* partial group before returning:
2897	* The events up to the failed event are scheduled out normally.
2898	*/
2899	for_each_sibling_event(event, group_event) {
2900	if (event == partial_group)
2901	break;
2902
2903	event_sched_out(event, ctx);
2904	}
2905	event_sched_out(group_event, ctx);
2906
2907	error:
2908	pmu->cancel_txn(pmu);
2909	return -EAGAIN;
2910	}
2911
2912	/*
2913	* Work out whether we can put this event group on the CPU now.
2914	*/
2915	static int group_can_go_on(struct perf_event *event, int can_add_hw)
2916	{
2917	struct perf_event_pmu_context *epc = event->pmu_ctx;
2918	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);
2919
2920	/*
2921	* Groups consisting entirely of software events can always go on.
2922	*/
2923	if (event->group_caps & PERF_EV_CAP_SOFTWARE)
2924	return 1;
2925	/*
2926	* If an exclusive group is already on, no other hardware
2927	* events can go on.
2928	*/
2929	if (cpc->exclusive)
2930	return 0;
2931	/*
2932	* If this group is exclusive and there are already
2933	* events on the CPU, it can't go on.
2934	*/
2935	if (event->attr.exclusive && !list_empty(get_event_list(event)))
2936	return 0;
2937	/*
2938	* Otherwise, try to add it if all previous groups were able
2939	* to go on.
2940	*/
2941	return can_add_hw;
2942	}
2943
2944	static void add_event_to_ctx(struct perf_event *event,
2945	struct perf_event_context *ctx)
2946	{
2947	list_add_event(event, ctx);
2948	perf_group_attach(event);
2949	}
2950
2951	static void task_ctx_sched_out(struct perf_event_context *ctx,
2952	struct pmu *pmu,
2953	enum event_type_t event_type)
2954	{
2955	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
2956
2957	if (!cpuctx->task_ctx)
2958	return;
2959
2960	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2961	return;
2962
2963	ctx_sched_out(ctx, pmu, event_type);
2964	}
2965
2966	static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
2967	struct perf_event_context *ctx,
2968	struct pmu *pmu,
2969	enum event_type_t event_type)
2970	{
2971	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED \| event_type);
2972	if (ctx)
2973	ctx_sched_in(ctx, pmu, EVENT_PINNED \| event_type);
2974	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE \| event_type);
2975	if (ctx)
2976	ctx_sched_in(ctx, pmu, EVENT_FLEXIBLE \| event_type);
2977	}
2978
2979	/*
2980	* We want to maintain the following priority of scheduling:
2981	* - CPU pinned (EVENT_CPU \| EVENT_PINNED)
2982	* - task pinned (EVENT_PINNED)
2983	* - CPU flexible (EVENT_CPU \| EVENT_FLEXIBLE)
2984	* - task flexible (EVENT_FLEXIBLE).
2985	*
2986	* In order to avoid unscheduling and scheduling back in everything every
2987	* time an event is added, only do it for the groups of equal priority and
2988	* below.
2989	*
2990	* This can be called after a batch operation on task events, in which case
2991	* event_type is a bit mask of the types of events involved. For CPU events,
2992	* event_type is only either EVENT_PINNED or EVENT_FLEXIBLE.
2993	*/
2994	static void ctx_resched(struct perf_cpu_context *cpuctx,
2995	struct perf_event_context *task_ctx,
2996	struct pmu *pmu, enum event_type_t event_type)
2997	{
2998	bool cpu_event = !!(event_type & EVENT_CPU);
2999	struct perf_event_pmu_context *epc;
3000
3001	/*
3002	* If pinned groups are involved, flexible groups also need to be
3003	* scheduled out.
3004	*/
3005	if (event_type & EVENT_PINNED)
3006	event_type \|= EVENT_FLEXIBLE;
3007
3008	event_type &= EVENT_ALL;
3009
3010	for_each_epc(epc, &cpuctx->ctx, pmu, 0)
3011	perf_pmu_disable(epc->pmu);
3012
3013	if (task_ctx) {
3014	for_each_epc(epc, task_ctx, pmu, 0)
3015	perf_pmu_disable(epc->pmu);
3016
3017	task_ctx_sched_out(task_ctx, pmu, event_type);
3018	}
3019
3020	/*
3021	* Decide which cpu ctx groups to schedule out based on the types
3022	* of events that caused rescheduling:
3023	* - EVENT_CPU: schedule out corresponding groups;
3024	* - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;
3025	* - otherwise, do nothing more.
3026	*/
3027	if (cpu_event)
3028	ctx_sched_out(&cpuctx->ctx, pmu, event_type);
3029	else if (event_type & EVENT_PINNED)
3030	ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);
3031
3032	perf_event_sched_in(cpuctx, task_ctx, pmu, 0);
3033
3034	for_each_epc(epc, &cpuctx->ctx, pmu, 0)
3035	perf_pmu_enable(epc->pmu);
3036
3037	if (task_ctx) {
3038	for_each_epc(epc, task_ctx, pmu, 0)
3039	perf_pmu_enable(epc->pmu);
3040	}
3041	}
3042
3043	void perf_pmu_resched(struct pmu *pmu)
3044	{
3045	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3046	struct perf_event_context *task_ctx = cpuctx->task_ctx;
3047
3048	perf_ctx_lock(cpuctx, task_ctx);
3049	ctx_resched(cpuctx, task_ctx, pmu, EVENT_ALL\|EVENT_CPU);
3050	perf_ctx_unlock(cpuctx, task_ctx);
3051	}
3052
3053	/*
3054	* Cross CPU call to install and enable a performance event
3055	*
3056	* Very similar to remote_function() + event_function() but cannot assume that
3057	* things like ctx->is_active and cpuctx->task_ctx are set.
3058	*/
3059	static int __perf_install_in_context(void *info)
3060	{
3061	struct perf_event *event = info;
3062	struct perf_event_context *ctx = event->ctx;
3063	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3064	struct perf_event_context *task_ctx = cpuctx->task_ctx;
3065	bool reprogram = true;
3066	int ret = 0;
3067
3068	raw_spin_lock(&cpuctx->ctx.lock);
3069	if (ctx->task) {
3070	raw_spin_lock(&ctx->lock);
3071	task_ctx = ctx;
3072
3073	reprogram = (ctx->task == current);
3074
3075	/*
3076	* If the task is running, it must be running on this CPU,
3077	* otherwise we cannot reprogram things.
3078	*
3079	* If its not running, we don't care, ctx->lock will
3080	* serialize against it becoming runnable.
3081	*/
3082	if (task_curr(ctx->task) && !reprogram) {
3083	ret = -ESRCH;
3084	goto unlock;
3085	}
3086
3087	WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);
3088	} else if (task_ctx) {
3089	raw_spin_lock(&task_ctx->lock);
3090	}
3091
3092	#ifdef CONFIG_CGROUP_PERF
3093	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
3094	/*
3095	* If the current cgroup doesn't match the event's
3096	* cgroup, we should not try to schedule it.
3097	*/
3098	struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
3099	reprogram = cgroup_is_descendant(cgrp->css.cgroup,
3100	event->cgrp->css.cgroup);
3101	}
3102	#endif
3103
3104	if (reprogram) {
3105	ctx_time_freeze(cpuctx, ctx);
3106	add_event_to_ctx(event, ctx);
3107	ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu,
3108	get_event_type(event));
3109	} else {
3110	add_event_to_ctx(event, ctx);
3111	}
3112
3113	unlock:
3114	perf_ctx_unlock(cpuctx, task_ctx);
3115
3116	return ret;
3117	}
3118
3119	static bool exclusive_event_installable(struct perf_event *event,
3120	struct perf_event_context *ctx);
3121
3122	/*
3123	* Attach a performance event to a context.
3124	*
3125	* Very similar to event_function_call, see comment there.
3126	*/
3127	static void
3128	perf_install_in_context(struct perf_event_context *ctx,
3129	struct perf_event *event,
3130	int cpu)
3131	{
3132	struct task_struct *task = READ_ONCE(ctx->task);
3133
3134	lockdep_assert_held(&ctx->mutex);
3135
3136	WARN_ON_ONCE(!exclusive_event_installable(event, ctx));
3137
3138	if (event->cpu != -1)
3139	WARN_ON_ONCE(event->cpu != cpu);
3140
3141	/*
3142	* Ensures that if we can observe event->ctx, both the event and ctx
3143	* will be 'complete'. See perf_iterate_sb_cpu().
3144	*/
3145	smp_store_release(&event->ctx, ctx);
3146
3147	/*
3148	* perf_event_attr::disabled events will not run and can be initialized
3149	* without IPI. Except when this is the first event for the context, in
3150	* that case we need the magic of the IPI to set ctx->is_active.
3151	*
3152	* The IOC_ENABLE that is sure to follow the creation of a disabled
3153	* event will issue the IPI and reprogram the hardware.
3154	*/
3155	if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF &&
3156	ctx->nr_events && !is_cgroup_event(event)) {
3157	raw_spin_lock_irq(&ctx->lock);
3158	if (ctx->task == TASK_TOMBSTONE) {
3159	raw_spin_unlock_irq(&ctx->lock);
3160	return;
3161	}
3162	add_event_to_ctx(event, ctx);
3163	raw_spin_unlock_irq(&ctx->lock);
3164	return;
3165	}
3166
3167	if (!task) {
3168	cpu_function_call(cpu, __perf_install_in_context, event);
3169	return;
3170	}
3171
3172	/*
3173	* Should not happen, we validate the ctx is still alive before calling.
3174	*/
3175	if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
3176	return;
3177
3178	/*
3179	* Installing events is tricky because we cannot rely on ctx->is_active
3180	* to be set in case this is the nr_events 0 -> 1 transition.
3181	*
3182	* Instead we use task_curr(), which tells us if the task is running.
3183	* However, since we use task_curr() outside of rq::lock, we can race
3184	* against the actual state. This means the result can be wrong.
3185	*
3186	* If we get a false positive, we retry, this is harmless.
3187	*
3188	* If we get a false negative, things are complicated. If we are after
3189	* perf_event_context_sched_in() ctx::lock will serialize us, and the
3190	* value must be correct. If we're before, it doesn't matter since
3191	* perf_event_context_sched_in() will program the counter.
3192	*
3193	* However, this hinges on the remote context switch having observed
3194	* our task->perf_event_ctxp[] store, such that it will in fact take
3195	* ctx::lock in perf_event_context_sched_in().
3196	*
3197	* We do this by task_function_call(), if the IPI fails to hit the task
3198	* we know any future context switch of task must see the
3199	* perf_event_ctpx[] store.
3200	*/
3201
3202	/*
3203	* This smp_mb() orders the task->perf_event_ctxp[] store with the
3204	* task_cpu() load, such that if the IPI then does not find the task
3205	* running, a future context switch of that task must observe the
3206	* store.
3207	*/
3208	smp_mb();
3209	again:
3210	if (!task_function_call(task, __perf_install_in_context, event))
3211	return;
3212
3213	raw_spin_lock_irq(&ctx->lock);
3214	task = ctx->task;
3215	if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
3216	/*
3217	* Cannot happen because we already checked above (which also
3218	* cannot happen), and we hold ctx->mutex, which serializes us
3219	* against perf_event_exit_task_context().
3220	*/
3221	raw_spin_unlock_irq(&ctx->lock);
3222	return;
3223	}
3224	/*
3225	* If the task is not running, ctx->lock will avoid it becoming so,
3226	* thus we can safely install the event.
3227	*/
3228	if (task_curr(task)) {
3229	raw_spin_unlock_irq(&ctx->lock);
3230	goto again;
3231	}
3232	add_event_to_ctx(event, ctx);
3233	raw_spin_unlock_irq(&ctx->lock);
3234	}
3235
3236	/*
3237	* Cross CPU call to enable a performance event
3238	*/
3239	static void __perf_event_enable(struct perf_event *event,
3240	struct perf_cpu_context *cpuctx,
3241	struct perf_event_context *ctx,
3242	void *info)
3243	{
3244	struct perf_event *leader = event->group_leader;
3245	struct perf_event_context *task_ctx;
3246
3247	if (event->state >= PERF_EVENT_STATE_INACTIVE \|\|
3248	event->state <= PERF_EVENT_STATE_ERROR)
3249	return;
3250
3251	ctx_time_freeze(cpuctx, ctx);
3252
3253	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
3254	perf_cgroup_event_enable(event, ctx);
3255
3256	if (!ctx->is_active)
3257	return;
3258
3259	if (!event_filter_match(event))
3260	return;
3261
3262	/*
3263	* If the event is in a group and isn't the group leader,
3264	* then don't put it on unless the group is on.
3265	*/
3266	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
3267	return;
3268
3269	task_ctx = cpuctx->task_ctx;
3270	if (ctx->task)
3271	WARN_ON_ONCE(task_ctx != ctx);
3272
3273	ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, get_event_type(event));
3274	}
3275
3276	/*
3277	* Enable an event.
3278	*
3279	* If event->ctx is a cloned context, callers must make sure that
3280	* every task struct that event->ctx->task could possibly point to
3281	* remains valid. This condition is satisfied when called through
3282	* perf_event_for_each_child or perf_event_for_each as described
3283	* for perf_event_disable.
3284	*/
3285	static void _perf_event_enable(struct perf_event *event)
3286	{
3287	struct perf_event_context *ctx = event->ctx;
3288
3289	raw_spin_lock_irq(&ctx->lock);
3290	if (event->state >= PERF_EVENT_STATE_INACTIVE \|\|
3291	event->state < PERF_EVENT_STATE_ERROR) {
3292	out:
3293	raw_spin_unlock_irq(&ctx->lock);
3294	return;
3295	}
3296
3297	/*
3298	* If the event is in error state, clear that first.
3299	*
3300	* That way, if we see the event in error state below, we know that it
3301	* has gone back into error state, as distinct from the task having
3302	* been scheduled away before the cross-call arrived.
3303	*/
3304	if (event->state == PERF_EVENT_STATE_ERROR) {
3305	/*
3306	* Detached SIBLING events cannot leave ERROR state.
3307	*/
3308	if (event->event_caps & PERF_EV_CAP_SIBLING &&
3309	event->group_leader == event)
3310	goto out;
3311
3312	event->state = PERF_EVENT_STATE_OFF;
3313	}
3314	raw_spin_unlock_irq(&ctx->lock);
3315
3316	event_function_call(event, __perf_event_enable, NULL);
3317	}
3318
3319	/*
3320	* See perf_event_disable();
3321	*/
3322	void perf_event_enable(struct perf_event *event)
3323	{
3324	struct perf_event_context *ctx;
3325
3326	ctx = perf_event_ctx_lock(event);
3327	_perf_event_enable(event);
3328	perf_event_ctx_unlock(event, ctx);
3329	}
3330	EXPORT_SYMBOL_GPL(perf_event_enable);
3331
3332	struct stop_event_data {
3333	struct perf_event *event;
3334	unsigned int restart;
3335	};
3336
3337	static int __perf_event_stop(void *info)
3338	{
3339	struct stop_event_data *sd = info;
3340	struct perf_event *event = sd->event;
3341
3342	/* if it's already INACTIVE, do nothing */
3343	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3344	return 0;
3345
3346	/* matches smp_wmb() in event_sched_in() */
3347	smp_rmb();
3348
3349	/*
3350	* There is a window with interrupts enabled before we get here,
3351	* so we need to check again lest we try to stop another CPU's event.
3352	*/
3353	if (READ_ONCE(event->oncpu) != smp_processor_id())
3354	return -EAGAIN;
3355
3356	event->pmu->stop(event, PERF_EF_UPDATE);
3357
3358	/*
3359	* May race with the actual stop (through perf_pmu_output_stop()),
3360	* but it is only used for events with AUX ring buffer, and such
3361	* events will refuse to restart because of rb::aux_mmap_count==0,
3362	* see comments in perf_aux_output_begin().
3363	*
3364	* Since this is happening on an event-local CPU, no trace is lost
3365	* while restarting.
3366	*/
3367	if (sd->restart)
3368	event->pmu->start(event, 0);
3369
3370	return 0;
3371	}
3372
3373	static int perf_event_stop(struct perf_event *event, int restart)
3374	{
3375	struct stop_event_data sd = {
3376	.event = event,
3377	.restart = restart,
3378	};
3379	int ret = 0;
3380
3381	do {
3382	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3383	return 0;
3384
3385	/* matches smp_wmb() in event_sched_in() */
3386	smp_rmb();
3387
3388	/*
3389	* We only want to restart ACTIVE events, so if the event goes
3390	* inactive here (event->oncpu==-1), there's nothing more to do;
3391	* fall through with ret==-ENXIO.
3392	*/
3393	ret = cpu_function_call(READ_ONCE(event->oncpu),
3394	__perf_event_stop, &sd);
3395	} while (ret == -EAGAIN);
3396
3397	return ret;
3398	}
3399
3400	/*
3401	* In order to contain the amount of racy and tricky in the address filter
3402	* configuration management, it is a two part process:
3403	*
3404	* (p1) when userspace mappings change as a result of (1) or (2) or (3) below,
3405	* we update the addresses of corresponding vmas in
3406	* event::addr_filter_ranges array and bump the event::addr_filters_gen;
3407	* (p2) when an event is scheduled in (pmu::add), it calls
3408	* perf_event_addr_filters_sync() which calls pmu::addr_filters_sync()
3409	* if the generation has changed since the previous call.
3410	*
3411	* If (p1) happens while the event is active, we restart it to force (p2).
3412	*
3413	* (1) perf_addr_filters_apply(): adjusting filters' offsets based on
3414	* pre-existing mappings, called once when new filters arrive via SET_FILTER
3415	* ioctl;
3416	* (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly
3417	* registered mapping, called for every new mmap(), with mm::mmap_lock down
3418	* for reading;
3419	* (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process
3420	* of exec.
3421	*/
3422	void perf_event_addr_filters_sync(struct perf_event *event)
3423	{
3424	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
3425
3426	if (!has_addr_filter(event))
3427	return;
3428
3429	raw_spin_lock(&ifh->lock);
3430	if (event->addr_filters_gen != event->hw.addr_filters_gen) {
3431	event->pmu->addr_filters_sync(event);
3432	event->hw.addr_filters_gen = event->addr_filters_gen;
3433	}
3434	raw_spin_unlock(&ifh->lock);
3435	}
3436	EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync);
3437
3438	static int _perf_event_refresh(struct perf_event *event, int refresh)
3439	{
3440	/*
3441	* not supported on inherited events
3442	*/
3443	if (event->attr.inherit \|\| !is_sampling_event(event))
3444	return -EINVAL;
3445
3446	atomic_add(refresh, &event->event_limit);
3447	_perf_event_enable(event);
3448
3449	return 0;
3450	}
3451
3452	/*
3453	* See perf_event_disable()
3454	*/
3455	int perf_event_refresh(struct perf_event *event, int refresh)
3456	{
3457	struct perf_event_context *ctx;
3458	int ret;
3459
3460	ctx = perf_event_ctx_lock(event);
3461	ret = _perf_event_refresh(event, refresh);
3462	perf_event_ctx_unlock(event, ctx);
3463
3464	return ret;
3465	}
3466	EXPORT_SYMBOL_GPL(perf_event_refresh);
3467
3468	static int perf_event_modify_breakpoint(struct perf_event *bp,
3469	struct perf_event_attr *attr)
3470	{
3471	int err;
3472
3473	_perf_event_disable(bp);
3474
3475	err = modify_user_hw_breakpoint_check(bp, attr, true);
3476
3477	if (!bp->attr.disabled)
3478	_perf_event_enable(bp);
3479
3480	return err;
3481	}
3482
3483	/*
3484	* Copy event-type-independent attributes that may be modified.
3485	*/
3486	static void perf_event_modify_copy_attr(struct perf_event_attr *to,
3487	const struct perf_event_attr *from)
3488	{
3489	to->sig_data = from->sig_data;
3490	}
3491
3492	static int perf_event_modify_attr(struct perf_event *event,
3493	struct perf_event_attr *attr)
3494	{
3495	int (func)(struct perf_event , struct perf_event_attr *);
3496	struct perf_event *child;
3497	int err;
3498
3499	if (event->attr.type != attr->type)
3500	return -EINVAL;
3501
3502	switch (event->attr.type) {
3503	case PERF_TYPE_BREAKPOINT:
3504	func = perf_event_modify_breakpoint;
3505	break;
3506	default:
3507	/* Place holder for future additions. */
3508	return -EOPNOTSUPP;
3509	}
3510
3511	WARN_ON_ONCE(event->ctx->parent_ctx);
3512
3513	mutex_lock(&event->child_mutex);
3514	/*
3515	* Event-type-independent attributes must be copied before event-type
3516	* modification, which will validate that final attributes match the
3517	* source attributes after all relevant attributes have been copied.
3518	*/
3519	perf_event_modify_copy_attr(&event->attr, attr);
3520	err = func(event, attr);
3521	if (err)
3522	goto out;
3523	list_for_each_entry(child, &event->child_list, child_list) {
3524	perf_event_modify_copy_attr(&child->attr, attr);
3525	err = func(child, attr);
3526	if (err)
3527	goto out;
3528	}
3529	out:
3530	mutex_unlock(&event->child_mutex);
3531	return err;
3532	}
3533
3534	static void __pmu_ctx_sched_out(struct perf_event_pmu_context *pmu_ctx,
3535	enum event_type_t event_type)
3536	{
3537	struct perf_event_context *ctx = pmu_ctx->ctx;
3538	struct perf_event event, tmp;
3539	struct pmu *pmu = pmu_ctx->pmu;
3540
3541	if (ctx->task && !(ctx->is_active & EVENT_ALL)) {
3542	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);
3543
3544	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
3545	cpc->task_epc = NULL;
3546	}
3547
3548	if (!(event_type & EVENT_ALL))
3549	return;
3550
3551	perf_pmu_disable(pmu);
3552	if (event_type & EVENT_PINNED) {
3553	list_for_each_entry_safe(event, tmp,
3554	&pmu_ctx->pinned_active,
3555	active_list)
3556	group_sched_out(event, ctx);
3557	}
3558
3559	if (event_type & EVENT_FLEXIBLE) {
3560	list_for_each_entry_safe(event, tmp,
3561	&pmu_ctx->flexible_active,
3562	active_list)
3563	group_sched_out(event, ctx);
3564	/*
3565	* Since we cleared EVENT_FLEXIBLE, also clear
3566	* rotate_necessary, is will be reset by
3567	* ctx_flexible_sched_in() when needed.
3568	*/
3569	pmu_ctx->rotate_necessary = 0;
3570	}
3571	perf_pmu_enable(pmu);
3572	}
3573
3574	/*
3575	* Be very careful with the @pmu argument since this will change ctx state.
3576	* The @pmu argument works for ctx_resched(), because that is symmetric in
3577	* ctx_sched_out() / ctx_sched_in() usage and the ctx state ends up invariant.
3578	*
3579	* However, if you were to be asymmetrical, you could end up with messed up
3580	* state, eg. ctx->is_active cleared even though most EPCs would still actually
3581	* be active.
3582	*/
3583	static void
3584	ctx_sched_out(struct perf_event_context ctx, struct pmu pmu, enum event_type_t event_type)
3585	{
3586	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3587	enum event_type_t active_type = event_type & ~EVENT_FLAGS;
3588	struct perf_event_pmu_context *pmu_ctx;
3589	int is_active = ctx->is_active;
3590
3591
3592	lockdep_assert_held(&ctx->lock);
3593
3594	if (likely(!ctx->nr_events)) {
3595	/*
3596	* See __perf_remove_from_context().
3597	*/
3598	WARN_ON_ONCE(ctx->is_active);
3599	if (ctx->task)
3600	WARN_ON_ONCE(cpuctx->task_ctx);
3601	return;
3602	}
3603
3604	/*
3605	* Always update time if it was set; not only when it changes.
3606	* Otherwise we can 'forget' to update time for any but the last
3607	* context we sched out. For example:
3608	*
3609	* ctx_sched_out(.event_type = EVENT_FLEXIBLE)
3610	* ctx_sched_out(.event_type = EVENT_PINNED)
3611	*
3612	* would only update time for the pinned events.
3613	*/
3614	__ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx, event_type);
3615
3616	/*
3617	* CPU-release for the below ->is_active store,
3618	* see __load_acquire() in perf_event_time_now()
3619	*/
3620	barrier();
3621	ctx->is_active &= ~active_type;
3622
3623	if (!(ctx->is_active & EVENT_ALL)) {
3624	/*
3625	* For FROZEN, preserve TIME\|FROZEN such that perf_event_time_now()
3626	* does not observe a hole. perf_ctx_unlock() will clean up.
3627	*/
3628	if (ctx->is_active & EVENT_FROZEN)
3629	ctx->is_active &= EVENT_TIME_FROZEN;
3630	else
3631	ctx->is_active = 0;
3632	}
3633
3634	if (ctx->task) {
3635	WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3636	if (!(ctx->is_active & EVENT_ALL))
3637	cpuctx->task_ctx = NULL;
3638	}
3639
3640	if (event_type & EVENT_GUEST) {
3641	/*
3642	* Schedule out all exclude_guest events of PMU
3643	* with PERF_PMU_CAP_MEDIATED_VPMU.
3644	*/
3645	is_active = EVENT_ALL;
3646	__update_context_guest_time(ctx, false);
3647	perf_cgroup_set_timestamp(cpuctx, true);
3648	barrier();
3649	} else {
3650	is_active ^= ctx->is_active; /* changed bits */
3651	}
3652
3653	for_each_epc(pmu_ctx, ctx, pmu, event_type)
3654	__pmu_ctx_sched_out(pmu_ctx, is_active);
3655	}
3656
3657	/*
3658	* Test whether two contexts are equivalent, i.e. whether they have both been
3659	* cloned from the same version of the same context.
3660	*
3661	* Equivalence is measured using a generation number in the context that is
3662	* incremented on each modification to it; see unclone_ctx(), list_add_event()
3663	* and list_del_event().
3664	*/
3665	static int context_equiv(struct perf_event_context *ctx1,
3666	struct perf_event_context *ctx2)
3667	{
3668	lockdep_assert_held(&ctx1->lock);
3669	lockdep_assert_held(&ctx2->lock);
3670
3671	/* Pinning disables the swap optimization */
3672	if (ctx1->pin_count \|\| ctx2->pin_count)
3673	return 0;
3674
3675	/* If ctx1 is the parent of ctx2 */
3676	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
3677	return 1;
3678
3679	/* If ctx2 is the parent of ctx1 */
3680	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
3681	return 1;
3682
3683	/*
3684	* If ctx1 and ctx2 have the same parent; we flatten the parent
3685	* hierarchy, see perf_event_init_context().
3686	*/
3687	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
3688	ctx1->parent_gen == ctx2->parent_gen)
3689	return 1;
3690
3691	/* Unmatched */
3692	return 0;
3693	}
3694
3695	static void __perf_event_sync_stat(struct perf_event *event,
3696	struct perf_event *next_event)
3697	{
3698	u64 value;
3699
3700	if (!event->attr.inherit_stat)
3701	return;
3702
3703	/*
3704	* Update the event value, we cannot use perf_event_read()
3705	* because we're in the middle of a context switch and have IRQs
3706	* disabled, which upsets smp_call_function_single(), however
3707	* we know the event must be on the current CPU, therefore we
3708	* don't need to use it.
3709	*/
3710	perf_pmu_read(event);
3711
3712	perf_event_update_time(event);
3713
3714	/*
3715	* In order to keep per-task stats reliable we need to flip the event
3716	* values when we flip the contexts.
3717	*/
3718	value = local64_read(&next_event->count);
3719	value = local64_xchg(&event->count, value);
3720	local64_set(&next_event->count, value);
3721
3722	swap(event->total_time_enabled, next_event->total_time_enabled);
3723	swap(event->total_time_running, next_event->total_time_running);
3724
3725	/*
3726	* Since we swizzled the values, update the user visible data too.
3727	*/
3728	perf_event_update_userpage(event);
3729	perf_event_update_userpage(next_event);
3730	}
3731
3732	static void perf_event_sync_stat(struct perf_event_context *ctx,
3733	struct perf_event_context *next_ctx)
3734	{
3735	struct perf_event event, next_event;
3736
3737	if (!ctx->nr_stat)
3738	return;
3739
3740	update_context_time(ctx);
3741
3742	event = list_first_entry(&ctx->event_list,
3743	struct perf_event, event_entry);
3744
3745	next_event = list_first_entry(&next_ctx->event_list,
3746	struct perf_event, event_entry);
3747
3748	while (&event->event_entry != &ctx->event_list &&
3749	&next_event->event_entry != &next_ctx->event_list) {
3750
3751	__perf_event_sync_stat(event, next_event);
3752
3753	event = list_next_entry(event, event_entry);
3754	next_event = list_next_entry(next_event, event_entry);
3755	}
3756	}
3757
3758	static void perf_ctx_sched_task_cb(struct perf_event_context *ctx,
3759	struct task_struct *task, bool sched_in)
3760	{
3761	struct perf_event_pmu_context *pmu_ctx;
3762	struct perf_cpu_pmu_context *cpc;
3763
3764	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
3765	cpc = this_cpc(pmu_ctx->pmu);
3766
3767	if (cpc->task_epc != pmu_ctx)
3768	continue;
3769
3770	if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)
3771	pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in);
3772	}
3773	}
3774
3775	static void
3776	perf_event_context_sched_out(struct task_struct task, struct task_struct next)
3777	{
3778	struct perf_event_context *ctx = task->perf_event_ctxp;
3779	struct perf_event_context *next_ctx;
3780	struct perf_event_context parent, next_parent;
3781	int do_switch = 1;
3782
3783	if (likely(!ctx))
3784	return;
3785
3786	rcu_read_lock();
3787	next_ctx = rcu_dereference(next->perf_event_ctxp);
3788	if (!next_ctx)
3789	goto unlock;
3790
3791	parent = rcu_dereference(ctx->parent_ctx);
3792	next_parent = rcu_dereference(next_ctx->parent_ctx);
3793
3794	/* If neither context have a parent context; they cannot be clones. */
3795	if (!parent && !next_parent)
3796	goto unlock;
3797
3798	if (next_parent == ctx \|\| next_ctx == parent \|\| next_parent == parent) {
3799	/*
3800	* Looks like the two contexts are clones, so we might be
3801	* able to optimize the context switch. We lock both
3802	* contexts and check that they are clones under the
3803	* lock (including re-checking that neither has been
3804	* uncloned in the meantime). It doesn't matter which
3805	* order we take the locks because no other cpu could
3806	* be trying to lock both of these tasks.
3807	*/
3808	raw_spin_lock(&ctx->lock);
3809	raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
3810	if (context_equiv(ctx, next_ctx)) {
3811
3812	perf_ctx_disable(ctx, 0);
3813
3814	/* PMIs are disabled; ctx->nr_no_switch_fast is stable. */
3815	if (local_read(&ctx->nr_no_switch_fast) \|\|
3816	local_read(&next_ctx->nr_no_switch_fast)) {
3817	/*
3818	* Must not swap out ctx when there's pending
3819	* events that rely on the ctx->task relation.
3820	*
3821	* Likewise, when a context contains inherit +
3822	* SAMPLE_READ events they should be switched
3823	* out using the slow path so that they are
3824	* treated as if they were distinct contexts.
3825	*/
3826	raw_spin_unlock(&next_ctx->lock);
3827	rcu_read_unlock();
3828	goto inside_switch;
3829	}
3830
3831	WRITE_ONCE(ctx->task, next);
3832	WRITE_ONCE(next_ctx->task, task);
3833
3834	perf_ctx_sched_task_cb(ctx, task, false);
3835
3836	perf_ctx_enable(ctx, 0);
3837
3838	/*
3839	* RCU_INIT_POINTER here is safe because we've not
3840	* modified the ctx and the above modification of
3841	* ctx->task is immaterial since this value is
3842	* always verified under ctx->lock which we're now
3843	* holding.
3844	*/
3845	RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx);
3846	RCU_INIT_POINTER(next->perf_event_ctxp, ctx);
3847
3848	do_switch = 0;
3849
3850	perf_event_sync_stat(ctx, next_ctx);
3851	}
3852	raw_spin_unlock(&next_ctx->lock);
3853	raw_spin_unlock(&ctx->lock);
3854	}
3855	unlock:
3856	rcu_read_unlock();
3857
3858	if (do_switch) {
3859	raw_spin_lock(&ctx->lock);
3860	perf_ctx_disable(ctx, 0);
3861
3862	inside_switch:
3863	perf_ctx_sched_task_cb(ctx, task, false);
3864	task_ctx_sched_out(ctx, NULL, EVENT_ALL);
3865
3866	perf_ctx_enable(ctx, 0);
3867	raw_spin_unlock(&ctx->lock);
3868	}
3869	}
3870
3871	static DEFINE_PER_CPU(struct list_head, sched_cb_list);
3872	static DEFINE_PER_CPU(int, perf_sched_cb_usages);
3873
3874	void perf_sched_cb_dec(struct pmu *pmu)
3875	{
3876	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);
3877
3878	this_cpu_dec(perf_sched_cb_usages);
3879	barrier();
3880
3881	if (!--cpc->sched_cb_usage)
3882	list_del(&cpc->sched_cb_entry);
3883	}
3884
3885
3886	void perf_sched_cb_inc(struct pmu *pmu)
3887	{
3888	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);
3889
3890	if (!cpc->sched_cb_usage++)
3891	list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
3892
3893	barrier();
3894	this_cpu_inc(perf_sched_cb_usages);
3895	}
3896
3897	/*
3898	* This function provides the context switch callback to the lower code
3899	* layer. It is invoked ONLY when the context switch callback is enabled.
3900	*
3901	* This callback is relevant even to per-cpu events; for example multi event
3902	* PEBS requires this to provide PID/TID information. This requires we flush
3903	* all queued PEBS records before we context switch to a new task.
3904	*/
3905	static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc,
3906	struct task_struct *task, bool sched_in)
3907	{
3908	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3909	struct pmu *pmu;
3910
3911	pmu = cpc->epc.pmu;
3912
3913	/* software PMUs will not have sched_task */
3914	if (WARN_ON_ONCE(!pmu->sched_task))
3915	return;
3916
3917	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
3918	perf_pmu_disable(pmu);
3919
3920	pmu->sched_task(&cpc->epc, task, sched_in);
3921
3922	perf_pmu_enable(pmu);
3923	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
3924	}
3925
3926	static void perf_pmu_sched_task(struct task_struct *prev,
3927	struct task_struct *next,
3928	bool sched_in)
3929	{
3930	struct perf_cpu_pmu_context cpc, cpc2;
3931
3932	if (prev == next)
3933	return;
3934
3935	list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {
3936	if (cpc->task_epc)
3937	continue;
3938
3939	__perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in);
3940	}
3941	}
3942
3943	static void perf_event_switch(struct task_struct *task,
3944	struct task_struct *next_prev, bool sched_in);
3945
3946	/*
3947	* Called from scheduler to remove the events of the current task,
3948	* with interrupts disabled.
3949	*
3950	* We stop each event and update the event value in event->count.
3951	*
3952	* This does not protect us against NMI, but disable()
3953	* sets the disabled bit in the control field of event _before_
3954	* accessing the event control register. If a NMI hits, then it will
3955	* not restart the event.
3956	*/
3957	void __perf_event_task_sched_out(struct task_struct *task,
3958	struct task_struct *next)
3959	{
3960	if (__this_cpu_read(perf_sched_cb_usages))
3961	perf_pmu_sched_task(task, next, false);
3962
3963	if (atomic_read(&nr_switch_events))
3964	perf_event_switch(task, next, false);
3965
3966	perf_event_context_sched_out(task, next);
3967
3968	/*
3969	* if cgroup events exist on this CPU, then we need
3970	* to check if we have to switch out PMU state.
3971	* cgroup event are system-wide mode only
3972	*/
3973	perf_cgroup_switch(next);
3974	}
3975
3976	static bool perf_less_group_idx(const void l, const void r, void __always_unused *args)
3977	{
3978	const struct perf_event le = (const struct perf_event **)l;
3979	const struct perf_event re = (const struct perf_event **)r;
3980
3981	return le->group_index < re->group_index;
3982	}
3983
3984	DEFINE_MIN_HEAP(struct perf_event *, perf_event_min_heap);
3985
3986	static const struct min_heap_callbacks perf_min_heap = {
3987	.less = perf_less_group_idx,
3988	.swp = NULL,
3989	};
3990
3991	static void __heap_add(struct perf_event_min_heap heap, struct perf_event event)
3992	{
3993	struct perf_event **itrs = heap->data;
3994
3995	if (event) {
3996	itrs[heap->nr] = event;
3997	heap->nr++;
3998	}
3999	}
4000
4001	static void __link_epc(struct perf_event_pmu_context *pmu_ctx)
4002	{
4003	struct perf_cpu_pmu_context *cpc;
4004
4005	if (!pmu_ctx->ctx->task)
4006	return;
4007
4008	cpc = this_cpc(pmu_ctx->pmu);
4009	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
4010	cpc->task_epc = pmu_ctx;
4011	}
4012
4013	static noinline int visit_groups_merge(struct perf_event_context *ctx,
4014	struct perf_event_groups *groups, int cpu,
4015	struct pmu *pmu,
4016	int (func)(struct perf_event , void *),
4017	void *data)
4018	{
4019	#ifdef CONFIG_CGROUP_PERF
4020	struct cgroup_subsys_state *css = NULL;
4021	#endif
4022	struct perf_cpu_context *cpuctx = NULL;
4023	/* Space for per CPU and/or any CPU event iterators. */
4024	struct perf_event *itrs[2];
4025	struct perf_event_min_heap event_heap;
4026	struct perf_event **evt;
4027	int ret;
4028
4029	if (pmu->filter && pmu->filter(pmu, cpu))
4030	return 0;
4031
4032	if (!ctx->task) {
4033	cpuctx = this_cpu_ptr(&perf_cpu_context);
4034	event_heap = (struct perf_event_min_heap){
4035	.data = cpuctx->heap,
4036	.nr = 0,
4037	.size = cpuctx->heap_size,
4038	};
4039
4040	lockdep_assert_held(&cpuctx->ctx.lock);
4041
4042	#ifdef CONFIG_CGROUP_PERF
4043	if (cpuctx->cgrp)
4044	css = &cpuctx->cgrp->css;
4045	#endif
4046	} else {
4047	event_heap = (struct perf_event_min_heap){
4048	.data = itrs,
4049	.nr = 0,
4050	.size = ARRAY_SIZE(itrs),
4051	};
4052	/* Events not within a CPU context may be on any CPU. */
4053	__heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL));
4054	}
4055	evt = event_heap.data;
4056
4057	__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, NULL));
4058
4059	#ifdef CONFIG_CGROUP_PERF
4060	for (; css; css = css->parent)
4061	__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup));
4062	#endif
4063
4064	if (event_heap.nr) {
4065	__link_epc((*evt)->pmu_ctx);
4066	perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu);
4067	}
4068
4069	min_heapify_all_inline(&event_heap, &perf_min_heap, NULL);
4070
4071	while (event_heap.nr) {
4072	ret = func(*evt, data);
4073	if (ret)
4074	return ret;
4075
4076	evt = perf_event_groups_next(evt, pmu);
4077	if (*evt)
4078	min_heap_sift_down_inline(&event_heap, 0, &perf_min_heap, NULL);
4079	else
4080	min_heap_pop_inline(&event_heap, &perf_min_heap, NULL);
4081	}
4082
4083	return 0;
4084	}
4085
4086	/*
4087	* Because the userpage is strictly per-event (there is no concept of context,
4088	* so there cannot be a context indirection), every userpage must be updated
4089	* when context time starts :-(
4090	*
4091	* IOW, we must not miss EVENT_TIME edges.
4092	*/
4093	static inline bool event_update_userpage(struct perf_event *event)
4094	{
4095	if (likely(!refcount_read(&event->mmap_count)))
4096	return false;
4097
4098	perf_event_update_time(event);
4099	perf_event_update_userpage(event);
4100
4101	return true;
4102	}
4103
4104	static inline void group_update_userpage(struct perf_event *group_event)
4105	{
4106	struct perf_event *event;
4107
4108	if (!event_update_userpage(group_event))
4109	return;
4110
4111	for_each_sibling_event(event, group_event)
4112	event_update_userpage(event);
4113	}
4114
4115	struct merge_sched_data {
4116	int can_add_hw;
4117	enum event_type_t event_type;
4118	};
4119
4120	static int merge_sched_in(struct perf_event event, void data)
4121	{
4122	struct perf_event_context *ctx = event->ctx;
4123	struct merge_sched_data *msd = data;
4124
4125	if (event->state <= PERF_EVENT_STATE_OFF)
4126	return 0;
4127
4128	if (!event_filter_match(event))
4129	return 0;
4130
4131	/*
4132	* Don't schedule in any host events from PMU with
4133	* PERF_PMU_CAP_MEDIATED_VPMU, while a guest is running.
4134	*/
4135	if (is_guest_mediated_pmu_loaded() &&
4136	event->pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU &&
4137	!(msd->event_type & EVENT_GUEST))
4138	return 0;
4139
4140	if (group_can_go_on(event, msd->can_add_hw)) {
4141	if (!group_sched_in(event, ctx))
4142	list_add_tail(&event->active_list, get_event_list(event));
4143	}
4144
4145	if (event->state == PERF_EVENT_STATE_INACTIVE) {
4146	msd->can_add_hw = 0;
4147	if (event->attr.pinned) {
4148	perf_cgroup_event_disable(event, ctx);
4149	perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
4150
4151	if (*perf_event_fasync(event))
4152	event->pending_kill = POLL_ERR;
4153
4154	event->pending_wakeup = 1;
4155	irq_work_queue(&event->pending_irq);
4156	} else {
4157	struct perf_cpu_pmu_context *cpc = this_cpc(event->pmu_ctx->pmu);
4158
4159	event->pmu_ctx->rotate_necessary = 1;
4160	perf_mux_hrtimer_restart(cpc);
4161	group_update_userpage(event);
4162	}
4163	}
4164
4165	return 0;
4166	}
4167
4168	static void pmu_groups_sched_in(struct perf_event_context *ctx,
4169	struct perf_event_groups *groups,
4170	struct pmu *pmu,
4171	enum event_type_t event_type)
4172	{
4173	struct merge_sched_data msd = {
4174	.can_add_hw = 1,
4175	.event_type = event_type,
4176	};
4177	visit_groups_merge(ctx, groups, smp_processor_id(), pmu,
4178	merge_sched_in, &msd);
4179	}
4180
4181	static void __pmu_ctx_sched_in(struct perf_event_pmu_context *pmu_ctx,
4182	enum event_type_t event_type)
4183	{
4184	struct perf_event_context *ctx = pmu_ctx->ctx;
4185
4186	if (event_type & EVENT_PINNED)
4187	pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu, event_type);
4188	if (event_type & EVENT_FLEXIBLE)
4189	pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu, event_type);
4190	}
4191
4192	static void
4193	ctx_sched_in(struct perf_event_context ctx, struct pmu pmu, enum event_type_t event_type)
4194	{
4195	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4196	enum event_type_t active_type = event_type & ~EVENT_FLAGS;
4197	struct perf_event_pmu_context *pmu_ctx;
4198	int is_active = ctx->is_active;
4199
4200	lockdep_assert_held(&ctx->lock);
4201
4202	if (likely(!ctx->nr_events))
4203	return;
4204
4205	if (!(is_active & EVENT_TIME)) {
4206	/* EVENT_TIME should be active while the guest runs */
4207	WARN_ON_ONCE(event_type & EVENT_GUEST);
4208	/* start ctx time */
4209	__update_context_time(ctx, false);
4210	perf_cgroup_set_timestamp(cpuctx, false);
4211	/*
4212	* CPU-release for the below ->is_active store,
4213	* see __load_acquire() in perf_event_time_now()
4214	*/
4215	barrier();
4216	}
4217
4218	ctx->is_active \|= active_type \| EVENT_TIME;
4219	if (ctx->task) {
4220	if (!(is_active & EVENT_ALL))
4221	cpuctx->task_ctx = ctx;
4222	else
4223	WARN_ON_ONCE(cpuctx->task_ctx != ctx);
4224	}
4225
4226	if (event_type & EVENT_GUEST) {
4227	/*
4228	* Schedule in the required exclude_guest events of PMU
4229	* with PERF_PMU_CAP_MEDIATED_VPMU.
4230	*/
4231	is_active = event_type & EVENT_ALL;
4232
4233	/*
4234	* Update ctx time to set the new start time for
4235	* the exclude_guest events.
4236	*/
4237	update_context_time(ctx);
4238	update_cgrp_time_from_cpuctx(cpuctx, false);
4239	barrier();
4240	} else {
4241	is_active ^= ctx->is_active; /* changed bits */
4242	}
4243
4244	/*
4245	* First go through the list and put on any pinned groups
4246	* in order to give them the best chance of going on.
4247	*/
4248	if (is_active & EVENT_PINNED) {
4249	for_each_epc(pmu_ctx, ctx, pmu, event_type)
4250	__pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED \| (event_type & EVENT_GUEST));
4251	}
4252
4253	/* Then walk through the lower prio flexible groups */
4254	if (is_active & EVENT_FLEXIBLE) {
4255	for_each_epc(pmu_ctx, ctx, pmu, event_type)
4256	__pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE \| (event_type & EVENT_GUEST));
4257	}
4258	}
4259
4260	static void perf_event_context_sched_in(struct task_struct *task)
4261	{
4262	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4263	struct perf_event_context *ctx;
4264
4265	rcu_read_lock();
4266	ctx = rcu_dereference(task->perf_event_ctxp);
4267	if (!ctx)
4268	goto rcu_unlock;
4269
4270	if (cpuctx->task_ctx == ctx) {
4271	perf_ctx_lock(cpuctx, ctx);
4272	perf_ctx_disable(ctx, 0);
4273
4274	perf_ctx_sched_task_cb(ctx, task, true);
4275
4276	perf_ctx_enable(ctx, 0);
4277	perf_ctx_unlock(cpuctx, ctx);
4278	goto rcu_unlock;
4279	}
4280
4281	perf_ctx_lock(cpuctx, ctx);
4282	/*
4283	* We must check ctx->nr_events while holding ctx->lock, such
4284	* that we serialize against perf_install_in_context().
4285	*/
4286	if (!ctx->nr_events)
4287	goto unlock;
4288
4289	perf_ctx_disable(ctx, 0);
4290	/*
4291	* We want to keep the following priority order:
4292	* cpu pinned (that don't need to move), task pinned,
4293	* cpu flexible, task flexible.
4294	*
4295	* However, if task's ctx is not carrying any pinned
4296	* events, no need to flip the cpuctx's events around.
4297	*/
4298	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {
4299	perf_ctx_disable(&cpuctx->ctx, 0);
4300	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE);
4301	}
4302
4303	perf_event_sched_in(cpuctx, ctx, NULL, 0);
4304
4305	perf_ctx_sched_task_cb(cpuctx->task_ctx, task, true);
4306
4307	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
4308	perf_ctx_enable(&cpuctx->ctx, 0);
4309
4310	perf_ctx_enable(ctx, 0);
4311
4312	unlock:
4313	perf_ctx_unlock(cpuctx, ctx);
4314	rcu_unlock:
4315	rcu_read_unlock();
4316	}
4317
4318	/*
4319	* Called from scheduler to add the events of the current task
4320	* with interrupts disabled.
4321	*
4322	* We restore the event value and then enable it.
4323	*
4324	* This does not protect us against NMI, but enable()
4325	* sets the enabled bit in the control field of event _before_
4326	* accessing the event control register. If a NMI hits, then it will
4327	* keep the event running.
4328	*/
4329	void __perf_event_task_sched_in(struct task_struct *prev,
4330	struct task_struct *task)
4331	{
4332	perf_event_context_sched_in(task);
4333
4334	if (atomic_read(&nr_switch_events))
4335	perf_event_switch(task, prev, true);
4336
4337	if (__this_cpu_read(perf_sched_cb_usages))
4338	perf_pmu_sched_task(prev, task, true);
4339	}
4340
4341	static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
4342	{
4343	u64 frequency = event->attr.sample_freq;
4344	u64 sec = NSEC_PER_SEC;
4345	u64 divisor, dividend;
4346
4347	int count_fls, nsec_fls, frequency_fls, sec_fls;
4348
4349	count_fls = fls64(count);
4350	nsec_fls = fls64(nsec);
4351	frequency_fls = fls64(frequency);
4352	sec_fls = 30;
4353
4354	/*
4355	* We got @count in @nsec, with a target of sample_freq HZ
4356	* the target period becomes:
4357	*
4358	* @count * 10^9
4359	* period = -------------------
4360	* @nsec * sample_freq
4361	*
4362	*/
4363
4364	/*
4365	* Reduce accuracy by one bit such that @a and @b converge
4366	* to a similar magnitude.
4367	*/
4368	#define REDUCE_FLS(a, b) \
4369	do { \
4370	if (a##_fls > b##_fls) { \
4371	a >>= 1; \
4372	a##_fls--; \
4373	} else { \
4374	b >>= 1; \
4375	b##_fls--; \
4376	} \
4377	} while (0)
4378
4379	/*
4380	* Reduce accuracy until either term fits in a u64, then proceed with
4381	* the other, so that finally we can do a u64/u64 division.
4382	*/
4383	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
4384	REDUCE_FLS(nsec, frequency);
4385	REDUCE_FLS(sec, count);
4386	}
4387
4388	if (count_fls + sec_fls > 64) {
4389	divisor = nsec * frequency;
4390
4391	while (count_fls + sec_fls > 64) {
4392	REDUCE_FLS(count, sec);
4393	divisor >>= 1;
4394	}
4395
4396	dividend = count * sec;
4397	} else {
4398	dividend = count * sec;
4399
4400	while (nsec_fls + frequency_fls > 64) {
4401	REDUCE_FLS(nsec, frequency);
4402	dividend >>= 1;
4403	}
4404
4405	divisor = nsec * frequency;
4406	}
4407
4408	if (!divisor)
4409	return dividend;
4410
4411	return div64_u64(dividend, divisor);
4412	}
4413
4414	static DEFINE_PER_CPU(int, perf_throttled_count);
4415	static DEFINE_PER_CPU(u64, perf_throttled_seq);
4416
4417	static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
4418	{
4419	struct hw_perf_event *hwc = &event->hw;
4420	s64 period, sample_period;
4421	s64 delta;
4422
4423	period = perf_calculate_period(event, nsec, count);
4424
4425	delta = (s64)(period - hwc->sample_period);
4426	if (delta >= 0)
4427	delta += 7;
4428	else
4429	delta -= 7;
4430	delta /= 8; /* low pass filter */
4431
4432	sample_period = hwc->sample_period + delta;
4433
4434	if (!sample_period)
4435	sample_period = 1;
4436
4437	hwc->sample_period = sample_period;
4438
4439	if (local64_read(&hwc->period_left) > 8*sample_period) {
4440	if (disable)
4441	event->pmu->stop(event, PERF_EF_UPDATE);
4442
4443	local64_set(&hwc->period_left, 0);
4444
4445	if (disable)
4446	event->pmu->start(event, PERF_EF_RELOAD);
4447	}
4448	}
4449
4450	static void perf_adjust_freq_unthr_events(struct list_head *event_list)
4451	{
4452	struct perf_event *event;
4453	struct hw_perf_event *hwc;
4454	u64 now, period = TICK_NSEC;
4455	s64 delta;
4456
4457	list_for_each_entry(event, event_list, active_list) {
4458	if (event->state != PERF_EVENT_STATE_ACTIVE)
4459	continue;
4460
4461	// XXX use visit thingy to avoid the -1,cpu match
4462	if (!event_filter_match(event))
4463	continue;
4464
4465	hwc = &event->hw;
4466
4467	if (hwc->interrupts == MAX_INTERRUPTS)
4468	perf_event_unthrottle_group(event, is_event_in_freq_mode(event));
4469
4470	if (!is_event_in_freq_mode(event))
4471	continue;
4472
4473	/*
4474	* stop the event and update event->count
4475	*/
4476	event->pmu->stop(event, PERF_EF_UPDATE);
4477
4478	now = local64_read(&event->count);
4479	delta = now - hwc->freq_count_stamp;
4480	hwc->freq_count_stamp = now;
4481
4482	/*
4483	* restart the event
4484	* reload only if value has changed
4485	* we have stopped the event so tell that
4486	* to perf_adjust_period() to avoid stopping it
4487	* twice.
4488	*/
4489	if (delta > 0)
4490	perf_adjust_period(event, period, delta, false);
4491
4492	event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
4493	}
4494	}
4495
4496	/*
4497	* combine freq adjustment with unthrottling to avoid two passes over the
4498	* events. At the same time, make sure, having freq events does not change
4499	* the rate of unthrottling as that would introduce bias.
4500	*/
4501	static void
4502	perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
4503	{
4504	struct perf_event_pmu_context *pmu_ctx;
4505
4506	/*
4507	* only need to iterate over all events iff:
4508	* - context have events in frequency mode (needs freq adjust)
4509	* - there are events to unthrottle on this cpu
4510	*/
4511	if (!(ctx->nr_freq \|\| unthrottle))
4512	return;
4513
4514	raw_spin_lock(&ctx->lock);
4515
4516	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
4517	if (!(pmu_ctx->nr_freq \|\| unthrottle))
4518	continue;
4519	if (!perf_pmu_ctx_is_active(pmu_ctx))
4520	continue;
4521	if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)
4522	continue;
4523
4524	perf_pmu_disable(pmu_ctx->pmu);
4525	perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);
4526	perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);
4527	perf_pmu_enable(pmu_ctx->pmu);
4528	}
4529
4530	raw_spin_unlock(&ctx->lock);
4531	}
4532
4533	/*
4534	* Move @event to the tail of the @ctx's elegible events.
4535	*/
4536	static void rotate_ctx(struct perf_event_context ctx, struct perf_event event)
4537	{
4538	/*
4539	* Rotate the first entry last of non-pinned groups. Rotation might be
4540	* disabled by the inheritance code.
4541	*/
4542	if (ctx->rotate_disable)
4543	return;
4544
4545	perf_event_groups_delete(&ctx->flexible_groups, event);
4546	perf_event_groups_insert(&ctx->flexible_groups, event);
4547	}
4548
4549	/* pick an event from the flexible_groups to rotate */
4550	static inline struct perf_event *
4551	ctx_event_to_rotate(struct perf_event_pmu_context *pmu_ctx)
4552	{
4553	struct perf_event *event;
4554	struct rb_node *node;
4555	struct rb_root *tree;
4556	struct __group_key key = {
4557	.pmu = pmu_ctx->pmu,
4558	};
4559
4560	/* pick the first active flexible event */
4561	event = list_first_entry_or_null(&pmu_ctx->flexible_active,
4562	struct perf_event, active_list);
4563	if (event)
4564	goto out;
4565
4566	/* if no active flexible event, pick the first event */
4567	tree = &pmu_ctx->ctx->flexible_groups.tree;
4568
4569	if (!pmu_ctx->ctx->task) {
4570	key.cpu = smp_processor_id();
4571
4572	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
4573	if (node)
4574	event = __node_2_pe(node);
4575	goto out;
4576	}
4577
4578	key.cpu = -1;
4579	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
4580	if (node) {
4581	event = __node_2_pe(node);
4582	goto out;
4583	}
4584
4585	key.cpu = smp_processor_id();
4586	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
4587	if (node)
4588	event = __node_2_pe(node);
4589
4590	out:
4591	/*
4592	* Unconditionally clear rotate_necessary; if ctx_flexible_sched_in()
4593	* finds there are unschedulable events, it will set it again.
4594	*/
4595	pmu_ctx->rotate_necessary = 0;
4596
4597	return event;
4598	}
4599
4600	static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc)
4601	{
4602	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4603	struct perf_event_pmu_context cpu_epc, task_epc = NULL;
4604	struct perf_event cpu_event = NULL, task_event = NULL;
4605	int cpu_rotate, task_rotate;
4606	struct pmu *pmu;
4607
4608	/*
4609	* Since we run this from IRQ context, nobody can install new
4610	* events, thus the event count values are stable.
4611	*/
4612
4613	cpu_epc = &cpc->epc;
4614	pmu = cpu_epc->pmu;
4615	task_epc = cpc->task_epc;
4616
4617	cpu_rotate = cpu_epc->rotate_necessary;
4618	task_rotate = task_epc ? task_epc->rotate_necessary : 0;
4619
4620	if (!(cpu_rotate \|\| task_rotate))
4621	return false;
4622
4623	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
4624	perf_pmu_disable(pmu);
4625
4626	if (task_rotate)
4627	task_event = ctx_event_to_rotate(task_epc);
4628	if (cpu_rotate)
4629	cpu_event = ctx_event_to_rotate(cpu_epc);
4630
4631	/*
4632	* As per the order given at ctx_resched() first 'pop' task flexible
4633	* and then, if needed CPU flexible.
4634	*/
4635	if (task_event \|\| (task_epc && cpu_event)) {
4636	update_context_time(task_epc->ctx);
4637	__pmu_ctx_sched_out(task_epc, EVENT_FLEXIBLE);
4638	}
4639
4640	if (cpu_event) {
4641	update_context_time(&cpuctx->ctx);
4642	update_cgrp_time_from_cpuctx(cpuctx, false);
4643	__pmu_ctx_sched_out(cpu_epc, EVENT_FLEXIBLE);
4644	rotate_ctx(&cpuctx->ctx, cpu_event);
4645	__pmu_ctx_sched_in(cpu_epc, EVENT_FLEXIBLE);
4646	}
4647
4648	if (task_event)
4649	rotate_ctx(task_epc->ctx, task_event);
4650
4651	if (task_event \|\| (task_epc && cpu_event))
4652	__pmu_ctx_sched_in(task_epc, EVENT_FLEXIBLE);
4653
4654	perf_pmu_enable(pmu);
4655	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
4656
4657	return true;
4658	}
4659
4660	void perf_event_task_tick(void)
4661	{
4662	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4663	struct perf_event_context *ctx;
4664	int throttled;
4665
4666	lockdep_assert_irqs_disabled();
4667
4668	__this_cpu_inc(perf_throttled_seq);
4669	throttled = __this_cpu_xchg(perf_throttled_count, 0);
4670	tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
4671
4672	perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled);
4673
4674	rcu_read_lock();
4675	ctx = rcu_dereference(current->perf_event_ctxp);
4676	if (ctx)
4677	perf_adjust_freq_unthr_context(ctx, !!throttled);
4678	rcu_read_unlock();
4679	}
4680
4681	static int event_enable_on_exec(struct perf_event *event,
4682	struct perf_event_context *ctx)
4683	{
4684	if (!event->attr.enable_on_exec)
4685	return 0;
4686
4687	event->attr.enable_on_exec = 0;
4688	if (event->state >= PERF_EVENT_STATE_INACTIVE)
4689	return 0;
4690
4691	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
4692
4693	return 1;
4694	}
4695
4696	/*
4697	* Enable all of a task's events that have been marked enable-on-exec.
4698	* This expects task == current.
4699	*/
4700	static void perf_event_enable_on_exec(struct perf_event_context *ctx)
4701	{
4702	struct perf_event_context *clone_ctx = NULL;
4703	enum event_type_t event_type = 0;
4704	struct perf_cpu_context *cpuctx;
4705	struct perf_event *event;
4706	unsigned long flags;
4707	int enabled = 0;
4708
4709	local_irq_save(flags);
4710	if (WARN_ON_ONCE(current->perf_event_ctxp != ctx))
4711	goto out;
4712
4713	if (!ctx->nr_events)
4714	goto out;
4715
4716	cpuctx = this_cpu_ptr(&perf_cpu_context);
4717	perf_ctx_lock(cpuctx, ctx);
4718	ctx_time_freeze(cpuctx, ctx);
4719
4720	list_for_each_entry(event, &ctx->event_list, event_entry) {
4721	enabled \|= event_enable_on_exec(event, ctx);
4722	event_type \|= get_event_type(event);
4723	}
4724
4725	/*
4726	* Unclone and reschedule this context if we enabled any event.
4727	*/
4728	if (enabled) {
4729	clone_ctx = unclone_ctx(ctx);
4730	ctx_resched(cpuctx, ctx, NULL, event_type);
4731	}
4732	perf_ctx_unlock(cpuctx, ctx);
4733
4734	out:
4735	local_irq_restore(flags);
4736
4737	if (clone_ctx)
4738	put_ctx(clone_ctx);
4739	}
4740
4741	static void perf_remove_from_owner(struct perf_event *event);
4742	static void perf_event_exit_event(struct perf_event *event,
4743	struct perf_event_context *ctx,
4744	struct task_struct *task,
4745	unsigned long detach_flags);
4746
4747	/*
4748	* Removes all events from the current task that have been marked
4749	* remove-on-exec, and feeds their values back to parent events.
4750	*/
4751	static void perf_event_remove_on_exec(struct perf_event_context *ctx)
4752	{
4753	struct perf_event_context *clone_ctx = NULL;
4754	struct perf_event event, next;
4755	unsigned long flags;
4756	bool modified = false;
4757
4758	mutex_lock(&ctx->mutex);
4759
4760	if (WARN_ON_ONCE(ctx->task != current))
4761	goto unlock;
4762
4763	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {
4764	if (!event->attr.remove_on_exec)
4765	continue;
4766
4767	if (!is_kernel_event(event))
4768	perf_remove_from_owner(event);
4769
4770	modified = true;
4771
4772	perf_event_exit_event(event, ctx, ctx->task, DETACH_GROUP);
4773	}
4774
4775	raw_spin_lock_irqsave(&ctx->lock, flags);
4776	if (modified)
4777	clone_ctx = unclone_ctx(ctx);
4778	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4779
4780	unlock:
4781	mutex_unlock(&ctx->mutex);
4782
4783	if (clone_ctx)
4784	put_ctx(clone_ctx);
4785	}
4786
4787	struct perf_read_data {
4788	struct perf_event *event;
4789	bool group;
4790	int ret;
4791	};
4792
4793	static inline const struct cpumask *perf_scope_cpu_topology_cpumask(unsigned int scope, int cpu);
4794
4795	static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)
4796	{
4797	int local_cpu = smp_processor_id();
4798	u16 local_pkg, event_pkg;
4799
4800	if ((unsigned)event_cpu >= nr_cpu_ids)
4801	return event_cpu;
4802
4803	if (event->group_caps & PERF_EV_CAP_READ_SCOPE) {
4804	const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(event->pmu->scope, event_cpu);
4805
4806	if (cpumask && cpumask_test_cpu(local_cpu, cpumask))
4807	return local_cpu;
4808	}
4809
4810	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {
4811	event_pkg = topology_physical_package_id(event_cpu);
4812	local_pkg = topology_physical_package_id(local_cpu);
4813
4814	if (event_pkg == local_pkg)
4815	return local_cpu;
4816	}
4817
4818	return event_cpu;
4819	}
4820
4821	/*
4822	* Cross CPU call to read the hardware event
4823	*/
4824	static void __perf_event_read(void *info)
4825	{
4826	struct perf_read_data *data = info;
4827	struct perf_event sub, event = data->event;
4828	struct perf_event_context *ctx = event->ctx;
4829	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4830	struct pmu *pmu;
4831
4832	/*
4833	* If this is a task context, we need to check whether it is
4834	* the current task context of this cpu. If not it has been
4835	* scheduled out before the smp call arrived. In that case
4836	* event->count would have been updated to a recent sample
4837	* when the event was scheduled out.
4838	*/
4839	if (ctx->task && cpuctx->task_ctx != ctx)
4840	return;
4841
4842	guard(raw_spinlock)(&ctx->lock);
4843	ctx_time_update_event(ctx, event);
4844
4845	perf_event_update_time(event);
4846	if (data->group)
4847	perf_event_update_sibling_time(event);
4848
4849	if (event->state != PERF_EVENT_STATE_ACTIVE)
4850	return;
4851
4852	if (!data->group) {
4853	perf_pmu_read(event);
4854	data->ret = 0;
4855	return;
4856	}
4857
4858	pmu = event->pmu_ctx->pmu;
4859	pmu->start_txn(pmu, PERF_PMU_TXN_READ);
4860
4861	perf_pmu_read(event);
4862	for_each_sibling_event(sub, event)
4863	perf_pmu_read(sub);
4864
4865	data->ret = pmu->commit_txn(pmu);
4866	}
4867
4868	static inline u64 perf_event_count(struct perf_event *event, bool self)
4869	{
4870	if (self)
4871	return local64_read(&event->count);
4872
4873	return local64_read(&event->count) + atomic64_read(&event->child_count);
4874	}
4875
4876	static void calc_timer_values(struct perf_event *event,
4877	u64 *now,
4878	u64 *enabled,
4879	u64 *running)
4880	{
4881	u64 ctx_time;
4882
4883	*now = perf_clock();
4884	ctx_time = perf_event_time_now(event, *now);
4885	__perf_update_times(event, ctx_time, enabled, running);
4886	}
4887
4888	/*
4889	* NMI-safe method to read a local event, that is an event that
4890	* is:
4891	* - either for the current task, or for this CPU
4892	* - does not have inherit set, for inherited task events
4893	* will not be local and we cannot read them atomically
4894	* - must not have a pmu::count method
4895	*/
4896	int perf_event_read_local(struct perf_event event, u64 value,
4897	u64 enabled, u64 running)
4898	{
4899	unsigned long flags;
4900	int event_oncpu;
4901	int event_cpu;
4902	int ret = 0;
4903
4904	/*
4905	* Disabling interrupts avoids all counter scheduling (context
4906	* switches, timer based rotation and IPIs).
4907	*/
4908	local_irq_save(flags);
4909
4910	/*
4911	* It must not be an event with inherit set, we cannot read
4912	* all child counters from atomic context.
4913	*/
4914	if (event->attr.inherit) {
4915	ret = -EOPNOTSUPP;
4916	goto out;
4917	}
4918
4919	/* If this is a per-task event, it must be for current */
4920	if ((event->attach_state & PERF_ATTACH_TASK) &&
4921	event->hw.target != current) {
4922	ret = -EINVAL;
4923	goto out;
4924	}
4925
4926	/*
4927	* Get the event CPU numbers, and adjust them to local if the event is
4928	* a per-package event that can be read locally
4929	*/
4930	event_oncpu = __perf_event_read_cpu(event, event->oncpu);
4931	event_cpu = __perf_event_read_cpu(event, event->cpu);
4932
4933	/* If this is a per-CPU event, it must be for this CPU */
4934	if (!(event->attach_state & PERF_ATTACH_TASK) &&
4935	event_cpu != smp_processor_id()) {
4936	ret = -EINVAL;
4937	goto out;
4938	}
4939
4940	/* If this is a pinned event it must be running on this CPU */
4941	if (event->attr.pinned && event_oncpu != smp_processor_id()) {
4942	ret = -EBUSY;
4943	goto out;
4944	}
4945
4946	/*
4947	* If the event is currently on this CPU, its either a per-task event,
4948	* or local to this CPU. Furthermore it means its ACTIVE (otherwise
4949	* oncpu == -1).
4950	*/
4951	if (event_oncpu == smp_processor_id())
4952	event->pmu->read(event);
4953
4954	*value = local64_read(&event->count);
4955	if (enabled \|\| running) {
4956	u64 __enabled, __running, __now;
4957
4958	calc_timer_values(event, &__now, &__enabled, &__running);
4959	if (enabled)
4960	*enabled = __enabled;
4961	if (running)
4962	*running = __running;
4963	}
4964	out:
4965	local_irq_restore(flags);
4966
4967	return ret;
4968	}
4969
4970	static int perf_event_read(struct perf_event *event, bool group)
4971	{
4972	enum perf_event_state state = READ_ONCE(event->state);
4973	int event_cpu, ret = 0;
4974
4975	/*
4976	* If event is enabled and currently active on a CPU, update the
4977	* value in the event structure:
4978	*/
4979	again:
4980	if (state == PERF_EVENT_STATE_ACTIVE) {
4981	struct perf_read_data data;
4982
4983	/*
4984	* Orders the ->state and ->oncpu loads such that if we see
4985	* ACTIVE we must also see the right ->oncpu.
4986	*
4987	* Matches the smp_wmb() from event_sched_in().
4988	*/
4989	smp_rmb();
4990
4991	event_cpu = READ_ONCE(event->oncpu);
4992	if ((unsigned)event_cpu >= nr_cpu_ids)
4993	return 0;
4994
4995	data = (struct perf_read_data){
4996	.event = event,
4997	.group = group,
4998	.ret = 0,
4999	};
5000
5001	preempt_disable();
5002	event_cpu = __perf_event_read_cpu(event, event_cpu);
5003
5004	/*
5005	* Purposely ignore the smp_call_function_single() return
5006	* value.
5007	*
5008	* If event_cpu isn't a valid CPU it means the event got
5009	* scheduled out and that will have updated the event count.
5010	*
5011	* Therefore, either way, we'll have an up-to-date event count
5012	* after this.
5013	*/
5014	(void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1);
5015	preempt_enable();
5016	ret = data.ret;
5017
5018	} else if (state == PERF_EVENT_STATE_INACTIVE) {
5019	struct perf_event_context *ctx = event->ctx;
5020	unsigned long flags;
5021
5022	raw_spin_lock_irqsave(&ctx->lock, flags);
5023	state = event->state;
5024	if (state != PERF_EVENT_STATE_INACTIVE) {
5025	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5026	goto again;
5027	}
5028
5029	/*
5030	* May read while context is not active (e.g., thread is
5031	* blocked), in that case we cannot update context time
5032	*/
5033	ctx_time_update_event(ctx, event);
5034
5035	perf_event_update_time(event);
5036	if (group)
5037	perf_event_update_sibling_time(event);
5038	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5039	}
5040
5041	return ret;
5042	}
5043
5044	/*
5045	* Initialize the perf_event context in a task_struct:
5046	*/
5047	static void __perf_event_init_context(struct perf_event_context *ctx)
5048	{
5049	raw_spin_lock_init(&ctx->lock);
5050	mutex_init(&ctx->mutex);
5051	INIT_LIST_HEAD(&ctx->pmu_ctx_list);
5052	perf_event_groups_init(&ctx->pinned_groups);
5053	perf_event_groups_init(&ctx->flexible_groups);
5054	INIT_LIST_HEAD(&ctx->event_list);
5055	refcount_set(&ctx->refcount, 1);
5056	}
5057
5058	static void
5059	__perf_init_event_pmu_context(struct perf_event_pmu_context epc, struct pmu pmu)
5060	{
5061	epc->pmu = pmu;
5062	INIT_LIST_HEAD(&epc->pmu_ctx_entry);
5063	INIT_LIST_HEAD(&epc->pinned_active);
5064	INIT_LIST_HEAD(&epc->flexible_active);
5065	atomic_set(&epc->refcount, 1);
5066	}
5067
5068	static struct perf_event_context *
5069	alloc_perf_context(struct task_struct *task)
5070	{
5071	struct perf_event_context *ctx;
5072
5073	ctx = kzalloc_obj(struct perf_event_context);
5074	if (!ctx)
5075	return NULL;
5076
5077	__perf_event_init_context(ctx);
5078	if (task)
5079	ctx->task = get_task_struct(task);
5080
5081	return ctx;
5082	}
5083
5084	static struct task_struct *
5085	find_lively_task_by_vpid(pid_t vpid)
5086	{
5087	struct task_struct *task;
5088
5089	rcu_read_lock();
5090	if (!vpid)
5091	task = current;
5092	else
5093	task = find_task_by_vpid(vpid);
5094	if (task)
5095	get_task_struct(task);
5096	rcu_read_unlock();
5097
5098	if (!task)
5099	return ERR_PTR(-ESRCH);
5100
5101	return task;
5102	}
5103
5104	/*
5105	* Returns a matching context with refcount and pincount.
5106	*/
5107	static struct perf_event_context *
5108	find_get_context(struct task_struct task, struct perf_event event)
5109	{
5110	struct perf_event_context ctx, clone_ctx = NULL;
5111	struct perf_cpu_context *cpuctx;
5112	unsigned long flags;
5113	int err;
5114
5115	if (!task) {
5116	/* Must be root to operate on a CPU event: */
5117	err = perf_allow_cpu();
5118	if (err)
5119	return ERR_PTR(err);
5120
5121	cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
5122	ctx = &cpuctx->ctx;
5123	get_ctx(ctx);
5124	raw_spin_lock_irqsave(&ctx->lock, flags);
5125	++ctx->pin_count;
5126	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5127
5128	return ctx;
5129	}
5130
5131	err = -EINVAL;
5132	retry:
5133	ctx = perf_lock_task_context(task, &flags);
5134	if (ctx) {
5135	clone_ctx = unclone_ctx(ctx);
5136	++ctx->pin_count;
5137
5138	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5139
5140	if (clone_ctx)
5141	put_ctx(clone_ctx);
5142	} else {
5143	ctx = alloc_perf_context(task);
5144	err = -ENOMEM;
5145	if (!ctx)
5146	goto errout;
5147
5148	err = 0;
5149	mutex_lock(&task->perf_event_mutex);
5150	/*
5151	* If it has already passed perf_event_exit_task().
5152	* we must see PF_EXITING, it takes this mutex too.
5153	*/
5154	if (task->flags & PF_EXITING)
5155	err = -ESRCH;
5156	else if (task->perf_event_ctxp)
5157	err = -EAGAIN;
5158	else {
5159	get_ctx(ctx);
5160	++ctx->pin_count;
5161	rcu_assign_pointer(task->perf_event_ctxp, ctx);
5162	}
5163	mutex_unlock(&task->perf_event_mutex);
5164
5165	if (unlikely(err)) {
5166	put_ctx(ctx);
5167
5168	if (err == -EAGAIN)
5169	goto retry;
5170	goto errout;
5171	}
5172	}
5173
5174	return ctx;
5175
5176	errout:
5177	return ERR_PTR(err);
5178	}
5179
5180	static struct perf_event_pmu_context *
5181	find_get_pmu_context(struct pmu pmu, struct perf_event_context ctx,
5182	struct perf_event *event)
5183	{
5184	struct perf_event_pmu_context new = NULL, pos = NULL, *epc;
5185
5186	if (!ctx->task) {
5187	/*
5188	* perf_pmu_migrate_context() / __perf_pmu_install_event()
5189	* relies on the fact that find_get_pmu_context() cannot fail
5190	* for CPU contexts.
5191	*/
5192	struct perf_cpu_pmu_context *cpc;
5193
5194	cpc = *per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);
5195	epc = &cpc->epc;
5196	raw_spin_lock_irq(&ctx->lock);
5197	if (!epc->ctx) {
5198	/*
5199	* One extra reference for the pmu; see perf_pmu_free().
5200	*/
5201	atomic_set(&epc->refcount, 2);
5202	epc->embedded = 1;
5203	list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);
5204	epc->ctx = ctx;
5205	} else {
5206	WARN_ON_ONCE(epc->ctx != ctx);
5207	atomic_inc(&epc->refcount);
5208	}
5209	raw_spin_unlock_irq(&ctx->lock);
5210	return epc;
5211	}
5212
5213	new = kzalloc_obj(*epc);
5214	if (!new)
5215	return ERR_PTR(-ENOMEM);
5216
5217	__perf_init_event_pmu_context(new, pmu);
5218
5219	/*
5220	* XXX
5221	*
5222	* lockdep_assert_held(&ctx->mutex);
5223	*
5224	* can't because perf_event_init_task() doesn't actually hold the
5225	* child_ctx->mutex.
5226	*/
5227
5228	raw_spin_lock_irq(&ctx->lock);
5229	list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) {
5230	if (epc->pmu == pmu) {
5231	WARN_ON_ONCE(epc->ctx != ctx);
5232	atomic_inc(&epc->refcount);
5233	goto found_epc;
5234	}
5235	/* Make sure the pmu_ctx_list is sorted by PMU type: */
5236	if (!pos && epc->pmu->type > pmu->type)
5237	pos = epc;
5238	}
5239
5240	epc = new;
5241	new = NULL;
5242
5243	if (!pos)
5244	list_add_tail(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);
5245	else
5246	list_add(&epc->pmu_ctx_entry, pos->pmu_ctx_entry.prev);
5247
5248	epc->ctx = ctx;
5249
5250	found_epc:
5251	raw_spin_unlock_irq(&ctx->lock);
5252	kfree(new);
5253
5254	return epc;
5255	}
5256
5257	static void get_pmu_ctx(struct perf_event_pmu_context *epc)
5258	{
5259	WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount));
5260	}
5261
5262	static void free_cpc_rcu(struct rcu_head *head)
5263	{
5264	struct perf_cpu_pmu_context *cpc =
5265	container_of(head, typeof(*cpc), epc.rcu_head);
5266
5267	kfree(cpc);
5268	}
5269
5270	static void free_epc_rcu(struct rcu_head *head)
5271	{
5272	struct perf_event_pmu_context epc = container_of(head, typeof(epc), rcu_head);
5273
5274	kfree(epc);
5275	}
5276
5277	static void put_pmu_ctx(struct perf_event_pmu_context *epc)
5278	{
5279	struct perf_event_context *ctx = epc->ctx;
5280	unsigned long flags;
5281
5282	/*
5283	* XXX
5284	*
5285	* lockdep_assert_held(&ctx->mutex);
5286	*
5287	* can't because of the call-site in _free_event()/put_event()
5288	* which isn't always called under ctx->mutex.
5289	*/
5290	if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags))
5291	return;
5292
5293	WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry));
5294
5295	list_del_init(&epc->pmu_ctx_entry);
5296	epc->ctx = NULL;
5297
5298	WARN_ON_ONCE(!list_empty(&epc->pinned_active));
5299	WARN_ON_ONCE(!list_empty(&epc->flexible_active));
5300
5301	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5302
5303	if (epc->embedded) {
5304	call_rcu(&epc->rcu_head, free_cpc_rcu);
5305	return;
5306	}
5307
5308	call_rcu(&epc->rcu_head, free_epc_rcu);
5309	}
5310
5311	static void perf_event_free_filter(struct perf_event *event);
5312
5313	static void free_event_rcu(struct rcu_head *head)
5314	{
5315	struct perf_event event = container_of(head, typeof(event), rcu_head);
5316
5317	if (event->ns)
5318	put_pid_ns(event->ns);
5319	perf_event_free_filter(event);
5320	kfree(event->addr_filter_ranges);
5321	kmem_cache_free(perf_event_cache, event);
5322	}
5323
5324	static void ring_buffer_attach(struct perf_event *event,
5325	struct perf_buffer *rb);
5326
5327	static void detach_sb_event(struct perf_event *event)
5328	{
5329	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
5330
5331	raw_spin_lock(&pel->lock);
5332	list_del_rcu(&event->sb_list);
5333	raw_spin_unlock(&pel->lock);
5334	}
5335
5336	static bool is_sb_event(struct perf_event *event)
5337	{
5338	struct perf_event_attr *attr = &event->attr;
5339
5340	if (event->parent)
5341	return false;
5342
5343	if (event->attach_state & PERF_ATTACH_TASK)
5344	return false;
5345
5346	if (attr->mmap \|\| attr->mmap_data \|\| attr->mmap2 \|\|
5347	attr->comm \|\| attr->comm_exec \|\|
5348	attr->task \|\| attr->ksymbol \|\|
5349	attr->context_switch \|\| attr->text_poke \|\|
5350	attr->bpf_event)
5351	return true;
5352
5353	return false;
5354	}
5355
5356	static void unaccount_pmu_sb_event(struct perf_event *event)
5357	{
5358	if (is_sb_event(event))
5359	detach_sb_event(event);
5360	}
5361
5362	#ifdef CONFIG_NO_HZ_FULL
5363	static DEFINE_SPINLOCK(nr_freq_lock);
5364	#endif
5365
5366	static void unaccount_freq_event_nohz(void)
5367	{
5368	#ifdef CONFIG_NO_HZ_FULL
5369	spin_lock(&nr_freq_lock);
5370	if (atomic_dec_and_test(&nr_freq_events))
5371	tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
5372	spin_unlock(&nr_freq_lock);
5373	#endif
5374	}
5375
5376	static void unaccount_freq_event(void)
5377	{
5378	if (tick_nohz_full_enabled())
5379	unaccount_freq_event_nohz();
5380	else
5381	atomic_dec(&nr_freq_events);
5382	}
5383
5384
5385	static struct perf_ctx_data *
5386	alloc_perf_ctx_data(struct kmem_cache *ctx_cache, bool global, gfp_t gfp_flags)
5387	{
5388	struct perf_ctx_data *cd;
5389
5390	cd = kzalloc_obj(*cd, gfp_flags);
5391	if (!cd)
5392	return NULL;
5393
5394	cd->data = kmem_cache_zalloc(ctx_cache, gfp_flags);
5395	if (!cd->data) {
5396	kfree(cd);
5397	return NULL;
5398	}
5399
5400	cd->global = global;
5401	cd->ctx_cache = ctx_cache;
5402	refcount_set(&cd->refcount, 1);
5403
5404	return cd;
5405	}
5406
5407	static void free_perf_ctx_data(struct perf_ctx_data *cd)
5408	{
5409	kmem_cache_free(cd->ctx_cache, cd->data);
5410	kfree(cd);
5411	}
5412
5413	static void __free_perf_ctx_data_rcu(struct rcu_head *rcu_head)
5414	{
5415	struct perf_ctx_data *cd;
5416
5417	cd = container_of(rcu_head, struct perf_ctx_data, rcu_head);
5418	free_perf_ctx_data(cd);
5419	}
5420
5421	static inline void perf_free_ctx_data_rcu(struct perf_ctx_data *cd)
5422	{
5423	call_rcu(&cd->rcu_head, __free_perf_ctx_data_rcu);
5424	}
5425
5426	static int
5427	attach_task_ctx_data(struct task_struct task, struct kmem_cache ctx_cache,
5428	bool global, gfp_t gfp_flags)
5429	{
5430	struct perf_ctx_data cd, old = NULL;
5431
5432	cd = alloc_perf_ctx_data(ctx_cache, global, gfp_flags);
5433	if (!cd)
5434	return -ENOMEM;
5435
5436	for (;;) {
5437	if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) {
5438	if (old)
5439	perf_free_ctx_data_rcu(old);
5440	/*
5441	* Above try_cmpxchg() pairs with try_cmpxchg() from
5442	* detach_task_ctx_data() such that
5443	* if we race with perf_event_exit_task(), we must
5444	* observe PF_EXITING.
5445	*/
5446	if (task->flags & PF_EXITING) {
5447	/* detach_task_ctx_data() may free it already */
5448	if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL))
5449	perf_free_ctx_data_rcu(cd);
5450	}
5451	return 0;
5452	}
5453
5454	if (!old) {
5455	/*
5456	* After seeing a dead @old, we raced with
5457	* removal and lost, try again to install @cd.
5458	*/
5459	continue;
5460	}
5461
5462	if (refcount_inc_not_zero(&old->refcount)) {
5463	if (global)
5464	old->global = true;
5465	free_perf_ctx_data(cd); /* unused */
5466	return 0;
5467	}
5468
5469	/*
5470	* @old is a dead object, refcount==0 is stable, try and
5471	* replace it with @cd.
5472	*/
5473	}
5474	return 0;
5475	}
5476
5477	static void __detach_global_ctx_data(void);
5478	DEFINE_STATIC_PERCPU_RWSEM(global_ctx_data_rwsem);
5479	static refcount_t global_ctx_data_ref;
5480
5481	static int
5482	attach_global_ctx_data(struct kmem_cache *ctx_cache)
5483	{
5484	struct task_struct g, p;
5485	struct perf_ctx_data *cd;
5486	int ret;
5487
5488	if (refcount_inc_not_zero(&global_ctx_data_ref))
5489	return 0;
5490
5491	guard(percpu_write)(&global_ctx_data_rwsem);
5492	if (refcount_inc_not_zero(&global_ctx_data_ref))
5493	return 0;
5494	again:
5495	/* Allocate everything */
5496	scoped_guard (rcu) {
5497	for_each_process_thread(g, p) {
5498	if (p->flags & PF_EXITING)
5499	continue;
5500	cd = rcu_dereference(p->perf_ctx_data);
5501	if (cd && !cd->global) {
5502	cd->global = 1;
5503	if (!refcount_inc_not_zero(&cd->refcount))
5504	cd = NULL;
5505	}
5506	if (!cd) {
5507	/*
5508	* Try to allocate context quickly before
5509	* traversing the whole thread list again.
5510	*/
5511	if (!attach_task_ctx_data(p, ctx_cache, true, GFP_NOWAIT))
5512	continue;
5513	get_task_struct(p);
5514	goto alloc;
5515	}
5516	}
5517	}
5518
5519	refcount_set(&global_ctx_data_ref, 1);
5520
5521	return 0;
5522	alloc:
5523	ret = attach_task_ctx_data(p, ctx_cache, true, GFP_KERNEL);
5524	put_task_struct(p);
5525	if (ret) {
5526	__detach_global_ctx_data();
5527	return ret;
5528	}
5529	goto again;
5530	}
5531
5532	static int
5533	attach_perf_ctx_data(struct perf_event *event)
5534	{
5535	struct task_struct *task = event->hw.target;
5536	struct kmem_cache *ctx_cache = event->pmu->task_ctx_cache;
5537	int ret;
5538
5539	if (!ctx_cache)
5540	return -ENOMEM;
5541
5542	if (task)
5543	return attach_task_ctx_data(task, ctx_cache, false, GFP_KERNEL);
5544
5545	ret = attach_global_ctx_data(ctx_cache);
5546	if (ret)
5547	return ret;
5548
5549	event->attach_state \|= PERF_ATTACH_GLOBAL_DATA;
5550	return 0;
5551	}
5552
5553	static void
5554	detach_task_ctx_data(struct task_struct *p)
5555	{
5556	struct perf_ctx_data *cd;
5557
5558	scoped_guard (rcu) {
5559	cd = rcu_dereference(p->perf_ctx_data);
5560	if (!cd \|\| !refcount_dec_and_test(&cd->refcount))
5561	return;
5562	}
5563
5564	/*
5565	* The old ctx_data may be lost because of the race.
5566	* Nothing is required to do for the case.
5567	* See attach_task_ctx_data().
5568	*/
5569	if (try_cmpxchg((struct perf_ctx_data **)&p->perf_ctx_data, &cd, NULL))
5570	perf_free_ctx_data_rcu(cd);
5571	}
5572
5573	static void __detach_global_ctx_data(void)
5574	{
5575	struct task_struct g, p;
5576	struct perf_ctx_data *cd;
5577
5578	scoped_guard (rcu) {
5579	for_each_process_thread(g, p) {
5580	cd = rcu_dereference(p->perf_ctx_data);
5581	if (cd && cd->global) {
5582	cd->global = 0;
5583	detach_task_ctx_data(p);
5584	}
5585	}
5586	}
5587	}
5588
5589	static void detach_global_ctx_data(void)
5590	{
5591	if (refcount_dec_not_one(&global_ctx_data_ref))
5592	return;
5593
5594	guard(percpu_write)(&global_ctx_data_rwsem);
5595	if (!refcount_dec_and_test(&global_ctx_data_ref))
5596	return;
5597
5598	/* remove everything */
5599	__detach_global_ctx_data();
5600	}
5601
5602	static void detach_perf_ctx_data(struct perf_event *event)
5603	{
5604	struct task_struct *task = event->hw.target;
5605
5606	event->attach_state &= ~PERF_ATTACH_TASK_DATA;
5607
5608	if (task)
5609	return detach_task_ctx_data(task);
5610
5611	if (event->attach_state & PERF_ATTACH_GLOBAL_DATA) {
5612	detach_global_ctx_data();
5613	event->attach_state &= ~PERF_ATTACH_GLOBAL_DATA;
5614	}
5615	}
5616
5617	static void unaccount_event(struct perf_event *event)
5618	{
5619	bool dec = false;
5620
5621	if (event->parent)
5622	return;
5623
5624	if (event->attach_state & (PERF_ATTACH_TASK \| PERF_ATTACH_SCHED_CB))
5625	dec = true;
5626	if (event->attr.mmap \|\| event->attr.mmap_data)
5627	atomic_dec(&nr_mmap_events);
5628	if (event->attr.build_id)
5629	atomic_dec(&nr_build_id_events);
5630	if (event->attr.comm)
5631	atomic_dec(&nr_comm_events);
5632	if (event->attr.namespaces)
5633	atomic_dec(&nr_namespaces_events);
5634	if (event->attr.cgroup)
5635	atomic_dec(&nr_cgroup_events);
5636	if (event->attr.task)
5637	atomic_dec(&nr_task_events);
5638	if (event->attr.freq)
5639	unaccount_freq_event();
5640	if (event->attr.context_switch) {
5641	dec = true;
5642	atomic_dec(&nr_switch_events);
5643	}
5644	if (is_cgroup_event(event))
5645	dec = true;
5646	if (has_branch_stack(event))
5647	dec = true;
5648	if (event->attr.ksymbol)
5649	atomic_dec(&nr_ksymbol_events);
5650	if (event->attr.bpf_event)
5651	atomic_dec(&nr_bpf_events);
5652	if (event->attr.text_poke)
5653	atomic_dec(&nr_text_poke_events);
5654
5655	if (dec) {
5656	if (!atomic_add_unless(&perf_sched_count, -1, 1))
5657	schedule_delayed_work(&perf_sched_work, HZ);
5658	}
5659
5660	unaccount_pmu_sb_event(event);
5661	}
5662
5663	static void perf_sched_delayed(struct work_struct *work)
5664	{
5665	mutex_lock(&perf_sched_mutex);
5666	if (atomic_dec_and_test(&perf_sched_count))
5667	static_branch_disable(&perf_sched_events);
5668	mutex_unlock(&perf_sched_mutex);
5669	}
5670
5671	/*
5672	* The following implement mutual exclusion of events on "exclusive" pmus
5673	* (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
5674	* at a time, so we disallow creating events that might conflict, namely:
5675	*
5676	* 1) cpu-wide events in the presence of per-task events,
5677	* 2) per-task events in the presence of cpu-wide events,
5678	* 3) two matching events on the same perf_event_context.
5679	*
5680	* The former two cases are handled in the allocation path (perf_event_alloc(),
5681	* _free_event()), the latter -- before the first perf_install_in_context().
5682	*/
5683	static int exclusive_event_init(struct perf_event *event)
5684	{
5685	struct pmu *pmu = event->pmu;
5686
5687	if (!is_exclusive_pmu(pmu))
5688	return 0;
5689
5690	/*
5691	* Prevent co-existence of per-task and cpu-wide events on the
5692	* same exclusive pmu.
5693	*
5694	* Negative pmu::exclusive_cnt means there are cpu-wide
5695	* events on this "exclusive" pmu, positive means there are
5696	* per-task events.
5697	*
5698	* Since this is called in perf_event_alloc() path, event::ctx
5699	* doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
5700	* to mean "per-task event", because unlike other attach states it
5701	* never gets cleared.
5702	*/
5703	if (event->attach_state & PERF_ATTACH_TASK) {
5704	if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
5705	return -EBUSY;
5706	} else {
5707	if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
5708	return -EBUSY;
5709	}
5710
5711	event->attach_state \|= PERF_ATTACH_EXCLUSIVE;
5712
5713	return 0;
5714	}
5715
5716	static void exclusive_event_destroy(struct perf_event *event)
5717	{
5718	struct pmu *pmu = event->pmu;
5719
5720	/* see comment in exclusive_event_init() */
5721	if (event->attach_state & PERF_ATTACH_TASK)
5722	atomic_dec(&pmu->exclusive_cnt);
5723	else
5724	atomic_inc(&pmu->exclusive_cnt);
5725
5726	event->attach_state &= ~PERF_ATTACH_EXCLUSIVE;
5727	}
5728
5729	static bool exclusive_event_match(struct perf_event e1, struct perf_event e2)
5730	{
5731	if ((e1->pmu == e2->pmu) &&
5732	(e1->cpu == e2->cpu \|\|
5733	e1->cpu == -1 \|\|
5734	e2->cpu == -1))
5735	return true;
5736	return false;
5737	}
5738
5739	static bool exclusive_event_installable(struct perf_event *event,
5740	struct perf_event_context *ctx)
5741	{
5742	struct perf_event *iter_event;
5743	struct pmu *pmu = event->pmu;
5744
5745	lockdep_assert_held(&ctx->mutex);
5746
5747	if (!is_exclusive_pmu(pmu))
5748	return true;
5749
5750	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
5751	if (exclusive_event_match(iter_event, event))
5752	return false;
5753	}
5754
5755	return true;
5756	}
5757
5758	static void perf_free_addr_filters(struct perf_event *event);
5759
5760	/* vs perf_event_alloc() error */
5761	static void __free_event(struct perf_event *event)
5762	{
5763	struct pmu *pmu = event->pmu;
5764
5765	security_perf_event_free(event);
5766
5767	if (event->attach_state & PERF_ATTACH_CALLCHAIN)
5768	put_callchain_buffers();
5769
5770	if (event->attach_state & PERF_ATTACH_EXCLUSIVE)
5771	exclusive_event_destroy(event);
5772
5773	if (is_cgroup_event(event))
5774	perf_detach_cgroup(event);
5775
5776	if (event->attach_state & PERF_ATTACH_TASK_DATA)
5777	detach_perf_ctx_data(event);
5778
5779	if (event->destroy)
5780	event->destroy(event);
5781
5782	/*
5783	* Must be after ->destroy(), due to uprobe_perf_close() using
5784	* hw.target.
5785	*/
5786	if (event->hw.target)
5787	put_task_struct(event->hw.target);
5788
5789	if (event->pmu_ctx) {
5790	/*
5791	* put_pmu_ctx() needs an event->ctx reference, because of
5792	* epc->ctx.
5793	*/
5794	WARN_ON_ONCE(!pmu);
5795	WARN_ON_ONCE(!event->ctx);
5796	WARN_ON_ONCE(event->pmu_ctx->ctx != event->ctx);
5797	put_pmu_ctx(event->pmu_ctx);
5798	}
5799
5800	/*
5801	* perf_event_free_task() relies on put_ctx() being 'last', in
5802	* particular all task references must be cleaned up.
5803	*/
5804	if (event->ctx)
5805	put_ctx(event->ctx);
5806
5807	if (pmu) {
5808	module_put(pmu->module);
5809	scoped_guard (spinlock, &pmu->events_lock) {
5810	list_del(&event->pmu_list);
5811	wake_up_var(pmu);
5812	}
5813	}
5814
5815	call_rcu(&event->rcu_head, free_event_rcu);
5816	}
5817
5818	static void mediated_pmu_unaccount_event(struct perf_event *event);
5819
5820	DEFINE_FREE(__free_event, struct perf_event *, if (_T) __free_event(_T))
5821
5822	/* vs perf_event_alloc() success */
5823	static void _free_event(struct perf_event *event)
5824	{
5825	irq_work_sync(&event->pending_irq);
5826	irq_work_sync(&event->pending_disable_irq);
5827
5828	unaccount_event(event);
5829	mediated_pmu_unaccount_event(event);
5830
5831	if (event->rb) {
5832	/*
5833	* Can happen when we close an event with re-directed output.
5834	*
5835	* Since we have a 0 refcount, perf_mmap_close() will skip
5836	* over us; possibly making our ring_buffer_put() the last.
5837	*/
5838	mutex_lock(&event->mmap_mutex);
5839	ring_buffer_attach(event, NULL);
5840	mutex_unlock(&event->mmap_mutex);
5841	}
5842
5843	perf_event_free_bpf_prog(event);
5844	perf_free_addr_filters(event);
5845
5846	__free_event(event);
5847	}
5848
5849	/*
5850	* Used to free events which have a known refcount of 1, such as in error paths
5851	* of inherited events.
5852	*/
5853	static void free_event(struct perf_event *event)
5854	{
5855	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
5856	"unexpected event refcount: %ld; ptr=%p\n",
5857	atomic_long_read(&event->refcount), event)) {
5858	/* leak to avoid use-after-free */
5859	return;
5860	}
5861
5862	_free_event(event);
5863	}
5864
5865	/*
5866	* Remove user event from the owner task.
5867	*/
5868	static void perf_remove_from_owner(struct perf_event *event)
5869	{
5870	struct task_struct *owner;
5871
5872	rcu_read_lock();
5873	/*
5874	* Matches the smp_store_release() in perf_event_exit_task(). If we
5875	* observe !owner it means the list deletion is complete and we can
5876	* indeed free this event, otherwise we need to serialize on
5877	* owner->perf_event_mutex.
5878	*/
5879	owner = READ_ONCE(event->owner);
5880	if (owner) {
5881	/*
5882	* Since delayed_put_task_struct() also drops the last
5883	* task reference we can safely take a new reference
5884	* while holding the rcu_read_lock().
5885	*/
5886	get_task_struct(owner);
5887	}
5888	rcu_read_unlock();
5889
5890	if (owner) {
5891	/*
5892	* If we're here through perf_event_exit_task() we're already
5893	* holding ctx->mutex which would be an inversion wrt. the
5894	* normal lock order.
5895	*
5896	* However we can safely take this lock because its the child
5897	* ctx->mutex.
5898	*/
5899	mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
5900
5901	/*
5902	* We have to re-check the event->owner field, if it is cleared
5903	* we raced with perf_event_exit_task(), acquiring the mutex
5904	* ensured they're done, and we can proceed with freeing the
5905	* event.
5906	*/
5907	if (event->owner) {
5908	list_del_init(&event->owner_entry);
5909	smp_store_release(&event->owner, NULL);
5910	}
5911	mutex_unlock(&owner->perf_event_mutex);
5912	put_task_struct(owner);
5913	}
5914	}
5915
5916	static void put_event(struct perf_event *event)
5917	{
5918	struct perf_event *parent;
5919
5920	if (!atomic_long_dec_and_test(&event->refcount))
5921	return;
5922
5923	parent = event->parent;
5924	_free_event(event);
5925
5926	/* Matches the refcount bump in inherit_event() */
5927	if (parent)
5928	put_event(parent);
5929	}
5930
5931	/*
5932	* Kill an event dead; while event:refcount will preserve the event
5933	* object, it will not preserve its functionality. Once the last 'user'
5934	* gives up the object, we'll destroy the thing.
5935	*/
5936	int perf_event_release_kernel(struct perf_event *event)
5937	{
5938	struct perf_event_context *ctx = event->ctx;
5939	struct perf_event child, tmp;
5940
5941	/*
5942	* If we got here through err_alloc: free_event(event); we will not
5943	* have attached to a context yet.
5944	*/
5945	if (!ctx) {
5946	WARN_ON_ONCE(event->attach_state &
5947	(PERF_ATTACH_CONTEXT\|PERF_ATTACH_GROUP));
5948	goto no_ctx;
5949	}
5950
5951	if (!is_kernel_event(event))
5952	perf_remove_from_owner(event);
5953
5954	ctx = perf_event_ctx_lock(event);
5955	WARN_ON_ONCE(ctx->parent_ctx);
5956
5957	/*
5958	* Mark this event as STATE_DEAD, there is no external reference to it
5959	* anymore.
5960	*
5961	* Anybody acquiring event->child_mutex after the below loop _must_
5962	* also see this, most importantly inherit_event() which will avoid
5963	* placing more children on the list.
5964	*
5965	* Thus this guarantees that we will in fact observe and kill _ALL_
5966	* child events.
5967	*/
5968	if (event->state > PERF_EVENT_STATE_REVOKED) {
5969	perf_remove_from_context(event, DETACH_GROUP\|DETACH_DEAD);
5970	} else {
5971	event->state = PERF_EVENT_STATE_DEAD;
5972	}
5973
5974	perf_event_ctx_unlock(event, ctx);
5975
5976	again:
5977	mutex_lock(&event->child_mutex);
5978	list_for_each_entry(child, &event->child_list, child_list) {
5979	/*
5980	* Cannot change, child events are not migrated, see the
5981	* comment with perf_event_ctx_lock_nested().
5982	*/
5983	ctx = READ_ONCE(child->ctx);
5984	/*
5985	* Since child_mutex nests inside ctx::mutex, we must jump
5986	* through hoops. We start by grabbing a reference on the ctx.
5987	*
5988	* Since the event cannot get freed while we hold the
5989	* child_mutex, the context must also exist and have a !0
5990	* reference count.
5991	*/
5992	get_ctx(ctx);
5993
5994	/*
5995	* Now that we have a ctx ref, we can drop child_mutex, and
5996	* acquire ctx::mutex without fear of it going away. Then we
5997	* can re-acquire child_mutex.
5998	*/
5999	mutex_unlock(&event->child_mutex);
6000	mutex_lock(&ctx->mutex);
6001	mutex_lock(&event->child_mutex);
6002
6003	/*
6004	* Now that we hold ctx::mutex and child_mutex, revalidate our
6005	* state, if child is still the first entry, it didn't get freed
6006	* and we can continue doing so.
6007	*/
6008	tmp = list_first_entry_or_null(&event->child_list,
6009	struct perf_event, child_list);
6010	if (tmp == child) {
6011	perf_remove_from_context(child, DETACH_GROUP \| DETACH_CHILD);
6012	} else {
6013	child = NULL;
6014	}
6015
6016	mutex_unlock(&event->child_mutex);
6017	mutex_unlock(&ctx->mutex);
6018
6019	if (child) {
6020	/* Last reference unless ->pending_task work is pending */
6021	put_event(child);
6022	}
6023	put_ctx(ctx);
6024
6025	goto again;
6026	}
6027	mutex_unlock(&event->child_mutex);
6028
6029	no_ctx:
6030	/*
6031	* Last reference unless ->pending_task work is pending on this event
6032	* or any of its children.
6033	*/
6034	put_event(event);
6035	return 0;
6036	}
6037	EXPORT_SYMBOL_GPL(perf_event_release_kernel);
6038
6039	/*
6040	* Called when the last reference to the file is gone.
6041	*/
6042	static int perf_release(struct inode inode, struct file file)
6043	{
6044	perf_event_release_kernel(file->private_data);
6045	return 0;
6046	}
6047
6048	static u64 __perf_event_read_value(struct perf_event event, u64 enabled, u64 *running)
6049	{
6050	struct perf_event *child;
6051	u64 total = 0;
6052
6053	*enabled = 0;
6054	*running = 0;
6055
6056	mutex_lock(&event->child_mutex);
6057
6058	(void)perf_event_read(event, false);
6059	total += perf_event_count(event, false);
6060
6061	*enabled += event->total_time_enabled +
6062	atomic64_read(&event->child_total_time_enabled);
6063	*running += event->total_time_running +
6064	atomic64_read(&event->child_total_time_running);
6065
6066	list_for_each_entry(child, &event->child_list, child_list) {
6067	(void)perf_event_read(child, false);
6068	total += perf_event_count(child, false);
6069	*enabled += child->total_time_enabled;
6070	*running += child->total_time_running;
6071	}
6072	mutex_unlock(&event->child_mutex);
6073
6074	return total;
6075	}
6076
6077	u64 perf_event_read_value(struct perf_event event, u64 enabled, u64 *running)
6078	{
6079	struct perf_event_context *ctx;
6080	u64 count;
6081
6082	ctx = perf_event_ctx_lock(event);
6083	count = __perf_event_read_value(event, enabled, running);
6084	perf_event_ctx_unlock(event, ctx);
6085
6086	return count;
6087	}
6088	EXPORT_SYMBOL_GPL(perf_event_read_value);
6089
6090	static int __perf_read_group_add(struct perf_event *leader,
6091	u64 read_format, u64 *values)
6092	{
6093	struct perf_event_context *ctx = leader->ctx;
6094	struct perf_event sub, parent;
6095	unsigned long flags;
6096	int n = 1; /* skip @nr */
6097	int ret;
6098
6099	ret = perf_event_read(leader, true);
6100	if (ret)
6101	return ret;
6102
6103	raw_spin_lock_irqsave(&ctx->lock, flags);
6104	/*
6105	* Verify the grouping between the parent and child (inherited)
6106	* events is still in tact.
6107	*
6108	* Specifically:
6109	* - leader->ctx->lock pins leader->sibling_list
6110	* - parent->child_mutex pins parent->child_list
6111	* - parent->ctx->mutex pins parent->sibling_list
6112	*
6113	* Because parent->ctx != leader->ctx (and child_list nests inside
6114	* ctx->mutex), group destruction is not atomic between children, also
6115	* see perf_event_release_kernel(). Additionally, parent can grow the
6116	* group.
6117	*
6118	* Therefore it is possible to have parent and child groups in a
6119	* different configuration and summing over such a beast makes no sense
6120	* what so ever.
6121	*
6122	* Reject this.
6123	*/
6124	parent = leader->parent;
6125	if (parent &&
6126	(parent->group_generation != leader->group_generation \|\|
6127	parent->nr_siblings != leader->nr_siblings)) {
6128	ret = -ECHILD;
6129	goto unlock;
6130	}
6131
6132	/*
6133	* Since we co-schedule groups, {enabled,running} times of siblings
6134	* will be identical to those of the leader, so we only publish one
6135	* set.
6136	*/
6137	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
6138	values[n++] += leader->total_time_enabled +
6139	atomic64_read(&leader->child_total_time_enabled);
6140	}
6141
6142	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
6143	values[n++] += leader->total_time_running +
6144	atomic64_read(&leader->child_total_time_running);
6145	}
6146
6147	/*
6148	* Write {count,id} tuples for every sibling.
6149	*/
6150	values[n++] += perf_event_count(leader, false);
6151	if (read_format & PERF_FORMAT_ID)
6152	values[n++] = primary_event_id(leader);
6153	if (read_format & PERF_FORMAT_LOST)
6154	values[n++] = atomic64_read(&leader->lost_samples);
6155
6156	for_each_sibling_event(sub, leader) {
6157	values[n++] += perf_event_count(sub, false);
6158	if (read_format & PERF_FORMAT_ID)
6159	values[n++] = primary_event_id(sub);
6160	if (read_format & PERF_FORMAT_LOST)
6161	values[n++] = atomic64_read(&sub->lost_samples);
6162	}
6163
6164	unlock:
6165	raw_spin_unlock_irqrestore(&ctx->lock, flags);
6166	return ret;
6167	}
6168
6169	static int perf_read_group(struct perf_event *event,
6170	u64 read_format, char __user *buf)
6171	{
6172	struct perf_event leader = event->group_leader, child;
6173	struct perf_event_context *ctx = leader->ctx;
6174	int ret;
6175	u64 *values;
6176
6177	lockdep_assert_held(&ctx->mutex);
6178
6179	values = kzalloc(event->read_size, GFP_KERNEL);
6180	if (!values)
6181	return -ENOMEM;
6182
6183	values[0] = 1 + leader->nr_siblings;
6184
6185	mutex_lock(&leader->child_mutex);
6186
6187	ret = __perf_read_group_add(leader, read_format, values);
6188	if (ret)
6189	goto unlock;
6190
6191	list_for_each_entry(child, &leader->child_list, child_list) {
6192	ret = __perf_read_group_add(child, read_format, values);
6193	if (ret)
6194	goto unlock;
6195	}
6196
6197	mutex_unlock(&leader->child_mutex);
6198
6199	ret = event->read_size;
6200	if (copy_to_user(buf, values, event->read_size))
6201	ret = -EFAULT;
6202	goto out;
6203
6204	unlock:
6205	mutex_unlock(&leader->child_mutex);
6206	out:
6207	kfree(values);
6208	return ret;
6209	}
6210
6211	static int perf_read_one(struct perf_event *event,
6212	u64 read_format, char __user *buf)
6213	{
6214	u64 enabled, running;
6215	u64 values[5];
6216	int n = 0;
6217
6218	values[n++] = __perf_event_read_value(event, &enabled, &running);
6219	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
6220	values[n++] = enabled;
6221	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
6222	values[n++] = running;
6223	if (read_format & PERF_FORMAT_ID)
6224	values[n++] = primary_event_id(event);
6225	if (read_format & PERF_FORMAT_LOST)
6226	values[n++] = atomic64_read(&event->lost_samples);
6227
6228	if (copy_to_user(buf, values, n * sizeof(u64)))
6229	return -EFAULT;
6230
6231	return n * sizeof(u64);
6232	}
6233
6234	static bool is_event_hup(struct perf_event *event)
6235	{
6236	bool no_children;
6237
6238	if (event->state > PERF_EVENT_STATE_EXIT)
6239	return false;
6240
6241	mutex_lock(&event->child_mutex);
6242	no_children = list_empty(&event->child_list);
6243	mutex_unlock(&event->child_mutex);
6244	return no_children;
6245	}
6246
6247	/*
6248	* Read the performance event - simple non blocking version for now
6249	*/
6250	static ssize_t
6251	__perf_read(struct perf_event event, char __user buf, size_t count)
6252	{
6253	u64 read_format = event->attr.read_format;
6254	int ret;
6255
6256	/*
6257	* Return end-of-file for a read on an event that is in
6258	* error state (i.e. because it was pinned but it couldn't be
6259	* scheduled on to the CPU at some point).
6260	*/
6261	if (event->state == PERF_EVENT_STATE_ERROR)
6262	return 0;
6263
6264	if (count < event->read_size)
6265	return -ENOSPC;
6266
6267	WARN_ON_ONCE(event->ctx->parent_ctx);
6268	if (read_format & PERF_FORMAT_GROUP)
6269	ret = perf_read_group(event, read_format, buf);
6270	else
6271	ret = perf_read_one(event, read_format, buf);
6272
6273	return ret;
6274	}
6275
6276	static ssize_t
6277	perf_read(struct file file, char __user buf, size_t count, loff_t *ppos)
6278	{
6279	struct perf_event *event = file->private_data;
6280	struct perf_event_context *ctx;
6281	int ret;
6282
6283	ret = security_perf_event_read(event);
6284	if (ret)
6285	return ret;
6286
6287	ctx = perf_event_ctx_lock(event);
6288	ret = __perf_read(event, buf, count);
6289	perf_event_ctx_unlock(event, ctx);
6290
6291	return ret;
6292	}
6293
6294	static __poll_t perf_poll(struct file file, poll_table wait)
6295	{
6296	struct perf_event *event = file->private_data;
6297	struct perf_buffer *rb;
6298	__poll_t events = EPOLLHUP;
6299
6300	if (event->state <= PERF_EVENT_STATE_REVOKED)
6301	return EPOLLERR;
6302
6303	poll_wait(file, &event->waitq, wait);
6304
6305	if (event->state <= PERF_EVENT_STATE_REVOKED)
6306	return EPOLLERR;
6307
6308	if (is_event_hup(event))
6309	return events;
6310
6311	if (unlikely(READ_ONCE(event->state) == PERF_EVENT_STATE_ERROR &&
6312	event->attr.pinned))
6313	return EPOLLERR;
6314
6315	/*
6316	* Pin the event->rb by taking event->mmap_mutex; otherwise
6317	* perf_event_set_output() can swizzle our rb and make us miss wakeups.
6318	*/
6319	mutex_lock(&event->mmap_mutex);
6320	rb = event->rb;
6321	if (rb)
6322	events = atomic_xchg(&rb->poll, 0);
6323	mutex_unlock(&event->mmap_mutex);
6324	return events;
6325	}
6326
6327	static void _perf_event_reset(struct perf_event *event)
6328	{
6329	(void)perf_event_read(event, false);
6330	local64_set(&event->count, 0);
6331	perf_event_update_userpage(event);
6332	}
6333
6334	/* Assume it's not an event with inherit set. */
6335	u64 perf_event_pause(struct perf_event *event, bool reset)
6336	{
6337	struct perf_event_context *ctx;
6338	u64 count;
6339
6340	ctx = perf_event_ctx_lock(event);
6341	WARN_ON_ONCE(event->attr.inherit);
6342	_perf_event_disable(event);
6343	count = local64_read(&event->count);
6344	if (reset)
6345	local64_set(&event->count, 0);
6346	perf_event_ctx_unlock(event, ctx);
6347
6348	return count;
6349	}
6350	EXPORT_SYMBOL_GPL(perf_event_pause);
6351
6352	#ifdef CONFIG_PERF_GUEST_MEDIATED_PMU
6353	static atomic_t nr_include_guest_events __read_mostly;
6354
6355	static atomic_t nr_mediated_pmu_vms __read_mostly;
6356	static DEFINE_MUTEX(perf_mediated_pmu_mutex);
6357
6358	/* !exclude_guest event of PMU with PERF_PMU_CAP_MEDIATED_VPMU */
6359	static inline bool is_include_guest_event(struct perf_event *event)
6360	{
6361	if (!event->pmu)
6362	return false;
6363
6364	if ((event->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) &&
6365	!event->attr.exclude_guest)
6366	return true;
6367
6368	return false;
6369	}
6370
6371	static int mediated_pmu_account_event(struct perf_event *event)
6372	{
6373	if (!is_include_guest_event(event))
6374	return 0;
6375
6376	if (atomic_inc_not_zero(&nr_include_guest_events))
6377	return 0;
6378
6379	guard(mutex)(&perf_mediated_pmu_mutex);
6380	if (atomic_read(&nr_mediated_pmu_vms))
6381	return -EOPNOTSUPP;
6382
6383	atomic_inc(&nr_include_guest_events);
6384	return 0;
6385	}
6386
6387	static void mediated_pmu_unaccount_event(struct perf_event *event)
6388	{
6389	if (!is_include_guest_event(event))
6390	return;
6391
6392	if (WARN_ON_ONCE(!atomic_read(&nr_include_guest_events)))
6393	return;
6394
6395	atomic_dec(&nr_include_guest_events);
6396	}
6397
6398	/*
6399	* Currently invoked at VM creation to
6400	* - Check whether there are existing !exclude_guest events of PMU with
6401	* PERF_PMU_CAP_MEDIATED_VPMU
6402	* - Set nr_mediated_pmu_vms to prevent !exclude_guest event creation on
6403	* PMUs with PERF_PMU_CAP_MEDIATED_VPMU
6404	*
6405	* No impact for the PMU without PERF_PMU_CAP_MEDIATED_VPMU. The perf
6406	* still owns all the PMU resources.
6407	*/
6408	int perf_create_mediated_pmu(void)
6409	{
6410	if (atomic_inc_not_zero(&nr_mediated_pmu_vms))
6411	return 0;
6412
6413	guard(mutex)(&perf_mediated_pmu_mutex);
6414	if (atomic_read(&nr_include_guest_events))
6415	return -EBUSY;
6416
6417	atomic_inc(&nr_mediated_pmu_vms);
6418	return 0;
6419	}
6420	EXPORT_SYMBOL_FOR_KVM(perf_create_mediated_pmu);
6421
6422	void perf_release_mediated_pmu(void)
6423	{
6424	if (WARN_ON_ONCE(!atomic_read(&nr_mediated_pmu_vms)))
6425	return;
6426
6427	atomic_dec(&nr_mediated_pmu_vms);
6428	}
6429	EXPORT_SYMBOL_FOR_KVM(perf_release_mediated_pmu);
6430
6431	/* When loading a guest's mediated PMU, schedule out all exclude_guest events. */
6432	void perf_load_guest_context(void)
6433	{
6434	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
6435
6436	lockdep_assert_irqs_disabled();
6437
6438	guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx);
6439
6440	if (WARN_ON_ONCE(__this_cpu_read(guest_ctx_loaded)))
6441	return;
6442
6443	perf_ctx_disable(&cpuctx->ctx, EVENT_GUEST);
6444	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_GUEST);
6445	if (cpuctx->task_ctx) {
6446	perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST);
6447	task_ctx_sched_out(cpuctx->task_ctx, NULL, EVENT_GUEST);
6448	}
6449
6450	perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST);
6451	if (cpuctx->task_ctx)
6452	perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST);
6453
6454	__this_cpu_write(guest_ctx_loaded, true);
6455	}
6456	EXPORT_SYMBOL_GPL(perf_load_guest_context);
6457
6458	void perf_put_guest_context(void)
6459	{
6460	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
6461
6462	lockdep_assert_irqs_disabled();
6463
6464	guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx);
6465
6466	if (WARN_ON_ONCE(!__this_cpu_read(guest_ctx_loaded)))
6467	return;
6468
6469	perf_ctx_disable(&cpuctx->ctx, EVENT_GUEST);
6470	if (cpuctx->task_ctx)
6471	perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST);
6472
6473	perf_event_sched_in(cpuctx, cpuctx->task_ctx, NULL, EVENT_GUEST);
6474
6475	if (cpuctx->task_ctx)
6476	perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST);
6477	perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST);
6478
6479	__this_cpu_write(guest_ctx_loaded, false);
6480	}
6481	EXPORT_SYMBOL_GPL(perf_put_guest_context);
6482	#else
6483	static int mediated_pmu_account_event(struct perf_event *event) { return 0; }
6484	static void mediated_pmu_unaccount_event(struct perf_event *event) {}
6485	#endif
6486
6487	/*
6488	* Holding the top-level event's child_mutex means that any
6489	* descendant process that has inherited this event will block
6490	* in perf_event_exit_event() if it goes to exit, thus satisfying the
6491	* task existence requirements of perf_event_enable/disable.
6492	*/
6493	static void perf_event_for_each_child(struct perf_event *event,
6494	void (func)(struct perf_event ))
6495	{
6496	struct perf_event *child;
6497
6498	WARN_ON_ONCE(event->ctx->parent_ctx);
6499
6500	mutex_lock(&event->child_mutex);
6501	func(event);
6502	list_for_each_entry(child, &event->child_list, child_list)
6503	func(child);
6504	mutex_unlock(&event->child_mutex);
6505	}
6506
6507	static void perf_event_for_each(struct perf_event *event,
6508	void (func)(struct perf_event ))
6509	{
6510	struct perf_event_context *ctx = event->ctx;
6511	struct perf_event *sibling;
6512
6513	lockdep_assert_held(&ctx->mutex);
6514
6515	event = event->group_leader;
6516
6517	perf_event_for_each_child(event, func);
6518	for_each_sibling_event(sibling, event)
6519	perf_event_for_each_child(sibling, func);
6520	}
6521
6522	static void __perf_event_period(struct perf_event *event,
6523	struct perf_cpu_context *cpuctx,
6524	struct perf_event_context *ctx,
6525	void *info)
6526	{
6527	u64 value = ((u64 )info);
6528	bool active;
6529
6530	if (event->attr.freq) {
6531	event->attr.sample_freq = value;
6532	} else {
6533	event->attr.sample_period = value;
6534	event->hw.sample_period = value;
6535	}
6536
6537	active = (event->state == PERF_EVENT_STATE_ACTIVE);
6538	if (active) {
6539	perf_pmu_disable(event->pmu);
6540	event->pmu->stop(event, PERF_EF_UPDATE);
6541	}
6542
6543	local64_set(&event->hw.period_left, 0);
6544
6545	if (active) {
6546	event->pmu->start(event, PERF_EF_RELOAD);
6547	/*
6548	* Once the period is force-reset, the event starts immediately.
6549	* But the event/group could be throttled. Unthrottle the
6550	* event/group now to avoid the next tick trying to unthrottle
6551	* while we already re-started the event/group.
6552	*/
6553	if (event->hw.interrupts == MAX_INTERRUPTS)
6554	perf_event_unthrottle_group(event, true);
6555	perf_pmu_enable(event->pmu);
6556	}
6557	}
6558
6559	static int perf_event_check_period(struct perf_event *event, u64 value)
6560	{
6561	return event->pmu->check_period(event, value);
6562	}
6563
6564	static int _perf_event_period(struct perf_event *event, u64 value)
6565	{
6566	if (!is_sampling_event(event))
6567	return -EINVAL;
6568
6569	if (!value)
6570	return -EINVAL;
6571
6572	if (event->attr.freq) {
6573	if (value > sysctl_perf_event_sample_rate)
6574	return -EINVAL;
6575	} else {
6576	if (perf_event_check_period(event, value))
6577	return -EINVAL;
6578	if (value & (1ULL << 63))
6579	return -EINVAL;
6580	}
6581
6582	event_function_call(event, __perf_event_period, &value);
6583
6584	return 0;
6585	}
6586
6587	int perf_event_period(struct perf_event *event, u64 value)
6588	{
6589	struct perf_event_context *ctx;
6590	int ret;
6591
6592	ctx = perf_event_ctx_lock(event);
6593	ret = _perf_event_period(event, value);
6594	perf_event_ctx_unlock(event, ctx);
6595
6596	return ret;
6597	}
6598	EXPORT_SYMBOL_GPL(perf_event_period);
6599
6600	static const struct file_operations perf_fops;
6601
6602	static inline bool is_perf_file(struct fd f)
6603	{
6604	return !fd_empty(f) && fd_file(f)->f_op == &perf_fops;
6605	}
6606
6607	static int perf_event_set_output(struct perf_event *event,
6608	struct perf_event *output_event);
6609	static int perf_event_set_filter(struct perf_event event, void __user arg);
6610	static int perf_copy_attr(struct perf_event_attr __user *uattr,
6611	struct perf_event_attr *attr);
6612	static int __perf_event_set_bpf_prog(struct perf_event *event,
6613	struct bpf_prog *prog,
6614	u64 bpf_cookie);
6615
6616	static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
6617	{
6618	void (func)(struct perf_event );
6619	u32 flags = arg;
6620
6621	if (event->state <= PERF_EVENT_STATE_REVOKED)
6622	return -ENODEV;
6623
6624	switch (cmd) {
6625	case PERF_EVENT_IOC_ENABLE:
6626	func = _perf_event_enable;
6627	break;
6628	case PERF_EVENT_IOC_DISABLE:
6629	func = _perf_event_disable;
6630	break;
6631	case PERF_EVENT_IOC_RESET:
6632	func = _perf_event_reset;
6633	break;
6634
6635	case PERF_EVENT_IOC_REFRESH:
6636	return _perf_event_refresh(event, arg);
6637
6638	case PERF_EVENT_IOC_PERIOD:
6639	{
6640	u64 value;
6641
6642	if (copy_from_user(&value, (u64 __user *)arg, sizeof(value)))
6643	return -EFAULT;
6644
6645	return _perf_event_period(event, value);
6646	}
6647	case PERF_EVENT_IOC_ID:
6648	{
6649	u64 id = primary_event_id(event);
6650
6651	if (copy_to_user((void __user *)arg, &id, sizeof(id)))
6652	return -EFAULT;
6653	return 0;
6654	}
6655
6656	case PERF_EVENT_IOC_SET_OUTPUT:
6657	{
6658	CLASS(fd, output)(arg); // arg == -1 => empty
6659	struct perf_event *output_event = NULL;
6660	if (arg != -1) {
6661	if (!is_perf_file(output))
6662	return -EBADF;
6663	output_event = fd_file(output)->private_data;
6664	}
6665	return perf_event_set_output(event, output_event);
6666	}
6667
6668	case PERF_EVENT_IOC_SET_FILTER:
6669	return perf_event_set_filter(event, (void __user *)arg);
6670
6671	case PERF_EVENT_IOC_SET_BPF:
6672	{
6673	struct bpf_prog *prog;
6674	int err;
6675
6676	prog = bpf_prog_get(arg);
6677	if (IS_ERR(prog))
6678	return PTR_ERR(prog);
6679
6680	err = __perf_event_set_bpf_prog(event, prog, 0);
6681	if (err) {
6682	bpf_prog_put(prog);
6683	return err;
6684	}
6685
6686	return 0;
6687	}
6688
6689	case PERF_EVENT_IOC_PAUSE_OUTPUT: {
6690	struct perf_buffer *rb;
6691
6692	rcu_read_lock();
6693	rb = rcu_dereference(event->rb);
6694	if (!rb \|\| !rb->nr_pages) {
6695	rcu_read_unlock();
6696	return -EINVAL;
6697	}
6698	rb_toggle_paused(rb, !!arg);
6699	rcu_read_unlock();
6700	return 0;
6701	}
6702
6703	case PERF_EVENT_IOC_QUERY_BPF:
6704	return perf_event_query_prog_array(event, (void __user *)arg);
6705
6706	case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: {
6707	struct perf_event_attr new_attr;
6708	int err = perf_copy_attr((struct perf_event_attr __user *)arg,
6709	&new_attr);
6710
6711	if (err)
6712	return err;
6713
6714	return perf_event_modify_attr(event, &new_attr);
6715	}
6716	default:
6717	return -ENOTTY;
6718	}
6719
6720	if (flags & PERF_IOC_FLAG_GROUP)
6721	perf_event_for_each(event, func);
6722	else
6723	perf_event_for_each_child(event, func);
6724
6725	return 0;
6726	}
6727
6728	static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
6729	{
6730	struct perf_event *event = file->private_data;
6731	struct perf_event_context *ctx;
6732	long ret;
6733
6734	/* Treat ioctl like writes as it is likely a mutating operation. */
6735	ret = security_perf_event_write(event);
6736	if (ret)
6737	return ret;
6738
6739	ctx = perf_event_ctx_lock(event);
6740	ret = _perf_ioctl(event, cmd, arg);
6741	perf_event_ctx_unlock(event, ctx);
6742
6743	return ret;
6744	}
6745
6746	#ifdef CONFIG_COMPAT
6747	static long perf_compat_ioctl(struct file *file, unsigned int cmd,
6748	unsigned long arg)
6749	{
6750	switch (_IOC_NR(cmd)) {
6751	case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
6752	case _IOC_NR(PERF_EVENT_IOC_ID):
6753	case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF):
6754	case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES):
6755	/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
6756	if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
6757	cmd &= ~IOCSIZE_MASK;
6758	cmd \|= sizeof(void *) << IOCSIZE_SHIFT;
6759	}
6760	break;
6761	}
6762	return perf_ioctl(file, cmd, arg);
6763	}
6764	#else
6765	# define perf_compat_ioctl NULL
6766	#endif
6767
6768	int perf_event_task_enable(void)
6769	{
6770	struct perf_event_context *ctx;
6771	struct perf_event *event;
6772
6773	mutex_lock(&current->perf_event_mutex);
6774	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
6775	ctx = perf_event_ctx_lock(event);
6776	perf_event_for_each_child(event, _perf_event_enable);
6777	perf_event_ctx_unlock(event, ctx);
6778	}
6779	mutex_unlock(&current->perf_event_mutex);
6780
6781	return 0;
6782	}
6783
6784	int perf_event_task_disable(void)
6785	{
6786	struct perf_event_context *ctx;
6787	struct perf_event *event;
6788
6789	mutex_lock(&current->perf_event_mutex);
6790	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
6791	ctx = perf_event_ctx_lock(event);
6792	perf_event_for_each_child(event, _perf_event_disable);
6793	perf_event_ctx_unlock(event, ctx);
6794	}
6795	mutex_unlock(&current->perf_event_mutex);
6796
6797	return 0;
6798	}
6799
6800	static int perf_event_index(struct perf_event *event)
6801	{
6802	if (event->hw.state & PERF_HES_STOPPED)
6803	return 0;
6804
6805	if (event->state != PERF_EVENT_STATE_ACTIVE)
6806	return 0;
6807
6808	return event->pmu->event_idx(event);
6809	}
6810
6811	static void perf_event_init_userpage(struct perf_event *event)
6812	{
6813	struct perf_event_mmap_page *userpg;
6814	struct perf_buffer *rb;
6815
6816	rcu_read_lock();
6817	rb = rcu_dereference(event->rb);
6818	if (!rb)
6819	goto unlock;
6820
6821	userpg = rb->user_page;
6822
6823	/* Allow new userspace to detect that bit 0 is deprecated */
6824	userpg->cap_bit0_is_deprecated = 1;
6825	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
6826	userpg->data_offset = PAGE_SIZE;
6827	userpg->data_size = perf_data_size(rb);
6828
6829	unlock:
6830	rcu_read_unlock();
6831	}
6832
6833	void __weak arch_perf_update_userpage(
6834	struct perf_event event, struct perf_event_mmap_page userpg, u64 now)
6835	{
6836	}
6837
6838	/*
6839	* Callers need to ensure there can be no nesting of this function, otherwise
6840	* the seqlock logic goes bad. We can not serialize this because the arch
6841	* code calls this from NMI context.
6842	*/
6843	void perf_event_update_userpage(struct perf_event *event)
6844	{
6845	struct perf_event_mmap_page *userpg;
6846	struct perf_buffer *rb;
6847	u64 enabled, running, now;
6848
6849	rcu_read_lock();
6850	rb = rcu_dereference(event->rb);
6851	if (!rb)
6852	goto unlock;
6853
6854	/*
6855	* Disable preemption to guarantee consistent time stamps are stored to
6856	* the user page.
6857	*/
6858	preempt_disable();
6859
6860	/*
6861	* Compute total_time_enabled, total_time_running based on snapshot
6862	* values taken when the event was last scheduled in.
6863	*
6864	* We cannot simply call update_context_time() because doing so would
6865	* lead to deadlock when called from NMI context.
6866	*/
6867	calc_timer_values(event, &now, &enabled, &running);
6868
6869	userpg = rb->user_page;
6870
6871	++userpg->lock;
6872	barrier();
6873	userpg->index = perf_event_index(event);
6874	userpg->offset = perf_event_count(event, false);
6875	if (userpg->index)
6876	userpg->offset -= local64_read(&event->hw.prev_count);
6877
6878	userpg->time_enabled = enabled +
6879	atomic64_read(&event->child_total_time_enabled);
6880
6881	userpg->time_running = running +
6882	atomic64_read(&event->child_total_time_running);
6883
6884	arch_perf_update_userpage(event, userpg, now);
6885
6886	barrier();
6887	++userpg->lock;
6888	preempt_enable();
6889	unlock:
6890	rcu_read_unlock();
6891	}
6892	EXPORT_SYMBOL_GPL(perf_event_update_userpage);
6893
6894	static void ring_buffer_attach(struct perf_event *event,
6895	struct perf_buffer *rb)
6896	{
6897	struct perf_buffer *old_rb = NULL;
6898	unsigned long flags;
6899
6900	WARN_ON_ONCE(event->parent);
6901
6902	if (event->rb) {
6903	/*
6904	* Should be impossible, we set this when removing
6905	* event->rb_entry and wait/clear when adding event->rb_entry.
6906	*/
6907	WARN_ON_ONCE(event->rcu_pending);
6908
6909	old_rb = event->rb;
6910	spin_lock_irqsave(&old_rb->event_lock, flags);
6911	list_del_rcu(&event->rb_entry);
6912	spin_unlock_irqrestore(&old_rb->event_lock, flags);
6913
6914	event->rcu_batches = get_state_synchronize_rcu();
6915	event->rcu_pending = 1;
6916	}
6917
6918	if (rb) {
6919	if (event->rcu_pending) {
6920	cond_synchronize_rcu(event->rcu_batches);
6921	event->rcu_pending = 0;
6922	}
6923
6924	spin_lock_irqsave(&rb->event_lock, flags);
6925	list_add_rcu(&event->rb_entry, &rb->event_list);
6926	spin_unlock_irqrestore(&rb->event_lock, flags);
6927	}
6928
6929	/*
6930	* Avoid racing with perf_mmap_close(AUX): stop the event
6931	* before swizzling the event::rb pointer; if it's getting
6932	* unmapped, its aux_mmap_count will be 0 and it won't
6933	* restart. See the comment in __perf_pmu_output_stop().
6934	*
6935	* Data will inevitably be lost when set_output is done in
6936	* mid-air, but then again, whoever does it like this is
6937	* not in for the data anyway.
6938	*/
6939	if (has_aux(event))
6940	perf_event_stop(event, 0);
6941
6942	rcu_assign_pointer(event->rb, rb);
6943
6944	if (old_rb) {
6945	ring_buffer_put(old_rb);
6946	/*
6947	* Since we detached before setting the new rb, so that we
6948	* could attach the new rb, we could have missed a wakeup.
6949	* Provide it now.
6950	*/
6951	wake_up_all(&event->waitq);
6952	}
6953	}
6954
6955	static void ring_buffer_wakeup(struct perf_event *event)
6956	{
6957	struct perf_buffer *rb;
6958
6959	if (event->parent)
6960	event = event->parent;
6961
6962	rcu_read_lock();
6963	rb = rcu_dereference(event->rb);
6964	if (rb) {
6965	list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
6966	wake_up_all(&event->waitq);
6967	}
6968	rcu_read_unlock();
6969	}
6970
6971	struct perf_buffer ring_buffer_get(struct perf_event event)
6972	{
6973	struct perf_buffer *rb;
6974
6975	if (event->parent)
6976	event = event->parent;
6977
6978	rcu_read_lock();
6979	rb = rcu_dereference(event->rb);
6980	if (rb) {
6981	if (!refcount_inc_not_zero(&rb->refcount))
6982	rb = NULL;
6983	}
6984	rcu_read_unlock();
6985
6986	return rb;
6987	}
6988
6989	void ring_buffer_put(struct perf_buffer *rb)
6990	{
6991	if (!refcount_dec_and_test(&rb->refcount))
6992	return;
6993
6994	WARN_ON_ONCE(!list_empty(&rb->event_list));
6995
6996	call_rcu(&rb->rcu_head, rb_free_rcu);
6997	}
6998
6999	typedef void (mapped_f)(struct perf_event event, struct mm_struct *mm);
7000
7001	#define get_mapped(event, func) \
7002	({ struct pmu *pmu; \
7003	mapped_f f = NULL; \
7004	guard(rcu)(); \
7005	pmu = READ_ONCE(event->pmu); \
7006	if (pmu) \
7007	f = pmu->func; \
7008	f; \
7009	})
7010
7011	static void perf_mmap_open(struct vm_area_struct *vma)
7012	{
7013	struct perf_event *event = vma->vm_file->private_data;
7014	mapped_f mapped = get_mapped(event, event_mapped);
7015
7016	refcount_inc(&event->mmap_count);
7017	refcount_inc(&event->rb->mmap_count);
7018
7019	if (vma_start_pgoff(vma))
7020	refcount_inc(&event->rb->aux_mmap_count);
7021
7022	if (mapped)
7023	mapped(event, vma->vm_mm);
7024	}
7025
7026	static void perf_pmu_output_stop(struct perf_event *event);
7027	static void perf_mmap_unaccount(struct vm_area_struct vma, struct perf_buffer rb);
7028
7029	/*
7030	* A buffer can be mmap()ed multiple times; either directly through the same
7031	* event, or through other events by use of perf_event_set_output().
7032	*
7033	* In order to undo the VM accounting done by perf_mmap() we need to destroy
7034	* the buffer here, where we still have a VM context. This means we need
7035	* to detach all events redirecting to us.
7036	*/
7037	static void perf_mmap_close(struct vm_area_struct *vma)
7038	{
7039	struct perf_event *event = vma->vm_file->private_data;
7040	mapped_f unmapped = get_mapped(event, event_unmapped);
7041	struct perf_buffer *rb = ring_buffer_get(event);
7042	struct user_struct *mmap_user = rb->mmap_user;
7043
7044	/* FIXIES vs perf_pmu_unregister() */
7045	if (unmapped)
7046	unmapped(event, vma->vm_mm);
7047
7048	/*
7049	* The AUX buffer is strictly a sub-buffer, serialize using aux_mutex
7050	* to avoid complications.
7051	*/
7052	if (rb_has_aux(rb) && vma_start_pgoff(vma) == rb->aux_pgoff &&
7053	refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {
7054	/*
7055	* Stop all AUX events that are writing to this buffer,
7056	* so that we can free its AUX pages and corresponding PMU
7057	* data. Note that after rb::aux_mmap_count dropped to zero,
7058	* they won't start any more (see perf_aux_output_begin()).
7059	*/
7060	perf_pmu_output_stop(event);
7061
7062	/* now it's safe to free the pages */
7063	atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->perf_mlock);
7064	atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);
7065
7066	/* this has to be the last one */
7067	rb_free_aux(rb);
7068	WARN_ON_ONCE(refcount_read(&rb->aux_refcount));
7069
7070	mutex_unlock(&rb->aux_mutex);
7071	}
7072
7073	/*
7074	* Drop references in reverse order of perf_mmap() to prevent
7075	* rb revival after rb->mmap_count reaches zero.
7076	*/
7077	if (refcount_dec_and_mutex_lock(&event->mmap_count,
7078	&event->mmap_mutex)) {
7079	ring_buffer_attach(event, NULL);
7080	mutex_unlock(&event->mmap_mutex);
7081	}
7082
7083	/* If there's still other mmap()s of this buffer, we're done. */
7084	if (!refcount_dec_and_test(&rb->mmap_count))
7085	goto out_put;
7086
7087	/*
7088	* No other mmap()s, detach from all other events that might redirect
7089	* into the now unreachable buffer. Somewhat complicated by the
7090	* fact that rb::event_lock otherwise nests inside mmap_mutex.
7091	*/
7092	again:
7093	rcu_read_lock();
7094	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
7095	if (!atomic_long_inc_not_zero(&event->refcount)) {
7096	/*
7097	* This event is en-route to free_event() which will
7098	* detach it and remove it from the list.
7099	*/
7100	continue;
7101	}
7102	rcu_read_unlock();
7103
7104	mutex_lock(&event->mmap_mutex);
7105	/*
7106	* Check we didn't race with perf_event_set_output() which can
7107	* swizzle the rb from under us while we were waiting to
7108	* acquire mmap_mutex.
7109	*
7110	* If we find a different rb; ignore this event, a next
7111	* iteration will no longer find it on the list. We have to
7112	* still restart the iteration to make sure we're not now
7113	* iterating the wrong list.
7114	*/
7115	if (event->rb == rb)
7116	ring_buffer_attach(event, NULL);
7117
7118	mutex_unlock(&event->mmap_mutex);
7119	put_event(event);
7120
7121	/*
7122	* Restart the iteration; either we're on the wrong list or
7123	* destroyed its integrity by doing a deletion.
7124	*/
7125	goto again;
7126	}
7127	rcu_read_unlock();
7128
7129	/*
7130	* It could be there's still a few 0-ref events on the list; they'll
7131	* get cleaned up by free_event() -- they'll also still have their
7132	* ref on the rb and will free it whenever they are done with it.
7133	*
7134	* Aside from that, this buffer is 'fully' detached and unmapped,
7135	* undo the VM accounting.
7136	*/
7137	perf_mmap_unaccount(vma, rb);
7138
7139	out_put:
7140	ring_buffer_put(rb); /* could be last */
7141	}
7142
7143	static vm_fault_t perf_mmap_pfn_mkwrite(struct vm_fault *vmf)
7144	{
7145	/* The first page is the user control page, others are read-only. */
7146	return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS;
7147	}
7148
7149	static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr)
7150	{
7151	/*
7152	* Forbid splitting perf mappings to prevent refcount leaks due to
7153	* the resulting non-matching offsets and sizes. See open()/close().
7154	*/
7155	return -EINVAL;
7156	}
7157
7158	static const struct vm_operations_struct perf_mmap_vmops = {
7159	.open = perf_mmap_open,
7160	.close = perf_mmap_close, /* non mergeable */
7161	.pfn_mkwrite = perf_mmap_pfn_mkwrite,
7162	.may_split = perf_mmap_may_split,
7163	};
7164
7165	static int map_range(struct perf_buffer rb, struct vm_area_struct vma)
7166	{
7167	unsigned long nr_pages = vma_pages(vma);
7168	int err = 0;
7169	unsigned long pagenum;
7170
7171	guard(mutex)(&rb->aux_mutex);
7172
7173	/*
7174	* We map this as a VM_PFNMAP VMA.
7175	*
7176	* This is not ideal as this is designed broadly for mappings of PFNs
7177	* referencing memory-mapped I/O ranges or non-system RAM i.e. for which
7178	* !pfn_valid(pfn).
7179	*
7180	* We are mapping kernel-allocated memory (memory we manage ourselves)
7181	* which would more ideally be mapped using vm_insert_page() or a
7182	* similar mechanism, that is as a VM_MIXEDMAP mapping.
7183	*
7184	* However this won't work here, because:
7185	*
7186	* 1. It uses vma->vm_page_prot, but this field has not been completely
7187	* setup at the point of the f_op->mmp() hook, so we are unable to
7188	* indicate that this should be mapped CoW in order that the
7189	* mkwrite() hook can be invoked to make the first page R/W and the
7190	* rest R/O as desired.
7191	*
7192	* 2. Anything other than a VM_PFNMAP of valid PFNs will result in
7193	* vm_normal_page() returning a struct page * pointer, which means
7194	* vm_ops->page_mkwrite() will be invoked rather than
7195	* vm_ops->pfn_mkwrite(), and this means we have to set page->mapping
7196	* to work around retry logic in the fault handler, however this
7197	* field is no longer allowed to be used within struct page.
7198	*
7199	* 3. Having a struct page * made available in the fault logic also
7200	* means that the page gets put on the rmap and becomes
7201	* inappropriately accessible and subject to map and ref counting.
7202	*
7203	* Ideally we would have a mechanism that could explicitly express our
7204	* desires, but this is not currently the case, so we instead use
7205	* VM_PFNMAP.
7206	*
7207	* We manage the lifetime of these mappings with internal refcounts (see
7208	* perf_mmap_open() and perf_mmap_close()) so we ensure the lifetime of
7209	* this mapping is maintained correctly.
7210	*/
7211	for (pagenum = 0; pagenum < nr_pages; pagenum++) {
7212	unsigned long va = vma->vm_start + PAGE_SIZE * pagenum;
7213	struct page *page = perf_mmap_to_page(rb,
7214	vma_start_pgoff(vma) + pagenum);
7215
7216	if (page == NULL) {
7217	err = -EINVAL;
7218	break;
7219	}
7220
7221	/* Map readonly, perf_mmap_pfn_mkwrite() called on write fault. */
7222	err = remap_pfn_range(vma, va, page_to_pfn(page), PAGE_SIZE,
7223	vm_get_page_prot(vma->vm_flags & ~VM_SHARED));
7224	if (err)
7225	break;
7226	}
7227
7228	#ifdef CONFIG_MMU
7229	/* Clear any partial mappings on error. */
7230	if (err)
7231	zap_vma_range(vma, vma->vm_start, nr_pages * PAGE_SIZE);
7232	#endif
7233
7234	return err;
7235	}
7236
7237	static bool perf_mmap_calc_limits(struct vm_area_struct vma, long user_extra, long *extra)
7238	{
7239	unsigned long user_locked, user_lock_limit, locked, lock_limit;
7240	struct user_struct *user = current_user();
7241
7242	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
7243	/* Increase the limit linearly with more CPUs */
7244	user_lock_limit *= num_online_cpus();
7245
7246	user_locked = atomic_long_read(&user->perf_mlock);
7247
7248	/*
7249	* sysctl_perf_event_mlock may have changed, so that
7250	* user->perf_mlock > user_lock_limit
7251	*/
7252	if (user_locked > user_lock_limit)
7253	user_locked = user_lock_limit;
7254	user_locked += *user_extra;
7255
7256	if (user_locked > user_lock_limit) {
7257	/*
7258	* charge perf_mlock until it hits user_lock_limit;
7259	* charge the rest from pinned_vm
7260	*/
7261	*extra = user_locked - user_lock_limit;
7262	user_extra -= extra;
7263	}
7264
7265	lock_limit = rlimit(RLIMIT_MEMLOCK);
7266	lock_limit >>= PAGE_SHIFT;
7267	locked = atomic64_read(&vma->vm_mm->pinned_vm) + *extra;
7268
7269	return locked <= lock_limit \|\| !perf_is_paranoid() \|\| capable(CAP_IPC_LOCK);
7270	}
7271
7272	static void perf_mmap_account(struct vm_area_struct *vma, long user_extra, long extra)
7273	{
7274	struct user_struct *user = current_user();
7275
7276	atomic_long_add(user_extra, &user->perf_mlock);
7277	atomic64_add(extra, &vma->vm_mm->pinned_vm);
7278	}
7279
7280	static void perf_mmap_unaccount(struct vm_area_struct vma, struct perf_buffer rb)
7281	{
7282	struct user_struct *user = rb->mmap_user;
7283
7284	atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked,
7285	&user->perf_mlock);
7286	atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm);
7287	}
7288
7289	static int perf_mmap_rb(struct vm_area_struct vma, struct perf_event event,
7290	unsigned long nr_pages)
7291	{
7292	long extra = 0, user_extra = nr_pages;
7293	struct perf_buffer *rb;
7294	int rb_flags = 0;
7295
7296	nr_pages -= 1;
7297
7298	/*
7299	* If we have rb pages ensure they're a power-of-two number, so we
7300	* can do bitmasks instead of modulo.
7301	*/
7302	if (nr_pages != 0 && !is_power_of_2(nr_pages))
7303	return -EINVAL;
7304
7305	WARN_ON_ONCE(event->ctx->parent_ctx);
7306
7307	if (event->rb) {
7308	if (data_page_nr(event->rb) != nr_pages)
7309	return -EINVAL;
7310
7311	/*
7312	* If this event doesn't have mmap_count, we're attempting to
7313	* create an alias of another event's mmap(); this would mean
7314	* both events will end up scribbling the same user_page;
7315	* which makes no sense.
7316	*/
7317	if (!refcount_read(&event->mmap_count))
7318	return -EBUSY;
7319
7320	if (refcount_inc_not_zero(&event->rb->mmap_count)) {
7321	/*
7322	* Success -- managed to mmap() the same buffer
7323	* multiple times.
7324	*/
7325	perf_mmap_account(vma, user_extra, extra);
7326	refcount_inc(&event->mmap_count);
7327	return 0;
7328	}
7329
7330	/*
7331	* Raced against perf_mmap_close()'s
7332	* refcount_dec_and_mutex_lock() remove the
7333	* event and continue as if !event->rb
7334	*/
7335	ring_buffer_attach(event, NULL);
7336	}
7337
7338	if (!perf_mmap_calc_limits(vma, &user_extra, &extra))
7339	return -EPERM;
7340
7341	if (vma->vm_flags & VM_WRITE)
7342	rb_flags \|= RING_BUFFER_WRITABLE;
7343
7344	rb = rb_alloc(nr_pages,
7345	event->attr.watermark ? event->attr.wakeup_watermark : 0,
7346	event->cpu, rb_flags);
7347
7348	if (!rb)
7349	return -ENOMEM;
7350
7351	rb->mmap_locked = extra;
7352
7353	ring_buffer_attach(event, rb);
7354
7355	perf_event_update_time(event);
7356	perf_event_init_userpage(event);
7357	perf_event_update_userpage(event);
7358
7359	perf_mmap_account(vma, user_extra, extra);
7360	refcount_set(&event->mmap_count, 1);
7361
7362	return 0;
7363	}
7364
7365	static int perf_mmap_aux(struct vm_area_struct vma, struct perf_event event,
7366	unsigned long nr_pages)
7367	{
7368	const pgoff_t pgoff_start = vma_start_pgoff(vma);
7369	long extra = 0, user_extra = nr_pages;
7370	u64 aux_offset, aux_size;
7371	struct perf_buffer *rb;
7372	int ret, rb_flags = 0;
7373
7374	rb = event->rb;
7375	if (!rb)
7376	return -EINVAL;
7377
7378	guard(mutex)(&rb->aux_mutex);
7379
7380	/*
7381	* AUX area mapping: if rb->aux_nr_pages != 0, it's already
7382	* mapped, all subsequent mappings should have the same size
7383	* and offset. Must be above the normal perf buffer.
7384	*/
7385	aux_offset = READ_ONCE(rb->user_page->aux_offset);
7386	aux_size = READ_ONCE(rb->user_page->aux_size);
7387
7388	if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
7389	return -EINVAL;
7390
7391	if (aux_offset != pgoff_start << PAGE_SHIFT)
7392	return -EINVAL;
7393
7394	/* already mapped with a different offset */
7395	if (rb_has_aux(rb) && rb->aux_pgoff != pgoff_start)
7396	return -EINVAL;
7397
7398	if (aux_size != nr_pages * PAGE_SIZE)
7399	return -EINVAL;
7400
7401	/* already mapped with a different size */
7402	if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
7403	return -EINVAL;
7404
7405	if (!is_power_of_2(nr_pages))
7406	return -EINVAL;
7407
7408	if (!refcount_inc_not_zero(&rb->mmap_count))
7409	return -EINVAL;
7410
7411	if (rb_has_aux(rb)) {
7412	refcount_inc(&rb->aux_mmap_count);
7413
7414	} else {
7415	if (!perf_mmap_calc_limits(vma, &user_extra, &extra)) {
7416	refcount_dec(&rb->mmap_count);
7417	return -EPERM;
7418	}
7419
7420	WARN_ON(!rb && event->rb);
7421
7422	if (vma->vm_flags & VM_WRITE)
7423	rb_flags \|= RING_BUFFER_WRITABLE;
7424
7425	ret = rb_alloc_aux(rb, event, pgoff_start, nr_pages,
7426	event->attr.aux_watermark, rb_flags);
7427	if (ret) {
7428	refcount_dec(&rb->mmap_count);
7429	return ret;
7430	}
7431
7432	refcount_set(&rb->aux_mmap_count, 1);
7433	rb->aux_mmap_locked = extra;
7434	}
7435
7436	perf_mmap_account(vma, user_extra, extra);
7437	refcount_inc(&event->mmap_count);
7438
7439	return 0;
7440	}
7441
7442	static int perf_mmap(struct file file, struct vm_area_struct vma)
7443	{
7444	struct perf_event *event = file->private_data;
7445	unsigned long vma_size, nr_pages;
7446	mapped_f mapped;
7447	int ret;
7448
7449	/*
7450	* Don't allow mmap() of inherited per-task counters. This would
7451	* create a performance issue due to all children writing to the
7452	* same rb.
7453	*/
7454	if (event->cpu == -1 && event->attr.inherit)
7455	return -EINVAL;
7456
7457	if (!(vma->vm_flags & VM_SHARED))
7458	return -EINVAL;
7459
7460	ret = security_perf_event_read(event);
7461	if (ret)
7462	return ret;
7463
7464	vma_size = vma->vm_end - vma->vm_start;
7465	nr_pages = vma_size / PAGE_SIZE;
7466
7467	if (nr_pages > INT_MAX)
7468	return -ENOMEM;
7469
7470	if (vma_size != PAGE_SIZE * nr_pages)
7471	return -EINVAL;
7472
7473	scoped_guard (mutex, &event->mmap_mutex) {
7474	/*
7475	* This relies on __pmu_detach_event() taking mmap_mutex after marking
7476	* the event REVOKED. Either we observe the state, or __pmu_detach_event()
7477	* will detach the rb created here.
7478	*/
7479	if (event->state <= PERF_EVENT_STATE_REVOKED)
7480	return -ENODEV;
7481
7482	if (!vma_start_pgoff(vma))
7483	ret = perf_mmap_rb(vma, event, nr_pages);
7484	else
7485	ret = perf_mmap_aux(vma, event, nr_pages);
7486	if (ret)
7487	return ret;
7488
7489	/*
7490	* Since pinned accounting is per vm we cannot allow fork() to copy our
7491	* vma.
7492	*/
7493	vm_flags_set(vma, VM_DONTCOPY \| VM_DONTEXPAND \| VM_DONTDUMP);
7494	vma->vm_ops = &perf_mmap_vmops;
7495
7496	mapped = get_mapped(event, event_mapped);
7497	if (mapped)
7498	mapped(event, vma->vm_mm);
7499
7500	/*
7501	* Try to map it into the page table. On fail undo the above,
7502	* as the callsite expects full cleanup in this case and
7503	* therefore does not invoke vmops::close().
7504	*/
7505	ret = map_range(event->rb, vma);
7506	if (likely(!ret))
7507	return 0;
7508
7509	/* Error path */
7510
7511	/*
7512	* If this is the first mmap(), then event->mmap_count should
7513	* be stable at 1. It is only modified by:
7514	* perf_mmap_{open,close}() and perf_mmap().
7515	*
7516	* The former are not possible because this mmap() hasn't been
7517	* successful yet, and the latter is serialized by
7518	* event->mmap_mutex which we still hold (note that mmap_lock
7519	* is not strictly sufficient here, because the event fd can
7520	* be passed to another process through trivial means like
7521	* fork(), leading to concurrent mmap() from different mm).
7522	*
7523	* Make sure to remove event->rb before releasing
7524	* event->mmap_mutex, such that any concurrent mmap() will not
7525	* attempt use this failed buffer.
7526	*/
7527	if (refcount_read(&event->mmap_count) == 1) {
7528	/*
7529	* Minimal perf_mmap_close(); there can't be AUX or
7530	* other events on account of this being the first.
7531	*/
7532	mapped = get_mapped(event, event_unmapped);
7533	if (mapped)
7534	mapped(event, vma->vm_mm);
7535	perf_mmap_unaccount(vma, event->rb);
7536	ring_buffer_attach(event, NULL); /* drops last rb->refcount */
7537	refcount_set(&event->mmap_count, 0);
7538	return ret;
7539	}
7540
7541	/*
7542	* Otherwise this is an already existing buffer, and there is
7543	* no race vs first exposure, so fall-through and call
7544	* perf_mmap_close().
7545	*/
7546	}
7547
7548	perf_mmap_close(vma);
7549	return ret;
7550	}
7551
7552	static int perf_fasync(int fd, struct file *filp, int on)
7553	{
7554	struct inode *inode = file_inode(filp);
7555	struct perf_event *event = filp->private_data;
7556	int retval;
7557
7558	if (event->state <= PERF_EVENT_STATE_REVOKED)
7559	return -ENODEV;
7560
7561	inode_lock(inode);
7562	retval = fasync_helper(fd, filp, on, &event->fasync);
7563	inode_unlock(inode);
7564
7565	if (retval < 0)
7566	return retval;
7567
7568	return 0;
7569	}
7570
7571	static void perf_show_fdinfo(struct seq_file m, struct file f)
7572	{
7573	struct perf_event *event = f->private_data;
7574	struct perf_event_context *ctx;
7575	struct mutex *child_mutex;
7576
7577	ctx = perf_event_ctx_lock(event);
7578	child_mutex = event->parent ? &event->parent->child_mutex : &event->child_mutex;
7579	mutex_lock(child_mutex);
7580
7581	seq_printf(m, "perf_event_attr.type:\t%u\n", event->orig_type);
7582	if (event->pmu)
7583	seq_printf(m, "pmu_type:\t%u\n", event->pmu->type);
7584	seq_printf(m, "perf_event_attr.config:\t0x%llx\n", (unsigned long long)event->attr.config);
7585	seq_printf(m, "perf_event_attr.config1:\t0x%llx\n",
7586	(unsigned long long)event->attr.config1);
7587	seq_printf(m, "perf_event_attr.config2:\t0x%llx\n",
7588	(unsigned long long)event->attr.config2);
7589	seq_printf(m, "perf_event_attr.config3:\t0x%llx\n",
7590	(unsigned long long)event->attr.config3);
7591	seq_printf(m, "perf_event_attr.config4:\t0x%llx\n",
7592	(unsigned long long)event->attr.config4);
7593
7594	mutex_unlock(child_mutex);
7595	perf_event_ctx_unlock(event, ctx);
7596	}
7597
7598	static const struct file_operations perf_fops = {
7599	.release = perf_release,
7600	.read = perf_read,
7601	.poll = perf_poll,
7602	.unlocked_ioctl = perf_ioctl,
7603	.compat_ioctl = perf_compat_ioctl,
7604	.mmap = perf_mmap,
7605	.fasync = perf_fasync,
7606	.show_fdinfo = perf_show_fdinfo,
7607	};
7608
7609	/*
7610	* Perf event wakeup
7611	*
7612	* If there's data, ensure we set the poll() state and publish everything
7613	* to user-space before waking everybody up.
7614	*/
7615
7616	void perf_event_wakeup(struct perf_event *event)
7617	{
7618	ring_buffer_wakeup(event);
7619
7620	if (event->pending_kill) {
7621	kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
7622	event->pending_kill = 0;
7623	}
7624	}
7625
7626	static void perf_sigtrap(struct perf_event *event)
7627	{
7628	/*
7629	* Both perf_pending_task() and perf_pending_irq() can race with the
7630	* task exiting or exec-ing. We can determine if such a race has
7631	* occurred by checking if perf_event_exit_task(), which will set
7632	* ctx->task to TASK_TOMBSTONE, has already been called.
7633	*/
7634	if (event->ctx->task == TASK_TOMBSTONE)
7635	return;
7636
7637	/*
7638	* We'd expect this to only occur if the irq_work is delayed and either
7639	* ctx->task or current has changed in the meantime. This can be the
7640	* case on architectures that do not implement arch_irq_work_raise().
7641	*/
7642	if (WARN_ON_ONCE(event->ctx->task != current))
7643	return;
7644
7645	send_sig_perf((void __user *)event->pending_addr,
7646	event->orig_type, event->attr.sig_data);
7647	}
7648
7649	/*
7650	* Deliver the pending work in-event-context or follow the context.
7651	*/
7652	static void __perf_pending_disable(struct perf_event *event)
7653	{
7654	int cpu = READ_ONCE(event->oncpu);
7655
7656	/*
7657	* If the event isn't running; we done. event_sched_out() will have
7658	* taken care of things.
7659	*/
7660	if (cpu < 0)
7661	return;
7662
7663	/*
7664	* Yay, we hit home and are in the context of the event.
7665	*/
7666	if (cpu == smp_processor_id()) {
7667	if (event->pending_disable) {
7668	event->pending_disable = 0;
7669	perf_event_disable_local(event);
7670	}
7671	return;
7672	}
7673
7674	/*
7675	* CPU-A CPU-B
7676	*
7677	* perf_event_disable_inatomic()
7678	* @pending_disable = 1;
7679	* irq_work_queue();
7680	*
7681	* sched-out
7682	* @pending_disable = 0;
7683	*
7684	* sched-in
7685	* perf_event_disable_inatomic()
7686	* @pending_disable = 1;
7687	* irq_work_queue(); // FAILS
7688	*
7689	* irq_work_run()
7690	* perf_pending_disable()
7691	*
7692	* But the event runs on CPU-B and wants disabling there.
7693	*/
7694	irq_work_queue_on(&event->pending_disable_irq, cpu);
7695	}
7696
7697	static void perf_pending_disable(struct irq_work *entry)
7698	{
7699	struct perf_event *event = container_of(entry, struct perf_event, pending_disable_irq);
7700	int rctx;
7701
7702	/*
7703	* If we 'fail' here, that's OK, it means recursion is already disabled
7704	* and we won't recurse 'further'.
7705	*/
7706	rctx = perf_swevent_get_recursion_context();
7707	__perf_pending_disable(event);
7708	if (rctx >= 0)
7709	perf_swevent_put_recursion_context(rctx);
7710	}
7711
7712	static void perf_pending_irq(struct irq_work *entry)
7713	{
7714	struct perf_event *event = container_of(entry, struct perf_event, pending_irq);
7715	int rctx;
7716
7717	/*
7718	* If we 'fail' here, that's OK, it means recursion is already disabled
7719	* and we won't recurse 'further'.
7720	*/
7721	rctx = perf_swevent_get_recursion_context();
7722
7723	/*
7724	* The wakeup isn't bound to the context of the event -- it can happen
7725	* irrespective of where the event is.
7726	*/
7727	if (event->pending_wakeup) {
7728	event->pending_wakeup = 0;
7729	perf_event_wakeup(event);
7730	}
7731
7732	if (rctx >= 0)
7733	perf_swevent_put_recursion_context(rctx);
7734	}
7735
7736	static void perf_pending_task(struct callback_head *head)
7737	{
7738	struct perf_event *event = container_of(head, struct perf_event, pending_task);
7739	int rctx;
7740
7741	/*
7742	* If we 'fail' here, that's OK, it means recursion is already disabled
7743	* and we won't recurse 'further'.
7744	*/
7745	rctx = perf_swevent_get_recursion_context();
7746
7747	if (event->pending_work) {
7748	event->pending_work = 0;
7749	perf_sigtrap(event);
7750	local_dec(&event->ctx->nr_no_switch_fast);
7751	}
7752	put_event(event);
7753
7754	if (rctx >= 0)
7755	perf_swevent_put_recursion_context(rctx);
7756	}
7757
7758	#ifdef CONFIG_GUEST_PERF_EVENTS
7759	struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
7760
7761	DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
7762	DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
7763	DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
7764	DEFINE_STATIC_CALL_RET0(__perf_guest_handle_mediated_pmi, *perf_guest_cbs->handle_mediated_pmi);
7765
7766	void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
7767	{
7768	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs)))
7769	return;
7770
7771	rcu_assign_pointer(perf_guest_cbs, cbs);
7772	static_call_update(__perf_guest_state, cbs->state);
7773	static_call_update(__perf_guest_get_ip, cbs->get_ip);
7774
7775	/* Implementing ->handle_intel_pt_intr is optional. */
7776	if (cbs->handle_intel_pt_intr)
7777	static_call_update(__perf_guest_handle_intel_pt_intr,
7778	cbs->handle_intel_pt_intr);
7779
7780	if (cbs->handle_mediated_pmi)
7781	static_call_update(__perf_guest_handle_mediated_pmi,
7782	cbs->handle_mediated_pmi);
7783	}
7784	EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
7785
7786	void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
7787	{
7788	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs))
7789	return;
7790
7791	rcu_assign_pointer(perf_guest_cbs, NULL);
7792	static_call_update(__perf_guest_state, (void *)&__static_call_return0);
7793	static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0);
7794	static_call_update(__perf_guest_handle_intel_pt_intr, (void *)&__static_call_return0);
7795	static_call_update(__perf_guest_handle_mediated_pmi, (void *)&__static_call_return0);
7796	synchronize_rcu();
7797	}
7798	EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
7799	#endif
7800
7801	static bool should_sample_guest(struct perf_event *event)
7802	{
7803	return !event->attr.exclude_guest && perf_guest_state();
7804	}
7805
7806	unsigned long perf_misc_flags(struct perf_event *event,
7807	struct pt_regs *regs)
7808	{
7809	if (should_sample_guest(event))
7810	return perf_arch_guest_misc_flags(regs);
7811
7812	return perf_arch_misc_flags(regs);
7813	}
7814
7815	unsigned long perf_instruction_pointer(struct perf_event *event,
7816	struct pt_regs *regs)
7817	{
7818	/*
7819	* Hardware skid can lead to a scenario where a PMI is
7820	* delivered after the CPU has already entered kernel mode.
7821	* In that case, user-space sampling must not expose kernel
7822	* register state.
7823	*/
7824	if (should_sample_guest(event)) {
7825	return event->attr.exclude_kernel &&
7826	!(perf_guest_state() & PERF_GUEST_USER) ?
7827	0 : perf_guest_get_ip();
7828	}
7829
7830	return event->attr.exclude_kernel && !user_mode(regs) ?
7831	0 : perf_arch_instruction_pointer(regs);
7832	}
7833
7834	static void
7835	perf_output_sample_regs(struct perf_output_handle *handle,
7836	struct pt_regs *regs, u64 mask)
7837	{
7838	int bit;
7839	DECLARE_BITMAP(_mask, 64);
7840
7841	bitmap_from_u64(_mask, mask);
7842	for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) {
7843	u64 val;
7844
7845	val = perf_reg_value(regs, bit);
7846	perf_output_put(handle, val);
7847	}
7848	}
7849
7850	static void perf_sample_regs_user(struct perf_regs *regs_user,
7851	struct pt_regs *regs)
7852	{
7853	if (user_mode(regs)) {
7854	regs_user->abi = perf_reg_abi(current);
7855	regs_user->regs = regs;
7856	} else if (is_user_task(current)) {
7857	perf_get_regs_user(regs_user, regs);
7858	} else {
7859	regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
7860	regs_user->regs = NULL;
7861	}
7862	}
7863
7864	static void perf_sample_regs_intr(struct perf_regs *regs_intr,
7865	struct pt_regs *regs,
7866	bool exclude_kernel)
7867	{
7868	/*
7869	* Hardware skid can lead to a scenario where a PMI is
7870	* delivered after the CPU has already entered kernel mode.
7871	* In that case, user-space sampling must not expose kernel
7872	* register state.
7873	*/
7874	if (exclude_kernel && !user_mode(regs)) {
7875	regs_intr->abi = PERF_SAMPLE_REGS_ABI_NONE;
7876	regs_intr->regs = NULL;
7877	} else {
7878	regs_intr->regs = regs;
7879	regs_intr->abi = perf_reg_abi(current);
7880	}
7881	}
7882
7883
7884	/*
7885	* Get remaining task size from user stack pointer.
7886	*
7887	* It'd be better to take stack vma map and limit this more
7888	* precisely, but there's no way to get it safely under interrupt,
7889	* so using TASK_SIZE as limit.
7890	*/
7891	static u64 perf_ustack_task_size(struct pt_regs *regs)
7892	{
7893	unsigned long addr = perf_user_stack_pointer(regs);
7894
7895	if (!addr \|\| addr >= TASK_SIZE)
7896	return 0;
7897
7898	return TASK_SIZE - addr;
7899	}
7900
7901	static u16
7902	perf_sample_ustack_size(u16 stack_size, u16 header_size,
7903	struct pt_regs *regs)
7904	{
7905	u64 task_size;
7906
7907	/* No regs, no stack pointer, no dump. */
7908	if (!regs)
7909	return 0;
7910
7911	/* No mm, no stack, no dump. */
7912	if (!current->mm)
7913	return 0;
7914
7915	/*
7916	* Check if we fit in with the requested stack size into the:
7917	* - TASK_SIZE
7918	* If we don't, we limit the size to the TASK_SIZE.
7919	*
7920	* - remaining sample size
7921	* If we don't, we customize the stack size to
7922	* fit in to the remaining sample size.
7923	*/
7924
7925	task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
7926	stack_size = min(stack_size, (u16) task_size);
7927
7928	/* Current header size plus static size and dynamic size. */
7929	header_size += 2 * sizeof(u64);
7930
7931	/* Do we fit in with the current stack dump size? */
7932	if ((u16) (header_size + stack_size) < header_size) {
7933	/*
7934	* If we overflow the maximum size for the sample,
7935	* we customize the stack dump size to fit in.
7936	*/
7937	stack_size = USHRT_MAX - header_size - sizeof(u64);
7938	stack_size = round_up(stack_size, sizeof(u64));
7939	}
7940
7941	return stack_size;
7942	}
7943
7944	static void
7945	perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
7946	struct pt_regs *regs)
7947	{
7948	/* Case of a kernel thread, nothing to dump */
7949	if (!regs) {
7950	u64 size = 0;
7951	perf_output_put(handle, size);
7952	} else {
7953	unsigned long sp;
7954	unsigned int rem;
7955	u64 dyn_size;
7956
7957	/*
7958	* We dump:
7959	* static size
7960	* - the size requested by user or the best one we can fit
7961	* in to the sample max size
7962	* data
7963	* - user stack dump data
7964	* dynamic size
7965	* - the actual dumped size
7966	*/
7967
7968	/* Static size. */
7969	perf_output_put(handle, dump_size);
7970
7971	/* Data. */
7972	sp = perf_user_stack_pointer(regs);
7973	rem = __output_copy_user(handle, (void *) sp, dump_size);
7974	dyn_size = dump_size - rem;
7975
7976	perf_output_skip(handle, rem);
7977
7978	/* Dynamic size. */
7979	perf_output_put(handle, dyn_size);
7980	}
7981	}
7982
7983	static unsigned long perf_prepare_sample_aux(struct perf_event *event,
7984	struct perf_sample_data *data,
7985	size_t size)
7986	{
7987	struct perf_event *sampler = event->aux_event;
7988	struct perf_buffer *rb;
7989
7990	data->aux_size = 0;
7991
7992	if (!sampler)
7993	goto out;
7994
7995	if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))
7996	goto out;
7997
7998	if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))
7999	goto out;
8000
8001	rb = ring_buffer_get(sampler);
8002	if (!rb)
8003	goto out;
8004
8005	/*
8006	* If this is an NMI hit inside sampling code, don't take
8007	* the sample. See also perf_aux_sample_output().
8008	*/
8009	if (READ_ONCE(rb->aux_in_sampling)) {
8010	data->aux_size = 0;
8011	} else {
8012	size = min_t(size_t, size, perf_aux_size(rb));
8013	data->aux_size = ALIGN(size, sizeof(u64));
8014	}
8015	ring_buffer_put(rb);
8016
8017	out:
8018	return data->aux_size;
8019	}
8020
8021	static long perf_pmu_snapshot_aux(struct perf_buffer *rb,
8022	struct perf_event *event,
8023	struct perf_output_handle *handle,
8024	unsigned long size)
8025	{
8026	unsigned long flags;
8027	long ret;
8028
8029	/*
8030	* Normal ->start()/->stop() callbacks run in IRQ mode in scheduler
8031	* paths. If we start calling them in NMI context, they may race with
8032	* the IRQ ones, that is, for example, re-starting an event that's just
8033	* been stopped, which is why we're using a separate callback that
8034	* doesn't change the event state.
8035	*
8036	* IRQs need to be disabled to prevent IPIs from racing with us.
8037	*/
8038	local_irq_save(flags);
8039	/*
8040	* Guard against NMI hits inside the critical section;
8041	* see also perf_prepare_sample_aux().
8042	*/
8043	WRITE_ONCE(rb->aux_in_sampling, 1);
8044	barrier();
8045
8046	ret = event->pmu->snapshot_aux(event, handle, size);
8047
8048	barrier();
8049	WRITE_ONCE(rb->aux_in_sampling, 0);
8050	local_irq_restore(flags);
8051
8052	return ret;
8053	}
8054
8055	static void perf_aux_sample_output(struct perf_event *event,
8056	struct perf_output_handle *handle,
8057	struct perf_sample_data *data)
8058	{
8059	struct perf_event *sampler = event->aux_event;
8060	struct perf_buffer *rb;
8061	unsigned long pad;
8062	long size;
8063
8064	if (WARN_ON_ONCE(!sampler \|\| !data->aux_size))
8065	return;
8066
8067	rb = ring_buffer_get(sampler);
8068	if (!rb)
8069	return;
8070
8071	size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);
8072
8073	/*
8074	* An error here means that perf_output_copy() failed (returned a
8075	* non-zero surplus that it didn't copy), which in its current
8076	* enlightened implementation is not possible. If that changes, we'd
8077	* like to know.
8078	*/
8079	if (WARN_ON_ONCE(size < 0))
8080	goto out_put;
8081
8082	/*
8083	* The pad comes from ALIGN()ing data->aux_size up to u64 in
8084	* perf_prepare_sample_aux(), so should not be more than that.
8085	*/
8086	pad = data->aux_size - size;
8087	if (WARN_ON_ONCE(pad >= sizeof(u64)))
8088	pad = 8;
8089
8090	if (pad) {
8091	u64 zero = 0;
8092	perf_output_copy(handle, &zero, pad);
8093	}
8094
8095	out_put:
8096	ring_buffer_put(rb);
8097	}
8098
8099	/*
8100	* A set of common sample data types saved even for non-sample records
8101	* when event->attr.sample_id_all is set.
8102	*/
8103	#define PERF_SAMPLE_ID_ALL (PERF_SAMPLE_TID \| PERF_SAMPLE_TIME \| \
8104	PERF_SAMPLE_ID \| PERF_SAMPLE_STREAM_ID \| \
8105	PERF_SAMPLE_CPU \| PERF_SAMPLE_IDENTIFIER)
8106
8107	static void __perf_event_header__init_id(struct perf_sample_data *data,
8108	struct perf_event *event,
8109	u64 sample_type)
8110	{
8111	data->type = event->attr.sample_type;
8112	data->sample_flags \|= data->type & PERF_SAMPLE_ID_ALL;
8113
8114	if (sample_type & PERF_SAMPLE_TID) {
8115	/* namespace issues */
8116	data->tid_entry.pid = perf_event_pid(event, current);
8117	data->tid_entry.tid = perf_event_tid(event, current);
8118	}
8119
8120	if (sample_type & PERF_SAMPLE_TIME)
8121	data->time = perf_event_clock(event);
8122
8123	if (sample_type & (PERF_SAMPLE_ID \| PERF_SAMPLE_IDENTIFIER))
8124	data->id = primary_event_id(event);
8125
8126	if (sample_type & PERF_SAMPLE_STREAM_ID)
8127	data->stream_id = event->id;
8128
8129	if (sample_type & PERF_SAMPLE_CPU) {
8130	data->cpu_entry.cpu = raw_smp_processor_id();
8131	data->cpu_entry.reserved = 0;
8132	}
8133	}
8134
8135	void perf_event_header__init(struct perf_event_header *header,
8136	struct perf_sample_data *data,
8137	u32 type, u16 misc, u16 size,
8138	struct perf_event *event)
8139	{
8140	header->type = type;
8141	header->misc = misc;
8142	header->size = size;
8143
8144	if (event->attr.sample_id_all) {
8145	header->size += event->id_header_size;
8146	__perf_event_header__init_id(data, event, event->attr.sample_type);
8147	}
8148	}
8149
8150	static void __perf_event__output_id_sample(struct perf_output_handle *handle,
8151	struct perf_sample_data *data)
8152	{
8153	u64 sample_type = data->type;
8154
8155	if (sample_type & PERF_SAMPLE_TID)
8156	perf_output_put(handle, data->tid_entry);
8157
8158	if (sample_type & PERF_SAMPLE_TIME)
8159	perf_output_put(handle, data->time);
8160
8161	if (sample_type & PERF_SAMPLE_ID)
8162	perf_output_put(handle, data->id);
8163
8164	if (sample_type & PERF_SAMPLE_STREAM_ID)
8165	perf_output_put(handle, data->stream_id);
8166
8167	if (sample_type & PERF_SAMPLE_CPU)
8168	perf_output_put(handle, data->cpu_entry);
8169
8170	if (sample_type & PERF_SAMPLE_IDENTIFIER)
8171	perf_output_put(handle, data->id);
8172	}
8173
8174	void perf_event__output_id_sample(struct perf_event *event,
8175	struct perf_output_handle *handle,
8176	struct perf_sample_data *sample)
8177	{
8178	if (event->attr.sample_id_all)
8179	__perf_event__output_id_sample(handle, sample);
8180	}
8181
8182	static void perf_output_read_one(struct perf_output_handle *handle,
8183	struct perf_event *event,
8184	u64 enabled, u64 running)
8185	{
8186	u64 read_format = event->attr.read_format;
8187	u64 values[5];
8188	int n = 0;
8189
8190	values[n++] = perf_event_count(event, has_inherit_and_sample_read(&event->attr));
8191	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
8192	values[n++] = enabled +
8193	atomic64_read(&event->child_total_time_enabled);
8194	}
8195	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
8196	values[n++] = running +
8197	atomic64_read(&event->child_total_time_running);
8198	}
8199	if (read_format & PERF_FORMAT_ID)
8200	values[n++] = primary_event_id(event);
8201	if (read_format & PERF_FORMAT_LOST)
8202	values[n++] = atomic64_read(&event->lost_samples);
8203
8204	__output_copy(handle, values, n * sizeof(u64));
8205	}
8206
8207	static void perf_output_read_group(struct perf_output_handle *handle,
8208	struct perf_event *event,
8209	u64 enabled, u64 running)
8210	{
8211	struct perf_event leader = event->group_leader, sub;
8212	u64 read_format = event->attr.read_format;
8213	unsigned long flags;
8214	u64 values[6];
8215	int n = 0;
8216	bool self = has_inherit_and_sample_read(&event->attr);
8217
8218	/*
8219	* Disabling interrupts avoids all counter scheduling
8220	* (context switches, timer based rotation and IPIs).
8221	*/
8222	local_irq_save(flags);
8223
8224	values[n++] = 1 + leader->nr_siblings;
8225
8226	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
8227	values[n++] = enabled;
8228
8229	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
8230	values[n++] = running;
8231
8232	if ((leader != event) && !handle->skip_read)
8233	perf_pmu_read(leader);
8234
8235	values[n++] = perf_event_count(leader, self);
8236	if (read_format & PERF_FORMAT_ID)
8237	values[n++] = primary_event_id(leader);
8238	if (read_format & PERF_FORMAT_LOST)
8239	values[n++] = atomic64_read(&leader->lost_samples);
8240
8241	__output_copy(handle, values, n * sizeof(u64));
8242
8243	for_each_sibling_event(sub, leader) {
8244	n = 0;
8245
8246	if ((sub != event) && !handle->skip_read)
8247	perf_pmu_read(sub);
8248
8249	values[n++] = perf_event_count(sub, self);
8250	if (read_format & PERF_FORMAT_ID)
8251	values[n++] = primary_event_id(sub);
8252	if (read_format & PERF_FORMAT_LOST)
8253	values[n++] = atomic64_read(&sub->lost_samples);
8254
8255	__output_copy(handle, values, n * sizeof(u64));
8256	}
8257
8258	local_irq_restore(flags);
8259	}
8260
8261	#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED\|\
8262	PERF_FORMAT_TOTAL_TIME_RUNNING)
8263
8264	/*
8265	* XXX PERF_SAMPLE_READ vs inherited events seems difficult.
8266	*
8267	* The problem is that its both hard and excessively expensive to iterate the
8268	* child list, not to mention that its impossible to IPI the children running
8269	* on another CPU, from interrupt/NMI context.
8270	*
8271	* Instead the combination of PERF_SAMPLE_READ and inherit will track per-thread
8272	* counts rather than attempting to accumulate some value across all children on
8273	* all cores.
8274	*/
8275	static void perf_output_read(struct perf_output_handle *handle,
8276	struct perf_event *event)
8277	{
8278	u64 enabled = 0, running = 0, now;
8279	u64 read_format = event->attr.read_format;
8280
8281	/*
8282	* Compute total_time_enabled, total_time_running based on snapshot
8283	* values taken when the event was last scheduled in.
8284	*
8285	* We cannot simply call update_context_time() because doing so would
8286	* lead to deadlock when called from NMI context.
8287	*/
8288	if (read_format & PERF_FORMAT_TOTAL_TIMES)
8289	calc_timer_values(event, &now, &enabled, &running);
8290
8291	if (event->attr.read_format & PERF_FORMAT_GROUP)
8292	perf_output_read_group(handle, event, enabled, running);
8293	else
8294	perf_output_read_one(handle, event, enabled, running);
8295	}
8296
8297	void perf_output_sample(struct perf_output_handle *handle,
8298	struct perf_event_header *header,
8299	struct perf_sample_data *data,
8300	struct perf_event *event)
8301	{
8302	u64 sample_type = data->type;
8303
8304	if (data->sample_flags & PERF_SAMPLE_READ)
8305	handle->skip_read = 1;
8306
8307	perf_output_put(handle, *header);
8308
8309	if (sample_type & PERF_SAMPLE_IDENTIFIER)
8310	perf_output_put(handle, data->id);
8311
8312	if (sample_type & PERF_SAMPLE_IP)
8313	perf_output_put(handle, data->ip);
8314
8315	if (sample_type & PERF_SAMPLE_TID)
8316	perf_output_put(handle, data->tid_entry);
8317
8318	if (sample_type & PERF_SAMPLE_TIME)
8319	perf_output_put(handle, data->time);
8320
8321	if (sample_type & PERF_SAMPLE_ADDR)
8322	perf_output_put(handle, data->addr);
8323
8324	if (sample_type & PERF_SAMPLE_ID)
8325	perf_output_put(handle, data->id);
8326
8327	if (sample_type & PERF_SAMPLE_STREAM_ID)
8328	perf_output_put(handle, data->stream_id);
8329
8330	if (sample_type & PERF_SAMPLE_CPU)
8331	perf_output_put(handle, data->cpu_entry);
8332
8333	if (sample_type & PERF_SAMPLE_PERIOD)
8334	perf_output_put(handle, data->period);
8335
8336	if (sample_type & PERF_SAMPLE_READ)
8337	perf_output_read(handle, event);
8338
8339	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
8340	int size = 1;
8341
8342	size += data->callchain->nr;
8343	size *= sizeof(u64);
8344	__output_copy(handle, data->callchain, size);
8345	}
8346
8347	if (sample_type & PERF_SAMPLE_RAW) {
8348	struct perf_raw_record *raw = data->raw;
8349
8350	if (raw) {
8351	struct perf_raw_frag *frag = &raw->frag;
8352
8353	perf_output_put(handle, raw->size);
8354	do {
8355	if (frag->copy) {
8356	__output_custom(handle, frag->copy,
8357	frag->data, frag->size);
8358	} else {
8359	__output_copy(handle, frag->data,
8360	frag->size);
8361	}
8362	if (perf_raw_frag_last(frag))
8363	break;
8364	frag = frag->next;
8365	} while (1);
8366	if (frag->pad)
8367	__output_skip(handle, NULL, frag->pad);
8368	} else {
8369	struct {
8370	u32 size;
8371	u32 data;
8372	} raw = {
8373	.size = sizeof(u32),
8374	.data = 0,
8375	};
8376	perf_output_put(handle, raw);
8377	}
8378	}
8379
8380	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
8381	if (data->br_stack) {
8382	size_t size;
8383
8384	size = data->br_stack->nr
8385	* sizeof(struct perf_branch_entry);
8386
8387	perf_output_put(handle, data->br_stack->nr);
8388	if (branch_sample_hw_index(event))
8389	perf_output_put(handle, data->br_stack->hw_idx);
8390	perf_output_copy(handle, data->br_stack->entries, size);
8391	/*
8392	* Add the extension space which is appended
8393	* right after the struct perf_branch_stack.
8394	*/
8395	if (data->br_stack_cntr) {
8396	size = data->br_stack->nr * sizeof(u64);
8397	perf_output_copy(handle, data->br_stack_cntr, size);
8398	}
8399	} else {
8400	/*
8401	* we always store at least the value of nr
8402	*/
8403	u64 nr = 0;
8404	perf_output_put(handle, nr);
8405	}
8406	}
8407
8408	if (sample_type & PERF_SAMPLE_REGS_USER) {
8409	u64 abi = data->regs_user.abi;
8410
8411	/*
8412	* If there are no regs to dump, notice it through
8413	* first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
8414	*/
8415	perf_output_put(handle, abi);
8416
8417	if (abi) {
8418	u64 mask = event->attr.sample_regs_user;
8419	perf_output_sample_regs(handle,
8420	data->regs_user.regs,
8421	mask);
8422	}
8423	}
8424
8425	if (sample_type & PERF_SAMPLE_STACK_USER) {
8426	perf_output_sample_ustack(handle,
8427	data->stack_user_size,
8428	data->regs_user.regs);
8429	}
8430
8431	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
8432	perf_output_put(handle, data->weight.full);
8433
8434	if (sample_type & PERF_SAMPLE_DATA_SRC)
8435	perf_output_put(handle, data->data_src.val);
8436
8437	if (sample_type & PERF_SAMPLE_TRANSACTION)
8438	perf_output_put(handle, data->txn);
8439
8440	if (sample_type & PERF_SAMPLE_REGS_INTR) {
8441	u64 abi = data->regs_intr.abi;
8442	/*
8443	* If there are no regs to dump, notice it through
8444	* first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
8445	*/
8446	perf_output_put(handle, abi);
8447
8448	if (abi) {
8449	u64 mask = event->attr.sample_regs_intr;
8450
8451	perf_output_sample_regs(handle,
8452	data->regs_intr.regs,
8453	mask);
8454	}
8455	}
8456
8457	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
8458	perf_output_put(handle, data->phys_addr);
8459
8460	if (sample_type & PERF_SAMPLE_CGROUP)
8461	perf_output_put(handle, data->cgroup);
8462
8463	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
8464	perf_output_put(handle, data->data_page_size);
8465
8466	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
8467	perf_output_put(handle, data->code_page_size);
8468
8469	if (sample_type & PERF_SAMPLE_AUX) {
8470	perf_output_put(handle, data->aux_size);
8471
8472	if (data->aux_size)
8473	perf_aux_sample_output(event, handle, data);
8474	}
8475
8476	if (!event->attr.watermark) {
8477	int wakeup_events = event->attr.wakeup_events;
8478
8479	if (wakeup_events) {
8480	struct perf_buffer *rb = handle->rb;
8481	int events = local_inc_return(&rb->events);
8482
8483	if (events >= wakeup_events) {
8484	local_sub(wakeup_events, &rb->events);
8485	local_inc(&rb->wakeup);
8486	}
8487	}
8488	}
8489	}
8490
8491	static u64 perf_virt_to_phys(u64 virt)
8492	{
8493	u64 phys_addr = 0;
8494
8495	if (!virt)
8496	return 0;
8497
8498	if (virt >= TASK_SIZE) {
8499	/* If it's vmalloc()d memory, leave phys_addr as 0 */
8500	if (virt_addr_valid((void *)(uintptr_t)virt) &&
8501	!(virt >= VMALLOC_START && virt < VMALLOC_END))
8502	phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
8503	} else {
8504	/*
8505	* Walking the pages tables for user address.
8506	* Interrupts are disabled, so it prevents any tear down
8507	* of the page tables.
8508	* Try IRQ-safe get_user_page_fast_only first.
8509	* If failed, leave phys_addr as 0.
8510	*/
8511	if (is_user_task(current)) {
8512	struct page *p;
8513
8514	pagefault_disable();
8515	if (get_user_page_fast_only(virt, 0, &p)) {
8516	phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
8517	put_page(p);
8518	}
8519	pagefault_enable();
8520	}
8521	}
8522
8523	return phys_addr;
8524	}
8525
8526	/*
8527	* Return the pagetable size of a given virtual address.
8528	*/
8529	static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr)
8530	{
8531	u64 size = 0;
8532
8533	#ifdef CONFIG_HAVE_GUP_FAST
8534	pgd_t *pgdp, pgd;
8535	p4d_t *p4dp, p4d;
8536	pud_t *pudp, pud;
8537	pmd_t *pmdp, pmd;
8538	pte_t *ptep, pte;
8539
8540	pgdp = pgd_offset(mm, addr);
8541	pgd = pgdp_get(pgdp);
8542	if (pgd_none(pgd))
8543	return 0;
8544
8545	if (pgd_leaf(pgd))
8546	return pgd_leaf_size(pgd);
8547
8548	p4dp = p4d_offset_lockless(pgdp, pgd, addr);
8549	p4d = p4dp_get(p4dp);
8550	if (!p4d_present(p4d))
8551	return 0;
8552
8553	if (p4d_leaf(p4d))
8554	return p4d_leaf_size(p4d);
8555
8556	pudp = pud_offset_lockless(p4dp, p4d, addr);
8557	pud = pudp_get(pudp);
8558	if (!pud_present(pud))
8559	return 0;
8560
8561	if (pud_leaf(pud))
8562	return pud_leaf_size(pud);
8563
8564	pmdp = pmd_offset_lockless(pudp, pud, addr);
8565	again:
8566	pmd = pmdp_get_lockless(pmdp);
8567	if (!pmd_present(pmd))
8568	return 0;
8569
8570	if (pmd_leaf(pmd))
8571	return pmd_leaf_size(pmd);
8572
8573	ptep = pte_offset_map(&pmd, addr);
8574	if (!ptep)
8575	goto again;
8576
8577	pte = ptep_get_lockless(ptep);
8578	if (pte_present(pte))
8579	size = __pte_leaf_size(pmd, pte);
8580	pte_unmap(ptep);
8581	#endif /* CONFIG_HAVE_GUP_FAST */
8582
8583	return size;
8584	}
8585
8586	static u64 perf_get_page_size(unsigned long addr)
8587	{
8588	struct mm_struct *mm;
8589	unsigned long flags;
8590	u64 size;
8591
8592	if (!addr)
8593	return 0;
8594
8595	/*
8596	* Software page-table walkers must disable IRQs,
8597	* which prevents any tear down of the page tables.
8598	*/
8599	local_irq_save(flags);
8600
8601	mm = current->mm;
8602	if (!mm) {
8603	/*
8604	* For kernel threads and the like, use init_mm so that
8605	* we can find kernel memory.
8606	*/
8607	mm = &init_mm;
8608	}
8609
8610	size = perf_get_pgtable_size(mm, addr);
8611
8612	local_irq_restore(flags);
8613
8614	return size;
8615	}
8616
8617	static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
8618
8619	static struct unwind_work perf_unwind_work;
8620
8621	struct perf_callchain_entry *
8622	perf_callchain(struct perf_event event, struct pt_regs regs)
8623	{
8624	bool kernel = !event->attr.exclude_callchain_kernel;
8625	bool user = !event->attr.exclude_callchain_user &&
8626	is_user_task(current);
8627	/* Disallow cross-task user callchains. */
8628	bool crosstask = event->ctx->task && event->ctx->task != current;
8629	bool defer_user = IS_ENABLED(CONFIG_UNWIND_USER) && user &&
8630	event->attr.defer_callchain;
8631	const u32 max_stack = event->attr.sample_max_stack;
8632	struct perf_callchain_entry *callchain;
8633	u64 defer_cookie;
8634
8635	if (!current->mm)
8636	user = false;
8637
8638	if (!kernel && !user)
8639	return &__empty_callchain;
8640
8641	if (!(user && defer_user && !crosstask &&
8642	unwind_deferred_request(&perf_unwind_work, &defer_cookie) >= 0))
8643	defer_cookie = 0;
8644
8645	callchain = get_perf_callchain(regs, kernel, user, max_stack,
8646	crosstask, true, defer_cookie);
8647
8648	return callchain ?: &__empty_callchain;
8649	}
8650
8651	static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d)
8652	{
8653	return d * !!(flags & s);
8654	}
8655
8656	void perf_prepare_sample(struct perf_sample_data *data,
8657	struct perf_event *event,
8658	struct pt_regs *regs)
8659	{
8660	u64 sample_type = event->attr.sample_type;
8661	u64 filtered_sample_type;
8662
8663	/*
8664	* Add the sample flags that are dependent to others. And clear the
8665	* sample flags that have already been done by the PMU driver.
8666	*/
8667	filtered_sample_type = sample_type;
8668	filtered_sample_type \|= __cond_set(sample_type, PERF_SAMPLE_CODE_PAGE_SIZE,
8669	PERF_SAMPLE_IP);
8670	filtered_sample_type \|= __cond_set(sample_type, PERF_SAMPLE_DATA_PAGE_SIZE \|
8671	PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR);
8672	filtered_sample_type \|= __cond_set(sample_type, PERF_SAMPLE_STACK_USER,
8673	PERF_SAMPLE_REGS_USER);
8674	filtered_sample_type &= ~data->sample_flags;
8675
8676	if (filtered_sample_type == 0) {
8677	/* Make sure it has the correct data->type for output */
8678	data->type = event->attr.sample_type;
8679	return;
8680	}
8681
8682	__perf_event_header__init_id(data, event, filtered_sample_type);
8683
8684	if (filtered_sample_type & PERF_SAMPLE_IP) {
8685	data->ip = perf_instruction_pointer(event, regs);
8686	data->sample_flags \|= PERF_SAMPLE_IP;
8687	}
8688
8689	if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN)
8690	perf_sample_save_callchain(data, event, regs);
8691
8692	if (filtered_sample_type & PERF_SAMPLE_RAW) {
8693	data->raw = NULL;
8694	data->dyn_size += sizeof(u64);
8695	data->sample_flags \|= PERF_SAMPLE_RAW;
8696	}
8697
8698	if (filtered_sample_type & PERF_SAMPLE_BRANCH_STACK) {
8699	data->br_stack = NULL;
8700	data->dyn_size += sizeof(u64);
8701	data->sample_flags \|= PERF_SAMPLE_BRANCH_STACK;
8702	}
8703
8704	if (filtered_sample_type & PERF_SAMPLE_REGS_USER)
8705	perf_sample_regs_user(&data->regs_user, regs);
8706
8707	/*
8708	* It cannot use the filtered_sample_type here as REGS_USER can be set
8709	* by STACK_USER (using __cond_set() above) and we don't want to update
8710	* the dyn_size if it's not requested by users.
8711	*/
8712	if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) {
8713	/* regs dump ABI info */
8714	int size = sizeof(u64);
8715
8716	if (data->regs_user.regs) {
8717	u64 mask = event->attr.sample_regs_user;
8718	size += hweight64(mask) * sizeof(u64);
8719	}
8720
8721	data->dyn_size += size;
8722	data->sample_flags \|= PERF_SAMPLE_REGS_USER;
8723	}
8724
8725	if (filtered_sample_type & PERF_SAMPLE_STACK_USER) {
8726	/*
8727	* Either we need PERF_SAMPLE_STACK_USER bit to be always
8728	* processed as the last one or have additional check added
8729	* in case new sample type is added, because we could eat
8730	* up the rest of the sample size.
8731	*/
8732	u16 stack_size = event->attr.sample_stack_user;
8733	u16 header_size = perf_sample_data_size(data, event);
8734	u16 size = sizeof(u64);
8735
8736	stack_size = perf_sample_ustack_size(stack_size, header_size,
8737	data->regs_user.regs);
8738
8739	/*
8740	* If there is something to dump, add space for the dump
8741	* itself and for the field that tells the dynamic size,
8742	* which is how many have been actually dumped.
8743	*/
8744	if (stack_size)
8745	size += sizeof(u64) + stack_size;
8746
8747	data->stack_user_size = stack_size;
8748	data->dyn_size += size;
8749	data->sample_flags \|= PERF_SAMPLE_STACK_USER;
8750	}
8751
8752	if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
8753	data->weight.full = 0;
8754	data->sample_flags \|= PERF_SAMPLE_WEIGHT_TYPE;
8755	}
8756
8757	if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) {
8758	data->data_src.val = PERF_MEM_NA;
8759	data->sample_flags \|= PERF_SAMPLE_DATA_SRC;
8760	}
8761
8762	if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) {
8763	data->txn = 0;
8764	data->sample_flags \|= PERF_SAMPLE_TRANSACTION;
8765	}
8766
8767	if (filtered_sample_type & PERF_SAMPLE_ADDR) {
8768	data->addr = 0;
8769	data->sample_flags \|= PERF_SAMPLE_ADDR;
8770	}
8771
8772	if (filtered_sample_type & PERF_SAMPLE_REGS_INTR) {
8773	/* regs dump ABI info */
8774	int size = sizeof(u64);
8775
8776	perf_sample_regs_intr(&data->regs_intr, regs,
8777	event->attr.exclude_kernel);
8778
8779	if (data->regs_intr.regs) {
8780	u64 mask = event->attr.sample_regs_intr;
8781
8782	size += hweight64(mask) * sizeof(u64);
8783	}
8784
8785	data->dyn_size += size;
8786	data->sample_flags \|= PERF_SAMPLE_REGS_INTR;
8787	}
8788
8789	if (filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) {
8790	data->phys_addr = perf_virt_to_phys(data->addr);
8791	data->sample_flags \|= PERF_SAMPLE_PHYS_ADDR;
8792	}
8793
8794	#ifdef CONFIG_CGROUP_PERF
8795	if (filtered_sample_type & PERF_SAMPLE_CGROUP) {
8796	struct cgroup *cgrp;
8797
8798	/* protected by RCU */
8799	cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;
8800	data->cgroup = cgroup_id(cgrp);
8801	data->sample_flags \|= PERF_SAMPLE_CGROUP;
8802	}
8803	#endif
8804
8805	/*
8806	* PERF_DATA_PAGE_SIZE requires PERF_SAMPLE_ADDR. If the user doesn't
8807	* require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,
8808	* but the value will not dump to the userspace.
8809	*/
8810	if (filtered_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) {
8811	data->data_page_size = perf_get_page_size(data->addr);
8812	data->sample_flags \|= PERF_SAMPLE_DATA_PAGE_SIZE;
8813	}
8814
8815	if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) {
8816	data->code_page_size = perf_get_page_size(data->ip);
8817	data->sample_flags \|= PERF_SAMPLE_CODE_PAGE_SIZE;
8818	}
8819
8820	if (filtered_sample_type & PERF_SAMPLE_AUX) {
8821	u64 size;
8822	u16 header_size = perf_sample_data_size(data, event);
8823
8824	header_size += sizeof(u64); /* size */
8825
8826	/*
8827	* Given the 16bit nature of header::size, an AUX sample can
8828	* easily overflow it, what with all the preceding sample bits.
8829	* Make sure this doesn't happen by using up to U16_MAX bytes
8830	* per sample in total (rounded down to 8 byte boundary).
8831	*/
8832	size = min_t(size_t, U16_MAX - header_size,
8833	event->attr.aux_sample_size);
8834	size = rounddown(size, 8);
8835	size = perf_prepare_sample_aux(event, data, size);
8836
8837	WARN_ON_ONCE(size + header_size > U16_MAX);
8838	data->dyn_size += size + sizeof(u64); /* size above */
8839	data->sample_flags \|= PERF_SAMPLE_AUX;
8840	}
8841	}
8842
8843	void perf_prepare_header(struct perf_event_header *header,
8844	struct perf_sample_data *data,
8845	struct perf_event *event,
8846	struct pt_regs *regs)
8847	{
8848	header->type = PERF_RECORD_SAMPLE;
8849	header->size = perf_sample_data_size(data, event);
8850	header->misc = perf_misc_flags(event, regs);
8851
8852	/*
8853	* If you're adding more sample types here, you likely need to do
8854	* something about the overflowing header::size, like repurpose the
8855	* lowest 3 bits of size, which should be always zero at the moment.
8856	* This raises a more important question, do we really need 512k sized
8857	* samples and why, so good argumentation is in order for whatever you
8858	* do here next.
8859	*/
8860	WARN_ON_ONCE(header->size & 7);
8861	}
8862
8863	static void __perf_event_aux_pause(struct perf_event *event, bool pause)
8864	{
8865	if (pause) {
8866	if (!event->hw.aux_paused) {
8867	event->hw.aux_paused = 1;
8868	event->pmu->stop(event, PERF_EF_PAUSE);
8869	}
8870	} else {
8871	if (event->hw.aux_paused) {
8872	event->hw.aux_paused = 0;
8873	event->pmu->start(event, PERF_EF_RESUME);
8874	}
8875	}
8876	}
8877
8878	static void perf_event_aux_pause(struct perf_event *event, bool pause)
8879	{
8880	struct perf_buffer *rb;
8881
8882	if (WARN_ON_ONCE(!event))
8883	return;
8884
8885	rb = ring_buffer_get(event);
8886	if (!rb)
8887	return;
8888
8889	scoped_guard (irqsave) {
8890	/*
8891	* Guard against self-recursion here. Another event could trip
8892	* this same from NMI context.
8893	*/
8894	if (READ_ONCE(rb->aux_in_pause_resume))
8895	break;
8896
8897	WRITE_ONCE(rb->aux_in_pause_resume, 1);
8898	barrier();
8899	__perf_event_aux_pause(event, pause);
8900	barrier();
8901	WRITE_ONCE(rb->aux_in_pause_resume, 0);
8902	}
8903	ring_buffer_put(rb);
8904	}
8905
8906	static __always_inline int
8907	__perf_event_output(struct perf_event *event,
8908	struct perf_sample_data *data,
8909	struct pt_regs *regs,
8910	int (output_begin)(struct perf_output_handle ,
8911	struct perf_sample_data *,
8912	struct perf_event *,
8913	unsigned int))
8914	{
8915	struct perf_output_handle handle;
8916	struct perf_event_header header;
8917	int err;
8918
8919	/* protect the callchain buffers */
8920	rcu_read_lock();
8921
8922	perf_prepare_sample(data, event, regs);
8923	perf_prepare_header(&header, data, event, regs);
8924
8925	err = output_begin(&handle, data, event, header.size);
8926	if (err)
8927	goto exit;
8928
8929	perf_output_sample(&handle, &header, data, event);
8930
8931	perf_output_end(&handle);
8932
8933	exit:
8934	rcu_read_unlock();
8935	return err;
8936	}
8937
8938	void
8939	perf_event_output_forward(struct perf_event *event,
8940	struct perf_sample_data *data,
8941	struct pt_regs *regs)
8942	{
8943	__perf_event_output(event, data, regs, perf_output_begin_forward);
8944	}
8945
8946	void
8947	perf_event_output_backward(struct perf_event *event,
8948	struct perf_sample_data *data,
8949	struct pt_regs *regs)
8950	{
8951	__perf_event_output(event, data, regs, perf_output_begin_backward);
8952	}
8953
8954	int
8955	perf_event_output(struct perf_event *event,
8956	struct perf_sample_data *data,
8957	struct pt_regs *regs)
8958	{
8959	return __perf_event_output(event, data, regs, perf_output_begin);
8960	}
8961
8962	/*
8963	* read event_id
8964	*/
8965
8966	struct perf_read_event {
8967	struct perf_event_header header;
8968
8969	u32 pid;
8970	u32 tid;
8971	};
8972
8973	static void
8974	perf_event_read_event(struct perf_event *event,
8975	struct task_struct *task)
8976	{
8977	struct perf_output_handle handle;
8978	struct perf_sample_data sample;
8979	struct perf_read_event read_event = {
8980	.pid = perf_event_pid(event, task),
8981	.tid = perf_event_tid(event, task),
8982	};
8983	int ret;
8984
8985	perf_event_header__init(&read_event.header, &sample,
8986	PERF_RECORD_READ,
8987	/* misc= */ 0,
8988	sizeof(read_event) + event->read_size,
8989	event);
8990	ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
8991	if (ret)
8992	return;
8993
8994	perf_output_put(&handle, read_event);
8995	perf_output_read(&handle, event);
8996	perf_event__output_id_sample(event, &handle, &sample);
8997
8998	perf_output_end(&handle);
8999	}
9000
9001	typedef void (perf_iterate_f)(struct perf_event event, void data);
9002
9003	static void
9004	perf_iterate_ctx(struct perf_event_context *ctx,
9005	perf_iterate_f output,
9006	void *data, bool all)
9007	{
9008	struct perf_event *event;
9009
9010	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
9011	if (!all) {
9012	if (event->state < PERF_EVENT_STATE_INACTIVE)
9013	continue;
9014	if (!event_filter_match(event))
9015	continue;
9016	}
9017
9018	output(event, data);
9019	}
9020	}
9021
9022	static void perf_iterate_sb_cpu(perf_iterate_f output, void *data)
9023	{
9024	struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events);
9025	struct perf_event *event;
9026
9027	list_for_each_entry_rcu(event, &pel->list, sb_list) {
9028	/*
9029	* Skip events that are not fully formed yet; ensure that
9030	* if we observe event->ctx, both event and ctx will be
9031	* complete enough. See perf_install_in_context().
9032	*/
9033	if (!smp_load_acquire(&event->ctx))
9034	continue;
9035
9036	if (event->state < PERF_EVENT_STATE_INACTIVE)
9037	continue;
9038	if (!event_filter_match(event))
9039	continue;
9040	output(event, data);
9041	}
9042	}
9043
9044	/*
9045	* Iterate all events that need to receive side-band events.
9046	*
9047	* For new callers; ensure that account_pmu_sb_event() includes
9048	* your event, otherwise it might not get delivered.
9049	*/
9050	static void
9051	perf_iterate_sb(perf_iterate_f output, void *data,
9052	struct perf_event_context *task_ctx)
9053	{
9054	struct perf_event_context *ctx;
9055
9056	rcu_read_lock();
9057	preempt_disable();
9058
9059	/*
9060	* If we have task_ctx != NULL we only notify the task context itself.
9061	* The task_ctx is set only for EXIT events before releasing task
9062	* context.
9063	*/
9064	if (task_ctx) {
9065	perf_iterate_ctx(task_ctx, output, data, false);
9066	goto done;
9067	}
9068
9069	perf_iterate_sb_cpu(output, data);
9070
9071	ctx = rcu_dereference(current->perf_event_ctxp);
9072	if (ctx)
9073	perf_iterate_ctx(ctx, output, data, false);
9074	done:
9075	preempt_enable();
9076	rcu_read_unlock();
9077	}
9078
9079	/*
9080	* Clear all file-based filters at exec, they'll have to be
9081	* re-instated when/if these objects are mmapped again.
9082	*/
9083	static void perf_event_addr_filters_exec(struct perf_event event, void data)
9084	{
9085	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
9086	struct perf_addr_filter *filter;
9087	unsigned int restart = 0, count = 0;
9088	unsigned long flags;
9089
9090	if (!has_addr_filter(event))
9091	return;
9092
9093	raw_spin_lock_irqsave(&ifh->lock, flags);
9094	list_for_each_entry(filter, &ifh->list, entry) {
9095	if (filter->path.dentry) {
9096	event->addr_filter_ranges[count].start = 0;
9097	event->addr_filter_ranges[count].size = 0;
9098	restart++;
9099	}
9100
9101	count++;
9102	}
9103
9104	if (restart)
9105	event->addr_filters_gen++;
9106	raw_spin_unlock_irqrestore(&ifh->lock, flags);
9107
9108	if (restart)
9109	perf_event_stop(event, 1);
9110	}
9111
9112	void perf_event_exec(void)
9113	{
9114	struct perf_event_context *ctx;
9115
9116	ctx = perf_pin_task_context(current);
9117	if (!ctx)
9118	return;
9119
9120	perf_event_enable_on_exec(ctx);
9121	perf_event_remove_on_exec(ctx);
9122	scoped_guard(rcu)
9123	perf_iterate_ctx(ctx, perf_event_addr_filters_exec, NULL, true);
9124
9125	perf_unpin_context(ctx);
9126	put_ctx(ctx);
9127	}
9128
9129	struct remote_output {
9130	struct perf_buffer *rb;
9131	int err;
9132	};
9133
9134	static void __perf_event_output_stop(struct perf_event event, void data)
9135	{
9136	struct perf_event *parent = event->parent;
9137	struct remote_output *ro = data;
9138	struct perf_buffer *rb = ro->rb;
9139	struct stop_event_data sd = {
9140	.event = event,
9141	};
9142
9143	if (!has_aux(event))
9144	return;
9145
9146	if (!parent)
9147	parent = event;
9148
9149	/*
9150	* In case of inheritance, it will be the parent that links to the
9151	* ring-buffer, but it will be the child that's actually using it.
9152	*
9153	* We are using event::rb to determine if the event should be stopped,
9154	* however this may race with ring_buffer_attach() (through set_output),
9155	* which will make us skip the event that actually needs to be stopped.
9156	* So ring_buffer_attach() has to stop an aux event before re-assigning
9157	* its rb pointer.
9158	*/
9159	if (rcu_dereference(parent->rb) == rb)
9160	ro->err = __perf_event_stop(&sd);
9161	}
9162
9163	static int __perf_pmu_output_stop(void *info)
9164	{
9165	struct perf_event *event = info;
9166	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
9167	struct remote_output ro = {
9168	.rb = event->rb,
9169	};
9170
9171	rcu_read_lock();
9172	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);
9173	if (cpuctx->task_ctx)
9174	perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,
9175	&ro, false);
9176	rcu_read_unlock();
9177
9178	return ro.err;
9179	}
9180
9181	static void perf_pmu_output_stop(struct perf_event *event)
9182	{
9183	struct perf_event *iter;
9184	int err, cpu;
9185
9186	restart:
9187	rcu_read_lock();
9188	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {
9189	/*
9190	* For per-CPU events, we need to make sure that neither they
9191	* nor their children are running; for cpu==-1 events it's
9192	* sufficient to stop the event itself if it's active, since
9193	* it can't have children.
9194	*/
9195	cpu = iter->cpu;
9196	if (cpu == -1)
9197	cpu = READ_ONCE(iter->oncpu);
9198
9199	if (cpu == -1)
9200	continue;
9201
9202	err = cpu_function_call(cpu, __perf_pmu_output_stop, event);
9203	if (err == -EAGAIN) {
9204	rcu_read_unlock();
9205	goto restart;
9206	}
9207	}
9208	rcu_read_unlock();
9209	}
9210
9211	/*
9212	* task tracking -- fork/exit
9213	*
9214	* enabled by: attr.comm \| attr.mmap \| attr.mmap2 \| attr.mmap_data \| attr.task
9215	*/
9216
9217	struct perf_task_event {
9218	struct task_struct *task;
9219	struct perf_event_context *task_ctx;
9220
9221	struct {
9222	struct perf_event_header header;
9223
9224	u32 pid;
9225	u32 ppid;
9226	u32 tid;
9227	u32 ptid;
9228	u64 time;
9229	} event_id;
9230	int new;
9231	};
9232
9233	static int perf_event_task_match(struct perf_event *event)
9234	{
9235	return event->attr.comm \|\| event->attr.mmap \|\|
9236	event->attr.mmap2 \|\| event->attr.mmap_data \|\|
9237	event->attr.task;
9238	}
9239
9240	static void perf_event_task_output(struct perf_event *event,
9241	void *data)
9242	{
9243	struct perf_task_event *task_event = data;
9244	struct perf_output_handle handle;
9245	struct perf_sample_data sample;
9246	struct task_struct *task = task_event->task;
9247	int ret;
9248
9249	if (!perf_event_task_match(event))
9250	return;
9251
9252	perf_event_header__init(&task_event->event_id.header, &sample,
9253	task_event->new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
9254	/* misc= */ 0,
9255	sizeof(task_event->event_id),
9256	event);
9257
9258	ret = perf_output_begin(&handle, &sample, event,
9259	task_event->event_id.header.size);
9260	if (ret)
9261	return;
9262
9263	task_event->event_id.pid = perf_event_pid(event, task);
9264	task_event->event_id.tid = perf_event_tid(event, task);
9265
9266	if (task_event->event_id.header.type == PERF_RECORD_EXIT) {
9267	task_event->event_id.ppid = perf_event_pid(event,
9268	task->real_parent);
9269	task_event->event_id.ptid = perf_event_pid(event,
9270	task->real_parent);
9271	} else { /* PERF_RECORD_FORK */
9272	task_event->event_id.ppid = perf_event_pid(event, current);
9273	task_event->event_id.ptid = perf_event_tid(event, current);
9274	}
9275
9276	task_event->event_id.time = perf_event_clock(event);
9277
9278	perf_output_put(&handle, task_event->event_id);
9279
9280	perf_event__output_id_sample(event, &handle, &sample);
9281
9282	perf_output_end(&handle);
9283	}
9284
9285	static void perf_event_task(struct task_struct *task,
9286	struct perf_event_context *task_ctx,
9287	int new)
9288	{
9289	struct perf_task_event task_event;
9290
9291	if (!atomic_read(&nr_comm_events) &&
9292	!atomic_read(&nr_mmap_events) &&
9293	!atomic_read(&nr_task_events))
9294	return;
9295
9296	task_event = (struct perf_task_event){
9297	.task = task,
9298	.task_ctx = task_ctx,
9299	.new = new,
9300	};
9301
9302	perf_iterate_sb(perf_event_task_output,
9303	&task_event,
9304	task_ctx);
9305	}
9306
9307	/*
9308	* Allocate data for a new task when profiling system-wide
9309	* events which require PMU specific data
9310	*/
9311	static void
9312	perf_event_alloc_task_data(struct task_struct *child,
9313	struct task_struct *parent)
9314	{
9315	struct kmem_cache *ctx_cache = NULL;
9316	struct perf_ctx_data *cd;
9317
9318	if (!refcount_read(&global_ctx_data_ref))
9319	return;
9320
9321	scoped_guard (rcu) {
9322	cd = rcu_dereference(parent->perf_ctx_data);
9323	if (cd)
9324	ctx_cache = cd->ctx_cache;
9325	}
9326
9327	if (!ctx_cache)
9328	return;
9329
9330	guard(percpu_read)(&global_ctx_data_rwsem);
9331	scoped_guard (rcu) {
9332	cd = rcu_dereference(child->perf_ctx_data);
9333	if (!cd) {
9334	/*
9335	* A system-wide event may be unaccount,
9336	* when attaching the perf_ctx_data.
9337	*/
9338	if (!refcount_read(&global_ctx_data_ref))
9339	return;
9340	goto attach;
9341	}
9342
9343	if (!cd->global) {
9344	cd->global = 1;
9345	refcount_inc(&cd->refcount);
9346	}
9347	}
9348
9349	return;
9350	attach:
9351	attach_task_ctx_data(child, ctx_cache, true, GFP_KERNEL);
9352	}
9353
9354	void perf_event_fork(struct task_struct *task)
9355	{
9356	perf_event_task(task, NULL, 1);
9357	perf_event_namespaces(task);
9358	perf_event_alloc_task_data(task, current);
9359	}
9360
9361	/*
9362	* comm tracking
9363	*/
9364
9365	struct perf_comm_event {
9366	struct task_struct *task;
9367	char *comm;
9368	int comm_size;
9369
9370	struct {
9371	struct perf_event_header header;
9372
9373	u32 pid;
9374	u32 tid;
9375	} event_id;
9376	bool exec;
9377	};
9378
9379	static int perf_event_comm_match(struct perf_event *event)
9380	{
9381	return event->attr.comm;
9382	}
9383
9384	static void perf_event_comm_output(struct perf_event *event,
9385	void *data)
9386	{
9387	struct perf_comm_event *comm_event = data;
9388	struct perf_output_handle handle;
9389	struct perf_sample_data sample;
9390	int ret;
9391
9392	if (!perf_event_comm_match(event))
9393	return;
9394
9395	perf_event_header__init(&comm_event->event_id.header, &sample,
9396	PERF_RECORD_COMM,
9397	comm_event->exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
9398	sizeof(comm_event->event_id) + comm_event->comm_size,
9399	event);
9400	ret = perf_output_begin(&handle, &sample, event,
9401	comm_event->event_id.header.size);
9402
9403	if (ret)
9404	return;
9405
9406	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
9407	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
9408
9409	perf_output_put(&handle, comm_event->event_id);
9410	__output_copy(&handle, comm_event->comm,
9411	comm_event->comm_size);
9412
9413	perf_event__output_id_sample(event, &handle, &sample);
9414
9415	perf_output_end(&handle);
9416	}
9417
9418	static void perf_event_comm_event(struct perf_comm_event *comm_event)
9419	{
9420	char comm[TASK_COMM_LEN];
9421	unsigned int size;
9422
9423	memset(comm, 0, sizeof(comm));
9424	strscpy(comm, comm_event->task->comm);
9425	size = ALIGN(strlen(comm)+1, sizeof(u64));
9426
9427	comm_event->comm = comm;
9428	comm_event->comm_size = size;
9429
9430	perf_iterate_sb(perf_event_comm_output,
9431	comm_event,
9432	NULL);
9433	}
9434
9435	void perf_event_comm(struct task_struct *task, bool exec)
9436	{
9437	struct perf_comm_event comm_event;
9438
9439	if (!atomic_read(&nr_comm_events))
9440	return;
9441
9442	comm_event = (struct perf_comm_event){
9443	.task = task,
9444	/* .comm */
9445	/* .comm_size */
9446	/* .event_id */
9447	.exec = exec,
9448	};
9449
9450	perf_event_comm_event(&comm_event);
9451	}
9452
9453	/*
9454	* namespaces tracking
9455	*/
9456
9457	struct perf_namespaces_event {
9458	struct task_struct *task;
9459
9460	struct {
9461	struct perf_event_header header;
9462
9463	u32 pid;
9464	u32 tid;
9465	u64 nr_namespaces;
9466	struct perf_ns_link_info link_info[NR_NAMESPACES];
9467	} event_id;
9468	};
9469
9470	static int perf_event_namespaces_match(struct perf_event *event)
9471	{
9472	return event->attr.namespaces;
9473	}
9474
9475	static void perf_event_namespaces_output(struct perf_event *event,
9476	void *data)
9477	{
9478	struct perf_namespaces_event *namespaces_event = data;
9479	struct perf_output_handle handle;
9480	struct perf_sample_data sample;
9481	int ret;
9482
9483	if (!perf_event_namespaces_match(event))
9484	return;
9485
9486	perf_event_header__init(&namespaces_event->event_id.header, &sample,
9487	PERF_RECORD_NAMESPACES,
9488	/* misc= */ 0,
9489	sizeof(namespaces_event->event_id),
9490	event);
9491	ret = perf_output_begin(&handle, &sample, event,
9492	namespaces_event->event_id.header.size);
9493	if (ret)
9494	return;
9495
9496	namespaces_event->event_id.pid = perf_event_pid(event,
9497	namespaces_event->task);
9498	namespaces_event->event_id.tid = perf_event_tid(event,
9499	namespaces_event->task);
9500
9501	perf_output_put(&handle, namespaces_event->event_id);
9502
9503	perf_event__output_id_sample(event, &handle, &sample);
9504
9505	perf_output_end(&handle);
9506	}
9507
9508	static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info,
9509	struct task_struct *task,
9510	const struct proc_ns_operations *ns_ops)
9511	{
9512	struct path ns_path;
9513	struct inode *ns_inode;
9514	int error;
9515
9516	error = ns_get_path(&ns_path, task, ns_ops);
9517	if (!error) {
9518	ns_inode = ns_path.dentry->d_inode;
9519	ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev);
9520	ns_link_info->ino = ns_inode->i_ino;
9521	path_put(&ns_path);
9522	}
9523	}
9524
9525	void perf_event_namespaces(struct task_struct *task)
9526	{
9527	struct perf_namespaces_event namespaces_event;
9528	struct perf_ns_link_info *ns_link_info;
9529
9530	if (!atomic_read(&nr_namespaces_events))
9531	return;
9532
9533	namespaces_event = (struct perf_namespaces_event){
9534	.task = task,
9535	.event_id = {
9536	/* .header */
9537	/* .pid */
9538	/* .tid */
9539	.nr_namespaces = NR_NAMESPACES,
9540	/* .link_info[NR_NAMESPACES] */
9541	},
9542	};
9543
9544	ns_link_info = namespaces_event.event_id.link_info;
9545
9546	perf_fill_ns_link_info(&ns_link_info[MNT_NS_INDEX],
9547	task, &mntns_operations);
9548
9549	#ifdef CONFIG_USER_NS
9550	perf_fill_ns_link_info(&ns_link_info[USER_NS_INDEX],
9551	task, &userns_operations);
9552	#endif
9553	#ifdef CONFIG_NET_NS
9554	perf_fill_ns_link_info(&ns_link_info[NET_NS_INDEX],
9555	task, &netns_operations);
9556	#endif
9557	#ifdef CONFIG_UTS_NS
9558	perf_fill_ns_link_info(&ns_link_info[UTS_NS_INDEX],
9559	task, &utsns_operations);
9560	#endif
9561	#ifdef CONFIG_IPC_NS
9562	perf_fill_ns_link_info(&ns_link_info[IPC_NS_INDEX],
9563	task, &ipcns_operations);
9564	#endif
9565	#ifdef CONFIG_PID_NS
9566	perf_fill_ns_link_info(&ns_link_info[PID_NS_INDEX],
9567	task, &pidns_operations);
9568	#endif
9569	#ifdef CONFIG_CGROUPS
9570	perf_fill_ns_link_info(&ns_link_info[CGROUP_NS_INDEX],
9571	task, &cgroupns_operations);
9572	#endif
9573
9574	perf_iterate_sb(perf_event_namespaces_output,
9575	&namespaces_event,
9576	NULL);
9577	}
9578
9579	/*
9580	* cgroup tracking
9581	*/
9582	#ifdef CONFIG_CGROUP_PERF
9583
9584	struct perf_cgroup_event {
9585	char *path;
9586	int path_size;
9587	struct {
9588	struct perf_event_header header;
9589	u64 id;
9590	char path[];
9591	} event_id;
9592	};
9593
9594	static int perf_event_cgroup_match(struct perf_event *event)
9595	{
9596	return event->attr.cgroup;
9597	}
9598
9599	static void perf_event_cgroup_output(struct perf_event event, void data)
9600	{
9601	struct perf_cgroup_event *cgroup_event = data;
9602	struct perf_output_handle handle;
9603	struct perf_sample_data sample;
9604	int ret;
9605	u16 size = sizeof(cgroup_event->event_id) + cgroup_event->path_size;
9606
9607	if (!perf_event_cgroup_match(event))
9608	return;
9609
9610	perf_event_header__init(&cgroup_event->event_id.header, &sample,
9611	PERF_RECORD_CGROUP, /* misc= */ 0, size,
9612	event);
9613	ret = perf_output_begin(&handle, &sample, event,
9614	cgroup_event->event_id.header.size);
9615	if (ret)
9616	return;
9617
9618	perf_output_put(&handle, cgroup_event->event_id);
9619	__output_copy(&handle, cgroup_event->path, cgroup_event->path_size);
9620
9621	perf_event__output_id_sample(event, &handle, &sample);
9622
9623	perf_output_end(&handle);
9624	}
9625
9626	static void perf_event_cgroup(struct cgroup *cgrp)
9627	{
9628	struct perf_cgroup_event cgroup_event;
9629	char path_enomem[16] = "//enomem";
9630	char *pathname;
9631	size_t size;
9632
9633	if (!atomic_read(&nr_cgroup_events))
9634	return;
9635
9636	cgroup_event = (struct perf_cgroup_event){
9637	.event_id = {
9638	.id = cgroup_id(cgrp),
9639	},
9640	};
9641
9642	pathname = kmalloc(PATH_MAX, GFP_KERNEL);
9643	if (pathname == NULL) {
9644	cgroup_event.path = path_enomem;
9645	} else {
9646	/* just to be sure to have enough space for alignment */
9647	cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64));
9648	cgroup_event.path = pathname;
9649	}
9650
9651	/*
9652	* Since our buffer works in 8 byte units we need to align our string
9653	* size to a multiple of 8. However, we must guarantee the tail end is
9654	* zero'd out to avoid leaking random bits to userspace.
9655	*/
9656	size = strlen(cgroup_event.path) + 1;
9657	while (!IS_ALIGNED(size, sizeof(u64)))
9658	cgroup_event.path[size++] = '\0';
9659
9660	cgroup_event.path_size = size;
9661
9662	perf_iterate_sb(perf_event_cgroup_output,
9663	&cgroup_event,
9664	NULL);
9665
9666	kfree(pathname);
9667	}
9668
9669	#endif
9670
9671	/*
9672	* mmap tracking
9673	*/
9674
9675	struct perf_mmap_event {
9676	struct vm_area_struct *vma;
9677
9678	const char *file_name;
9679	int file_size;
9680	int maj, min;
9681	u64 ino;
9682	u64 ino_generation;
9683	u32 prot, flags;
9684	u8 build_id[BUILD_ID_SIZE_MAX];
9685	u32 build_id_size;
9686
9687	struct {
9688	struct perf_event_header header;
9689
9690	u32 pid;
9691	u32 tid;
9692	u64 start;
9693	u64 len;
9694	u64 pgoff;
9695	} event_id;
9696	};
9697
9698	static int perf_event_mmap_match(struct perf_event *event,
9699	void *data)
9700	{
9701	struct perf_mmap_event *mmap_event = data;
9702	struct vm_area_struct *vma = mmap_event->vma;
9703	int executable = vma->vm_flags & VM_EXEC;
9704
9705	return (!executable && event->attr.mmap_data) \|\|
9706	(executable && (event->attr.mmap \|\| event->attr.mmap2));
9707	}
9708
9709	static void perf_event_mmap_output(struct perf_event *event,
9710	void *data)
9711	{
9712	struct perf_mmap_event *mmap_event = data;
9713	struct perf_output_handle handle;
9714	struct perf_sample_data sample;
9715	int size = sizeof(mmap_event->event_id) + mmap_event->file_size;
9716	u32 type = PERF_RECORD_MMAP;
9717	u16 misc = PERF_RECORD_MISC_USER;
9718	bool use_build_id = false;
9719	int ret;
9720
9721	if (!perf_event_mmap_match(event, data))
9722	return;
9723
9724	if (event->attr.mmap2) {
9725	type = PERF_RECORD_MMAP2;
9726	size += sizeof(mmap_event->maj);
9727	size += sizeof(mmap_event->min);
9728	size += sizeof(mmap_event->ino);
9729	size += sizeof(mmap_event->ino_generation);
9730	size += sizeof(mmap_event->prot);
9731	size += sizeof(mmap_event->flags);
9732	use_build_id = event->attr.build_id && mmap_event->build_id_size;
9733	if (use_build_id)
9734	misc \|= PERF_RECORD_MISC_MMAP_BUILD_ID;
9735	}
9736	if (!(mmap_event->vma->vm_flags & VM_EXEC))
9737	misc \|= PERF_RECORD_MISC_MMAP_DATA;
9738
9739	perf_event_header__init(&mmap_event->event_id.header, &sample,
9740	type, misc, size, event);
9741	ret = perf_output_begin(&handle, &sample, event,
9742	mmap_event->event_id.header.size);
9743	if (ret)
9744	return;
9745
9746	mmap_event->event_id.pid = perf_event_pid(event, current);
9747	mmap_event->event_id.tid = perf_event_tid(event, current);
9748
9749	perf_output_put(&handle, mmap_event->event_id);
9750
9751	if (event->attr.mmap2) {
9752	if (use_build_id) {
9753	u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 };
9754
9755	__output_copy(&handle, size, 4);
9756	__output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX);
9757	} else {
9758	perf_output_put(&handle, mmap_event->maj);
9759	perf_output_put(&handle, mmap_event->min);
9760	perf_output_put(&handle, mmap_event->ino);
9761	perf_output_put(&handle, mmap_event->ino_generation);
9762	}
9763	perf_output_put(&handle, mmap_event->prot);
9764	perf_output_put(&handle, mmap_event->flags);
9765	}
9766
9767	__output_copy(&handle, mmap_event->file_name,
9768	mmap_event->file_size);
9769
9770	perf_event__output_id_sample(event, &handle, &sample);
9771
9772	perf_output_end(&handle);
9773	}
9774
9775	static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
9776	{
9777	struct vm_area_struct *vma = mmap_event->vma;
9778	struct file *file = vma->vm_file;
9779	int maj = 0, min = 0;
9780	u64 ino = 0, gen = 0;
9781	u32 prot = 0, flags = 0;
9782	unsigned int size;
9783	char tmp[16];
9784	char *buf = NULL;
9785	char *name = NULL;
9786
9787	if (vma->vm_flags & VM_READ)
9788	prot \|= PROT_READ;
9789	if (vma->vm_flags & VM_WRITE)
9790	prot \|= PROT_WRITE;
9791	if (vma->vm_flags & VM_EXEC)
9792	prot \|= PROT_EXEC;
9793
9794	if (vma->vm_flags & VM_MAYSHARE)
9795	flags = MAP_SHARED;
9796	else
9797	flags = MAP_PRIVATE;
9798
9799	if (vma->vm_flags & VM_LOCKED)
9800	flags \|= MAP_LOCKED;
9801	if (is_vm_hugetlb_page(vma))
9802	flags \|= MAP_HUGETLB;
9803
9804	if (file) {
9805	const struct inode *inode;
9806	dev_t dev;
9807
9808	buf = kmalloc(PATH_MAX, GFP_KERNEL);
9809	if (!buf) {
9810	name = "//enomem";
9811	goto cpy_name;
9812	}
9813	/*
9814	* d_path() works from the end of the rb backwards, so we
9815	* need to add enough zero bytes after the string to handle
9816	* the 64bit alignment we do later.
9817	*/
9818	name = d_path(file_user_path(file), buf, PATH_MAX - sizeof(u64));
9819	if (IS_ERR(name)) {
9820	name = "//toolong";
9821	goto cpy_name;
9822	}
9823	inode = file_user_inode(vma->vm_file);
9824	dev = inode->i_sb->s_dev;
9825	ino = inode->i_ino;
9826	gen = inode->i_generation;
9827	maj = MAJOR(dev);
9828	min = MINOR(dev);
9829
9830	goto got_name;
9831	} else {
9832	if (vma->vm_ops && vma->vm_ops->name)
9833	name = (char *) vma->vm_ops->name(vma);
9834	if (!name)
9835	name = (char *)arch_vma_name(vma);
9836	if (!name) {
9837	if (vma_is_initial_heap(vma))
9838	name = "[heap]";
9839	else if (vma_is_initial_stack(vma))
9840	name = "[stack]";
9841	else
9842	name = "//anon";
9843	}
9844	}
9845
9846	cpy_name:
9847	strscpy(tmp, name);
9848	name = tmp;
9849	got_name:
9850	/*
9851	* Since our buffer works in 8 byte units we need to align our string
9852	* size to a multiple of 8. However, we must guarantee the tail end is
9853	* zero'd out to avoid leaking random bits to userspace.
9854	*/
9855	size = strlen(name)+1;
9856	while (!IS_ALIGNED(size, sizeof(u64)))
9857	name[size++] = '\0';
9858
9859	mmap_event->file_name = name;
9860	mmap_event->file_size = size;
9861	mmap_event->maj = maj;
9862	mmap_event->min = min;
9863	mmap_event->ino = ino;
9864	mmap_event->ino_generation = gen;
9865	mmap_event->prot = prot;
9866	mmap_event->flags = flags;
9867
9868	if (atomic_read(&nr_build_id_events))
9869	build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size);
9870
9871	perf_iterate_sb(perf_event_mmap_output,
9872	mmap_event,
9873	NULL);
9874
9875	kfree(buf);
9876	}
9877
9878	/*
9879	* Check whether inode and address range match filter criteria.
9880	*/
9881	static bool perf_addr_filter_match(struct perf_addr_filter *filter,
9882	struct file *file, unsigned long offset,
9883	unsigned long size)
9884	{
9885	/* d_inode(NULL) won't be equal to any mapped user-space file */
9886	if (!filter->path.dentry)
9887	return false;
9888
9889	if (d_inode(filter->path.dentry) != file_user_inode(file))
9890	return false;
9891
9892	if (filter->offset > offset + size)
9893	return false;
9894
9895	if (filter->offset + filter->size < offset)
9896	return false;
9897
9898	return true;
9899	}
9900
9901	static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter,
9902	struct vm_area_struct *vma,
9903	struct perf_addr_filter_range *fr)
9904	{
9905	unsigned long vma_size = vma->vm_end - vma->vm_start;
9906	unsigned long off = vma_start_pgoff(vma) << PAGE_SHIFT;
9907	struct file *file = vma->vm_file;
9908
9909	if (!perf_addr_filter_match(filter, file, off, vma_size))
9910	return false;
9911
9912	if (filter->offset < off) {
9913	fr->start = vma->vm_start;
9914	fr->size = min(vma_size, filter->size - (off - filter->offset));
9915	} else {
9916	fr->start = vma->vm_start + filter->offset - off;
9917	fr->size = min(vma->vm_end - fr->start, filter->size);
9918	}
9919
9920	return true;
9921	}
9922
9923	static void __perf_addr_filters_adjust(struct perf_event event, void data)
9924	{
9925	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
9926	struct vm_area_struct *vma = data;
9927	struct perf_addr_filter *filter;
9928	unsigned int restart = 0, count = 0;
9929	unsigned long flags;
9930
9931	if (!has_addr_filter(event))
9932	return;
9933
9934	if (!vma->vm_file)
9935	return;
9936
9937	raw_spin_lock_irqsave(&ifh->lock, flags);
9938	list_for_each_entry(filter, &ifh->list, entry) {
9939	if (perf_addr_filter_vma_adjust(filter, vma,
9940	&event->addr_filter_ranges[count]))
9941	restart++;
9942
9943	count++;
9944	}
9945
9946	if (restart)
9947	event->addr_filters_gen++;
9948	raw_spin_unlock_irqrestore(&ifh->lock, flags);
9949
9950	if (restart)
9951	perf_event_stop(event, 1);
9952	}
9953
9954	/*
9955	* Adjust all task's events' filters to the new vma
9956	*/
9957	static void perf_addr_filters_adjust(struct vm_area_struct *vma)
9958	{
9959	struct perf_event_context *ctx;
9960
9961	/*
9962	* Data tracing isn't supported yet and as such there is no need
9963	* to keep track of anything that isn't related to executable code:
9964	*/
9965	if (!(vma->vm_flags & VM_EXEC))
9966	return;
9967
9968	rcu_read_lock();
9969	ctx = rcu_dereference(current->perf_event_ctxp);
9970	if (ctx)
9971	perf_iterate_ctx(ctx, __perf_addr_filters_adjust, vma, true);
9972	rcu_read_unlock();
9973	}
9974
9975	void perf_event_mmap(struct vm_area_struct *vma)
9976	{
9977	struct perf_mmap_event mmap_event;
9978
9979	if (!atomic_read(&nr_mmap_events))
9980	return;
9981
9982	mmap_event = (struct perf_mmap_event){
9983	.vma = vma,
9984	/* .file_name */
9985	/* .file_size */
9986	.event_id = {
9987	/* .header */
9988	/* .pid */
9989	/* .tid */
9990	.start = vma->vm_start,
9991	.len = vma->vm_end - vma->vm_start,
9992	.pgoff = (u64)vma_start_pgoff(vma) << PAGE_SHIFT,
9993	},
9994	/* .maj (attr_mmap2 only) */
9995	/* .min (attr_mmap2 only) */
9996	/* .ino (attr_mmap2 only) */
9997	/* .ino_generation (attr_mmap2 only) */
9998	/* .prot (attr_mmap2 only) */
9999	/* .flags (attr_mmap2 only) */
10000	};
10001
10002	perf_addr_filters_adjust(vma);
10003	perf_event_mmap_event(&mmap_event);
10004	}
10005
10006	void perf_event_aux_event(struct perf_event *event, unsigned long head,
10007	unsigned long size, u64 flags)
10008	{
10009	struct perf_output_handle handle;
10010	struct perf_sample_data sample;
10011	struct perf_aux_event {
10012	struct perf_event_header header;
10013	u64 offset;
10014	u64 size;
10015	u64 flags;
10016	} rec = {
10017	.offset = head,
10018	.size = size,
10019	.flags = flags,
10020	};
10021	int ret;
10022
10023	perf_event_header__init(&rec.header, &sample,
10024	PERF_RECORD_AUX, /* misc= */ 0, sizeof(rec),
10025	event);
10026	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
10027
10028	if (ret)
10029	return;
10030
10031	perf_output_put(&handle, rec);
10032	perf_event__output_id_sample(event, &handle, &sample);
10033
10034	perf_output_end(&handle);
10035	}
10036
10037	/*
10038	* Lost/dropped samples logging
10039	*/
10040	void perf_log_lost_samples(struct perf_event *event, u64 lost)
10041	{
10042	struct perf_output_handle handle;
10043	struct perf_sample_data sample;
10044	int ret;
10045
10046	struct {
10047	struct perf_event_header header;
10048	u64 lost;
10049	} lost_samples_event = {
10050	.lost = lost,
10051	};
10052
10053	perf_event_header__init(&lost_samples_event.header, &sample,
10054	PERF_RECORD_LOST_SAMPLES,
10055	/* misc= */ 0,
10056	sizeof(lost_samples_event),
10057	event);
10058
10059	ret = perf_output_begin(&handle, &sample, event,
10060	lost_samples_event.header.size);
10061	if (ret)
10062	return;
10063
10064	perf_output_put(&handle, lost_samples_event);
10065	perf_event__output_id_sample(event, &handle, &sample);
10066	perf_output_end(&handle);
10067	}
10068
10069	/*
10070	* context_switch tracking
10071	*/
10072
10073	struct perf_switch_event {
10074	struct task_struct *task;
10075	struct task_struct *next_prev;
10076
10077	struct {
10078	struct perf_event_header header;
10079	u32 next_prev_pid;
10080	u32 next_prev_tid;
10081	} event_id;
10082	bool sched_in;
10083	bool preempt;
10084	};
10085
10086	static int perf_event_switch_match(struct perf_event *event)
10087	{
10088	return event->attr.context_switch;
10089	}
10090
10091	static void perf_event_switch_output(struct perf_event event, void data)
10092	{
10093	struct perf_switch_event *se = data;
10094	struct perf_output_handle handle;
10095	struct perf_sample_data sample;
10096	u32 type;
10097	u16 misc;
10098	u16 size;
10099	int ret;
10100
10101	if (!perf_event_switch_match(event))
10102	return;
10103
10104	/* Only CPU-wide events are allowed to see next/prev pid/tid */
10105	if (event->ctx->task) {
10106	type = PERF_RECORD_SWITCH;
10107	size = sizeof(se->event_id.header);
10108	} else {
10109	type = PERF_RECORD_SWITCH_CPU_WIDE;
10110	size = sizeof(se->event_id);
10111	se->event_id.next_prev_pid =
10112	perf_event_pid(event, se->next_prev);
10113	se->event_id.next_prev_tid =
10114	perf_event_tid(event, se->next_prev);
10115	}
10116	misc = se->sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT;
10117	if (se->preempt)
10118	misc \|= PERF_RECORD_MISC_SWITCH_OUT_PREEMPT;
10119
10120	perf_event_header__init(&se->event_id.header, &sample,
10121	type, misc, size, event);
10122
10123	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
10124	if (ret)
10125	return;
10126
10127	if (event->ctx->task)
10128	perf_output_put(&handle, se->event_id.header);
10129	else
10130	perf_output_put(&handle, se->event_id);
10131
10132	perf_event__output_id_sample(event, &handle, &sample);
10133
10134	perf_output_end(&handle);
10135	}
10136
10137	static void perf_event_switch(struct task_struct *task,
10138	struct task_struct *next_prev, bool sched_in)
10139	{
10140	struct perf_switch_event switch_event;
10141
10142	/* N.B. caller checks nr_switch_events != 0 */
10143
10144	switch_event = (struct perf_switch_event){
10145	.task = task,
10146	.next_prev = next_prev,
10147	/* .event_id */
10148	.sched_in = sched_in,
10149	.preempt = !sched_in && task_is_runnable(task),
10150	};
10151
10152	perf_iterate_sb(perf_event_switch_output, &switch_event, NULL);
10153	}
10154
10155	/*
10156	* IRQ throttle logging
10157	*/
10158
10159	static void perf_log_throttle(struct perf_event *event, int enable)
10160	{
10161	struct perf_output_handle handle;
10162	struct perf_sample_data sample;
10163	int ret;
10164
10165	struct {
10166	struct perf_event_header header;
10167	u64 time;
10168	u64 id;
10169	u64 stream_id;
10170	} throttle_event = {
10171	.time = perf_event_clock(event),
10172	.id = primary_event_id(event),
10173	.stream_id = event->id,
10174	};
10175
10176	perf_event_header__init(&throttle_event.header, &sample,
10177	enable ? PERF_RECORD_UNTHROTTLE
10178	: PERF_RECORD_THROTTLE,
10179	/* misc= */ 0,
10180	sizeof(throttle_event),
10181	event);
10182
10183	ret = perf_output_begin(&handle, &sample, event,
10184	throttle_event.header.size);
10185	if (ret)
10186	return;
10187
10188	perf_output_put(&handle, throttle_event);
10189	perf_event__output_id_sample(event, &handle, &sample);
10190	perf_output_end(&handle);
10191	}
10192
10193	/*
10194	* ksymbol register/unregister tracking
10195	*/
10196
10197	struct perf_ksymbol_event {
10198	const char *name;
10199	int name_len;
10200	struct {
10201	struct perf_event_header header;
10202	u64 addr;
10203	u32 len;
10204	u16 ksym_type;
10205	u16 flags;
10206	} event_id;
10207	};
10208
10209	static int perf_event_ksymbol_match(struct perf_event *event)
10210	{
10211	return event->attr.ksymbol;
10212	}
10213
10214	static void perf_event_ksymbol_output(struct perf_event event, void data)
10215	{
10216	struct perf_ksymbol_event *ksymbol_event = data;
10217	struct perf_output_handle handle;
10218	struct perf_sample_data sample;
10219	int ret;
10220	u16 size = sizeof(ksymbol_event->event_id) + ksymbol_event->name_len;
10221
10222	if (!perf_event_ksymbol_match(event))
10223	return;
10224
10225	perf_event_header__init(&ksymbol_event->event_id.header, &sample,
10226	PERF_RECORD_KSYMBOL, /* misc= */ 0, size,
10227	event);
10228	ret = perf_output_begin(&handle, &sample, event,
10229	ksymbol_event->event_id.header.size);
10230	if (ret)
10231	return;
10232
10233	perf_output_put(&handle, ksymbol_event->event_id);
10234	__output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len);
10235	perf_event__output_id_sample(event, &handle, &sample);
10236
10237	perf_output_end(&handle);
10238	}
10239
10240	void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister,
10241	const char *sym)
10242	{
10243	struct perf_ksymbol_event ksymbol_event;
10244	char name[KSYM_NAME_LEN];
10245	u16 flags = 0;
10246	int name_len;
10247
10248	if (!atomic_read(&nr_ksymbol_events))
10249	return;
10250
10251	if (ksym_type >= PERF_RECORD_KSYMBOL_TYPE_MAX \|\|
10252	ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN)
10253	goto err;
10254
10255	strscpy(name, sym);
10256	name_len = strlen(name) + 1;
10257	while (!IS_ALIGNED(name_len, sizeof(u64)))
10258	name[name_len++] = '\0';
10259	BUILD_BUG_ON(KSYM_NAME_LEN % sizeof(u64));
10260
10261	if (unregister)
10262	flags \|= PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER;
10263
10264	ksymbol_event = (struct perf_ksymbol_event){
10265	.name = name,
10266	.name_len = name_len,
10267	.event_id = {
10268	.addr = addr,
10269	.len = len,
10270	.ksym_type = ksym_type,
10271	.flags = flags,
10272	},
10273	};
10274
10275	perf_iterate_sb(perf_event_ksymbol_output, &ksymbol_event, NULL);
10276	return;
10277	err:
10278	WARN_ONCE(1, "%s: Invalid KSYMBOL type 0x%x\n", __func__, ksym_type);
10279	}
10280
10281	/*
10282	* bpf program load/unload tracking
10283	*/
10284
10285	struct perf_bpf_event {
10286	struct bpf_prog *prog;
10287	struct {
10288	struct perf_event_header header;
10289	u16 type;
10290	u16 flags;
10291	u32 id;
10292	u8 tag[BPF_TAG_SIZE];
10293	} event_id;
10294	};
10295
10296	static int perf_event_bpf_match(struct perf_event *event)
10297	{
10298	return event->attr.bpf_event;
10299	}
10300
10301	static void perf_event_bpf_output(struct perf_event event, void data)
10302	{
10303	struct perf_bpf_event *bpf_event = data;
10304	struct perf_output_handle handle;
10305	struct perf_sample_data sample;
10306	int ret;
10307
10308	if (!perf_event_bpf_match(event))
10309	return;
10310
10311	perf_event_header__init(&bpf_event->event_id.header, &sample,
10312	PERF_RECORD_BPF_EVENT,
10313	/* misc= */ 0,
10314	sizeof(bpf_event->event_id),
10315	event);
10316	ret = perf_output_begin(&handle, &sample, event,
10317	bpf_event->event_id.header.size);
10318	if (ret)
10319	return;
10320
10321	perf_output_put(&handle, bpf_event->event_id);
10322	perf_event__output_id_sample(event, &handle, &sample);
10323
10324	perf_output_end(&handle);
10325	}
10326
10327	static void perf_event_bpf_emit_ksymbols(struct bpf_prog *prog,
10328	enum perf_bpf_event_type type)
10329	{
10330	bool unregister = type == PERF_BPF_EVENT_PROG_UNLOAD;
10331	int i;
10332
10333	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF,
10334	(u64)(unsigned long)prog->bpf_func,
10335	prog->jited_len, unregister,
10336	prog->aux->ksym.name);
10337
10338	for (i = 1; i < prog->aux->func_cnt; i++) {
10339	struct bpf_prog *subprog = prog->aux->func[i];
10340
10341	perf_event_ksymbol(
10342	PERF_RECORD_KSYMBOL_TYPE_BPF,
10343	(u64)(unsigned long)subprog->bpf_func,
10344	subprog->jited_len, unregister,
10345	subprog->aux->ksym.name);
10346	}
10347	}
10348
10349	void perf_event_bpf_event(struct bpf_prog *prog,
10350	enum perf_bpf_event_type type,
10351	u16 flags)
10352	{
10353	struct perf_bpf_event bpf_event;
10354
10355	switch (type) {
10356	case PERF_BPF_EVENT_PROG_LOAD:
10357	case PERF_BPF_EVENT_PROG_UNLOAD:
10358	if (atomic_read(&nr_ksymbol_events))
10359	perf_event_bpf_emit_ksymbols(prog, type);
10360	break;
10361	default:
10362	return;
10363	}
10364
10365	if (!atomic_read(&nr_bpf_events))
10366	return;
10367
10368	bpf_event = (struct perf_bpf_event){
10369	.prog = prog,
10370	.event_id = {
10371	.type = type,
10372	.flags = flags,
10373	.id = prog->aux->id,
10374	},
10375	};
10376
10377	BUILD_BUG_ON(BPF_TAG_SIZE % sizeof(u64));
10378
10379	memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);
10380	perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL);
10381	}
10382
10383	struct perf_callchain_deferred_event {
10384	struct unwind_stacktrace *trace;
10385	struct {
10386	struct perf_event_header header;
10387	u64 cookie;
10388	u64 nr;
10389	u64 ips[];
10390	} event;
10391	};
10392
10393	static void perf_callchain_deferred_output(struct perf_event event, void data)
10394	{
10395	struct perf_callchain_deferred_event *deferred_event = data;
10396	struct perf_output_handle handle;
10397	struct perf_sample_data sample;
10398	int ret;
10399	u16 size = sizeof(deferred_event->event) + (deferred_event->trace->nr * sizeof(u64));
10400
10401	if (!event->attr.defer_output)
10402	return;
10403
10404	/* XXX do we really need sample_id_all for this ??? */
10405	perf_event_header__init(&deferred_event->event.header, &sample,
10406	PERF_RECORD_CALLCHAIN_DEFERRED,
10407	PERF_RECORD_MISC_USER,
10408	size,
10409	event);
10410
10411	ret = perf_output_begin(&handle, &sample, event,
10412	deferred_event->event.header.size);
10413	if (ret)
10414	return;
10415
10416	perf_output_put(&handle, deferred_event->event);
10417	for (int i = 0; i < deferred_event->trace->nr; i++) {
10418	u64 entry = deferred_event->trace->entries[i];
10419	perf_output_put(&handle, entry);
10420	}
10421	perf_event__output_id_sample(event, &handle, &sample);
10422
10423	perf_output_end(&handle);
10424	}
10425
10426	static void perf_unwind_deferred_callback(struct unwind_work *work,
10427	struct unwind_stacktrace *trace, u64 cookie)
10428	{
10429	struct perf_callchain_deferred_event deferred_event = {
10430	.trace = trace,
10431	.event = {
10432	.cookie = cookie,
10433	.nr = trace->nr,
10434	},
10435	};
10436
10437	perf_iterate_sb(perf_callchain_deferred_output, &deferred_event, NULL);
10438	}
10439
10440	struct perf_text_poke_event {
10441	const void *old_bytes;
10442	const void *new_bytes;
10443	u16 tot;
10444	u16 pad;
10445	u16 old_len;
10446	u16 new_len;
10447
10448	struct {
10449	struct perf_event_header header;
10450
10451	u64 addr;
10452	} event_id;
10453	};
10454
10455	static int perf_event_text_poke_match(struct perf_event *event)
10456	{
10457	return event->attr.text_poke;
10458	}
10459
10460	static void perf_event_text_poke_output(struct perf_event event, void data)
10461	{
10462	struct perf_text_poke_event *text_poke_event = data;
10463	struct perf_output_handle handle;
10464	struct perf_sample_data sample;
10465	u16 size = sizeof(text_poke_event->event_id) + text_poke_event->tot + text_poke_event->pad;
10466	u64 padding = 0;
10467	int ret;
10468
10469	if (!perf_event_text_poke_match(event))
10470	return;
10471
10472	perf_event_header__init(&text_poke_event->event_id.header, &sample,
10473	PERF_RECORD_TEXT_POKE,
10474	PERF_RECORD_MISC_KERNEL,
10475	size,
10476	event);
10477
10478	ret = perf_output_begin(&handle, &sample, event,
10479	text_poke_event->event_id.header.size);
10480	if (ret)
10481	return;
10482
10483	perf_output_put(&handle, text_poke_event->event_id);
10484	perf_output_put(&handle, text_poke_event->old_len);
10485	perf_output_put(&handle, text_poke_event->new_len);
10486
10487	__output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);
10488	__output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);
10489
10490	if (text_poke_event->pad)
10491	__output_copy(&handle, &padding, text_poke_event->pad);
10492
10493	perf_event__output_id_sample(event, &handle, &sample);
10494
10495	perf_output_end(&handle);
10496	}
10497
10498	void perf_event_text_poke(const void addr, const void old_bytes,
10499	size_t old_len, const void *new_bytes, size_t new_len)
10500	{
10501	struct perf_text_poke_event text_poke_event;
10502	size_t tot, pad;
10503
10504	if (!atomic_read(&nr_text_poke_events))
10505	return;
10506
10507	tot = sizeof(text_poke_event.old_len) + old_len;
10508	tot += sizeof(text_poke_event.new_len) + new_len;
10509	pad = ALIGN(tot, sizeof(u64)) - tot;
10510
10511	text_poke_event = (struct perf_text_poke_event){
10512	.old_bytes = old_bytes,
10513	.new_bytes = new_bytes,
10514	.tot = tot,
10515	.pad = pad,
10516	.old_len = old_len,
10517	.new_len = new_len,
10518	.event_id = {
10519	.addr = (unsigned long)addr,
10520	},
10521	};
10522
10523	perf_iterate_sb(perf_event_text_poke_output, &text_poke_event, NULL);
10524	}
10525
10526	void perf_event_itrace_started(struct perf_event *event)
10527	{
10528	WRITE_ONCE(event->attach_state, event->attach_state \| PERF_ATTACH_ITRACE);
10529	}
10530
10531	static void perf_log_itrace_start(struct perf_event *event)
10532	{
10533	struct perf_output_handle handle;
10534	struct perf_sample_data sample;
10535	struct perf_aux_event {
10536	struct perf_event_header header;
10537	u32 pid;
10538	u32 tid;
10539	} rec;
10540	int ret;
10541
10542	if (event->parent)
10543	event = event->parent;
10544
10545	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) \|\|
10546	event->attach_state & PERF_ATTACH_ITRACE)
10547	return;
10548
10549	rec.pid = perf_event_pid(event, current);
10550	rec.tid = perf_event_tid(event, current);
10551
10552	perf_event_header__init(&rec.header, &sample,
10553	PERF_RECORD_ITRACE_START, /* misc= */ 0, sizeof(rec),
10554	event);
10555	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
10556
10557	if (ret)
10558	return;
10559
10560	perf_output_put(&handle, rec);
10561	perf_event__output_id_sample(event, &handle, &sample);
10562
10563	perf_output_end(&handle);
10564	}
10565
10566	void perf_report_aux_output_id(struct perf_event *event, u64 hw_id)
10567	{
10568	struct perf_output_handle handle;
10569	struct perf_sample_data sample;
10570	struct perf_aux_event {
10571	struct perf_event_header header;
10572	u64 hw_id;
10573	} rec;
10574	int ret;
10575
10576	if (event->parent)
10577	event = event->parent;
10578
10579	rec.hw_id = hw_id;
10580	perf_event_header__init(&rec.header, &sample,
10581	PERF_RECORD_AUX_OUTPUT_HW_ID, /* misc= */ 0,
10582	sizeof(rec), event);
10583	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
10584
10585	if (ret)
10586	return;
10587
10588	perf_output_put(&handle, rec);
10589	perf_event__output_id_sample(event, &handle, &sample);
10590
10591	perf_output_end(&handle);
10592	}
10593	EXPORT_SYMBOL_GPL(perf_report_aux_output_id);
10594
10595	static int
10596	__perf_event_account_interrupt(struct perf_event *event, int throttle)
10597	{
10598	struct hw_perf_event *hwc = &event->hw;
10599	int ret = 0;
10600	u64 seq;
10601
10602	seq = __this_cpu_read(perf_throttled_seq);
10603	if (seq != hwc->interrupts_seq) {
10604	hwc->interrupts_seq = seq;
10605	hwc->interrupts = 1;
10606	} else {
10607	hwc->interrupts++;
10608	}
10609
10610	if (unlikely(throttle && hwc->interrupts >= max_samples_per_tick)) {
10611	__this_cpu_inc(perf_throttled_count);
10612	tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
10613	perf_event_throttle_group(event);
10614	ret = 1;
10615	}
10616
10617	if (event->attr.freq) {
10618	u64 now = perf_clock();
10619	s64 delta = now - hwc->freq_time_stamp;
10620
10621	hwc->freq_time_stamp = now;
10622
10623	if (delta > 0 && delta < 2*TICK_NSEC)
10624	perf_adjust_period(event, delta, hwc->last_period, true);
10625	}
10626
10627	return ret;
10628	}
10629
10630	int perf_event_account_interrupt(struct perf_event *event)
10631	{
10632	return __perf_event_account_interrupt(event, 1);
10633	}
10634
10635	static inline bool sample_is_allowed(struct perf_event event, struct pt_regs regs)
10636	{
10637	/*
10638	* Due to interrupt latency (AKA "skid"), we may enter the
10639	* kernel before taking an overflow, even if the PMU is only
10640	* counting user events.
10641	*/
10642	if (event->attr.exclude_kernel && !user_mode(regs))
10643	return false;
10644
10645	return true;
10646	}
10647
10648	#ifdef CONFIG_BPF_SYSCALL
10649	static int bpf_overflow_handler(struct perf_event *event,
10650	struct perf_sample_data *data,
10651	struct pt_regs *regs)
10652	{
10653	struct bpf_perf_event_data_kern ctx = {
10654	.data = data,
10655	.event = event,
10656	};
10657	struct bpf_prog *prog;
10658	int ret = 0;
10659
10660	ctx.regs = perf_arch_bpf_user_pt_regs(regs);
10661	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
10662	goto out;
10663	rcu_read_lock();
10664	prog = READ_ONCE(event->prog);
10665	if (prog) {
10666	perf_prepare_sample(data, event, regs);
10667	ret = bpf_prog_run(prog, &ctx);
10668	}
10669	rcu_read_unlock();
10670	out:
10671	__this_cpu_dec(bpf_prog_active);
10672
10673	return ret;
10674	}
10675
10676	static inline int perf_event_set_bpf_handler(struct perf_event *event,
10677	struct bpf_prog *prog,
10678	u64 bpf_cookie)
10679	{
10680	if (event->overflow_handler_context)
10681	/* hw breakpoint or kernel counter */
10682	return -EINVAL;
10683
10684	if (event->prog)
10685	return -EEXIST;
10686
10687	if (prog->type != BPF_PROG_TYPE_PERF_EVENT)
10688	return -EINVAL;
10689
10690	if (event->attr.precise_ip &&
10691	prog->call_get_stack &&
10692	(!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) \|\|
10693	event->attr.exclude_callchain_kernel \|\|
10694	event->attr.exclude_callchain_user)) {
10695	/*
10696	* On perf_event with precise_ip, calling bpf_get_stack()
10697	* may trigger unwinder warnings and occasional crashes.
10698	* bpf_get_[stack\|stackid] works around this issue by using
10699	* callchain attached to perf_sample_data. If the
10700	* perf_event does not full (kernel and user) callchain
10701	* attached to perf_sample_data, do not allow attaching BPF
10702	* program that calls bpf_get_[stack\|stackid].
10703	*/
10704	return -EPROTO;
10705	}
10706
10707	event->prog = prog;
10708	event->bpf_cookie = bpf_cookie;
10709	return 0;
10710	}
10711
10712	static inline void perf_event_free_bpf_handler(struct perf_event *event)
10713	{
10714	struct bpf_prog *prog = event->prog;
10715
10716	if (!prog)
10717	return;
10718
10719	event->prog = NULL;
10720	bpf_prog_put(prog);
10721	}
10722	#else
10723	static inline int bpf_overflow_handler(struct perf_event *event,
10724	struct perf_sample_data *data,
10725	struct pt_regs *regs)
10726	{
10727	return 1;
10728	}
10729
10730	static inline int perf_event_set_bpf_handler(struct perf_event *event,
10731	struct bpf_prog *prog,
10732	u64 bpf_cookie)
10733	{
10734	return -EOPNOTSUPP;
10735	}
10736
10737	static inline void perf_event_free_bpf_handler(struct perf_event *event)
10738	{
10739	}
10740	#endif
10741
10742	/*
10743	* Generic event overflow handling, sampling.
10744	*/
10745
10746	static int __perf_event_overflow(struct perf_event *event,
10747	int throttle, struct perf_sample_data *data,
10748	struct pt_regs *regs)
10749	{
10750	int events = atomic_read(&event->event_limit);
10751	int ret = 0;
10752
10753	/*
10754	* Non-sampling counters might still use the PMI to fold short
10755	* hardware counters, ignore those.
10756	*/
10757	if (unlikely(!is_sampling_event(event)))
10758	return 0;
10759
10760	ret = __perf_event_account_interrupt(event, throttle);
10761
10762	if (event->attr.aux_pause)
10763	perf_event_aux_pause(event->aux_event, true);
10764
10765	if (event->prog && event->prog->type == BPF_PROG_TYPE_PERF_EVENT &&
10766	!bpf_overflow_handler(event, data, regs))
10767	goto out;
10768
10769	/*
10770	* XXX event_limit might not quite work as expected on inherited
10771	* events
10772	*/
10773
10774	event->pending_kill = POLL_IN;
10775	if (events && atomic_dec_and_test(&event->event_limit)) {
10776	ret = 1;
10777	event->pending_kill = POLL_HUP;
10778	perf_event_disable_inatomic(event);
10779	event->pmu->stop(event, 0);
10780	}
10781
10782	if (event->attr.sigtrap) {
10783	/*
10784	* The desired behaviour of sigtrap vs invalid samples is a bit
10785	* tricky; on the one hand, one should not loose the SIGTRAP if
10786	* it is the first event, on the other hand, we should also not
10787	* trigger the WARN or override the data address.
10788	*/
10789	bool valid_sample = sample_is_allowed(event, regs);
10790	unsigned int pending_id = 1;
10791	enum task_work_notify_mode notify_mode;
10792
10793	if (regs)
10794	pending_id = hash32_ptr((void *)instruction_pointer(regs)) ?: 1;
10795
10796	notify_mode = in_nmi() ? TWA_NMI_CURRENT : TWA_RESUME;
10797
10798	if (!event->pending_work &&
10799	!task_work_add(current, &event->pending_task, notify_mode)) {
10800	event->pending_work = pending_id;
10801	local_inc(&event->ctx->nr_no_switch_fast);
10802	WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
10803
10804	event->pending_addr = 0;
10805	if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))
10806	event->pending_addr = data->addr;
10807
10808	} else if (event->attr.exclude_kernel && valid_sample) {
10809	/*
10810	* Should not be able to return to user space without
10811	* consuming pending_work; with exceptions:
10812	*
10813	* 1. Where !exclude_kernel, events can overflow again
10814	* in the kernel without returning to user space.
10815	*
10816	* 2. Events that can overflow again before the IRQ-
10817	* work without user space progress (e.g. hrtimer).
10818	* To approximate progress (with false negatives),
10819	* check 32-bit hash of the current IP.
10820	*/
10821	WARN_ON_ONCE(event->pending_work != pending_id);
10822	}
10823	}
10824
10825	READ_ONCE(event->overflow_handler)(event, data, regs);
10826
10827	if (*perf_event_fasync(event) && event->pending_kill) {
10828	event->pending_wakeup = 1;
10829	irq_work_queue(&event->pending_irq);
10830	}
10831	out:
10832	if (event->attr.aux_resume)
10833	perf_event_aux_pause(event->aux_event, false);
10834
10835	return ret;
10836	}
10837
10838	int perf_event_overflow(struct perf_event *event,
10839	struct perf_sample_data *data,
10840	struct pt_regs *regs)
10841	{
10842	/*
10843	* Entry point from hardware PMI, interrupts should be disabled here.
10844	* This serializes us against perf_event_remove_from_context() in
10845	* things like perf_event_release_kernel().
10846	*/
10847	lockdep_assert_irqs_disabled();
10848
10849	return __perf_event_overflow(event, 1, data, regs);
10850	}
10851
10852	/*
10853	* Generic software event infrastructure
10854	*/
10855
10856	struct swevent_htable {
10857	struct swevent_hlist *swevent_hlist;
10858	struct mutex hlist_mutex;
10859	int hlist_refcount;
10860	};
10861	static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
10862
10863	/*
10864	* We directly increment event->count and keep a second value in
10865	* event->hw.period_left to count intervals. This period event
10866	* is kept in the range [-sample_period, 0] so that we can use the
10867	* sign as trigger.
10868	*/
10869
10870	u64 perf_swevent_set_period(struct perf_event *event)
10871	{
10872	struct hw_perf_event *hwc = &event->hw;
10873	u64 period = hwc->last_period;
10874	u64 nr, offset;
10875	s64 old, val;
10876
10877	hwc->last_period = hwc->sample_period;
10878
10879	old = local64_read(&hwc->period_left);
10880	do {
10881	val = old;
10882	if (val < 0)
10883	return 0;
10884
10885	nr = div64_u64(period + val, period);
10886	offset = nr * period;
10887	val -= offset;
10888	} while (!local64_try_cmpxchg(&hwc->period_left, &old, val));
10889
10890	return nr;
10891	}
10892
10893	static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
10894	struct perf_sample_data *data,
10895	struct pt_regs *regs)
10896	{
10897	struct hw_perf_event *hwc = &event->hw;
10898	int throttle = 0;
10899
10900	if (!overflow)
10901	overflow = perf_swevent_set_period(event);
10902
10903	if (hwc->interrupts == MAX_INTERRUPTS)
10904	return;
10905
10906	for (; overflow; overflow--) {
10907	if (__perf_event_overflow(event, throttle,
10908	data, regs)) {
10909	/*
10910	* We inhibit the overflow from happening when
10911	* hwc->interrupts == MAX_INTERRUPTS.
10912	*/
10913	break;
10914	}
10915	throttle = 1;
10916	}
10917	}
10918
10919	static void perf_swevent_event(struct perf_event *event, u64 nr,
10920	struct perf_sample_data *data,
10921	struct pt_regs *regs)
10922	{
10923	struct hw_perf_event *hwc = &event->hw;
10924
10925	/*
10926	* This is:
10927	* - software preempt
10928	* - tracepoint preempt
10929	* - tp_target_task irq (ctx->lock)
10930	* - uprobes preempt/irq
10931	* - kprobes preempt/irq
10932	* - hw_breakpoint irq
10933	*
10934	* Any of these are sufficient to hold off RCU and thus ensure @event
10935	* exists.
10936	*/
10937	lockdep_assert_preemption_disabled();
10938	local64_add(nr, &event->count);
10939
10940	if (!regs)
10941	return;
10942
10943	if (!is_sampling_event(event))
10944	return;
10945
10946	/*
10947	* Serialize against event_function_call() IPIs like normal overflow
10948	* event handling. Specifically, must not allow
10949	* perf_event_release_kernel() -> perf_remove_from_context() to make
10950	* progress and 'release' the event from under us.
10951	*/
10952	guard(irqsave)();
10953	if (event->state != PERF_EVENT_STATE_ACTIVE)
10954	return;
10955
10956	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
10957	data->period = nr;
10958	return perf_swevent_overflow(event, 1, data, regs);
10959	} else
10960	data->period = event->hw.last_period;
10961
10962	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
10963	return perf_swevent_overflow(event, 1, data, regs);
10964
10965	if (local64_add_negative(nr, &hwc->period_left))
10966	return;
10967
10968	perf_swevent_overflow(event, 0, data, regs);
10969	}
10970
10971	int perf_exclude_event(struct perf_event event, struct pt_regs regs)
10972	{
10973	if (event->hw.state & PERF_HES_STOPPED)
10974	return 1;
10975
10976	if (regs) {
10977	if (event->attr.exclude_user && user_mode(regs))
10978	return 1;
10979
10980	if (event->attr.exclude_kernel && !user_mode(regs))
10981	return 1;
10982	}
10983
10984	return 0;
10985	}
10986
10987	static int perf_swevent_match(struct perf_event *event,
10988	enum perf_type_id type,
10989	u32 event_id,
10990	struct perf_sample_data *data,
10991	struct pt_regs *regs)
10992	{
10993	if (event->attr.type != type)
10994	return 0;
10995
10996	if (event->attr.config != event_id)
10997	return 0;
10998
10999	if (perf_exclude_event(event, regs))
11000	return 0;
11001
11002	return 1;
11003	}
11004
11005	static inline u64 swevent_hash(u64 type, u32 event_id)
11006	{
11007	u64 val = event_id \| (type << 32);
11008
11009	return hash_64(val, SWEVENT_HLIST_BITS);
11010	}
11011
11012	static inline struct hlist_head *
11013	__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
11014	{
11015	u64 hash = swevent_hash(type, event_id);
11016
11017	return &hlist->heads[hash];
11018	}
11019
11020	/* For the read side: events when they trigger */
11021	static inline struct hlist_head *
11022	find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
11023	{
11024	struct swevent_hlist *hlist;
11025
11026	hlist = rcu_dereference(swhash->swevent_hlist);
11027	if (!hlist)
11028	return NULL;
11029
11030	return __find_swevent_head(hlist, type, event_id);
11031	}
11032
11033	/* For the event head insertion and removal in the hlist */
11034	static inline struct hlist_head *
11035	find_swevent_head(struct swevent_htable swhash, struct perf_event event)
11036	{
11037	struct swevent_hlist *hlist;
11038	u32 event_id = event->attr.config;
11039	u64 type = event->attr.type;
11040
11041	/*
11042	* Event scheduling is always serialized against hlist allocation
11043	* and release. Which makes the protected version suitable here.
11044	* The context lock guarantees that.
11045	*/
11046	hlist = rcu_dereference_protected(swhash->swevent_hlist,
11047	lockdep_is_held(&event->ctx->lock));
11048	if (!hlist)
11049	return NULL;
11050
11051	return __find_swevent_head(hlist, type, event_id);
11052	}
11053
11054	static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
11055	u64 nr,
11056	struct perf_sample_data *data,
11057	struct pt_regs *regs)
11058	{
11059	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
11060	struct perf_event *event;
11061	struct hlist_head *head;
11062
11063	rcu_read_lock();
11064	head = find_swevent_head_rcu(swhash, type, event_id);
11065	if (!head)
11066	goto end;
11067
11068	hlist_for_each_entry_rcu(event, head, hlist_entry) {
11069	if (perf_swevent_match(event, type, event_id, data, regs))
11070	perf_swevent_event(event, nr, data, regs);
11071	}
11072	end:
11073	rcu_read_unlock();
11074	}
11075
11076	DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]);
11077
11078	int perf_swevent_get_recursion_context(void)
11079	{
11080	return get_recursion_context(current->perf_recursion);
11081	}
11082	EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
11083
11084	void perf_swevent_put_recursion_context(int rctx)
11085	{
11086	put_recursion_context(current->perf_recursion, rctx);
11087	}
11088
11089	void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
11090	{
11091	struct perf_sample_data data;
11092
11093	if (WARN_ON_ONCE(!regs))
11094	return;
11095
11096	perf_sample_data_init(&data, addr, 0);
11097	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
11098	}
11099
11100	void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
11101	{
11102	int rctx;
11103
11104	preempt_disable_notrace();
11105	rctx = perf_swevent_get_recursion_context();
11106	if (unlikely(rctx < 0))
11107	goto fail;
11108
11109	___perf_sw_event(event_id, nr, regs, addr);
11110
11111	perf_swevent_put_recursion_context(rctx);
11112	fail:
11113	preempt_enable_notrace();
11114	}
11115
11116	static void perf_swevent_read(struct perf_event *event)
11117	{
11118	}
11119
11120	static int perf_swevent_add(struct perf_event *event, int flags)
11121	{
11122	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
11123	struct hw_perf_event *hwc = &event->hw;
11124	struct hlist_head *head;
11125
11126	if (is_sampling_event(event)) {
11127	hwc->last_period = hwc->sample_period;
11128	perf_swevent_set_period(event);
11129	}
11130
11131	hwc->state = !(flags & PERF_EF_START);
11132
11133	head = find_swevent_head(swhash, event);
11134	if (WARN_ON_ONCE(!head))
11135	return -EINVAL;
11136
11137	hlist_add_head_rcu(&event->hlist_entry, head);
11138	perf_event_update_userpage(event);
11139
11140	return 0;
11141	}
11142
11143	static void perf_swevent_del(struct perf_event *event, int flags)
11144	{
11145	hlist_del_rcu(&event->hlist_entry);
11146	}
11147
11148	static void perf_swevent_start(struct perf_event *event, int flags)
11149	{
11150	event->hw.state = 0;
11151	}
11152
11153	static void perf_swevent_stop(struct perf_event *event, int flags)
11154	{
11155	event->hw.state = PERF_HES_STOPPED;
11156	}
11157
11158	/* Deref the hlist from the update side */
11159	static inline struct swevent_hlist *
11160	swevent_hlist_deref(struct swevent_htable *swhash)
11161	{
11162	return rcu_dereference_protected(swhash->swevent_hlist,
11163	lockdep_is_held(&swhash->hlist_mutex));
11164	}
11165
11166	static void swevent_hlist_release(struct swevent_htable *swhash)
11167	{
11168	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
11169
11170	if (!hlist)
11171	return;
11172
11173	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
11174	kfree_rcu(hlist, rcu_head);
11175	}
11176
11177	static void swevent_hlist_put_cpu(int cpu)
11178	{
11179	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
11180
11181	mutex_lock(&swhash->hlist_mutex);
11182
11183	if (!--swhash->hlist_refcount)
11184	swevent_hlist_release(swhash);
11185
11186	mutex_unlock(&swhash->hlist_mutex);
11187	}
11188
11189	static void swevent_hlist_put(void)
11190	{
11191	int cpu;
11192
11193	for_each_possible_cpu(cpu)
11194	swevent_hlist_put_cpu(cpu);
11195	}
11196
11197	static int swevent_hlist_get_cpu(int cpu)
11198	{
11199	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
11200	int err = 0;
11201
11202	mutex_lock(&swhash->hlist_mutex);
11203	if (!swevent_hlist_deref(swhash) &&
11204	cpumask_test_cpu(cpu, perf_online_mask)) {
11205	struct swevent_hlist *hlist;
11206
11207	hlist = kzalloc_obj(*hlist);
11208	if (!hlist) {
11209	err = -ENOMEM;
11210	goto exit;
11211	}
11212	rcu_assign_pointer(swhash->swevent_hlist, hlist);
11213	}
11214	swhash->hlist_refcount++;
11215	exit:
11216	mutex_unlock(&swhash->hlist_mutex);
11217
11218	return err;
11219	}
11220
11221	static int swevent_hlist_get(void)
11222	{
11223	int err, cpu, failed_cpu;
11224
11225	mutex_lock(&pmus_lock);
11226	for_each_possible_cpu(cpu) {
11227	err = swevent_hlist_get_cpu(cpu);
11228	if (err) {
11229	failed_cpu = cpu;
11230	goto fail;
11231	}
11232	}
11233	mutex_unlock(&pmus_lock);
11234	return 0;
11235	fail:
11236	for_each_possible_cpu(cpu) {
11237	if (cpu == failed_cpu)
11238	break;
11239	swevent_hlist_put_cpu(cpu);
11240	}
11241	mutex_unlock(&pmus_lock);
11242	return err;
11243	}
11244
11245	struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
11246
11247	static void sw_perf_event_destroy(struct perf_event *event)
11248	{
11249	u64 event_id = event->attr.config;
11250
11251	WARN_ON(event->parent);
11252
11253	static_key_slow_dec(&perf_swevent_enabled[event_id]);
11254	swevent_hlist_put();
11255	}
11256
11257	static struct pmu perf_cpu_clock; /* fwd declaration */
11258	static struct pmu perf_task_clock;
11259
11260	static int perf_swevent_init(struct perf_event *event)
11261	{
11262	u64 event_id = event->attr.config;
11263
11264	if (event->attr.type != PERF_TYPE_SOFTWARE)
11265	return -ENOENT;
11266
11267	/*
11268	* no branch sampling for software events
11269	*/
11270	if (has_branch_stack(event))
11271	return -EOPNOTSUPP;
11272
11273	switch (event_id) {
11274	case PERF_COUNT_SW_CPU_CLOCK:
11275	event->attr.type = perf_cpu_clock.type;
11276	return -ENOENT;
11277	case PERF_COUNT_SW_TASK_CLOCK:
11278	event->attr.type = perf_task_clock.type;
11279	return -ENOENT;
11280
11281	default:
11282	break;
11283	}
11284
11285	if (event_id >= PERF_COUNT_SW_MAX)
11286	return -ENOENT;
11287
11288	if (!event->parent) {
11289	int err;
11290
11291	err = swevent_hlist_get();
11292	if (err)
11293	return err;
11294
11295	static_key_slow_inc(&perf_swevent_enabled[event_id]);
11296	event->destroy = sw_perf_event_destroy;
11297	}
11298
11299	return 0;
11300	}
11301
11302	static struct pmu perf_swevent = {
11303	.task_ctx_nr = perf_sw_context,
11304
11305	.capabilities = PERF_PMU_CAP_NO_NMI,
11306
11307	.event_init = perf_swevent_init,
11308	.add = perf_swevent_add,
11309	.del = perf_swevent_del,
11310	.start = perf_swevent_start,
11311	.stop = perf_swevent_stop,
11312	.read = perf_swevent_read,
11313	};
11314
11315	#ifdef CONFIG_EVENT_TRACING
11316
11317	static void tp_perf_event_destroy(struct perf_event *event)
11318	{
11319	perf_trace_destroy(event);
11320	}
11321
11322	static int perf_tp_event_init(struct perf_event *event)
11323	{
11324	int err;
11325
11326	if (event->attr.type != PERF_TYPE_TRACEPOINT)
11327	return -ENOENT;
11328
11329	/*
11330	* no branch sampling for tracepoint events
11331	*/
11332	if (has_branch_stack(event))
11333	return -EOPNOTSUPP;
11334
11335	err = perf_trace_init(event);
11336	if (err)
11337	return err;
11338
11339	event->destroy = tp_perf_event_destroy;
11340
11341	return 0;
11342	}
11343
11344	static struct pmu perf_tracepoint = {
11345	.task_ctx_nr = perf_sw_context,
11346
11347	.event_init = perf_tp_event_init,
11348	.add = perf_trace_add,
11349	.del = perf_trace_del,
11350	.start = perf_swevent_start,
11351	.stop = perf_swevent_stop,
11352	.read = perf_swevent_read,
11353	};
11354
11355	static int perf_tp_filter_match(struct perf_event *event,
11356	struct perf_raw_record *raw)
11357	{
11358	void *record = raw->frag.data;
11359
11360	/* only top level events have filters set */
11361	if (event->parent)
11362	event = event->parent;
11363
11364	if (likely(!event->filter) \|\| filter_match_preds(event->filter, record))
11365	return 1;
11366	return 0;
11367	}
11368
11369	static int perf_tp_event_match(struct perf_event *event,
11370	struct perf_raw_record *raw,
11371	struct pt_regs *regs)
11372	{
11373	if (event->hw.state & PERF_HES_STOPPED)
11374	return 0;
11375	/*
11376	* If exclude_kernel, only trace user-space tracepoints (uprobes)
11377	*/
11378	if (event->attr.exclude_kernel && !user_mode(regs))
11379	return 0;
11380
11381	if (!perf_tp_filter_match(event, raw))
11382	return 0;
11383
11384	return 1;
11385	}
11386
11387	void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
11388	struct trace_event_call *call, u64 count,
11389	struct pt_regs regs, struct hlist_head head,
11390	struct task_struct *task)
11391	{
11392	if (bpf_prog_array_valid(call)) {
11393	(struct pt_regs *)raw_data = regs;
11394	if (!trace_call_bpf(call, raw_data) \|\| hlist_empty(head)) {
11395	perf_swevent_put_recursion_context(rctx);
11396	return;
11397	}
11398	}
11399	perf_tp_event(call->event.type, count, raw_data, size, regs, head,
11400	rctx, task);
11401	}
11402	EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit);
11403
11404	static void __perf_tp_event_target_task(u64 count, void *record,
11405	struct pt_regs *regs,
11406	struct perf_sample_data *data,
11407	struct perf_raw_record *raw,
11408	struct perf_event *event)
11409	{
11410	struct trace_entry *entry = record;
11411
11412	if (event->attr.config != entry->type)
11413	return;
11414	/* Cannot deliver synchronous signal to other task. */
11415	if (event->attr.sigtrap)
11416	return;
11417	if (perf_tp_event_match(event, raw, regs)) {
11418	perf_sample_data_init(data, 0, 0);
11419	perf_sample_save_raw_data(data, event, raw);
11420	perf_swevent_event(event, count, data, regs);
11421	}
11422	}
11423
11424	static void perf_tp_event_target_task(u64 count, void *record,
11425	struct pt_regs *regs,
11426	struct perf_sample_data *data,
11427	struct perf_raw_record *raw,
11428	struct perf_event_context *ctx)
11429	{
11430	unsigned int cpu = smp_processor_id();
11431	struct pmu *pmu = &perf_tracepoint;
11432	struct perf_event event, sibling;
11433
11434	perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {
11435	__perf_tp_event_target_task(count, record, regs, data, raw, event);
11436	for_each_sibling_event(sibling, event)
11437	__perf_tp_event_target_task(count, record, regs, data, raw, sibling);
11438	}
11439
11440	perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {
11441	__perf_tp_event_target_task(count, record, regs, data, raw, event);
11442	for_each_sibling_event(sibling, event)
11443	__perf_tp_event_target_task(count, record, regs, data, raw, sibling);
11444	}
11445	}
11446
11447	void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
11448	struct pt_regs regs, struct hlist_head head, int rctx,
11449	struct task_struct *task)
11450	{
11451	struct perf_sample_data data;
11452	struct perf_event *event;
11453
11454	/*
11455	* Per being a tracepoint, this runs with preemption disabled.
11456	*/
11457	lockdep_assert_preemption_disabled();
11458
11459	struct perf_raw_record raw = {
11460	.frag = {
11461	.size = entry_size,
11462	.data = record,
11463	},
11464	};
11465
11466	perf_trace_buf_update(record, event_type);
11467
11468	hlist_for_each_entry_rcu(event, head, hlist_entry) {
11469	if (perf_tp_event_match(event, &raw, regs)) {
11470	/*
11471	* Here use the same on-stack perf_sample_data,
11472	* some members in data are event-specific and
11473	* need to be re-computed for different sweveents.
11474	* Re-initialize data->sample_flags safely to avoid
11475	* the problem that next event skips preparing data
11476	* because data->sample_flags is set.
11477	*/
11478	perf_sample_data_init(&data, 0, 0);
11479	perf_sample_save_raw_data(&data, event, &raw);
11480	perf_swevent_event(event, count, &data, regs);
11481	}
11482	}
11483
11484	/*
11485	* If we got specified a target task, also iterate its context and
11486	* deliver this event there too.
11487	*/
11488	if (task && task != current) {
11489	struct perf_event_context *ctx;
11490
11491	rcu_read_lock();
11492	ctx = rcu_dereference(task->perf_event_ctxp);
11493	if (!ctx)
11494	goto unlock;
11495
11496	raw_spin_lock(&ctx->lock);
11497	perf_tp_event_target_task(count, record, regs, &data, &raw, ctx);
11498	raw_spin_unlock(&ctx->lock);
11499	unlock:
11500	rcu_read_unlock();
11501	}
11502
11503	perf_swevent_put_recursion_context(rctx);
11504	}
11505	EXPORT_SYMBOL_GPL(perf_tp_event);
11506
11507	#if defined(CONFIG_KPROBE_EVENTS) \|\| defined(CONFIG_UPROBE_EVENTS)
11508	/*
11509	* Flags in config, used by dynamic PMU kprobe and uprobe
11510	* The flags should match following PMU_FORMAT_ATTR().
11511	*
11512	* PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe
11513	* if not set, create kprobe/uprobe
11514	*
11515	* The following values specify a reference counter (or semaphore in the
11516	* terminology of tools like dtrace, systemtap, etc.) Userspace Statically
11517	* Defined Tracepoints (USDT). Currently, we use 40 bit for the offset.
11518	*
11519	* PERF_UPROBE_REF_CTR_OFFSET_BITS # of bits in config as th offset
11520	* PERF_UPROBE_REF_CTR_OFFSET_SHIFT # of bits to shift left
11521	*/
11522	enum perf_probe_config {
11523	PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */
11524	PERF_UPROBE_REF_CTR_OFFSET_BITS = 32,
11525	PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
11526	};
11527
11528	PMU_FORMAT_ATTR(retprobe, "config:0");
11529	#endif
11530
11531	#ifdef CONFIG_KPROBE_EVENTS
11532	static struct attribute *kprobe_attrs[] = {
11533	&format_attr_retprobe.attr,
11534	NULL,
11535	};
11536
11537	static struct attribute_group kprobe_format_group = {
11538	.name = "format",
11539	.attrs = kprobe_attrs,
11540	};
11541
11542	static const struct attribute_group *kprobe_attr_groups[] = {
11543	&kprobe_format_group,
11544	NULL,
11545	};
11546
11547	static int perf_kprobe_event_init(struct perf_event *event);
11548	static struct pmu perf_kprobe = {
11549	.task_ctx_nr = perf_sw_context,
11550	.event_init = perf_kprobe_event_init,
11551	.add = perf_trace_add,
11552	.del = perf_trace_del,
11553	.start = perf_swevent_start,
11554	.stop = perf_swevent_stop,
11555	.read = perf_swevent_read,
11556	.attr_groups = kprobe_attr_groups,
11557	};
11558
11559	static int perf_kprobe_event_init(struct perf_event *event)
11560	{
11561	int err;
11562	bool is_retprobe;
11563
11564	if (event->attr.type != perf_kprobe.type)
11565	return -ENOENT;
11566
11567	if (!perfmon_capable())
11568	return -EACCES;
11569
11570	/*
11571	* no branch sampling for probe events
11572	*/
11573	if (has_branch_stack(event))
11574	return -EOPNOTSUPP;
11575
11576	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
11577	err = perf_kprobe_init(event, is_retprobe);
11578	if (err)
11579	return err;
11580
11581	event->destroy = perf_kprobe_destroy;
11582
11583	return 0;
11584	}
11585	#endif /* CONFIG_KPROBE_EVENTS */
11586
11587	#ifdef CONFIG_UPROBE_EVENTS
11588	PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
11589
11590	static struct attribute *uprobe_attrs[] = {
11591	&format_attr_retprobe.attr,
11592	&format_attr_ref_ctr_offset.attr,
11593	NULL,
11594	};
11595
11596	static struct attribute_group uprobe_format_group = {
11597	.name = "format",
11598	.attrs = uprobe_attrs,
11599	};
11600
11601	static const struct attribute_group *uprobe_attr_groups[] = {
11602	&uprobe_format_group,
11603	NULL,
11604	};
11605
11606	static int perf_uprobe_event_init(struct perf_event *event);
11607	static struct pmu perf_uprobe = {
11608	.task_ctx_nr = perf_sw_context,
11609	.event_init = perf_uprobe_event_init,
11610	.add = perf_trace_add,
11611	.del = perf_trace_del,
11612	.start = perf_swevent_start,
11613	.stop = perf_swevent_stop,
11614	.read = perf_swevent_read,
11615	.attr_groups = uprobe_attr_groups,
11616	};
11617
11618	static int perf_uprobe_event_init(struct perf_event *event)
11619	{
11620	int err;
11621	unsigned long ref_ctr_offset;
11622	bool is_retprobe;
11623
11624	if (event->attr.type != perf_uprobe.type)
11625	return -ENOENT;
11626
11627	if (!capable(CAP_SYS_ADMIN))
11628	return -EACCES;
11629
11630	/*
11631	* no branch sampling for probe events
11632	*/
11633	if (has_branch_stack(event))
11634	return -EOPNOTSUPP;
11635
11636	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
11637	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
11638	err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);
11639	if (err)
11640	return err;
11641
11642	event->destroy = perf_uprobe_destroy;
11643
11644	return 0;
11645	}
11646	#endif /* CONFIG_UPROBE_EVENTS */
11647
11648	static inline void perf_tp_register(void)
11649	{
11650	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
11651	#ifdef CONFIG_KPROBE_EVENTS
11652	perf_pmu_register(&perf_kprobe, "kprobe", -1);
11653	#endif
11654	#ifdef CONFIG_UPROBE_EVENTS
11655	perf_pmu_register(&perf_uprobe, "uprobe", -1);
11656	#endif
11657	}
11658
11659	static void perf_event_free_filter(struct perf_event *event)
11660	{
11661	ftrace_profile_free_filter(event);
11662	}
11663
11664	/*
11665	* returns true if the event is a tracepoint, or a kprobe/upprobe created
11666	* with perf_event_open()
11667	*/
11668	static inline bool perf_event_is_tracing(struct perf_event *event)
11669	{
11670	if (event->pmu == &perf_tracepoint)
11671	return true;
11672	#ifdef CONFIG_KPROBE_EVENTS
11673	if (event->pmu == &perf_kprobe)
11674	return true;
11675	#endif
11676	#ifdef CONFIG_UPROBE_EVENTS
11677	if (event->pmu == &perf_uprobe)
11678	return true;
11679	#endif
11680	return false;
11681	}
11682
11683	static int __perf_event_set_bpf_prog(struct perf_event *event,
11684	struct bpf_prog *prog,
11685	u64 bpf_cookie)
11686	{
11687	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;
11688
11689	if (event->state <= PERF_EVENT_STATE_REVOKED)
11690	return -ENODEV;
11691
11692	if (!perf_event_is_tracing(event))
11693	return perf_event_set_bpf_handler(event, prog, bpf_cookie);
11694
11695	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;
11696	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;
11697	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
11698	is_syscall_tp = is_syscall_trace_event(event->tp_event);
11699	if (!is_kprobe && !is_uprobe && !is_tracepoint && !is_syscall_tp)
11700	/* bpf programs can only be attached to u/kprobe or tracepoint */
11701	return -EINVAL;
11702
11703	if (((is_kprobe \|\| is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) \|\|
11704	(is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) \|\|
11705	(is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))
11706	return -EINVAL;
11707
11708	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->sleepable && !is_uprobe)
11709	/* only uprobe programs are allowed to be sleepable */
11710	return -EINVAL;
11711
11712	if (prog->type == BPF_PROG_TYPE_TRACEPOINT && prog->sleepable) {
11713	/*
11714	* Sleepable tracepoint programs can only attach to faultable
11715	* tracepoints. Currently only syscall tracepoints are faultable.
11716	*/
11717	if (!is_syscall_tp)
11718	return -EINVAL;
11719	}
11720
11721	/* Kprobe override only works for kprobes, not uprobes. */
11722	if (prog->kprobe_override && !is_kprobe)
11723	return -EINVAL;
11724
11725	/* Writing to context allowed only for uprobes. */
11726	if (prog->aux->kprobe_write_ctx && !is_uprobe)
11727	return -EINVAL;
11728
11729	if (is_tracepoint \|\| is_syscall_tp) {
11730	int off = trace_event_get_offsets(event->tp_event);
11731
11732	if (prog->aux->max_ctx_offset > off)
11733	return -EACCES;
11734	}
11735
11736	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
11737	}
11738
11739	int perf_event_set_bpf_prog(struct perf_event *event,
11740	struct bpf_prog *prog,
11741	u64 bpf_cookie)
11742	{
11743	struct perf_event_context *ctx;
11744	int ret;
11745
11746	ctx = perf_event_ctx_lock(event);
11747	ret = __perf_event_set_bpf_prog(event, prog, bpf_cookie);
11748	perf_event_ctx_unlock(event, ctx);
11749
11750	return ret;
11751	}
11752
11753	void perf_event_free_bpf_prog(struct perf_event *event)
11754	{
11755	if (!event->prog)
11756	return;
11757
11758	if (!perf_event_is_tracing(event)) {
11759	perf_event_free_bpf_handler(event);
11760	return;
11761	}
11762	perf_event_detach_bpf_prog(event);
11763	}
11764
11765	#else
11766
11767	static inline void perf_tp_register(void)
11768	{
11769	}
11770
11771	static void perf_event_free_filter(struct perf_event *event)
11772	{
11773	}
11774
11775	static int __perf_event_set_bpf_prog(struct perf_event *event,
11776	struct bpf_prog *prog,
11777	u64 bpf_cookie)
11778	{
11779	return -ENOENT;
11780	}
11781
11782	int perf_event_set_bpf_prog(struct perf_event *event,
11783	struct bpf_prog *prog,
11784	u64 bpf_cookie)
11785	{
11786	return -ENOENT;
11787	}
11788
11789	void perf_event_free_bpf_prog(struct perf_event *event)
11790	{
11791	}
11792	#endif /* CONFIG_EVENT_TRACING */
11793
11794	#ifdef CONFIG_HAVE_HW_BREAKPOINT
11795	void perf_bp_event(struct perf_event bp, void data)
11796	{
11797	struct perf_sample_data sample;
11798	struct pt_regs *regs = data;
11799
11800	/*
11801	* Exception context, will have interrupts disabled.
11802	*/
11803	lockdep_assert_irqs_disabled();
11804
11805	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
11806
11807	if (!bp->hw.state && !perf_exclude_event(bp, regs))
11808	perf_swevent_event(bp, 1, &sample, regs);
11809	}
11810	#endif
11811
11812	/*
11813	* Allocate a new address filter
11814	*/
11815	static struct perf_addr_filter *
11816	perf_addr_filter_new(struct perf_event event, struct list_head filters)
11817	{
11818	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);
11819	struct perf_addr_filter *filter;
11820
11821	filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node);
11822	if (!filter)
11823	return NULL;
11824
11825	INIT_LIST_HEAD(&filter->entry);
11826	list_add_tail(&filter->entry, filters);
11827
11828	return filter;
11829	}
11830
11831	static void free_filters_list(struct list_head *filters)
11832	{
11833	struct perf_addr_filter filter, iter;
11834
11835	list_for_each_entry_safe(filter, iter, filters, entry) {
11836	path_put(&filter->path);
11837	list_del(&filter->entry);
11838	kfree(filter);
11839	}
11840	}
11841
11842	/*
11843	* Free existing address filters and optionally install new ones
11844	*/
11845	static void perf_addr_filters_splice(struct perf_event *event,
11846	struct list_head *head)
11847	{
11848	unsigned long flags;
11849	LIST_HEAD(list);
11850
11851	if (!has_addr_filter(event))
11852	return;
11853
11854	/* don't bother with children, they don't have their own filters */
11855	if (event->parent)
11856	return;
11857
11858	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);
11859
11860	list_splice_init(&event->addr_filters.list, &list);
11861	if (head)
11862	list_splice(head, &event->addr_filters.list);
11863
11864	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);
11865
11866	free_filters_list(&list);
11867	}
11868
11869	static void perf_free_addr_filters(struct perf_event *event)
11870	{
11871	/*
11872	* Used during free paths, there is no concurrency.
11873	*/
11874	if (list_empty(&event->addr_filters.list))
11875	return;
11876
11877	perf_addr_filters_splice(event, NULL);
11878	}
11879
11880	/*
11881	* Scan through mm's vmas and see if one of them matches the
11882	* @filter; if so, adjust filter's address range.
11883	* Called with mm::mmap_lock down for reading.
11884	*/
11885	static void perf_addr_filter_apply(struct perf_addr_filter *filter,
11886	struct mm_struct *mm,
11887	struct perf_addr_filter_range *fr)
11888	{
11889	struct vm_area_struct *vma;
11890	VMA_ITERATOR(vmi, mm, 0);
11891
11892	for_each_vma(vmi, vma) {
11893	if (!vma->vm_file)
11894	continue;
11895
11896	if (perf_addr_filter_vma_adjust(filter, vma, fr))
11897	return;
11898	}
11899	}
11900
11901	/*
11902	* Update event's address range filters based on the
11903	* task's existing mappings, if any.
11904	*/
11905	static void perf_event_addr_filters_apply(struct perf_event *event)
11906	{
11907	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
11908	struct task_struct *task = READ_ONCE(event->ctx->task);
11909	struct perf_addr_filter *filter;
11910	struct mm_struct *mm = NULL;
11911	unsigned int count = 0;
11912	unsigned long flags;
11913
11914	/*
11915	* We may observe TASK_TOMBSTONE, which means that the event tear-down
11916	* will stop on the parent's child_mutex that our caller is also holding
11917	*/
11918	if (task == TASK_TOMBSTONE)
11919	return;
11920
11921	if (ifh->nr_file_filters) {
11922	mm = get_task_mm(task);
11923	if (!mm)
11924	goto restart;
11925
11926	mmap_read_lock(mm);
11927	}
11928
11929	raw_spin_lock_irqsave(&ifh->lock, flags);
11930	list_for_each_entry(filter, &ifh->list, entry) {
11931	if (filter->path.dentry) {
11932	/*
11933	* Adjust base offset if the filter is associated to a
11934	* binary that needs to be mapped:
11935	*/
11936	event->addr_filter_ranges[count].start = 0;
11937	event->addr_filter_ranges[count].size = 0;
11938
11939	perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);
11940	} else {
11941	event->addr_filter_ranges[count].start = filter->offset;
11942	event->addr_filter_ranges[count].size = filter->size;
11943	}
11944
11945	count++;
11946	}
11947
11948	event->addr_filters_gen++;
11949	raw_spin_unlock_irqrestore(&ifh->lock, flags);
11950
11951	if (ifh->nr_file_filters) {
11952	mmap_read_unlock(mm);
11953
11954	mmput(mm);
11955	}
11956
11957	restart:
11958	perf_event_stop(event, 1);
11959	}
11960
11961	/*
11962	* Address range filtering: limiting the data to certain
11963	* instruction address ranges. Filters are ioctl()ed to us from
11964	* userspace as ascii strings.
11965	*
11966	* Filter string format:
11967	*
11968	* ACTION RANGE_SPEC
11969	* where ACTION is one of the
11970	* * "filter": limit the trace to this region
11971	* * "start": start tracing from this address
11972	* * "stop": stop tracing at this address/region;
11973	* RANGE_SPEC is
11974	* * for kernel addresses: <start address>[/<size>]
11975	* * for object files: <start address>[/<size>]@</path/to/object/file>
11976	*
11977	* if <size> is not specified or is zero, the range is treated as a single
11978	* address; not valid for ACTION=="filter".
11979	*/
11980	enum {
11981	IF_ACT_NONE = -1,
11982	IF_ACT_FILTER,
11983	IF_ACT_START,
11984	IF_ACT_STOP,
11985	IF_SRC_FILE,
11986	IF_SRC_KERNEL,
11987	IF_SRC_FILEADDR,
11988	IF_SRC_KERNELADDR,
11989	};
11990
11991	enum {
11992	IF_STATE_ACTION = 0,
11993	IF_STATE_SOURCE,
11994	IF_STATE_END,
11995	};
11996
11997	static const match_table_t if_tokens = {
11998	{ IF_ACT_FILTER, "filter" },
11999	{ IF_ACT_START, "start" },
12000	{ IF_ACT_STOP, "stop" },
12001	{ IF_SRC_FILE, "%u/%u@%s" },
12002	{ IF_SRC_KERNEL, "%u/%u" },
12003	{ IF_SRC_FILEADDR, "%u@%s" },
12004	{ IF_SRC_KERNELADDR, "%u" },
12005	{ IF_ACT_NONE, NULL },
12006	};
12007
12008	/*
12009	* Address filter string parser
12010	*/
12011	static int
12012	perf_event_parse_addr_filter(struct perf_event event, char fstr,
12013	struct list_head *filters)
12014	{
12015	struct perf_addr_filter *filter = NULL;
12016	char start, orig, *filename = NULL;
12017	substring_t args[MAX_OPT_ARGS];
12018	int state = IF_STATE_ACTION, token;
12019	unsigned int kernel = 0;
12020	int ret = -EINVAL;
12021
12022	orig = fstr = kstrdup(fstr, GFP_KERNEL);
12023	if (!fstr)
12024	return -ENOMEM;
12025
12026	while ((start = strsep(&fstr, " ,\n")) != NULL) {
12027	static const enum perf_addr_filter_action_t actions[] = {
12028	[IF_ACT_FILTER] = PERF_ADDR_FILTER_ACTION_FILTER,
12029	[IF_ACT_START] = PERF_ADDR_FILTER_ACTION_START,
12030	[IF_ACT_STOP] = PERF_ADDR_FILTER_ACTION_STOP,
12031	};
12032	ret = -EINVAL;
12033
12034	if (!*start)
12035	continue;
12036
12037	/* filter definition begins */
12038	if (state == IF_STATE_ACTION) {
12039	filter = perf_addr_filter_new(event, filters);
12040	if (!filter)
12041	goto fail;
12042	}
12043
12044	token = match_token(start, if_tokens, args);
12045	switch (token) {
12046	case IF_ACT_FILTER:
12047	case IF_ACT_START:
12048	case IF_ACT_STOP:
12049	if (state != IF_STATE_ACTION)
12050	goto fail;
12051
12052	filter->action = actions[token];
12053	state = IF_STATE_SOURCE;
12054	break;
12055
12056	case IF_SRC_KERNELADDR:
12057	case IF_SRC_KERNEL:
12058	kernel = 1;
12059	fallthrough;
12060
12061	case IF_SRC_FILEADDR:
12062	case IF_SRC_FILE:
12063	if (state != IF_STATE_SOURCE)
12064	goto fail;
12065
12066	*args[0].to = 0;
12067	ret = kstrtoul(args[0].from, 0, &filter->offset);
12068	if (ret)
12069	goto fail;
12070
12071	if (token == IF_SRC_KERNEL \|\| token == IF_SRC_FILE) {
12072	*args[1].to = 0;
12073	ret = kstrtoul(args[1].from, 0, &filter->size);
12074	if (ret)
12075	goto fail;
12076	}
12077
12078	if (token == IF_SRC_FILE \|\| token == IF_SRC_FILEADDR) {
12079	int fpos = token == IF_SRC_FILE ? 2 : 1;
12080
12081	kfree(filename);
12082	filename = match_strdup(&args[fpos]);
12083	if (!filename) {
12084	ret = -ENOMEM;
12085	goto fail;
12086	}
12087	}
12088
12089	state = IF_STATE_END;
12090	break;
12091
12092	default:
12093	goto fail;
12094	}
12095
12096	/*
12097	* Filter definition is fully parsed, validate and install it.
12098	* Make sure that it doesn't contradict itself or the event's
12099	* attribute.
12100	*/
12101	if (state == IF_STATE_END) {
12102	ret = -EINVAL;
12103
12104	/*
12105	* ACTION "filter" must have a non-zero length region
12106	* specified.
12107	*/
12108	if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&
12109	!filter->size)
12110	goto fail;
12111
12112	if (!kernel) {
12113	if (!filename)
12114	goto fail;
12115
12116	/*
12117	* For now, we only support file-based filters
12118	* in per-task events; doing so for CPU-wide
12119	* events requires additional context switching
12120	* trickery, since same object code will be
12121	* mapped at different virtual addresses in
12122	* different processes.
12123	*/
12124	ret = -EOPNOTSUPP;
12125	if (!event->ctx->task)
12126	goto fail;
12127
12128	/* look up the path and grab its inode */
12129	ret = kern_path(filename, LOOKUP_FOLLOW,
12130	&filter->path);
12131	if (ret)
12132	goto fail;
12133
12134	ret = -EINVAL;
12135	if (!filter->path.dentry \|\|
12136	!S_ISREG(d_inode(filter->path.dentry)
12137	->i_mode))
12138	goto fail;
12139
12140	event->addr_filters.nr_file_filters++;
12141	}
12142
12143	/* ready to consume more filters */
12144	kfree(filename);
12145	filename = NULL;
12146	state = IF_STATE_ACTION;
12147	filter = NULL;
12148	kernel = 0;
12149	}
12150	}
12151
12152	if (state != IF_STATE_ACTION)
12153	goto fail;
12154
12155	kfree(filename);
12156	kfree(orig);
12157
12158	return 0;
12159
12160	fail:
12161	kfree(filename);
12162	free_filters_list(filters);
12163	kfree(orig);
12164
12165	return ret;
12166	}
12167
12168	static int
12169	perf_event_set_addr_filter(struct perf_event event, char filter_str)
12170	{
12171	LIST_HEAD(filters);
12172	int ret;
12173
12174	/*
12175	* Since this is called in perf_ioctl() path, we're already holding
12176	* ctx::mutex.
12177	*/
12178	lockdep_assert_held(&event->ctx->mutex);
12179
12180	if (WARN_ON_ONCE(event->parent))
12181	return -EINVAL;
12182
12183	ret = perf_event_parse_addr_filter(event, filter_str, &filters);
12184	if (ret)
12185	goto fail_clear_files;
12186
12187	ret = event->pmu->addr_filters_validate(&filters);
12188	if (ret)
12189	goto fail_free_filters;
12190
12191	/* remove existing filters, if any */
12192	perf_addr_filters_splice(event, &filters);
12193
12194	/* install new filters */
12195	perf_event_for_each_child(event, perf_event_addr_filters_apply);
12196
12197	return ret;
12198
12199	fail_free_filters:
12200	free_filters_list(&filters);
12201
12202	fail_clear_files:
12203	event->addr_filters.nr_file_filters = 0;
12204
12205	return ret;
12206	}
12207
12208	static int perf_event_set_filter(struct perf_event event, void __user arg)
12209	{
12210	int ret = -EINVAL;
12211	char *filter_str;
12212
12213	filter_str = strndup_user(arg, PAGE_SIZE);
12214	if (IS_ERR(filter_str))
12215	return PTR_ERR(filter_str);
12216
12217	#ifdef CONFIG_EVENT_TRACING
12218	if (perf_event_is_tracing(event)) {
12219	struct perf_event_context *ctx = event->ctx;
12220
12221	/*
12222	* Beware, here be dragons!!
12223	*
12224	* the tracepoint muck will deadlock against ctx->mutex, but
12225	* the tracepoint stuff does not actually need it. So
12226	* temporarily drop ctx->mutex. As per perf_event_ctx_lock() we
12227	* already have a reference on ctx.
12228	*
12229	* This can result in event getting moved to a different ctx,
12230	* but that does not affect the tracepoint state.
12231	*/
12232	mutex_unlock(&ctx->mutex);
12233	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
12234	mutex_lock(&ctx->mutex);
12235	} else
12236	#endif
12237	if (has_addr_filter(event))
12238	ret = perf_event_set_addr_filter(event, filter_str);
12239
12240	kfree(filter_str);
12241	return ret;
12242	}
12243
12244	/*
12245	* hrtimer based swevent callback
12246	*/
12247
12248	static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
12249	{
12250	enum hrtimer_restart ret = HRTIMER_RESTART;
12251	struct perf_sample_data data;
12252	struct pt_regs *regs;
12253	struct perf_event *event;
12254	u64 period;
12255
12256	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
12257
12258	if (event->state != PERF_EVENT_STATE_ACTIVE \|\|
12259	event->hw.state & PERF_HES_STOPPED)
12260	return HRTIMER_NORESTART;
12261
12262	event->pmu->read(event);
12263
12264	perf_sample_data_init(&data, 0, event->hw.last_period);
12265	regs = get_irq_regs();
12266
12267	if (regs && !perf_exclude_event(event, regs)) {
12268	if (!(event->attr.exclude_idle && is_idle_task(current)))
12269	if (perf_event_overflow(event, &data, regs))
12270	ret = HRTIMER_NORESTART;
12271	}
12272
12273	period = max_t(u64, 10000, event->hw.sample_period);
12274	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
12275
12276	return ret;
12277	}
12278
12279	static void perf_swevent_start_hrtimer(struct perf_event *event)
12280	{
12281	struct hw_perf_event *hwc = &event->hw;
12282	s64 period;
12283
12284	if (!is_sampling_event(event))
12285	return;
12286
12287	period = local64_read(&hwc->period_left);
12288	if (period) {
12289	if (period < 0)
12290	period = 10000;
12291
12292	local64_set(&hwc->period_left, 0);
12293	} else {
12294	period = max_t(u64, 10000, hwc->sample_period);
12295	}
12296	hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),
12297	HRTIMER_MODE_REL_PINNED_HARD);
12298	}
12299
12300	static void perf_swevent_cancel_hrtimer(struct perf_event *event)
12301	{
12302	struct hw_perf_event *hwc = &event->hw;
12303
12304	/*
12305	* Careful: this function can be triggered in the hrtimer handler,
12306	* for cpu-clock events, so hrtimer_cancel() would cause a
12307	* deadlock.
12308	*
12309	* So use hrtimer_try_to_cancel() to try to stop the hrtimer,
12310	* and the cpu-clock handler also sets the PERF_HES_STOPPED flag,
12311	* which guarantees that perf_swevent_hrtimer() will stop the
12312	* hrtimer once it sees the PERF_HES_STOPPED flag.
12313	*/
12314	if (is_sampling_event(event) && (hwc->interrupts != MAX_INTERRUPTS)) {
12315	ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
12316	local64_set(&hwc->period_left, ktime_to_ns(remaining));
12317
12318	hrtimer_try_to_cancel(&hwc->hrtimer);
12319	}
12320	}
12321
12322	static void perf_swevent_destroy_hrtimer(struct perf_event *event)
12323	{
12324	hrtimer_cancel(&event->hw.hrtimer);
12325	}
12326
12327	static void perf_swevent_init_hrtimer(struct perf_event *event)
12328	{
12329	struct hw_perf_event *hwc = &event->hw;
12330
12331	if (!is_sampling_event(event))
12332	return;
12333
12334	hrtimer_setup(&hwc->hrtimer, perf_swevent_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
12335	event->destroy = perf_swevent_destroy_hrtimer;
12336
12337	/*
12338	* Since hrtimers have a fixed rate, we can do a static freq->period
12339	* mapping and avoid the whole period adjust feedback stuff.
12340	*/
12341	if (event->attr.freq) {
12342	long freq = event->attr.sample_freq;
12343
12344	event->attr.sample_period = NSEC_PER_SEC / freq;
12345	hwc->sample_period = event->attr.sample_period;
12346	local64_set(&hwc->period_left, hwc->sample_period);
12347	hwc->last_period = hwc->sample_period;
12348	event->attr.freq = 0;
12349	}
12350	}
12351
12352	/*
12353	* Software event: cpu wall time clock
12354	*/
12355
12356	static void cpu_clock_event_update(struct perf_event *event)
12357	{
12358	s64 prev;
12359	u64 now;
12360
12361	now = local_clock();
12362	prev = local64_xchg(&event->hw.prev_count, now);
12363	local64_add(now - prev, &event->count);
12364	}
12365
12366	static void cpu_clock_event_start(struct perf_event *event, int flags)
12367	{
12368	event->hw.state = 0;
12369	local64_set(&event->hw.prev_count, local_clock());
12370	perf_swevent_start_hrtimer(event);
12371	}
12372
12373	static void cpu_clock_event_stop(struct perf_event *event, int flags)
12374	{
12375	event->hw.state = PERF_HES_STOPPED;
12376	perf_swevent_cancel_hrtimer(event);
12377	if (flags & PERF_EF_UPDATE)
12378	cpu_clock_event_update(event);
12379	}
12380
12381	static int cpu_clock_event_add(struct perf_event *event, int flags)
12382	{
12383	if (flags & PERF_EF_START)
12384	cpu_clock_event_start(event, flags);
12385	perf_event_update_userpage(event);
12386
12387	return 0;
12388	}
12389
12390	static void cpu_clock_event_del(struct perf_event *event, int flags)
12391	{
12392	cpu_clock_event_stop(event, PERF_EF_UPDATE);
12393	}
12394
12395	static void cpu_clock_event_read(struct perf_event *event)
12396	{
12397	cpu_clock_event_update(event);
12398	}
12399
12400	static int cpu_clock_event_init(struct perf_event *event)
12401	{
12402	if (event->attr.type != perf_cpu_clock.type)
12403	return -ENOENT;
12404
12405	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
12406	return -ENOENT;
12407
12408	/*
12409	* no branch sampling for software events
12410	*/
12411	if (has_branch_stack(event))
12412	return -EOPNOTSUPP;
12413
12414	perf_swevent_init_hrtimer(event);
12415
12416	return 0;
12417	}
12418
12419	static struct pmu perf_cpu_clock = {
12420	.task_ctx_nr = perf_sw_context,
12421
12422	.capabilities = PERF_PMU_CAP_NO_NMI,
12423	.dev = PMU_NULL_DEV,
12424
12425	.event_init = cpu_clock_event_init,
12426	.add = cpu_clock_event_add,
12427	.del = cpu_clock_event_del,
12428	.start = cpu_clock_event_start,
12429	.stop = cpu_clock_event_stop,
12430	.read = cpu_clock_event_read,
12431	};
12432
12433	/*
12434	* Software event: task time clock
12435	*/
12436
12437	static void task_clock_event_update(struct perf_event *event, u64 now)
12438	{
12439	u64 prev;
12440	s64 delta;
12441
12442	prev = local64_xchg(&event->hw.prev_count, now);
12443	delta = now - prev;
12444	local64_add(delta, &event->count);
12445	}
12446
12447	static void task_clock_event_start(struct perf_event *event, int flags)
12448	{
12449	event->hw.state = 0;
12450	local64_set(&event->hw.prev_count, event->ctx->time.time);
12451	perf_swevent_start_hrtimer(event);
12452	}
12453
12454	static void task_clock_event_stop(struct perf_event *event, int flags)
12455	{
12456	event->hw.state = PERF_HES_STOPPED;
12457	perf_swevent_cancel_hrtimer(event);
12458	if (flags & PERF_EF_UPDATE)
12459	task_clock_event_update(event, event->ctx->time.time);
12460	}
12461
12462	static int task_clock_event_add(struct perf_event *event, int flags)
12463	{
12464	if (flags & PERF_EF_START)
12465	task_clock_event_start(event, flags);
12466	perf_event_update_userpage(event);
12467
12468	return 0;
12469	}
12470
12471	static void task_clock_event_del(struct perf_event *event, int flags)
12472	{
12473	task_clock_event_stop(event, PERF_EF_UPDATE);
12474	}
12475
12476	static void task_clock_event_read(struct perf_event *event)
12477	{
12478	u64 now = perf_clock();
12479	u64 delta = now - event->ctx->time.stamp;
12480	u64 time = event->ctx->time.time + delta;
12481
12482	task_clock_event_update(event, time);
12483	}
12484
12485	static int task_clock_event_init(struct perf_event *event)
12486	{
12487	if (event->attr.type != perf_task_clock.type)
12488	return -ENOENT;
12489
12490	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
12491	return -ENOENT;
12492
12493	/*
12494	* no branch sampling for software events
12495	*/
12496	if (has_branch_stack(event))
12497	return -EOPNOTSUPP;
12498
12499	perf_swevent_init_hrtimer(event);
12500
12501	return 0;
12502	}
12503
12504	static struct pmu perf_task_clock = {
12505	.task_ctx_nr = perf_sw_context,
12506
12507	.capabilities = PERF_PMU_CAP_NO_NMI,
12508	.dev = PMU_NULL_DEV,
12509
12510	.event_init = task_clock_event_init,
12511	.add = task_clock_event_add,
12512	.del = task_clock_event_del,
12513	.start = task_clock_event_start,
12514	.stop = task_clock_event_stop,
12515	.read = task_clock_event_read,
12516	};
12517
12518	static void perf_pmu_nop_void(struct pmu *pmu)
12519	{
12520	}
12521
12522	static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags)
12523	{
12524	}
12525
12526	static int perf_pmu_nop_int(struct pmu *pmu)
12527	{
12528	return 0;
12529	}
12530
12531	static int perf_event_nop_int(struct perf_event *event, u64 value)
12532	{
12533	return 0;
12534	}
12535
12536	static DEFINE_PER_CPU(unsigned int, nop_txn_flags);
12537
12538	static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags)
12539	{
12540	__this_cpu_write(nop_txn_flags, flags);
12541
12542	if (flags & ~PERF_PMU_TXN_ADD)
12543	return;
12544
12545	perf_pmu_disable(pmu);
12546	}
12547
12548	static int perf_pmu_commit_txn(struct pmu *pmu)
12549	{
12550	unsigned int flags = __this_cpu_read(nop_txn_flags);
12551
12552	__this_cpu_write(nop_txn_flags, 0);
12553
12554	if (flags & ~PERF_PMU_TXN_ADD)
12555	return 0;
12556
12557	perf_pmu_enable(pmu);
12558	return 0;
12559	}
12560
12561	static void perf_pmu_cancel_txn(struct pmu *pmu)
12562	{
12563	unsigned int flags = __this_cpu_read(nop_txn_flags);
12564
12565	__this_cpu_write(nop_txn_flags, 0);
12566
12567	if (flags & ~PERF_PMU_TXN_ADD)
12568	return;
12569
12570	perf_pmu_enable(pmu);
12571	}
12572
12573	static int perf_event_idx_default(struct perf_event *event)
12574	{
12575	return 0;
12576	}
12577
12578	/*
12579	* Let userspace know that this PMU supports address range filtering:
12580	*/
12581	static ssize_t nr_addr_filters_show(struct device *dev,
12582	struct device_attribute *attr,
12583	char *page)
12584	{
12585	struct pmu *pmu = dev_get_drvdata(dev);
12586
12587	return sysfs_emit(page, "%d\n", pmu->nr_addr_filters);
12588	}
12589	DEVICE_ATTR_RO(nr_addr_filters);
12590
12591	static struct idr pmu_idr;
12592
12593	static ssize_t
12594	type_show(struct device dev, struct device_attribute attr, char *page)
12595	{
12596	struct pmu *pmu = dev_get_drvdata(dev);
12597
12598	return sysfs_emit(page, "%d\n", pmu->type);
12599	}
12600	static DEVICE_ATTR_RO(type);
12601
12602	static ssize_t
12603	perf_event_mux_interval_ms_show(struct device *dev,
12604	struct device_attribute *attr,
12605	char *page)
12606	{
12607	struct pmu *pmu = dev_get_drvdata(dev);
12608
12609	return sysfs_emit(page, "%d\n", pmu->hrtimer_interval_ms);
12610	}
12611
12612	static DEFINE_MUTEX(mux_interval_mutex);
12613
12614	static ssize_t
12615	perf_event_mux_interval_ms_store(struct device *dev,
12616	struct device_attribute *attr,
12617	const char *buf, size_t count)
12618	{
12619	struct pmu *pmu = dev_get_drvdata(dev);
12620	int timer, cpu, ret;
12621
12622	ret = kstrtoint(buf, 0, &timer);
12623	if (ret)
12624	return ret;
12625
12626	if (timer < 1)
12627	return -EINVAL;
12628
12629	/* same value, noting to do */
12630	if (timer == pmu->hrtimer_interval_ms)
12631	return count;
12632
12633	mutex_lock(&mux_interval_mutex);
12634	pmu->hrtimer_interval_ms = timer;
12635
12636	/* update all cpuctx for this PMU */
12637	cpus_read_lock();
12638	for_each_online_cpu(cpu) {
12639	struct perf_cpu_pmu_context *cpc;
12640	cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu);
12641	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
12642
12643	cpu_function_call(cpu, perf_mux_hrtimer_restart_ipi, cpc);
12644	}
12645	cpus_read_unlock();
12646	mutex_unlock(&mux_interval_mutex);
12647
12648	return count;
12649	}
12650	static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
12651
12652	static inline const struct cpumask *perf_scope_cpu_topology_cpumask(unsigned int scope, int cpu)
12653	{
12654	switch (scope) {
12655	case PERF_PMU_SCOPE_CORE:
12656	return topology_sibling_cpumask(cpu);
12657	case PERF_PMU_SCOPE_DIE:
12658	return topology_die_cpumask(cpu);
12659	case PERF_PMU_SCOPE_CLUSTER:
12660	return topology_cluster_cpumask(cpu);
12661	case PERF_PMU_SCOPE_PKG:
12662	return topology_core_cpumask(cpu);
12663	case PERF_PMU_SCOPE_SYS_WIDE:
12664	return cpu_online_mask;
12665	}
12666
12667	return NULL;
12668	}
12669
12670	static inline struct cpumask *perf_scope_cpumask(unsigned int scope)
12671	{
12672	switch (scope) {
12673	case PERF_PMU_SCOPE_CORE:
12674	return perf_online_core_mask;
12675	case PERF_PMU_SCOPE_DIE:
12676	return perf_online_die_mask;
12677	case PERF_PMU_SCOPE_CLUSTER:
12678	return perf_online_cluster_mask;
12679	case PERF_PMU_SCOPE_PKG:
12680	return perf_online_pkg_mask;
12681	case PERF_PMU_SCOPE_SYS_WIDE:
12682	return perf_online_sys_mask;
12683	}
12684
12685	return NULL;
12686	}
12687
12688	static ssize_t cpumask_show(struct device dev, struct device_attribute attr,
12689	char *buf)
12690	{
12691	struct pmu *pmu = dev_get_drvdata(dev);
12692	struct cpumask *mask = perf_scope_cpumask(pmu->scope);
12693
12694	if (mask)
12695	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(mask));
12696	return 0;
12697	}
12698
12699	static DEVICE_ATTR_RO(cpumask);
12700
12701	static struct attribute *pmu_dev_attrs[] = {
12702	&dev_attr_type.attr,
12703	&dev_attr_perf_event_mux_interval_ms.attr,
12704	&dev_attr_nr_addr_filters.attr,
12705	&dev_attr_cpumask.attr,
12706	NULL,
12707	};
12708
12709	static umode_t pmu_dev_is_visible(struct kobject kobj, struct attribute a, int n)
12710	{
12711	struct device *dev = kobj_to_dev(kobj);
12712	struct pmu *pmu = dev_get_drvdata(dev);
12713
12714	if (n == 2 && !pmu->nr_addr_filters)
12715	return 0;
12716
12717	/* cpumask */
12718	if (n == 3 && pmu->scope == PERF_PMU_SCOPE_NONE)
12719	return 0;
12720
12721	return a->mode;
12722	}
12723
12724	static struct attribute_group pmu_dev_attr_group = {
12725	.is_visible = pmu_dev_is_visible,
12726	.attrs = pmu_dev_attrs,
12727	};
12728
12729	static const struct attribute_group *pmu_dev_groups[] = {
12730	&pmu_dev_attr_group,
12731	NULL,
12732	};
12733
12734	static int pmu_bus_running;
12735	static const struct bus_type pmu_bus = {
12736	.name = "event_source",
12737	.dev_groups = pmu_dev_groups,
12738	};
12739
12740	static void pmu_dev_release(struct device *dev)
12741	{
12742	kfree(dev);
12743	}
12744
12745	static int pmu_dev_alloc(struct pmu *pmu)
12746	{
12747	int ret = -ENOMEM;
12748
12749	pmu->dev = kzalloc_obj(struct device);
12750	if (!pmu->dev)
12751	goto out;
12752
12753	pmu->dev->groups = pmu->attr_groups;
12754	device_initialize(pmu->dev);
12755
12756	dev_set_drvdata(pmu->dev, pmu);
12757	pmu->dev->bus = &pmu_bus;
12758	pmu->dev->parent = pmu->parent;
12759	pmu->dev->release = pmu_dev_release;
12760
12761	ret = dev_set_name(pmu->dev, "%s", pmu->name);
12762	if (ret)
12763	goto free_dev;
12764
12765	ret = device_add(pmu->dev);
12766	if (ret)
12767	goto free_dev;
12768
12769	if (pmu->attr_update) {
12770	ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);
12771	if (ret)
12772	goto del_dev;
12773	}
12774
12775	out:
12776	return ret;
12777
12778	del_dev:
12779	device_del(pmu->dev);
12780
12781	free_dev:
12782	put_device(pmu->dev);
12783	pmu->dev = NULL;
12784	goto out;
12785	}
12786
12787	static struct lock_class_key cpuctx_mutex;
12788	static struct lock_class_key cpuctx_lock;
12789
12790	static bool idr_cmpxchg(struct idr idr, unsigned long id, void old, void *new)
12791	{
12792	void tmp, val = idr_find(idr, id);
12793
12794	if (val != old)
12795	return false;
12796
12797	tmp = idr_replace(idr, new, id);
12798	if (IS_ERR(tmp))
12799	return false;
12800
12801	WARN_ON_ONCE(tmp != val);
12802	return true;
12803	}
12804
12805	static void perf_pmu_free(struct pmu *pmu)
12806	{
12807	if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) {
12808	if (pmu->nr_addr_filters)
12809	device_remove_file(pmu->dev, &dev_attr_nr_addr_filters);
12810	device_del(pmu->dev);
12811	put_device(pmu->dev);
12812	}
12813
12814	if (pmu->cpu_pmu_context) {
12815	int cpu;
12816
12817	for_each_possible_cpu(cpu) {
12818	struct perf_cpu_pmu_context *cpc;
12819
12820	cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu);
12821	if (!cpc)
12822	continue;
12823	if (cpc->epc.embedded) {
12824	/* refcount managed */
12825	put_pmu_ctx(&cpc->epc);
12826	continue;
12827	}
12828	kfree(cpc);
12829	}
12830	free_percpu(pmu->cpu_pmu_context);
12831	}
12832	}
12833
12834	DEFINE_FREE(pmu_unregister, struct pmu *, if (_T) perf_pmu_free(_T))
12835
12836	int perf_pmu_register(struct pmu _pmu, const char name, int type)
12837	{
12838	int cpu, max = PERF_TYPE_MAX;
12839
12840	struct pmu *pmu __free(pmu_unregister) = _pmu;
12841	guard(mutex)(&pmus_lock);
12842
12843	if (WARN_ONCE(!name, "Can not register anonymous pmu.\n"))
12844	return -EINVAL;
12845
12846	if (WARN_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE,
12847	"Can not register a pmu with an invalid scope.\n"))
12848	return -EINVAL;
12849
12850	pmu->name = name;
12851
12852	if (type >= 0)
12853	max = type;
12854
12855	CLASS(idr_alloc, pmu_type)(&pmu_idr, NULL, max, 0, GFP_KERNEL);
12856	if (pmu_type.id < 0)
12857	return pmu_type.id;
12858
12859	WARN_ON(type >= 0 && pmu_type.id != type);
12860
12861	pmu->type = pmu_type.id;
12862	atomic_set(&pmu->exclusive_cnt, 0);
12863
12864	if (pmu_bus_running && !pmu->dev) {
12865	int ret = pmu_dev_alloc(pmu);
12866	if (ret)
12867	return ret;
12868	}
12869
12870	pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context *);
12871	if (!pmu->cpu_pmu_context)
12872	return -ENOMEM;
12873
12874	for_each_possible_cpu(cpu) {
12875	struct perf_cpu_pmu_context *cpc =
12876	kmalloc_node(sizeof(struct perf_cpu_pmu_context),
12877	GFP_KERNEL \| __GFP_ZERO,
12878	cpu_to_node(cpu));
12879
12880	if (!cpc)
12881	return -ENOMEM;
12882
12883	*per_cpu_ptr(pmu->cpu_pmu_context, cpu) = cpc;
12884	__perf_init_event_pmu_context(&cpc->epc, pmu);
12885	__perf_mux_hrtimer_init(cpc, cpu);
12886	}
12887
12888	if (!pmu->start_txn) {
12889	if (pmu->pmu_enable) {
12890	/*
12891	* If we have pmu_enable/pmu_disable calls, install
12892	* transaction stubs that use that to try and batch
12893	* hardware accesses.
12894	*/
12895	pmu->start_txn = perf_pmu_start_txn;
12896	pmu->commit_txn = perf_pmu_commit_txn;
12897	pmu->cancel_txn = perf_pmu_cancel_txn;
12898	} else {
12899	pmu->start_txn = perf_pmu_nop_txn;
12900	pmu->commit_txn = perf_pmu_nop_int;
12901	pmu->cancel_txn = perf_pmu_nop_void;
12902	}
12903	}
12904
12905	if (!pmu->pmu_enable) {
12906	pmu->pmu_enable = perf_pmu_nop_void;
12907	pmu->pmu_disable = perf_pmu_nop_void;
12908	}
12909
12910	if (!pmu->check_period)
12911	pmu->check_period = perf_event_nop_int;
12912
12913	if (!pmu->event_idx)
12914	pmu->event_idx = perf_event_idx_default;
12915
12916	INIT_LIST_HEAD(&pmu->events);
12917	spin_lock_init(&pmu->events_lock);
12918
12919	/*
12920	* Now that the PMU is complete, make it visible to perf_try_init_event().
12921	*/
12922	if (!idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu))
12923	return -EINVAL;
12924	list_add_rcu(&pmu->entry, &pmus);
12925
12926	take_idr_id(pmu_type);
12927	_pmu = no_free_ptr(pmu); // let it rip
12928	return 0;
12929	}
12930	EXPORT_SYMBOL_GPL(perf_pmu_register);
12931
12932	static void __pmu_detach_event(struct pmu pmu, struct perf_event event,
12933	struct perf_event_context *ctx)
12934	{
12935	/*
12936	* De-schedule the event and mark it REVOKED.
12937	*/
12938	perf_event_exit_event(event, ctx, ctx->task, DETACH_REVOKE);
12939
12940	/*
12941	* All _free_event() bits that rely on event->pmu:
12942	*
12943	* Notably, perf_mmap() relies on the ordering here.
12944	*/
12945	scoped_guard (mutex, &event->mmap_mutex) {
12946	WARN_ON_ONCE(pmu->event_unmapped);
12947	/*
12948	* Mostly an empty lock sequence, such that perf_mmap(), which
12949	* relies on mmap_mutex, is sure to observe the state change.
12950	*/
12951	}
12952
12953	perf_event_free_bpf_prog(event);
12954	perf_free_addr_filters(event);
12955
12956	if (event->destroy) {
12957	event->destroy(event);
12958	event->destroy = NULL;
12959	}
12960
12961	if (event->pmu_ctx) {
12962	put_pmu_ctx(event->pmu_ctx);
12963	event->pmu_ctx = NULL;
12964	}
12965
12966	exclusive_event_destroy(event);
12967	module_put(pmu->module);
12968
12969	mediated_pmu_unaccount_event(event);
12970	event->pmu = NULL; /* force fault instead of UAF */
12971	}
12972
12973	static void pmu_detach_event(struct pmu pmu, struct perf_event event)
12974	{
12975	struct perf_event_context *ctx;
12976
12977	ctx = perf_event_ctx_lock(event);
12978	__pmu_detach_event(pmu, event, ctx);
12979	perf_event_ctx_unlock(event, ctx);
12980
12981	scoped_guard (spinlock, &pmu->events_lock)
12982	list_del(&event->pmu_list);
12983	}
12984
12985	static struct perf_event pmu_get_event(struct pmu pmu)
12986	{
12987	struct perf_event *event;
12988
12989	guard(spinlock)(&pmu->events_lock);
12990	list_for_each_entry(event, &pmu->events, pmu_list) {
12991	if (atomic_long_inc_not_zero(&event->refcount))
12992	return event;
12993	}
12994
12995	return NULL;
12996	}
12997
12998	static bool pmu_empty(struct pmu *pmu)
12999	{
13000	guard(spinlock)(&pmu->events_lock);
13001	return list_empty(&pmu->events);
13002	}
13003
13004	static void pmu_detach_events(struct pmu *pmu)
13005	{
13006	struct perf_event *event;
13007
13008	for (;;) {
13009	event = pmu_get_event(pmu);
13010	if (!event)
13011	break;
13012
13013	pmu_detach_event(pmu, event);
13014	put_event(event);
13015	}
13016
13017	/*
13018	* wait for pending _free_event()s
13019	*/
13020	wait_var_event(pmu, pmu_empty(pmu));
13021	}
13022
13023	int perf_pmu_unregister(struct pmu *pmu)
13024	{
13025	scoped_guard (mutex, &pmus_lock) {
13026	if (!idr_cmpxchg(&pmu_idr, pmu->type, pmu, NULL))
13027	return -EINVAL;
13028
13029	list_del_rcu(&pmu->entry);
13030	}
13031
13032	/*
13033	* We dereference the pmu list under both SRCU and regular RCU, so
13034	* synchronize against both of those.
13035	*
13036	* Notably, the entirety of event creation, from perf_init_event()
13037	* (which will now fail, because of the above) until
13038	* perf_install_in_context() should be under SRCU such that
13039	* this synchronizes against event creation. This avoids trying to
13040	* detach events that are not fully formed.
13041	*/
13042	synchronize_srcu(&pmus_srcu);
13043	synchronize_rcu();
13044
13045	if (pmu->event_unmapped && !pmu_empty(pmu)) {
13046	/*
13047	* Can't force remove events when pmu::event_unmapped()
13048	* is used in perf_mmap_close().
13049	*/
13050	guard(mutex)(&pmus_lock);
13051	idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu);
13052	list_add_rcu(&pmu->entry, &pmus);
13053	return -EBUSY;
13054	}
13055
13056	scoped_guard (mutex, &pmus_lock)
13057	idr_remove(&pmu_idr, pmu->type);
13058
13059	/*
13060	* PMU is removed from the pmus list, so no new events will
13061	* be created, now take care of the existing ones.
13062	*/
13063	pmu_detach_events(pmu);
13064
13065	/*
13066	* PMU is unused, make it go away.
13067	*/
13068	perf_pmu_free(pmu);
13069	return 0;
13070	}
13071	EXPORT_SYMBOL_GPL(perf_pmu_unregister);
13072
13073	static inline bool has_extended_regs(struct perf_event *event)
13074	{
13075	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) \|\|
13076	(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);
13077	}
13078
13079	static int perf_try_init_event(struct pmu pmu, struct perf_event event)
13080	{
13081	struct perf_event_context *ctx = NULL;
13082	int ret;
13083
13084	if (!try_module_get(pmu->module))
13085	return -ENODEV;
13086
13087	/*
13088	* A number of pmu->event_init() methods iterate the sibling_list to,
13089	* for example, validate if the group fits on the PMU. Therefore,
13090	* if this is a sibling event, acquire the ctx->mutex to protect
13091	* the sibling_list.
13092	*/
13093	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {
13094	/*
13095	* This ctx->mutex can nest when we're called through
13096	* inheritance. See the perf_event_ctx_lock_nested() comment.
13097	*/
13098	ctx = perf_event_ctx_lock_nested(event->group_leader,
13099	SINGLE_DEPTH_NESTING);
13100	BUG_ON(!ctx);
13101	}
13102
13103	event->pmu = pmu;
13104	ret = pmu->event_init(event);
13105
13106	if (ctx)
13107	perf_event_ctx_unlock(event->group_leader, ctx);
13108
13109	if (ret)
13110	goto err_pmu;
13111
13112	if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&
13113	has_extended_regs(event)) {
13114	ret = -EOPNOTSUPP;
13115	goto err_destroy;
13116	}
13117
13118	if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&
13119	event_has_any_exclude_flag(event)) {
13120	ret = -EINVAL;
13121	goto err_destroy;
13122	}
13123
13124	if (pmu->scope != PERF_PMU_SCOPE_NONE && event->cpu >= 0) {
13125	const struct cpumask *cpumask;
13126	struct cpumask *pmu_cpumask;
13127	int cpu;
13128
13129	cpumask = perf_scope_cpu_topology_cpumask(pmu->scope, event->cpu);
13130	pmu_cpumask = perf_scope_cpumask(pmu->scope);
13131
13132	ret = -ENODEV;
13133	if (!pmu_cpumask \|\| !cpumask)
13134	goto err_destroy;
13135
13136	cpu = cpumask_any_and(pmu_cpumask, cpumask);
13137	if (cpu >= nr_cpu_ids)
13138	goto err_destroy;
13139
13140	event->event_caps \|= PERF_EV_CAP_READ_SCOPE;
13141	}
13142
13143	return 0;
13144
13145	err_destroy:
13146	if (event->destroy) {
13147	event->destroy(event);
13148	event->destroy = NULL;
13149	}
13150
13151	err_pmu:
13152	event->pmu = NULL;
13153	module_put(pmu->module);
13154	return ret;
13155	}
13156
13157	static struct pmu perf_init_event(struct perf_event event)
13158	{
13159	bool extended_type = false;
13160	struct pmu *pmu;
13161	int type, ret;
13162
13163	guard(srcu)(&pmus_srcu); /* pmu idr/list access */
13164
13165	/*
13166	* Save original type before calling pmu->event_init() since certain
13167	* pmus overwrites event->attr.type to forward event to another pmu.
13168	*/
13169	event->orig_type = event->attr.type;
13170
13171	/* Try parent's PMU first: */
13172	if (event->parent && event->parent->pmu) {
13173	pmu = event->parent->pmu;
13174	ret = perf_try_init_event(pmu, event);
13175	if (!ret)
13176	return pmu;
13177	}
13178
13179	/*
13180	* PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
13181	* are often aliases for PERF_TYPE_RAW.
13182	*/
13183	type = event->attr.type;
13184	if (type == PERF_TYPE_HARDWARE \|\| type == PERF_TYPE_HW_CACHE) {
13185	type = event->attr.config >> PERF_PMU_TYPE_SHIFT;
13186	if (!type) {
13187	type = PERF_TYPE_RAW;
13188	} else {
13189	extended_type = true;
13190	event->attr.config &= PERF_HW_EVENT_MASK;
13191	}
13192	}
13193
13194	again:
13195	scoped_guard (rcu)
13196	pmu = idr_find(&pmu_idr, type);
13197	if (pmu) {
13198	if (event->attr.type != type && type != PERF_TYPE_RAW &&
13199	!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))
13200	return ERR_PTR(-ENOENT);
13201
13202	ret = perf_try_init_event(pmu, event);
13203	if (ret == -ENOENT && event->attr.type != type && !extended_type) {
13204	type = event->attr.type;
13205	goto again;
13206	}
13207
13208	if (ret)
13209	return ERR_PTR(ret);
13210
13211	return pmu;
13212	}
13213
13214	list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) {
13215	ret = perf_try_init_event(pmu, event);
13216	if (!ret)
13217	return pmu;
13218
13219	if (ret != -ENOENT)
13220	return ERR_PTR(ret);
13221	}
13222
13223	return ERR_PTR(-ENOENT);
13224	}
13225
13226	static void attach_sb_event(struct perf_event *event)
13227	{
13228	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
13229
13230	raw_spin_lock(&pel->lock);
13231	list_add_rcu(&event->sb_list, &pel->list);
13232	raw_spin_unlock(&pel->lock);
13233	}
13234
13235	/*
13236	* We keep a list of all !task (and therefore per-cpu) events
13237	* that need to receive side-band records.
13238	*
13239	* This avoids having to scan all the various PMU per-cpu contexts
13240	* looking for them.
13241	*/
13242	static void account_pmu_sb_event(struct perf_event *event)
13243	{
13244	if (is_sb_event(event))
13245	attach_sb_event(event);
13246	}
13247
13248	/* Freq events need the tick to stay alive (see perf_event_task_tick). */
13249	static void account_freq_event_nohz(void)
13250	{
13251	#ifdef CONFIG_NO_HZ_FULL
13252	/* Lock so we don't race with concurrent unaccount */
13253	spin_lock(&nr_freq_lock);
13254	if (atomic_inc_return(&nr_freq_events) == 1)
13255	tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS);
13256	spin_unlock(&nr_freq_lock);
13257	#endif
13258	}
13259
13260	static void account_freq_event(void)
13261	{
13262	if (tick_nohz_full_enabled())
13263	account_freq_event_nohz();
13264	else
13265	atomic_inc(&nr_freq_events);
13266	}
13267
13268
13269	static void account_event(struct perf_event *event)
13270	{
13271	bool inc = false;
13272
13273	if (event->parent)
13274	return;
13275
13276	if (event->attach_state & (PERF_ATTACH_TASK \| PERF_ATTACH_SCHED_CB))
13277	inc = true;
13278	if (event->attr.mmap \|\| event->attr.mmap_data)
13279	atomic_inc(&nr_mmap_events);
13280	if (event->attr.build_id)
13281	atomic_inc(&nr_build_id_events);
13282	if (event->attr.comm)
13283	atomic_inc(&nr_comm_events);
13284	if (event->attr.namespaces)
13285	atomic_inc(&nr_namespaces_events);
13286	if (event->attr.cgroup)
13287	atomic_inc(&nr_cgroup_events);
13288	if (event->attr.task)
13289	atomic_inc(&nr_task_events);
13290	if (event->attr.freq)
13291	account_freq_event();
13292	if (event->attr.context_switch) {
13293	atomic_inc(&nr_switch_events);
13294	inc = true;
13295	}
13296	if (has_branch_stack(event))
13297	inc = true;
13298	if (is_cgroup_event(event))
13299	inc = true;
13300	if (event->attr.ksymbol)
13301	atomic_inc(&nr_ksymbol_events);
13302	if (event->attr.bpf_event)
13303	atomic_inc(&nr_bpf_events);
13304	if (event->attr.text_poke)
13305	atomic_inc(&nr_text_poke_events);
13306
13307	if (inc) {
13308	/*
13309	* We need the mutex here because static_branch_enable()
13310	* must complete before the perf_sched_count increment
13311	* becomes visible.
13312	*/
13313	if (atomic_inc_not_zero(&perf_sched_count))
13314	goto enabled;
13315
13316	mutex_lock(&perf_sched_mutex);
13317	if (!atomic_read(&perf_sched_count)) {
13318	static_branch_enable(&perf_sched_events);
13319	/*
13320	* Guarantee that all CPUs observe they key change and
13321	* call the perf scheduling hooks before proceeding to
13322	* install events that need them.
13323	*/
13324	synchronize_rcu();
13325	}
13326	/*
13327	* Now that we have waited for the sync_sched(), allow further
13328	* increments to by-pass the mutex.
13329	*/
13330	atomic_inc(&perf_sched_count);
13331	mutex_unlock(&perf_sched_mutex);
13332	}
13333	enabled:
13334
13335	account_pmu_sb_event(event);
13336	}
13337
13338	/*
13339	* Allocate and initialize an event structure
13340	*/
13341	static struct perf_event *
13342	perf_event_alloc(struct perf_event_attr *attr, int cpu,
13343	struct task_struct *task,
13344	struct perf_event *group_leader,
13345	struct perf_event *parent_event,
13346	perf_overflow_handler_t overflow_handler,
13347	void *context, int cgroup_fd)
13348	{
13349	struct pmu *pmu;
13350	struct hw_perf_event *hwc;
13351	long err = -EINVAL;
13352	int node;
13353
13354	if ((unsigned)cpu >= nr_cpu_ids) {
13355	if (!task \|\| cpu != -1)
13356	return ERR_PTR(-EINVAL);
13357	}
13358	if (attr->sigtrap && !task) {
13359	/* Requires a task: avoid signalling random tasks. */
13360	return ERR_PTR(-EINVAL);
13361	}
13362
13363	node = (cpu >= 0) ? cpu_to_node(cpu) : -1;
13364	struct perf_event *event __free(__free_event) =
13365	kmem_cache_alloc_node(perf_event_cache, GFP_KERNEL \| __GFP_ZERO, node);
13366	if (!event)
13367	return ERR_PTR(-ENOMEM);
13368
13369	/*
13370	* Single events are their own group leaders, with an
13371	* empty sibling list:
13372	*/
13373	if (!group_leader)
13374	group_leader = event;
13375
13376	mutex_init(&event->child_mutex);
13377	INIT_LIST_HEAD(&event->child_list);
13378
13379	INIT_LIST_HEAD(&event->event_entry);
13380	INIT_LIST_HEAD(&event->sibling_list);
13381	INIT_LIST_HEAD(&event->active_list);
13382	init_event_group(event);
13383	INIT_LIST_HEAD(&event->rb_entry);
13384	INIT_LIST_HEAD(&event->active_entry);
13385	INIT_LIST_HEAD(&event->addr_filters.list);
13386	INIT_HLIST_NODE(&event->hlist_entry);
13387	INIT_LIST_HEAD(&event->pmu_list);
13388
13389
13390	init_waitqueue_head(&event->waitq);
13391	init_irq_work(&event->pending_irq, perf_pending_irq);
13392	event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable);
13393	init_task_work(&event->pending_task, perf_pending_task);
13394
13395	mutex_init(&event->mmap_mutex);
13396	raw_spin_lock_init(&event->addr_filters.lock);
13397
13398	atomic_long_set(&event->refcount, 1);
13399	event->cpu = cpu;
13400	event->attr = *attr;
13401	event->group_leader = group_leader;
13402	event->pmu = NULL;
13403	event->oncpu = -1;
13404
13405	event->parent = parent_event;
13406
13407	event->ns = get_pid_ns(task_active_pid_ns(current));
13408	event->id = atomic64_inc_return(&perf_event_id);
13409
13410	event->state = PERF_EVENT_STATE_INACTIVE;
13411
13412	if (parent_event)
13413	event->event_caps = parent_event->event_caps;
13414
13415	if (task) {
13416	event->attach_state = PERF_ATTACH_TASK;
13417	/*
13418	* XXX pmu::event_init needs to know what task to account to
13419	* and we cannot use the ctx information because we need the
13420	* pmu before we get a ctx.
13421	*/
13422	event->hw.target = get_task_struct(task);
13423	}
13424
13425	event->clock = &local_clock;
13426	if (parent_event)
13427	event->clock = parent_event->clock;
13428
13429	if (!overflow_handler && parent_event) {
13430	overflow_handler = parent_event->overflow_handler;
13431	context = parent_event->overflow_handler_context;
13432	#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
13433	if (parent_event->prog) {
13434	struct bpf_prog *prog = parent_event->prog;
13435
13436	bpf_prog_inc(prog);
13437	event->prog = prog;
13438	}
13439	#endif
13440	}
13441
13442	if (overflow_handler) {
13443	event->overflow_handler = overflow_handler;
13444	event->overflow_handler_context = context;
13445	} else if (is_write_backward(event)){
13446	event->overflow_handler = perf_event_output_backward;
13447	event->overflow_handler_context = NULL;
13448	} else {
13449	event->overflow_handler = perf_event_output_forward;
13450	event->overflow_handler_context = NULL;
13451	}
13452
13453	perf_event__state_init(event);
13454
13455	pmu = NULL;
13456
13457	hwc = &event->hw;
13458	hwc->sample_period = attr->sample_period;
13459	if (is_event_in_freq_mode(event))
13460	hwc->sample_period = 1;
13461	hwc->last_period = hwc->sample_period;
13462
13463	local64_set(&hwc->period_left, hwc->sample_period);
13464
13465	/*
13466	* We do not support PERF_SAMPLE_READ on inherited events unless
13467	* PERF_SAMPLE_TID is also selected, which allows inherited events to
13468	* collect per-thread samples.
13469	* See perf_output_read().
13470	*/
13471	if (has_inherit_and_sample_read(attr) && !(attr->sample_type & PERF_SAMPLE_TID))
13472	return ERR_PTR(-EINVAL);
13473
13474	if (!has_branch_stack(event))
13475	event->attr.branch_sample_type = 0;
13476
13477	pmu = perf_init_event(event);
13478	if (IS_ERR(pmu))
13479	return (void*)pmu;
13480
13481	/*
13482	* The PERF_ATTACH_TASK_DATA is set in the event_init()->hw_config().
13483	* The attach should be right after the perf_init_event().
13484	* Otherwise, the __free_event() would mistakenly detach the non-exist
13485	* perf_ctx_data because of the other errors between them.
13486	*/
13487	if (event->attach_state & PERF_ATTACH_TASK_DATA) {
13488	err = attach_perf_ctx_data(event);
13489	if (err)
13490	return ERR_PTR(err);
13491	}
13492
13493	/*
13494	* Disallow uncore-task events. Similarly, disallow uncore-cgroup
13495	* events (they don't make sense as the cgroup will be different
13496	* on other CPUs in the uncore mask).
13497	*/
13498	if (pmu->task_ctx_nr == perf_invalid_context && (task \|\| cgroup_fd != -1))
13499	return ERR_PTR(-EINVAL);
13500
13501	if (event->attr.aux_output &&
13502	(!(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT) \|\|
13503	event->attr.aux_pause \|\| event->attr.aux_resume))
13504	return ERR_PTR(-EOPNOTSUPP);
13505
13506	if (event->attr.aux_pause && event->attr.aux_resume)
13507	return ERR_PTR(-EINVAL);
13508
13509	if (event->attr.aux_start_paused) {
13510	if (!(pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE))
13511	return ERR_PTR(-EOPNOTSUPP);
13512	event->hw.aux_paused = 1;
13513	}
13514
13515	if (cgroup_fd != -1) {
13516	err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);
13517	if (err)
13518	return ERR_PTR(err);
13519	}
13520
13521	err = exclusive_event_init(event);
13522	if (err)
13523	return ERR_PTR(err);
13524
13525	if (has_addr_filter(event)) {
13526	event->addr_filter_ranges = kzalloc_objs(struct perf_addr_filter_range,
13527	pmu->nr_addr_filters);
13528	if (!event->addr_filter_ranges)
13529	return ERR_PTR(-ENOMEM);
13530
13531	/*
13532	* Clone the parent's vma offsets: they are valid until exec()
13533	* even if the mm is not shared with the parent.
13534	*/
13535	if (event->parent) {
13536	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
13537
13538	raw_spin_lock_irq(&ifh->lock);
13539	memcpy(event->addr_filter_ranges,
13540	event->parent->addr_filter_ranges,
13541	pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));
13542	raw_spin_unlock_irq(&ifh->lock);
13543	}
13544
13545	/* force hw sync on the address filters */
13546	event->addr_filters_gen = 1;
13547	}
13548
13549	if (!event->parent) {
13550	if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
13551	err = get_callchain_buffers(attr->sample_max_stack);
13552	if (err)
13553	return ERR_PTR(err);
13554	event->attach_state \|= PERF_ATTACH_CALLCHAIN;
13555	}
13556	}
13557
13558	err = security_perf_event_alloc(event);
13559	if (err)
13560	return ERR_PTR(err);
13561
13562	err = mediated_pmu_account_event(event);
13563	if (err)
13564	return ERR_PTR(err);
13565
13566	/* symmetric to unaccount_event() in _free_event() */
13567	account_event(event);
13568
13569	/*
13570	* Event creation should be under SRCU, see perf_pmu_unregister().
13571	*/
13572	lockdep_assert_held(&pmus_srcu);
13573	scoped_guard (spinlock, &pmu->events_lock)
13574	list_add(&event->pmu_list, &pmu->events);
13575
13576	return_ptr(event);
13577	}
13578
13579	static int perf_copy_attr(struct perf_event_attr __user *uattr,
13580	struct perf_event_attr *attr)
13581	{
13582	u32 size;
13583	int ret;
13584
13585	/* Zero the full structure, so that a short copy will be nice. */
13586	memset(attr, 0, sizeof(*attr));
13587
13588	ret = get_user(size, &uattr->size);
13589	if (ret)
13590	return ret;
13591
13592	/* ABI compatibility quirk: */
13593	if (!size)
13594	size = PERF_ATTR_SIZE_VER0;
13595	if (size < PERF_ATTR_SIZE_VER0 \|\| size > PAGE_SIZE)
13596	goto err_size;
13597
13598	ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size);
13599	if (ret) {
13600	if (ret == -E2BIG)
13601	goto err_size;
13602	return ret;
13603	}
13604
13605	attr->size = size;
13606
13607	if (attr->__reserved_1 \|\| attr->__reserved_2 \|\| attr->__reserved_3)
13608	return -EINVAL;
13609
13610	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
13611	return -EINVAL;
13612
13613	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
13614	return -EINVAL;
13615
13616	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
13617	u64 mask = attr->branch_sample_type;
13618
13619	/* only using defined bits */
13620	if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
13621	return -EINVAL;
13622
13623	/* at least one branch bit must be set */
13624	if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
13625	return -EINVAL;
13626
13627	/* propagate priv level, when not set for branch */
13628	if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
13629
13630	/* exclude_kernel checked on syscall entry */
13631	if (!attr->exclude_kernel)
13632	mask \|= PERF_SAMPLE_BRANCH_KERNEL;
13633
13634	if (!attr->exclude_user)
13635	mask \|= PERF_SAMPLE_BRANCH_USER;
13636
13637	if (!attr->exclude_hv)
13638	mask \|= PERF_SAMPLE_BRANCH_HV;
13639	/*
13640	* adjust user setting (for HW filter setup)
13641	*/
13642	attr->branch_sample_type = mask;
13643	}
13644	/* privileged levels capture (kernel, hv): check permissions */
13645	if (mask & PERF_SAMPLE_BRANCH_PERM_PLM) {
13646	ret = perf_allow_kernel();
13647	if (ret)
13648	return ret;
13649	}
13650	}
13651
13652	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
13653	ret = perf_reg_validate(attr->sample_regs_user);
13654	if (ret)
13655	return ret;
13656	}
13657
13658	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
13659	if (!arch_perf_have_user_stack_dump())
13660	return -ENOSYS;
13661
13662	/*
13663	* We have __u32 type for the size, but so far
13664	* we can only use __u16 as maximum due to the
13665	* __u16 sample size limit.
13666	*/
13667	if (attr->sample_stack_user >= USHRT_MAX)
13668	return -EINVAL;
13669	else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
13670	return -EINVAL;
13671	}
13672
13673	if (!attr->sample_max_stack)
13674	attr->sample_max_stack = sysctl_perf_event_max_stack;
13675
13676	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
13677	ret = perf_reg_validate(attr->sample_regs_intr);
13678
13679	#ifndef CONFIG_CGROUP_PERF
13680	if (attr->sample_type & PERF_SAMPLE_CGROUP)
13681	return -EINVAL;
13682	#endif
13683	if ((attr->sample_type & PERF_SAMPLE_WEIGHT) &&
13684	(attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT))
13685	return -EINVAL;
13686
13687	if (!attr->inherit && attr->inherit_thread)
13688	return -EINVAL;
13689
13690	if (attr->remove_on_exec && attr->enable_on_exec)
13691	return -EINVAL;
13692
13693	if (attr->sigtrap && !attr->remove_on_exec)
13694	return -EINVAL;
13695
13696	out:
13697	return ret;
13698
13699	err_size:
13700	put_user(sizeof(*attr), &uattr->size);
13701	ret = -E2BIG;
13702	goto out;
13703	}
13704
13705	static void mutex_lock_double(struct mutex a, struct mutex b)
13706	{
13707	if (b < a)
13708	swap(a, b);
13709
13710	mutex_lock(a);
13711	mutex_lock_nested(b, SINGLE_DEPTH_NESTING);
13712	}
13713
13714	static int
13715	perf_event_set_output(struct perf_event event, struct perf_event output_event)
13716	{
13717	struct perf_buffer *rb = NULL;
13718	int ret = -EINVAL;
13719
13720	if (!output_event) {
13721	mutex_lock(&event->mmap_mutex);
13722	goto set;
13723	}
13724
13725	/* don't allow circular references */
13726	if (event == output_event)
13727	goto out;
13728
13729	/*
13730	* Don't allow cross-cpu buffers
13731	*/
13732	if (output_event->cpu != event->cpu)
13733	goto out;
13734
13735	/*
13736	* If its not a per-cpu rb, it must be the same task.
13737	*/
13738	if (output_event->cpu == -1 && output_event->hw.target != event->hw.target)
13739	goto out;
13740
13741	/*
13742	* Mixing clocks in the same buffer is trouble you don't need.
13743	*/
13744	if (output_event->clock != event->clock)
13745	goto out;
13746
13747	/*
13748	* Either writing ring buffer from beginning or from end.
13749	* Mixing is not allowed.
13750	*/
13751	if (is_write_backward(output_event) != is_write_backward(event))
13752	goto out;
13753
13754	/*
13755	* If both events generate aux data, they must be on the same PMU
13756	*/
13757	if (has_aux(event) && has_aux(output_event) &&
13758	event->pmu != output_event->pmu)
13759	goto out;
13760
13761	/*
13762	* Hold both mmap_mutex to serialize against perf_mmap_close(). Since
13763	* output_event is already on rb->event_list, and the list iteration
13764	* restarts after every removal, it is guaranteed this new event is
13765	* observed OR if output_event is already removed, it's guaranteed we
13766	* observe !rb->mmap_count.
13767	*/
13768	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);
13769	set:
13770	/* Can't redirect output if we've got an active mmap() */
13771	if (refcount_read(&event->mmap_count))
13772	goto unlock;
13773
13774	if (output_event) {
13775	if (output_event->state <= PERF_EVENT_STATE_REVOKED)
13776	goto unlock;
13777
13778	/* get the rb we want to redirect to */
13779	rb = ring_buffer_get(output_event);
13780	if (!rb)
13781	goto unlock;
13782
13783	/* did we race against perf_mmap_close() */
13784	if (!refcount_read(&rb->mmap_count)) {
13785	ring_buffer_put(rb);
13786	goto unlock;
13787	}
13788	}
13789
13790	ring_buffer_attach(event, rb);
13791
13792	ret = 0;
13793	unlock:
13794	mutex_unlock(&event->mmap_mutex);
13795	if (output_event)
13796	mutex_unlock(&output_event->mmap_mutex);
13797
13798	out:
13799	return ret;
13800	}
13801
13802	static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)
13803	{
13804	bool nmi_safe = false;
13805
13806	switch (clk_id) {
13807	case CLOCK_MONOTONIC:
13808	event->clock = &ktime_get_mono_fast_ns;
13809	nmi_safe = true;
13810	break;
13811
13812	case CLOCK_MONOTONIC_RAW:
13813	event->clock = &ktime_get_raw_fast_ns;
13814	nmi_safe = true;
13815	break;
13816
13817	case CLOCK_REALTIME:
13818	event->clock = &ktime_get_real_ns;
13819	break;
13820
13821	case CLOCK_BOOTTIME:
13822	event->clock = &ktime_get_boottime_ns;
13823	break;
13824
13825	case CLOCK_TAI:
13826	event->clock = &ktime_get_clocktai_ns;
13827	break;
13828
13829	default:
13830	return -EINVAL;
13831	}
13832
13833	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))
13834	return -EINVAL;
13835
13836	return 0;
13837	}
13838
13839	static bool
13840	perf_check_permission(struct perf_event_attr attr, struct task_struct task)
13841	{
13842	unsigned int ptrace_mode = PTRACE_MODE_READ_REALCREDS;
13843	bool is_capable = perfmon_capable();
13844
13845	if (attr->sigtrap) {
13846	/*
13847	* perf_event_attr::sigtrap sends signals to the other task.
13848	* Require the current task to also have CAP_KILL.
13849	*/
13850	rcu_read_lock();
13851	is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL);
13852	rcu_read_unlock();
13853
13854	/*
13855	* If the required capabilities aren't available, checks for
13856	* ptrace permissions: upgrade to ATTACH, since sending signals
13857	* can effectively change the target task.
13858	*/
13859	ptrace_mode = PTRACE_MODE_ATTACH_REALCREDS;
13860	}
13861
13862	/*
13863	* Preserve ptrace permission check for backwards compatibility. The
13864	* ptrace check also includes checks that the current task and other
13865	* task have matching uids, and is therefore not done here explicitly.
13866	*/
13867	return is_capable \|\| ptrace_may_access(task, ptrace_mode);
13868	}
13869
13870	/**
13871	* sys_perf_event_open - open a performance event, associate it to a task/cpu
13872	*
13873	* @attr_uptr: event_id type attributes for monitoring/sampling
13874	* @pid: target pid
13875	* @cpu: target cpu
13876	* @group_fd: group leader event fd
13877	* @flags: perf event open flags
13878	*/
13879	SYSCALL_DEFINE5(perf_event_open,
13880	struct perf_event_attr __user *, attr_uptr,
13881	pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
13882	{
13883	struct perf_event group_leader = NULL, output_event = NULL;
13884	struct perf_event_pmu_context *pmu_ctx;
13885	struct perf_event event, sibling;
13886	struct perf_event_attr attr;
13887	struct perf_event_context *ctx;
13888	struct file *event_file = NULL;
13889	struct task_struct *task = NULL;
13890	struct pmu *pmu;
13891	int event_fd;
13892	int move_group = 0;
13893	int err;
13894	int f_flags = O_RDWR;
13895	int cgroup_fd = -1;
13896
13897	/* for future expandability... */
13898	if (flags & ~PERF_FLAG_ALL)
13899	return -EINVAL;
13900
13901	err = perf_copy_attr(attr_uptr, &attr);
13902	if (err)
13903	return err;
13904
13905	/* Do we allow access to perf_event_open(2) ? */
13906	err = security_perf_event_open(PERF_SECURITY_OPEN);
13907	if (err)
13908	return err;
13909
13910	if (!attr.exclude_kernel \|\| attr.text_poke \|\|
13911	((attr.sample_type & PERF_SAMPLE_CALLCHAIN) &&
13912	!attr.exclude_callchain_kernel)) {
13913	err = perf_allow_kernel();
13914	if (err)
13915	return err;
13916	}
13917
13918	if (attr.namespaces) {
13919	if (!perfmon_capable())
13920	return -EACCES;
13921	}
13922
13923	if (attr.freq) {
13924	if (attr.sample_freq > sysctl_perf_event_sample_rate)
13925	return -EINVAL;
13926	} else {
13927	if (attr.sample_period & (1ULL << 63))
13928	return -EINVAL;
13929	}
13930
13931	/* Only privileged users can get physical addresses */
13932	if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR)) {
13933	err = perf_allow_kernel();
13934	if (err)
13935	return err;
13936	}
13937
13938	/* REGS_INTR can leak data, lockdown must prevent this */
13939	if (attr.sample_type & PERF_SAMPLE_REGS_INTR) {
13940	err = security_locked_down(LOCKDOWN_PERF);
13941	if (err)
13942	return err;
13943	}
13944
13945	/*
13946	* In cgroup mode, the pid argument is used to pass the fd
13947	* opened to the cgroup directory in cgroupfs. The cpu argument
13948	* designates the cpu on which to monitor threads from that
13949	* cgroup.
13950	*/
13951	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 \|\| cpu == -1))
13952	return -EINVAL;
13953
13954	if (flags & PERF_FLAG_FD_CLOEXEC)
13955	f_flags \|= O_CLOEXEC;
13956
13957	event_fd = get_unused_fd_flags(f_flags);
13958	if (event_fd < 0)
13959	return event_fd;
13960
13961	/*
13962	* Event creation should be under SRCU, see perf_pmu_unregister().
13963	*/
13964	guard(srcu)(&pmus_srcu);
13965
13966	CLASS(fd, group)(group_fd); // group_fd == -1 => empty
13967	if (group_fd != -1) {
13968	if (!is_perf_file(group)) {
13969	err = -EBADF;
13970	goto err_fd;
13971	}
13972	group_leader = fd_file(group)->private_data;
13973	if (group_leader->state <= PERF_EVENT_STATE_EXIT) {
13974	err = -ENODEV;
13975	goto err_fd;
13976	}
13977	if (flags & PERF_FLAG_FD_OUTPUT)
13978	output_event = group_leader;
13979	if (flags & PERF_FLAG_FD_NO_GROUP)
13980	group_leader = NULL;
13981	}
13982
13983	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
13984	task = find_lively_task_by_vpid(pid);
13985	if (IS_ERR(task)) {
13986	err = PTR_ERR(task);
13987	goto err_fd;
13988	}
13989	}
13990
13991	if (task && group_leader &&
13992	group_leader->attr.inherit != attr.inherit) {
13993	err = -EINVAL;
13994	goto err_task;
13995	}
13996
13997	if (flags & PERF_FLAG_PID_CGROUP)
13998	cgroup_fd = pid;
13999
14000	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
14001	NULL, NULL, cgroup_fd);
14002	if (IS_ERR(event)) {
14003	err = PTR_ERR(event);
14004	goto err_task;
14005	}
14006
14007	if (is_sampling_event(event)) {
14008	if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
14009	err = -EOPNOTSUPP;
14010	goto err_alloc;
14011	}
14012	}
14013
14014	/*
14015	* Special case software events and allow them to be part of
14016	* any hardware group.
14017	*/
14018	pmu = event->pmu;
14019
14020	if (attr.use_clockid) {
14021	err = perf_event_set_clock(event, attr.clockid);
14022	if (err)
14023	goto err_alloc;
14024	}
14025
14026	if (pmu->task_ctx_nr == perf_sw_context)
14027	event->event_caps \|= PERF_EV_CAP_SOFTWARE;
14028
14029	if (task) {
14030	err = down_read_interruptible(&task->signal->exec_update_lock);
14031	if (err)
14032	goto err_alloc;
14033
14034	/*
14035	* We must hold exec_update_lock across this and any potential
14036	* perf_install_in_context() call for this new event to
14037	* serialize against exec() altering our credentials (and the
14038	* perf_event_exit_task() that could imply).
14039	*/
14040	err = -EACCES;
14041	if (!perf_check_permission(&attr, task))
14042	goto err_cred;
14043	}
14044
14045	/*
14046	* Get the target context (task or percpu):
14047	*/
14048	ctx = find_get_context(task, event);
14049	if (IS_ERR(ctx)) {
14050	err = PTR_ERR(ctx);
14051	goto err_cred;
14052	}
14053
14054	mutex_lock(&ctx->mutex);
14055
14056	if (ctx->task == TASK_TOMBSTONE) {
14057	err = -ESRCH;
14058	goto err_locked;
14059	}
14060
14061	if (!task) {
14062	/*
14063	* Check if the @cpu we're creating an event for is online.
14064	*
14065	* We use the perf_cpu_context::ctx::mutex to serialize against
14066	* the hotplug notifiers. See perf_event_{init,exit}_cpu().
14067	*/
14068	struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
14069
14070	if (!cpuctx->online) {
14071	err = -ENODEV;
14072	goto err_locked;
14073	}
14074	}
14075
14076	if (group_leader) {
14077	err = -EINVAL;
14078
14079	/*
14080	* Do not allow a recursive hierarchy (this new sibling
14081	* becoming part of another group-sibling):
14082	*/
14083	if (group_leader->group_leader != group_leader)
14084	goto err_locked;
14085
14086	/* All events in a group should have the same clock */
14087	if (group_leader->clock != event->clock)
14088	goto err_locked;
14089
14090	/*
14091	* Make sure we're both events for the same CPU;
14092	* grouping events for different CPUs is broken; since
14093	* you can never concurrently schedule them anyhow.
14094	*/
14095	if (group_leader->cpu != event->cpu)
14096	goto err_locked;
14097
14098	/*
14099	* Make sure we're both on the same context; either task or cpu.
14100	*/
14101	if (group_leader->ctx != ctx)
14102	goto err_locked;
14103
14104	/* Recheck under ctx::mutex to serialize against remove-on-exec. */
14105	if (group_leader->state <= PERF_EVENT_STATE_EXIT) {
14106	err = -ENODEV;
14107	goto err_locked;
14108	}
14109
14110	/*
14111	* Only a group leader can be exclusive or pinned
14112	*/
14113	if (attr.exclusive \|\| attr.pinned)
14114	goto err_locked;
14115
14116	if (is_software_event(event) &&
14117	!in_software_context(group_leader)) {
14118	/*
14119	* If the event is a sw event, but the group_leader
14120	* is on hw context.
14121	*
14122	* Allow the addition of software events to hw
14123	* groups, this is safe because software events
14124	* never fail to schedule.
14125	*
14126	* Note the comment that goes with struct
14127	* perf_event_pmu_context.
14128	*/
14129	pmu = group_leader->pmu_ctx->pmu;
14130	} else if (!is_software_event(event)) {
14131	if (is_software_event(group_leader) &&
14132	(group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {
14133	/*
14134	* In case the group is a pure software group, and we
14135	* try to add a hardware event, move the whole group to
14136	* the hardware context.
14137	*/
14138	move_group = 1;
14139	}
14140
14141	/* Don't allow group of multiple hw events from different pmus */
14142	if (!in_software_context(group_leader) &&
14143	group_leader->pmu_ctx->pmu != pmu)
14144	goto err_locked;
14145	}
14146	}
14147
14148	/*
14149	* Now that we're certain of the pmu; find the pmu_ctx.
14150	*/
14151	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
14152	if (IS_ERR(pmu_ctx)) {
14153	err = PTR_ERR(pmu_ctx);
14154	goto err_locked;
14155	}
14156	event->pmu_ctx = pmu_ctx;
14157
14158	if (output_event) {
14159	err = perf_event_set_output(event, output_event);
14160	if (err)
14161	goto err_context;
14162	}
14163
14164	if (!perf_event_validate_size(event)) {
14165	err = -E2BIG;
14166	goto err_context;
14167	}
14168
14169	if (perf_need_aux_event(event) && !perf_get_aux_event(event, group_leader)) {
14170	err = -EINVAL;
14171	goto err_context;
14172	}
14173
14174	/*
14175	* Must be under the same ctx::mutex as perf_install_in_context(),
14176	* because we need to serialize with concurrent event creation.
14177	*/
14178	if (!exclusive_event_installable(event, ctx)) {
14179	err = -EBUSY;
14180	goto err_context;
14181	}
14182
14183	WARN_ON_ONCE(ctx->parent_ctx);
14184
14185	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, f_flags);
14186	if (IS_ERR(event_file)) {
14187	err = PTR_ERR(event_file);
14188	event_file = NULL;
14189	goto err_context;
14190	}
14191
14192	/*
14193	* This is the point on no return; we cannot fail hereafter. This is
14194	* where we start modifying current state.
14195	*/
14196
14197	if (move_group) {
14198	perf_remove_from_context(group_leader, 0);
14199	put_pmu_ctx(group_leader->pmu_ctx);
14200
14201	for_each_sibling_event(sibling, group_leader) {
14202	perf_remove_from_context(sibling, 0);
14203	put_pmu_ctx(sibling->pmu_ctx);
14204	}
14205
14206	/*
14207	* Install the group siblings before the group leader.
14208	*
14209	* Because a group leader will try and install the entire group
14210	* (through the sibling list, which is still in-tact), we can
14211	* end up with siblings installed in the wrong context.
14212	*
14213	* By installing siblings first we NO-OP because they're not
14214	* reachable through the group lists.
14215	*/
14216	for_each_sibling_event(sibling, group_leader) {
14217	sibling->pmu_ctx = pmu_ctx;
14218	get_pmu_ctx(pmu_ctx);
14219	perf_event__state_init(sibling);
14220	perf_install_in_context(ctx, sibling, sibling->cpu);
14221	}
14222
14223	/*
14224	* Removing from the context ends up with disabled
14225	* event. What we want here is event in the initial
14226	* startup state, ready to be add into new context.
14227	*/
14228	group_leader->pmu_ctx = pmu_ctx;
14229	get_pmu_ctx(pmu_ctx);
14230	perf_event__state_init(group_leader);
14231	perf_install_in_context(ctx, group_leader, group_leader->cpu);
14232	}
14233
14234	/*
14235	* Precalculate sample_data sizes; do while holding ctx::mutex such
14236	* that we're serialized against further additions and before
14237	* perf_install_in_context() which is the point the event is active and
14238	* can use these values.
14239	*/
14240	perf_event__header_size(event);
14241	perf_event__id_header_size(event);
14242
14243	event->owner = current;
14244
14245	perf_install_in_context(ctx, event, event->cpu);
14246	perf_unpin_context(ctx);
14247
14248	mutex_unlock(&ctx->mutex);
14249
14250	if (task) {
14251	up_read(&task->signal->exec_update_lock);
14252	put_task_struct(task);
14253	}
14254
14255	mutex_lock(&current->perf_event_mutex);
14256	list_add_tail(&event->owner_entry, &current->perf_event_list);
14257	mutex_unlock(&current->perf_event_mutex);
14258
14259	/*
14260	* File reference in group guarantees that group_leader has been
14261	* kept alive until we place the new event on the sibling_list.
14262	* This ensures destruction of the group leader will find
14263	* the pointer to itself in perf_group_detach().
14264	*/
14265	fd_install(event_fd, event_file);
14266	return event_fd;
14267
14268	err_context:
14269	put_pmu_ctx(event->pmu_ctx);
14270	event->pmu_ctx = NULL; /* _free_event() */
14271	err_locked:
14272	mutex_unlock(&ctx->mutex);
14273	perf_unpin_context(ctx);
14274	put_ctx(ctx);
14275	err_cred:
14276	if (task)
14277	up_read(&task->signal->exec_update_lock);
14278	err_alloc:
14279	put_event(event);
14280	err_task:
14281	if (task)
14282	put_task_struct(task);
14283	err_fd:
14284	put_unused_fd(event_fd);
14285	return err;
14286	}
14287
14288	/**
14289	* perf_event_create_kernel_counter
14290	*
14291	* @attr: attributes of the counter to create
14292	* @cpu: cpu in which the counter is bound
14293	* @task: task to profile (NULL for percpu)
14294	* @overflow_handler: callback to trigger when we hit the event
14295	* @context: context data could be used in overflow_handler callback
14296	*/
14297	struct perf_event *
14298	perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
14299	struct task_struct *task,
14300	perf_overflow_handler_t overflow_handler,
14301	void *context)
14302	{
14303	struct perf_event_pmu_context *pmu_ctx;
14304	struct perf_event_context *ctx;
14305	struct perf_event *event;
14306	struct pmu *pmu;
14307	int err;
14308
14309	/*
14310	* Grouping is not supported for kernel events, neither is 'AUX',
14311	* make sure the caller's intentions are adjusted.
14312	*/
14313	if (attr->aux_output \|\| attr->aux_action)
14314	return ERR_PTR(-EINVAL);
14315
14316	/*
14317	* Event creation should be under SRCU, see perf_pmu_unregister().
14318	*/
14319	guard(srcu)(&pmus_srcu);
14320
14321	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
14322	overflow_handler, context, -1);
14323	if (IS_ERR(event)) {
14324	err = PTR_ERR(event);
14325	goto err;
14326	}
14327
14328	/* Mark owner so we could distinguish it from user events. */
14329	event->owner = TASK_TOMBSTONE;
14330	pmu = event->pmu;
14331
14332	if (pmu->task_ctx_nr == perf_sw_context)
14333	event->event_caps \|= PERF_EV_CAP_SOFTWARE;
14334
14335	/*
14336	* Get the target context (task or percpu):
14337	*/
14338	ctx = find_get_context(task, event);
14339	if (IS_ERR(ctx)) {
14340	err = PTR_ERR(ctx);
14341	goto err_alloc;
14342	}
14343
14344	WARN_ON_ONCE(ctx->parent_ctx);
14345	mutex_lock(&ctx->mutex);
14346	if (ctx->task == TASK_TOMBSTONE) {
14347	err = -ESRCH;
14348	goto err_unlock;
14349	}
14350
14351	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
14352	if (IS_ERR(pmu_ctx)) {
14353	err = PTR_ERR(pmu_ctx);
14354	goto err_unlock;
14355	}
14356	event->pmu_ctx = pmu_ctx;
14357
14358	if (!task) {
14359	/*
14360	* Check if the @cpu we're creating an event for is online.
14361	*
14362	* We use the perf_cpu_context::ctx::mutex to serialize against
14363	* the hotplug notifiers. See perf_event_{init,exit}_cpu().
14364	*/
14365	struct perf_cpu_context *cpuctx =
14366	container_of(ctx, struct perf_cpu_context, ctx);
14367	if (!cpuctx->online) {
14368	err = -ENODEV;
14369	goto err_pmu_ctx;
14370	}
14371	}
14372
14373	if (!exclusive_event_installable(event, ctx)) {
14374	err = -EBUSY;
14375	goto err_pmu_ctx;
14376	}
14377
14378	perf_install_in_context(ctx, event, event->cpu);
14379	perf_unpin_context(ctx);
14380	mutex_unlock(&ctx->mutex);
14381
14382	return event;
14383
14384	err_pmu_ctx:
14385	put_pmu_ctx(pmu_ctx);
14386	event->pmu_ctx = NULL; /* _free_event() */
14387	err_unlock:
14388	mutex_unlock(&ctx->mutex);
14389	perf_unpin_context(ctx);
14390	put_ctx(ctx);
14391	err_alloc:
14392	put_event(event);
14393	err:
14394	return ERR_PTR(err);
14395	}
14396	EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
14397
14398	static void __perf_pmu_remove(struct perf_event_context ctx, struct pmu pmu,
14399	struct list_head *events)
14400	{
14401	struct perf_event event, tmp;
14402
14403	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) {
14404	if (event->pmu_ctx->pmu != pmu)
14405	continue;
14406
14407	perf_remove_from_context(event, 0);
14408	put_pmu_ctx(event->pmu_ctx);
14409	list_add(&event->migrate_entry, events);
14410	}
14411	}
14412
14413	static void __perf_pmu_install_event(struct pmu *pmu,
14414	struct perf_event_context *ctx,
14415	int cpu, struct perf_event *event)
14416	{
14417	struct perf_event_pmu_context *epc;
14418	struct perf_event_context *old_ctx = event->ctx;
14419
14420	get_ctx(ctx); /* normally find_get_context() */
14421
14422	event->cpu = cpu;
14423	epc = find_get_pmu_context(pmu, ctx, event);
14424	event->pmu_ctx = epc;
14425
14426	if (event->state >= PERF_EVENT_STATE_OFF)
14427	event->state = PERF_EVENT_STATE_INACTIVE;
14428	perf_install_in_context(ctx, event, cpu);
14429
14430	/*
14431	* Now that event->ctx is updated and visible, put the old ctx.
14432	*/
14433	put_ctx(old_ctx);
14434	}
14435
14436	static void __perf_pmu_install(struct perf_event_context *ctx,
14437	int cpu, struct pmu pmu, struct list_head events)
14438	{
14439	struct perf_event event, tmp;
14440
14441	/*
14442	* Re-instate events in 2 passes.
14443	*
14444	* Skip over group leaders and only install siblings on this first
14445	* pass, siblings will not get enabled without a leader, however a
14446	* leader will enable its siblings, even if those are still on the old
14447	* context.
14448	*/
14449	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
14450	if (event->group_leader == event)
14451	continue;
14452
14453	list_del(&event->migrate_entry);
14454	__perf_pmu_install_event(pmu, ctx, cpu, event);
14455	}
14456
14457	/*
14458	* Once all the siblings are setup properly, install the group leaders
14459	* to make it go.
14460	*/
14461	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
14462	list_del(&event->migrate_entry);
14463	__perf_pmu_install_event(pmu, ctx, cpu, event);
14464	}
14465	}
14466
14467	void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
14468	{
14469	struct perf_event_context src_ctx, dst_ctx;
14470	LIST_HEAD(events);
14471
14472	/*
14473	* Since per-cpu context is persistent, no need to grab an extra
14474	* reference.
14475	*/
14476	src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;
14477	dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;
14478
14479	/*
14480	* See perf_event_ctx_lock() for comments on the details
14481	* of swizzling perf_event::ctx.
14482	*/
14483	mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);
14484
14485	__perf_pmu_remove(src_ctx, pmu, &events);
14486
14487	if (!list_empty(&events)) {
14488	/*
14489	* Wait for the events to quiesce before re-instating them.
14490	*/
14491	synchronize_rcu();
14492
14493	__perf_pmu_install(dst_ctx, dst_cpu, pmu, &events);
14494	}
14495
14496	mutex_unlock(&dst_ctx->mutex);
14497	mutex_unlock(&src_ctx->mutex);
14498	}
14499	EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
14500
14501	static void sync_child_event(struct perf_event *child_event,
14502	struct task_struct *task)
14503	{
14504	struct perf_event *parent_event = child_event->parent;
14505	u64 child_val;
14506
14507	if (child_event->attr.inherit_stat) {
14508	if (task && task != TASK_TOMBSTONE)
14509	perf_event_read_event(child_event, task);
14510	}
14511
14512	child_val = perf_event_count(child_event, false);
14513
14514	/*
14515	* Add back the child's count to the parent's count:
14516	*/
14517	atomic64_add(child_val, &parent_event->child_count);
14518	atomic64_add(child_event->total_time_enabled,
14519	&parent_event->child_total_time_enabled);
14520	atomic64_add(child_event->total_time_running,
14521	&parent_event->child_total_time_running);
14522	}
14523
14524	static void
14525	perf_event_exit_event(struct perf_event *event,
14526	struct perf_event_context *ctx,
14527	struct task_struct *task,
14528	unsigned long detach_flags)
14529	{
14530	struct perf_event *parent_event = event->parent;
14531	unsigned int attach_state;
14532
14533	detach_flags \|= DETACH_EXIT;
14534
14535	if (parent_event) {
14536	/*
14537	* Do not destroy the 'original' grouping; because of the
14538	* context switch optimization the original events could've
14539	* ended up in a random child task.
14540	*
14541	* If we were to destroy the original group, all group related
14542	* operations would cease to function properly after this
14543	* random child dies.
14544	*
14545	* Do destroy all inherited groups, we don't care about those
14546	* and being thorough is better.
14547	*/
14548	detach_flags \|= DETACH_GROUP \| DETACH_CHILD;
14549	mutex_lock(&parent_event->child_mutex);
14550	/* PERF_ATTACH_ITRACE might be set concurrently */
14551	attach_state = READ_ONCE(event->attach_state);
14552
14553	if (attach_state & PERF_ATTACH_CHILD)
14554	sync_child_event(event, task);
14555	}
14556
14557	if (detach_flags & DETACH_REVOKE)
14558	detach_flags \|= DETACH_GROUP;
14559
14560	perf_remove_from_context(event, detach_flags);
14561	/*
14562	* Child events can be freed.
14563	*/
14564	if (parent_event) {
14565	mutex_unlock(&parent_event->child_mutex);
14566
14567	/*
14568	* Match the refcount initialization. Make sure it doesn't happen
14569	* twice if pmu_detach_event() calls it on an already exited task.
14570	*/
14571	if (attach_state & PERF_ATTACH_CHILD) {
14572	/*
14573	* Kick perf_poll() for is_event_hup();
14574	*/
14575	perf_event_wakeup(parent_event);
14576	/*
14577	* pmu_detach_event() will have an extra refcount.
14578	* perf_pending_task() might have one too.
14579	*/
14580	put_event(event);
14581	}
14582
14583	return;
14584	}
14585
14586	/*
14587	* Parent events are governed by their filedesc, retain them.
14588	*/
14589	perf_event_wakeup(event);
14590	}
14591
14592	static void perf_event_exit_task_context(struct task_struct *task, bool exit)
14593	{
14594	struct perf_event_context ctx, clone_ctx = NULL;
14595	struct perf_event child_event, next;
14596
14597	ctx = perf_pin_task_context(task);
14598	if (!ctx)
14599	return;
14600
14601	/*
14602	* In order to reduce the amount of tricky in ctx tear-down, we hold
14603	* ctx::mutex over the entire thing. This serializes against almost
14604	* everything that wants to access the ctx.
14605	*
14606	* The exception is sys_perf_event_open() /
14607	* perf_event_create_kernel_count() which does find_get_context()
14608	* without ctx::mutex (it cannot because of the move_group double mutex
14609	* lock thing). See the comments in perf_install_in_context().
14610	*/
14611	mutex_lock(&ctx->mutex);
14612
14613	/*
14614	* In a single ctx::lock section, de-schedule the events and detach the
14615	* context from the task such that we cannot ever get it scheduled back
14616	* in.
14617	*/
14618	raw_spin_lock_irq(&ctx->lock);
14619	if (exit)
14620	task_ctx_sched_out(ctx, NULL, EVENT_ALL);
14621
14622	/*
14623	* Now that the context is inactive, destroy the task <-> ctx relation
14624	* and mark the context dead.
14625	*/
14626	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);
14627	put_ctx(ctx); /* cannot be last */
14628	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
14629	put_task_struct(task); /* cannot be last */
14630
14631	clone_ctx = unclone_ctx(ctx);
14632	raw_spin_unlock_irq(&ctx->lock);
14633
14634	if (clone_ctx)
14635	put_ctx(clone_ctx);
14636
14637	/*
14638	* Report the task dead after unscheduling the events so that we
14639	* won't get any samples after PERF_RECORD_EXIT. We can however still
14640	* get a few PERF_RECORD_READ events.
14641	*/
14642	if (exit)
14643	perf_event_task(task, ctx, 0);
14644
14645	list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry)
14646	perf_event_exit_event(child_event, ctx, exit ? task : NULL, 0);
14647
14648	mutex_unlock(&ctx->mutex);
14649
14650	if (!exit) {
14651	/*
14652	* perf_event_release_kernel() could still have a reference on
14653	* this context. In that case we must wait for these events to
14654	* have been freed (in particular all their references to this
14655	* task must've been dropped).
14656	*
14657	* Without this copy_process() will unconditionally free this
14658	* task (irrespective of its reference count) and
14659	* _free_event()'s put_task_struct(event->hw.target) will be a
14660	* use-after-free.
14661	*
14662	* Wait for all events to drop their context reference.
14663	*/
14664	wait_var_event(&ctx->refcount,
14665	refcount_read(&ctx->refcount) == 1);
14666	}
14667	put_ctx(ctx);
14668	}
14669
14670	/*
14671	* When a task exits, feed back event values to parent events.
14672	*
14673	* Can be called with exec_update_lock held when called from
14674	* setup_new_exec().
14675	*/
14676	void perf_event_exit_task(struct task_struct *task)
14677	{
14678	struct perf_event event, tmp;
14679
14680	WARN_ON_ONCE(task != current);
14681
14682	mutex_lock(&task->perf_event_mutex);
14683	list_for_each_entry_safe(event, tmp, &task->perf_event_list,
14684	owner_entry) {
14685	list_del_init(&event->owner_entry);
14686
14687	/*
14688	* Ensure the list deletion is visible before we clear
14689	* the owner, closes a race against perf_release() where
14690	* we need to serialize on the owner->perf_event_mutex.
14691	*/
14692	smp_store_release(&event->owner, NULL);
14693	}
14694	mutex_unlock(&task->perf_event_mutex);
14695
14696	perf_event_exit_task_context(task, true);
14697
14698	/*
14699	* The perf_event_exit_task_context calls perf_event_task
14700	* with task's task_ctx, which generates EXIT events for
14701	* task contexts and sets task->perf_event_ctxp[] to NULL.
14702	* At this point we need to send EXIT events to cpu contexts.
14703	*/
14704	perf_event_task(task, NULL, 0);
14705
14706	/*
14707	* Detach the perf_ctx_data for the system-wide event.
14708	*
14709	* Done without holding global_ctx_data_rwsem; typically
14710	* attach_global_ctx_data() will skip over this task, but otherwise
14711	* attach_task_ctx_data() will observe PF_EXITING.
14712	*/
14713	detach_task_ctx_data(task);
14714	}
14715
14716	/*
14717	* Free a context as created by inheritance by perf_event_init_task() below,
14718	* used by fork() in case of fail.
14719	*
14720	* Even though the task has never lived, the context and events have been
14721	* exposed through the child_list, so we must take care tearing it all down.
14722	*/
14723	void perf_event_free_task(struct task_struct *task)
14724	{
14725	perf_event_exit_task_context(task, false);
14726	}
14727
14728	void perf_event_delayed_put(struct task_struct *task)
14729	{
14730	WARN_ON_ONCE(task->perf_event_ctxp);
14731	}
14732
14733	struct file *perf_event_get(unsigned int fd)
14734	{
14735	struct file *file = fget(fd);
14736	if (!file)
14737	return ERR_PTR(-EBADF);
14738
14739	if (file->f_op != &perf_fops) {
14740	fput(file);
14741	return ERR_PTR(-EBADF);
14742	}
14743
14744	return file;
14745	}
14746
14747	const struct perf_event perf_get_event(struct file file)
14748	{
14749	if (file->f_op != &perf_fops)
14750	return ERR_PTR(-EINVAL);
14751
14752	return file->private_data;
14753	}
14754
14755	const struct perf_event_attr perf_event_attrs(struct perf_event event)
14756	{
14757	if (!event)
14758	return ERR_PTR(-EINVAL);
14759
14760	return &event->attr;
14761	}
14762
14763	int perf_allow_kernel(void)
14764	{
14765	if (sysctl_perf_event_paranoid > 1 && !perfmon_capable())
14766	return -EACCES;
14767
14768	return security_perf_event_open(PERF_SECURITY_KERNEL);
14769	}
14770	EXPORT_SYMBOL_GPL(perf_allow_kernel);
14771
14772	int perf_allow_cpu(void)
14773	{
14774	if (sysctl_perf_event_paranoid > 0 && !perfmon_capable())
14775	return -EACCES;
14776
14777	return security_perf_event_open(PERF_SECURITY_CPU);
14778	}
14779	EXPORT_SYMBOL_GPL(perf_allow_cpu);
14780
14781	int perf_allow_tracepoint(void)
14782	{
14783	if (sysctl_perf_event_paranoid > -1 && !perfmon_capable())
14784	return -EPERM;
14785
14786	return security_perf_event_open(PERF_SECURITY_TRACEPOINT);
14787	}
14788	EXPORT_SYMBOL_GPL(perf_allow_tracepoint);
14789
14790	/*
14791	* Inherit an event from parent task to child task.
14792	*
14793	* Returns:
14794	* - valid pointer on success
14795	* - NULL for orphaned events
14796	* - IS_ERR() on error
14797	*/
14798	static struct perf_event *
14799	inherit_event(struct perf_event *parent_event,
14800	struct task_struct *parent,
14801	struct perf_event_context *parent_ctx,
14802	struct task_struct *child,
14803	struct perf_event *group_leader,
14804	struct perf_event_context *child_ctx)
14805	{
14806	enum perf_event_state parent_state = parent_event->state;
14807	struct perf_event_pmu_context *pmu_ctx;
14808	struct perf_event *child_event;
14809	unsigned long flags;
14810
14811	/*
14812	* Instead of creating recursive hierarchies of events,
14813	* we link inherited events back to the original parent,
14814	* which has a filp for sure, which we use as the reference
14815	* count:
14816	*/
14817	if (parent_event->parent)
14818	parent_event = parent_event->parent;
14819
14820	if (parent_event->state <= PERF_EVENT_STATE_REVOKED)
14821	return NULL;
14822
14823	/*
14824	* Event creation should be under SRCU, see perf_pmu_unregister().
14825	*/
14826	guard(srcu)(&pmus_srcu);
14827
14828	child_event = perf_event_alloc(&parent_event->attr,
14829	parent_event->cpu,
14830	child,
14831	group_leader, parent_event,
14832	NULL, NULL, -1);
14833	if (IS_ERR(child_event))
14834	return child_event;
14835
14836	get_ctx(child_ctx);
14837	child_event->ctx = child_ctx;
14838
14839	pmu_ctx = find_get_pmu_context(parent_event->pmu_ctx->pmu, child_ctx, child_event);
14840	if (IS_ERR(pmu_ctx)) {
14841	free_event(child_event);
14842	return ERR_CAST(pmu_ctx);
14843	}
14844	child_event->pmu_ctx = pmu_ctx;
14845
14846	/*
14847	* is_orphaned_event() and list_add_tail(&parent_event->child_list)
14848	* must be under the same lock in order to serialize against
14849	* perf_event_release_kernel(), such that either we must observe
14850	* is_orphaned_event() or they will observe us on the child_list.
14851	*/
14852	mutex_lock(&parent_event->child_mutex);
14853	if (is_orphaned_event(parent_event) \|\|
14854	!atomic_long_inc_not_zero(&parent_event->refcount)) {
14855	mutex_unlock(&parent_event->child_mutex);
14856	free_event(child_event);
14857	return NULL;
14858	}
14859
14860	/*
14861	* Make the child state follow the state of the parent event,
14862	* not its attr.disabled bit. We hold the parent's mutex,
14863	* so we won't race with perf_event_{en, dis}able_family.
14864	*/
14865	if (parent_state >= PERF_EVENT_STATE_INACTIVE)
14866	child_event->state = PERF_EVENT_STATE_INACTIVE;
14867	else
14868	child_event->state = PERF_EVENT_STATE_OFF;
14869
14870	if (parent_event->attr.freq) {
14871	u64 sample_period = parent_event->hw.sample_period;
14872	struct hw_perf_event *hwc = &child_event->hw;
14873
14874	hwc->sample_period = sample_period;
14875	hwc->last_period = sample_period;
14876
14877	local64_set(&hwc->period_left, sample_period);
14878	}
14879
14880	child_event->overflow_handler = parent_event->overflow_handler;
14881	child_event->overflow_handler_context
14882	= parent_event->overflow_handler_context;
14883
14884	/*
14885	* Precalculate sample_data sizes
14886	*/
14887	perf_event__header_size(child_event);
14888	perf_event__id_header_size(child_event);
14889
14890	/*
14891	* Link it up in the child's context:
14892	*/
14893	raw_spin_lock_irqsave(&child_ctx->lock, flags);
14894	add_event_to_ctx(child_event, child_ctx);
14895	child_event->attach_state \|= PERF_ATTACH_CHILD;
14896	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
14897
14898	/*
14899	* Link this into the parent event's child list
14900	*/
14901	list_add_tail(&child_event->child_list, &parent_event->child_list);
14902	mutex_unlock(&parent_event->child_mutex);
14903
14904	return child_event;
14905	}
14906
14907	/*
14908	* Inherits an event group.
14909	*
14910	* This will quietly suppress orphaned events; !inherit_event() is not an error.
14911	* This matches with perf_event_release_kernel() removing all child events.
14912	*
14913	* Returns:
14914	* - 0 on success
14915	* - <0 on error
14916	*/
14917	static int inherit_group(struct perf_event *parent_event,
14918	struct task_struct *parent,
14919	struct perf_event_context *parent_ctx,
14920	struct task_struct *child,
14921	struct perf_event_context *child_ctx)
14922	{
14923	struct perf_event *leader;
14924	struct perf_event *sub;
14925	struct perf_event *child_ctr;
14926
14927	leader = inherit_event(parent_event, parent, parent_ctx,
14928	child, NULL, child_ctx);
14929	if (IS_ERR(leader))
14930	return PTR_ERR(leader);
14931	/*
14932	* @leader can be NULL here because of is_orphaned_event(). In this
14933	* case inherit_event() will create individual events, similar to what
14934	* perf_group_detach() would do anyway.
14935	*/
14936	for_each_sibling_event(sub, parent_event) {
14937	child_ctr = inherit_event(sub, parent, parent_ctx,
14938	child, leader, child_ctx);
14939	if (IS_ERR(child_ctr))
14940	return PTR_ERR(child_ctr);
14941
14942	if (sub->aux_event == parent_event && child_ctr &&
14943	!perf_get_aux_event(child_ctr, leader))
14944	return -EINVAL;
14945	}
14946	if (leader)
14947	leader->group_generation = parent_event->group_generation;
14948	return 0;
14949	}
14950
14951	/*
14952	* Creates the child task context and tries to inherit the event-group.
14953	*
14954	* Clears @inherited_all on !attr.inherited or error. Note that we'll leave
14955	* inherited_all set when we 'fail' to inherit an orphaned event; this is
14956	* consistent with perf_event_release_kernel() removing all child events.
14957	*
14958	* Returns:
14959	* - 0 on success
14960	* - <0 on error
14961	*/
14962	static int
14963	inherit_task_group(struct perf_event event, struct task_struct parent,
14964	struct perf_event_context *parent_ctx,
14965	struct task_struct *child,
14966	u64 clone_flags, int *inherited_all)
14967	{
14968	struct perf_event_context *child_ctx;
14969	int ret;
14970
14971	if (!event->attr.inherit \|\|
14972	(event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) \|\|
14973	/* Do not inherit if sigtrap and signal handlers were cleared. */
14974	(event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {
14975	*inherited_all = 0;
14976	return 0;
14977	}
14978
14979	child_ctx = child->perf_event_ctxp;
14980	if (!child_ctx) {
14981	/*
14982	* This is executed from the parent task context, so
14983	* inherit events that have been marked for cloning.
14984	* First allocate and initialize a context for the
14985	* child.
14986	*/
14987	child_ctx = alloc_perf_context(child);
14988	if (!child_ctx)
14989	return -ENOMEM;
14990
14991	child->perf_event_ctxp = child_ctx;
14992	}
14993
14994	ret = inherit_group(event, parent, parent_ctx, child, child_ctx);
14995	if (ret)
14996	*inherited_all = 0;
14997
14998	return ret;
14999	}
15000
15001	/*
15002	* Initialize the perf_event context in task_struct
15003	*/
15004	static int perf_event_init_context(struct task_struct *child, u64 clone_flags)
15005	{
15006	struct perf_event_context child_ctx, parent_ctx;
15007	struct perf_event_context *cloned_ctx;
15008	struct perf_event *event;
15009	struct task_struct *parent = current;
15010	int inherited_all = 1;
15011	unsigned long flags;
15012	int ret = 0;
15013
15014	if (likely(!parent->perf_event_ctxp))
15015	return 0;
15016
15017	/*
15018	* If the parent's context is a clone, pin it so it won't get
15019	* swapped under us.
15020	*/
15021	parent_ctx = perf_pin_task_context(parent);
15022	if (!parent_ctx)
15023	return 0;
15024
15025	/*
15026	* No need to check if parent_ctx != NULL here; since we saw
15027	* it non-NULL earlier, the only reason for it to become NULL
15028	* is if we exit, and since we're currently in the middle of
15029	* a fork we can't be exiting at the same time.
15030	*/
15031
15032	/*
15033	* Lock the parent list. No need to lock the child - not PID
15034	* hashed yet and not running, so nobody can access it.
15035	*/
15036	mutex_lock(&parent_ctx->mutex);
15037
15038	/*
15039	* We dont have to disable NMIs - we are only looking at
15040	* the list, not manipulating it:
15041	*/
15042	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {
15043	ret = inherit_task_group(event, parent, parent_ctx,
15044	child, clone_flags, &inherited_all);
15045	if (ret)
15046	goto out_unlock;
15047	}
15048
15049	/*
15050	* We can't hold ctx->lock when iterating the ->flexible_group list due
15051	* to allocations, but we need to prevent rotation because
15052	* rotate_ctx() will change the list from interrupt context.
15053	*/
15054	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
15055	parent_ctx->rotate_disable = 1;
15056	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
15057
15058	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {
15059	ret = inherit_task_group(event, parent, parent_ctx,
15060	child, clone_flags, &inherited_all);
15061	if (ret)
15062	goto out_unlock;
15063	}
15064
15065	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
15066	parent_ctx->rotate_disable = 0;
15067
15068	child_ctx = child->perf_event_ctxp;
15069
15070	if (child_ctx && inherited_all) {
15071	/*
15072	* Mark the child context as a clone of the parent
15073	* context, or of whatever the parent is a clone of.
15074	*
15075	* Note that if the parent is a clone, the holding of
15076	* parent_ctx->lock avoids it from being uncloned.
15077	*/
15078	cloned_ctx = parent_ctx->parent_ctx;
15079	if (cloned_ctx) {
15080	child_ctx->parent_ctx = cloned_ctx;
15081	child_ctx->parent_gen = parent_ctx->parent_gen;
15082	} else {
15083	child_ctx->parent_ctx = parent_ctx;
15084	child_ctx->parent_gen = parent_ctx->generation;
15085	}
15086	get_ctx(child_ctx->parent_ctx);
15087	}
15088
15089	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
15090	out_unlock:
15091	mutex_unlock(&parent_ctx->mutex);
15092
15093	perf_unpin_context(parent_ctx);
15094	put_ctx(parent_ctx);
15095
15096	return ret;
15097	}
15098
15099	/*
15100	* Initialize the perf_event context in task_struct
15101	*/
15102	int perf_event_init_task(struct task_struct *child, u64 clone_flags)
15103	{
15104	int ret;
15105
15106	memset(child->perf_recursion, 0, sizeof(child->perf_recursion));
15107	child->perf_event_ctxp = NULL;
15108	mutex_init(&child->perf_event_mutex);
15109	INIT_LIST_HEAD(&child->perf_event_list);
15110	child->perf_ctx_data = NULL;
15111
15112	ret = perf_event_init_context(child, clone_flags);
15113	if (ret) {
15114	perf_event_free_task(child);
15115	return ret;
15116	}
15117
15118	return 0;
15119	}
15120
15121	static void __init perf_event_init_all_cpus(void)
15122	{
15123	struct swevent_htable *swhash;
15124	struct perf_cpu_context *cpuctx;
15125	int cpu;
15126
15127	zalloc_cpumask_var(&perf_online_mask, GFP_KERNEL);
15128	zalloc_cpumask_var(&perf_online_core_mask, GFP_KERNEL);
15129	zalloc_cpumask_var(&perf_online_die_mask, GFP_KERNEL);
15130	zalloc_cpumask_var(&perf_online_cluster_mask, GFP_KERNEL);
15131	zalloc_cpumask_var(&perf_online_pkg_mask, GFP_KERNEL);
15132	zalloc_cpumask_var(&perf_online_sys_mask, GFP_KERNEL);
15133
15134
15135	for_each_possible_cpu(cpu) {
15136	swhash = &per_cpu(swevent_htable, cpu);
15137	mutex_init(&swhash->hlist_mutex);
15138
15139	INIT_LIST_HEAD(&per_cpu(pmu_sb_events.list, cpu));
15140	raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu));
15141
15142	INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));
15143
15144	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
15145	__perf_event_init_context(&cpuctx->ctx);
15146	lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
15147	lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
15148	cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);
15149	cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);
15150	cpuctx->heap = cpuctx->heap_default;
15151	}
15152	}
15153
15154	static void perf_swevent_init_cpu(unsigned int cpu)
15155	{
15156	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
15157
15158	mutex_lock(&swhash->hlist_mutex);
15159	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {
15160	struct swevent_hlist *hlist;
15161
15162	hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
15163	WARN_ON(!hlist);
15164	rcu_assign_pointer(swhash->swevent_hlist, hlist);
15165	}
15166	mutex_unlock(&swhash->hlist_mutex);
15167	}
15168
15169	#if defined CONFIG_HOTPLUG_CPU \|\| defined CONFIG_KEXEC_CORE
15170	static void __perf_event_exit_context(void *__info)
15171	{
15172	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
15173	struct perf_event_context *ctx = __info;
15174	struct perf_event *event;
15175
15176	raw_spin_lock(&ctx->lock);
15177	ctx_sched_out(ctx, NULL, EVENT_TIME);
15178	list_for_each_entry(event, &ctx->event_list, event_entry)
15179	__perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);
15180	raw_spin_unlock(&ctx->lock);
15181	}
15182
15183	static void perf_event_clear_cpumask(unsigned int cpu)
15184	{
15185	int target[PERF_PMU_MAX_SCOPE];
15186	unsigned int scope;
15187	struct pmu *pmu;
15188
15189	cpumask_clear_cpu(cpu, perf_online_mask);
15190
15191	for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) {
15192	const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(scope, cpu);
15193	struct cpumask *pmu_cpumask = perf_scope_cpumask(scope);
15194
15195	target[scope] = -1;
15196	if (WARN_ON_ONCE(!pmu_cpumask \|\| !cpumask))
15197	continue;
15198
15199	if (!cpumask_test_and_clear_cpu(cpu, pmu_cpumask))
15200	continue;
15201	target[scope] = cpumask_any_but(cpumask, cpu);
15202	if (target[scope] < nr_cpu_ids)
15203	cpumask_set_cpu(target[scope], pmu_cpumask);
15204	}
15205
15206	/* migrate */
15207	list_for_each_entry(pmu, &pmus, entry) {
15208	if (pmu->scope == PERF_PMU_SCOPE_NONE \|\|
15209	WARN_ON_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE))
15210	continue;
15211
15212	if (target[pmu->scope] >= 0 && target[pmu->scope] < nr_cpu_ids)
15213	perf_pmu_migrate_context(pmu, cpu, target[pmu->scope]);
15214	}
15215	}
15216
15217	static void perf_event_exit_cpu_context(int cpu)
15218	{
15219	struct perf_cpu_context *cpuctx;
15220	struct perf_event_context *ctx;
15221
15222	// XXX simplify cpuctx->online
15223	mutex_lock(&pmus_lock);
15224	/*
15225	* Clear the cpumasks, and migrate to other CPUs if possible.
15226	* Must be invoked before the __perf_event_exit_context.
15227	*/
15228	perf_event_clear_cpumask(cpu);
15229	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
15230	ctx = &cpuctx->ctx;
15231
15232	mutex_lock(&ctx->mutex);
15233	if (ctx->nr_events)
15234	smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
15235	cpuctx->online = 0;
15236	mutex_unlock(&ctx->mutex);
15237	mutex_unlock(&pmus_lock);
15238	}
15239	#else
15240
15241	static void perf_event_exit_cpu_context(int cpu) { }
15242
15243	#endif
15244
15245	static void perf_event_setup_cpumask(unsigned int cpu)
15246	{
15247	struct cpumask *pmu_cpumask;
15248	unsigned int scope;
15249
15250	/*
15251	* Early boot stage, the cpumask hasn't been set yet.
15252	* The perf_online_<domain>_masks includes the first CPU of each domain.
15253	* Always unconditionally set the boot CPU for the perf_online_<domain>_masks.
15254	*/
15255	if (cpumask_empty(perf_online_mask)) {
15256	for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) {
15257	pmu_cpumask = perf_scope_cpumask(scope);
15258	if (WARN_ON_ONCE(!pmu_cpumask))
15259	continue;
15260	cpumask_set_cpu(cpu, pmu_cpumask);
15261	}
15262	goto end;
15263	}
15264
15265	for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) {
15266	const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(scope, cpu);
15267
15268	pmu_cpumask = perf_scope_cpumask(scope);
15269
15270	if (WARN_ON_ONCE(!pmu_cpumask \|\| !cpumask))
15271	continue;
15272
15273	if (!cpumask_empty(cpumask) &&
15274	cpumask_any_and(pmu_cpumask, cpumask) >= nr_cpu_ids)
15275	cpumask_set_cpu(cpu, pmu_cpumask);
15276	}
15277	end:
15278	cpumask_set_cpu(cpu, perf_online_mask);
15279	}
15280
15281	int perf_event_init_cpu(unsigned int cpu)
15282	{
15283	struct perf_cpu_context *cpuctx;
15284	struct perf_event_context *ctx;
15285
15286	perf_swevent_init_cpu(cpu);
15287
15288	mutex_lock(&pmus_lock);
15289	perf_event_setup_cpumask(cpu);
15290	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
15291	ctx = &cpuctx->ctx;
15292
15293	mutex_lock(&ctx->mutex);
15294	cpuctx->online = 1;
15295	mutex_unlock(&ctx->mutex);
15296	mutex_unlock(&pmus_lock);
15297
15298	return 0;
15299	}
15300
15301	int perf_event_exit_cpu(unsigned int cpu)
15302	{
15303	perf_event_exit_cpu_context(cpu);
15304	return 0;
15305	}
15306
15307	static int
15308	perf_reboot(struct notifier_block notifier, unsigned long val, void v)
15309	{
15310	int cpu;
15311
15312	for_each_online_cpu(cpu)
15313	perf_event_exit_cpu(cpu);
15314
15315	return NOTIFY_OK;
15316	}
15317
15318	/*
15319	* Run the perf reboot notifier at the very last possible moment so that
15320	* the generic watchdog code runs as long as possible.
15321	*/
15322	static struct notifier_block perf_reboot_notifier = {
15323	.notifier_call = perf_reboot,
15324	.priority = INT_MIN,
15325	};
15326
15327	void __init perf_event_init(void)
15328	{
15329	int ret;
15330
15331	idr_init(&pmu_idr);
15332
15333	unwind_deferred_init(&perf_unwind_work,
15334	perf_unwind_deferred_callback);
15335
15336	perf_event_init_all_cpus();
15337	init_srcu_struct(&pmus_srcu);
15338	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
15339	perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1);
15340	perf_pmu_register(&perf_task_clock, "task_clock", -1);
15341	perf_tp_register();
15342	perf_event_init_cpu(smp_processor_id());
15343	register_reboot_notifier(&perf_reboot_notifier);
15344
15345	ret = init_hw_breakpoint();
15346	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
15347
15348	perf_event_cache = KMEM_CACHE(perf_event, SLAB_PANIC);
15349
15350	/*
15351	* Build time assertion that we keep the data_head at the intended
15352	* location. IOW, validation we got the __reserved[] size right.
15353	*/
15354	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
15355	!= 1024);
15356	}
15357
15358	ssize_t perf_event_sysfs_show(struct device dev, struct device_attribute attr,
15359	char *page)
15360	{
15361	struct perf_pmu_events_attr *pmu_attr =
15362	container_of(attr, struct perf_pmu_events_attr, attr);
15363
15364	if (pmu_attr->event_str)
15365	return sprintf(page, "%s\n", pmu_attr->event_str);
15366
15367	return 0;
15368	}
15369	EXPORT_SYMBOL_GPL(perf_event_sysfs_show);
15370
15371	static int __init perf_event_sysfs_init(void)
15372	{
15373	struct pmu *pmu;
15374	int ret;
15375
15376	mutex_lock(&pmus_lock);
15377
15378	ret = bus_register(&pmu_bus);
15379	if (ret)
15380	goto unlock;
15381
15382	list_for_each_entry(pmu, &pmus, entry) {
15383	if (pmu->dev)
15384	continue;
15385
15386	ret = pmu_dev_alloc(pmu);
15387	WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
15388	}
15389	pmu_bus_running = 1;
15390	ret = 0;
15391
15392	unlock:
15393	mutex_unlock(&pmus_lock);
15394
15395	return ret;
15396	}
15397	device_initcall(perf_event_sysfs_init);
15398
15399	#ifdef CONFIG_CGROUP_PERF
15400	static struct cgroup_subsys_state *
15401	perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
15402	{
15403	struct perf_cgroup *jc;
15404
15405	jc = kzalloc_obj(*jc);
15406	if (!jc)
15407	return ERR_PTR(-ENOMEM);
15408
15409	jc->info = alloc_percpu(struct perf_cgroup_info);
15410	if (!jc->info) {
15411	kfree(jc);
15412	return ERR_PTR(-ENOMEM);
15413	}
15414
15415	return &jc->css;
15416	}
15417
15418	static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
15419	{
15420	struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
15421
15422	free_percpu(jc->info);
15423	kfree(jc);
15424	}
15425
15426	static int perf_cgroup_css_online(struct cgroup_subsys_state *css)
15427	{
15428	perf_event_cgroup(css->cgroup);
15429	return 0;
15430	}
15431
15432	static int __perf_cgroup_move(void *info)
15433	{
15434	struct task_struct *task = info;
15435
15436	preempt_disable();
15437	perf_cgroup_switch(task);
15438	preempt_enable();
15439
15440	return 0;
15441	}
15442
15443	static void perf_cgroup_attach(struct cgroup_taskset *tset)
15444	{
15445	struct task_struct *task;
15446	struct cgroup_subsys_state *css;
15447
15448	cgroup_taskset_for_each(task, css, tset)
15449	task_function_call(task, __perf_cgroup_move, task);
15450	}
15451
15452	struct cgroup_subsys perf_event_cgrp_subsys = {
15453	.css_alloc = perf_cgroup_css_alloc,
15454	.css_free = perf_cgroup_css_free,
15455	.css_online = perf_cgroup_css_online,
15456	.attach = perf_cgroup_attach,
15457	/*
15458	* Implicitly enable on dfl hierarchy so that perf events can
15459	* always be filtered by cgroup2 path as long as perf_event
15460	* controller is not mounted on a legacy hierarchy.
15461	*/
15462	.implicit_on_dfl = true,
15463	.threaded = true,
15464	};
15465	#endif /* CONFIG_CGROUP_PERF */
15466
15467	DEFINE_STATIC_CALL_RET0(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);