linux - kernel/kprobes.c

3089 lines · 76.2 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Kernel Probes (KProbes)
4	*
5	* Copyright (C) IBM Corporation, 2002, 2004
6	*
7	* 2002-Oct Created by Vamsi Krishna S <[email protected]> Kernel
8	* Probes initial implementation (includes suggestions from
9	* Rusty Russell).
10	* 2004-Aug Updated by Prasanna S Panchamukhi <[email protected]> with
11	* hlists and exceptions notifier as suggested by Andi Kleen.
12	* 2004-July Suparna Bhattacharya <[email protected]> added jumper probes
13	* interface to access function arguments.
14	* 2004-Sep Prasanna S Panchamukhi <[email protected]> Changed Kprobes
15	* exceptions notifier to be first on the priority list.
16	* 2005-May Hien Nguyen <[email protected]>, Jim Keniston
17	* <[email protected]> and Prasanna S Panchamukhi
18	* <[email protected]> added function-return probes.
19	*/
20
21	#define pr_fmt(fmt) "kprobes: " fmt
22
23	#include <linux/kprobes.h>
24	#include <linux/hash.h>
25	#include <linux/init.h>
26	#include <linux/slab.h>
27	#include <linux/stddef.h>
28	#include <linux/export.h>
29	#include <linux/kallsyms.h>
30	#include <linux/freezer.h>
31	#include <linux/seq_file.h>
32	#include <linux/debugfs.h>
33	#include <linux/sysctl.h>
34	#include <linux/kdebug.h>
35	#include <linux/kthread.h>
36	#include <linux/memory.h>
37	#include <linux/ftrace.h>
38	#include <linux/cpu.h>
39	#include <linux/jump_label.h>
40	#include <linux/static_call.h>
41	#include <linux/perf_event.h>
42	#include <linux/execmem.h>
43	#include <linux/cleanup.h>
44	#include <linux/wait.h>
45	#include <linux/wait_bit.h>
46
47	#include <asm/sections.h>
48	#include <asm/cacheflush.h>
49	#include <asm/errno.h>
50	#include <linux/uaccess.h>
51
52	#define KPROBE_HASH_BITS 6
53	#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
54
55	#if !defined(CONFIG_OPTPROBES) \|\| !defined(CONFIG_SYSCTL)
56	#define kprobe_sysctls_init() do { } while (0)
57	#endif
58
59	static int kprobes_initialized;
60	/* kprobe_table can be accessed by
61	* - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held.
62	* Or
63	* - RCU hlist traversal under disabling preempt (breakpoint handlers)
64	*/
65	static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
66
67	/* NOTE: change this value only with 'kprobe_mutex' held */
68	static bool kprobes_all_disarmed;
69
70	/* This protects 'kprobe_table' and 'optimizing_list' */
71	static DEFINE_MUTEX(kprobe_mutex);
72	static DEFINE_PER_CPU(struct kprobe *, kprobe_instance);
73
74	kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
75	unsigned int __unused)
76	{
77	return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
78	}
79
80	/*
81	* Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where
82	* kprobes can not probe.
83	*/
84	static LIST_HEAD(kprobe_blacklist);
85
86	#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
87	/*
88	* 'kprobe::ainsn.insn' points to the copy of the instruction to be
89	* single-stepped. x86_64, POWER4 and above have no-exec support and
90	* stepping on the instruction on a vmalloced/kmalloced/data page
91	* is a recipe for disaster
92	*/
93	struct kprobe_insn_page {
94	struct list_head list;
95	kprobe_opcode_t insns; / Page of instruction slots */
96	struct kprobe_insn_cache *cache;
97	int nused;
98	int ngarbage;
99	char slot_used[];
100	};
101
102	static int slots_per_page(struct kprobe_insn_cache *c)
103	{
104	return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
105	}
106
107	enum kprobe_slot_state {
108	SLOT_CLEAN = 0,
109	SLOT_DIRTY = 1,
110	SLOT_USED = 2,
111	};
112
113	void __weak *alloc_insn_page(void)
114	{
115	/*
116	* Use execmem_alloc() so this page is within +/- 2GB of where the
117	* kernel image and loaded module images reside. This is required
118	* for most of the architectures.
119	* (e.g. x86-64 needs this to handle the %rip-relative fixups.)
120	*/
121	return execmem_alloc(EXECMEM_KPROBES, PAGE_SIZE);
122	}
123
124	static void free_insn_page(void *page)
125	{
126	execmem_free(page);
127	}
128
129	struct kprobe_insn_cache kprobe_insn_slots = {
130	.mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
131	.alloc = alloc_insn_page,
132	.free = free_insn_page,
133	.sym = KPROBE_INSN_PAGE_SYM,
134	.pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
135	.insn_size = MAX_INSN_SIZE,
136	.nr_garbage = 0,
137	};
138	static int collect_garbage_slots(struct kprobe_insn_cache *c);
139
140	/**
141	* __get_insn_slot - Find a slot on an executable page for an instruction.
142	* @c: Pointer to kprobe instruction cache
143	*
144	* Description: Locates available slot on existing executable pages,
145	* allocates an executable page if there's no room on existing ones.
146	* Return: Pointer to instruction slot on success, NULL on failure.
147	*/
148	kprobe_opcode_t __get_insn_slot(struct kprobe_insn_cache c)
149	{
150	struct kprobe_insn_page *kip;
151
152	/* Since the slot array is not protected by rcu, we need a mutex */
153	guard(mutex)(&c->mutex);
154	do {
155	guard(rcu)();
156	list_for_each_entry_rcu(kip, &c->pages, list) {
157	if (kip->nused < slots_per_page(c)) {
158	int i;
159
160	for (i = 0; i < slots_per_page(c); i++) {
161	if (kip->slot_used[i] == SLOT_CLEAN) {
162	kip->slot_used[i] = SLOT_USED;
163	kip->nused++;
164	return kip->insns + (i * c->insn_size);
165	}
166	}
167	/* kip->nused is broken. Fix it. */
168	kip->nused = slots_per_page(c);
169	WARN_ON(1);
170	}
171	}
172	/* If there are any garbage slots, collect it and try again. */
173	} while (c->nr_garbage && collect_garbage_slots(c) == 0);
174
175	/* All out of space. Need to allocate a new page. */
176	kip = kmalloc_flex(*kip, slot_used, slots_per_page(c));
177	if (!kip)
178	return NULL;
179
180	kip->insns = c->alloc();
181	if (!kip->insns) {
182	kfree(kip);
183	return NULL;
184	}
185	INIT_LIST_HEAD(&kip->list);
186	memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
187	kip->slot_used[0] = SLOT_USED;
188	kip->nused = 1;
189	kip->ngarbage = 0;
190	kip->cache = c;
191	list_add_rcu(&kip->list, &c->pages);
192
193	/* Record the perf ksymbol register event after adding the page */
194	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns,
195	PAGE_SIZE, false, c->sym);
196
197	return kip->insns;
198	}
199
200	/* Return true if all garbages are collected, otherwise false. */
201	static bool collect_one_slot(struct kprobe_insn_page *kip, int idx)
202	{
203	kip->slot_used[idx] = SLOT_CLEAN;
204	kip->nused--;
205	if (kip->nused != 0)
206	return false;
207
208	/*
209	* Page is no longer in use. Free it unless
210	* it's the last one. We keep the last one
211	* so as not to have to set it up again the
212	* next time somebody inserts a probe.
213	*/
214	if (!list_is_singular(&kip->list)) {
215	/*
216	* Record perf ksymbol unregister event before removing
217	* the page.
218	*/
219	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
220	(unsigned long)kip->insns, PAGE_SIZE, true,
221	kip->cache->sym);
222	list_del_rcu(&kip->list);
223	synchronize_rcu();
224	kip->cache->free(kip->insns);
225	kfree(kip);
226	}
227	return true;
228	}
229
230	static int collect_garbage_slots(struct kprobe_insn_cache *c)
231	{
232	struct kprobe_insn_page kip, next;
233
234	/* Ensure no-one is interrupted on the garbages */
235	synchronize_rcu();
236
237	list_for_each_entry_safe(kip, next, &c->pages, list) {
238	int i;
239
240	if (kip->ngarbage == 0)
241	continue;
242	kip->ngarbage = 0; /* we will collect all garbages */
243	for (i = 0; i < slots_per_page(c); i++) {
244	if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
245	break;
246	}
247	}
248	c->nr_garbage = 0;
249	return 0;
250	}
251
252	static long __find_insn_page(struct kprobe_insn_cache *c,
253	kprobe_opcode_t slot, struct kprobe_insn_page *pkip)
254	{
255	struct kprobe_insn_page *kip = NULL;
256	long idx;
257
258	guard(rcu)();
259	list_for_each_entry_rcu(kip, &c->pages, list) {
260	idx = ((long)slot - (long)kip->insns) /
261	(c->insn_size * sizeof(kprobe_opcode_t));
262	if (idx >= 0 && idx < slots_per_page(c)) {
263	*pkip = kip;
264	return idx;
265	}
266	}
267	/* Could not find this slot. */
268	WARN_ON(1);
269	*pkip = NULL;
270	return -1;
271	}
272
273	void __free_insn_slot(struct kprobe_insn_cache *c,
274	kprobe_opcode_t *slot, int dirty)
275	{
276	struct kprobe_insn_page *kip = NULL;
277	long idx;
278
279	guard(mutex)(&c->mutex);
280	idx = __find_insn_page(c, slot, &kip);
281	/* Mark and sweep: this may sleep */
282	if (kip) {
283	/* Check double free */
284	WARN_ON(kip->slot_used[idx] != SLOT_USED);
285	if (dirty) {
286	kip->slot_used[idx] = SLOT_DIRTY;
287	kip->ngarbage++;
288	if (++c->nr_garbage > slots_per_page(c))
289	collect_garbage_slots(c);
290	} else {
291	collect_one_slot(kip, idx);
292	}
293	}
294	}
295
296	/*
297	* Check given address is on the page of kprobe instruction slots.
298	* This will be used for checking whether the address on a stack
299	* is on a text area or not.
300	*/
301	bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
302	{
303	struct kprobe_insn_page *kip;
304	bool ret = false;
305
306	rcu_read_lock();
307	list_for_each_entry_rcu(kip, &c->pages, list) {
308	if (addr >= (unsigned long)kip->insns &&
309	addr < (unsigned long)kip->insns + PAGE_SIZE) {
310	ret = true;
311	break;
312	}
313	}
314	rcu_read_unlock();
315
316	return ret;
317	}
318
319	int kprobe_cache_get_kallsym(struct kprobe_insn_cache c, unsigned int symnum,
320	unsigned long value, char type, char *sym)
321	{
322	struct kprobe_insn_page *kip;
323	int ret = -ERANGE;
324
325	rcu_read_lock();
326	list_for_each_entry_rcu(kip, &c->pages, list) {
327	if ((*symnum)--)
328	continue;
329	strscpy(sym, c->sym, KSYM_NAME_LEN);
330	*type = 't';
331	*value = (unsigned long)kip->insns;
332	ret = 0;
333	break;
334	}
335	rcu_read_unlock();
336
337	return ret;
338	}
339
340	#ifdef CONFIG_OPTPROBES
341	void __weak *alloc_optinsn_page(void)
342	{
343	return alloc_insn_page();
344	}
345
346	void __weak free_optinsn_page(void *page)
347	{
348	free_insn_page(page);
349	}
350
351	/* For optimized_kprobe buffer */
352	struct kprobe_insn_cache kprobe_optinsn_slots = {
353	.mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
354	.alloc = alloc_optinsn_page,
355	.free = free_optinsn_page,
356	.sym = KPROBE_OPTINSN_PAGE_SYM,
357	.pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
358	/* .insn_size is initialized later */
359	.nr_garbage = 0,
360	};
361	#endif /* CONFIG_OPTPROBES */
362	#endif /* __ARCH_WANT_KPROBES_INSN_SLOT */
363
364	/* We have preemption disabled.. so it is safe to use __ versions */
365	static inline void set_kprobe_instance(struct kprobe *kp)
366	{
367	__this_cpu_write(kprobe_instance, kp);
368	}
369
370	static inline void reset_kprobe_instance(void)
371	{
372	__this_cpu_write(kprobe_instance, NULL);
373	}
374
375	/*
376	* This routine is called either:
377	* - under the 'kprobe_mutex' - during kprobe_[un]register().
378	* OR
379	* - with preemption disabled - from architecture specific code.
380	*/
381	struct kprobe get_kprobe(void addr)
382	{
383	struct hlist_head *head;
384	struct kprobe *p;
385
386	head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
387	hlist_for_each_entry_rcu(p, head, hlist,
388	lockdep_is_held(&kprobe_mutex)) {
389	if (p->addr == addr)
390	return p;
391	}
392
393	return NULL;
394	}
395	NOKPROBE_SYMBOL(get_kprobe);
396
397	static int aggr_pre_handler(struct kprobe p, struct pt_regs regs);
398
399	/* Return true if 'p' is an aggregator */
400	static inline bool kprobe_aggrprobe(struct kprobe *p)
401	{
402	return p->pre_handler == aggr_pre_handler;
403	}
404
405	/* Return true if 'p' is unused */
406	static inline bool kprobe_unused(struct kprobe *p)
407	{
408	return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
409	list_empty(&p->list);
410	}
411
412	/* Keep all fields in the kprobe consistent. */
413	static inline void copy_kprobe(struct kprobe ap, struct kprobe p)
414	{
415	memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
416	memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
417	}
418
419	#ifdef CONFIG_OPTPROBES
420	/* NOTE: This is protected by 'kprobe_mutex'. */
421	static bool kprobes_allow_optimization;
422
423	/*
424	* Call all 'kprobe::pre_handler' on the list, but ignores its return value.
425	* This must be called from arch-dep optimized caller.
426	*/
427	void opt_pre_handler(struct kprobe p, struct pt_regs regs)
428	{
429	struct kprobe *kp;
430
431	list_for_each_entry_rcu(kp, &p->list, list) {
432	if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
433	set_kprobe_instance(kp);
434	kp->pre_handler(kp, regs);
435	}
436	reset_kprobe_instance();
437	}
438	}
439	NOKPROBE_SYMBOL(opt_pre_handler);
440
441	/* Free optimized instructions and optimized_kprobe */
442	static void free_aggr_kprobe(struct kprobe *p)
443	{
444	struct optimized_kprobe *op;
445
446	op = container_of(p, struct optimized_kprobe, kp);
447	arch_remove_optimized_kprobe(op);
448	arch_remove_kprobe(p);
449	kfree(op);
450	}
451
452	/* Return true if the kprobe is ready for optimization. */
453	static inline int kprobe_optready(struct kprobe *p)
454	{
455	struct optimized_kprobe *op;
456
457	if (kprobe_aggrprobe(p)) {
458	op = container_of(p, struct optimized_kprobe, kp);
459	return arch_prepared_optinsn(&op->optinsn);
460	}
461
462	return 0;
463	}
464
465	/* Return true if the kprobe is disarmed. Note: p must be on hash list */
466	bool kprobe_disarmed(struct kprobe *p)
467	{
468	struct optimized_kprobe *op;
469
470	/* If kprobe is not aggr/opt probe, just return kprobe is disabled */
471	if (!kprobe_aggrprobe(p))
472	return kprobe_disabled(p);
473
474	op = container_of(p, struct optimized_kprobe, kp);
475
476	return kprobe_disabled(p) && list_empty(&op->list);
477	}
478
479	/* Return true if the probe is queued on (un)optimizing lists */
480	static bool kprobe_queued(struct kprobe *p)
481	{
482	struct optimized_kprobe *op;
483
484	if (kprobe_aggrprobe(p)) {
485	op = container_of(p, struct optimized_kprobe, kp);
486	if (!list_empty(&op->list))
487	return true;
488	}
489	return false;
490	}
491
492	/*
493	* Return an optimized kprobe whose optimizing code replaces
494	* instructions including 'addr' (exclude breakpoint).
495	*/
496	static struct kprobe get_optimized_kprobe(kprobe_opcode_t addr)
497	{
498	int i;
499	struct kprobe *p;
500	struct optimized_kprobe *op;
501
502	/* Don't check i == 0, since that is a breakpoint case. */
503	for (i = 1; i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) {
504	p = get_kprobe(addr - i);
505	/* A disabled probe can have prepared, but inactive, optinsns. */
506	if (!p \|\| !kprobe_optready(p) \|\| kprobe_disarmed(p))
507	continue;
508
509	op = container_of(p, struct optimized_kprobe, kp);
510	if (arch_within_optimized_kprobe(op, addr))
511	return p;
512	}
513
514	return NULL;
515	}
516
517	/* Optimization staging list, protected by 'kprobe_mutex' */
518	static LIST_HEAD(optimizing_list);
519	static LIST_HEAD(unoptimizing_list);
520	static LIST_HEAD(freeing_list);
521
522	static void optimize_kprobe(struct kprobe *p);
523	static struct task_struct *kprobe_optimizer_task;
524	static wait_queue_head_t kprobe_optimizer_wait;
525	static atomic_t optimizer_state;
526	enum {
527	OPTIMIZER_ST_IDLE = 0,
528	OPTIMIZER_ST_KICKED = 1,
529	OPTIMIZER_ST_FLUSHING = 2,
530	};
531
532	/* Bumped at the end of each kprobe_optimizer() pass, under 'kprobe_mutex' */
533	static unsigned long optimizer_passes;
534
535	#define OPTIMIZE_DELAY 5
536
537	/*
538	* Optimize (replace a breakpoint with a jump) kprobes listed on
539	* 'optimizing_list'.
540	*/
541	static void do_optimize_kprobes(void)
542	{
543	lockdep_assert_held(&text_mutex);
544	/*
545	* The optimization/unoptimization refers 'online_cpus' via
546	* stop_machine() and cpu-hotplug modifies the 'online_cpus'.
547	* And same time, 'text_mutex' will be held in cpu-hotplug and here.
548	* This combination can cause a deadlock (cpu-hotplug tries to lock
549	* 'text_mutex' but stop_machine() can not be done because
550	* the 'online_cpus' has been changed)
551	* To avoid this deadlock, caller must have locked cpu-hotplug
552	* for preventing cpu-hotplug outside of 'text_mutex' locking.
553	*/
554	lockdep_assert_cpus_held();
555
556	/* Optimization never be done when disarmed */
557	if (kprobes_all_disarmed \|\| !kprobes_allow_optimization \|\|
558	list_empty(&optimizing_list))
559	return;
560
561	arch_optimize_kprobes(&optimizing_list);
562	}
563
564	/*
565	* Unoptimize (replace a jump with a breakpoint and remove the breakpoint
566	* if need) kprobes listed on 'unoptimizing_list'.
567	*/
568	static void do_unoptimize_kprobes(void)
569	{
570	struct optimized_kprobe op, tmp;
571
572	lockdep_assert_held(&text_mutex);
573	/* See comment in do_optimize_kprobes() */
574	lockdep_assert_cpus_held();
575
576	if (!list_empty(&unoptimizing_list))
577	arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
578
579	/* Loop on 'freeing_list' for disarming and removing from kprobe hash list */
580	list_for_each_entry_safe(op, tmp, &freeing_list, list) {
581	/* Switching from detour code to origin */
582	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
583	/* Disarm probes if marked disabled and not gone */
584	if (kprobe_disabled(&op->kp) && !kprobe_gone(&op->kp))
585	arch_disarm_kprobe(&op->kp);
586	if (kprobe_unused(&op->kp)) {
587	/*
588	* Remove unused probes from hash list. After waiting
589	* for synchronization, these probes are reclaimed.
590	* (reclaiming is done by do_free_cleaned_kprobes().)
591	*/
592	hlist_del_rcu(&op->kp.hlist);
593	} else
594	list_del_init(&op->list);
595	}
596	}
597
598	/* Reclaim all kprobes on the 'freeing_list' */
599	static void do_free_cleaned_kprobes(void)
600	{
601	struct optimized_kprobe op, tmp;
602
603	list_for_each_entry_safe(op, tmp, &freeing_list, list) {
604	list_del_init(&op->list);
605	if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) {
606	/*
607	* This must not happen, but if there is a kprobe
608	* still in use, keep it on kprobes hash list.
609	*/
610	continue;
611	}
612
613	/*
614	* The aggregator was holding back another probe while it sat on the
615	* unoptimizing/freeing lists. Now that the aggregator has been fully
616	* reverted we can safely retry the optimization of that sibling.
617	*/
618
619	struct kprobe *_p = get_optimized_kprobe(op->kp.addr);
620	if (unlikely(_p))
621	optimize_kprobe(_p);
622
623	free_aggr_kprobe(&op->kp);
624	}
625	}
626
627	static void kick_kprobe_optimizer(void);
628
629	/* Kprobe jump optimizer */
630	static void kprobe_optimizer(void)
631	{
632	guard(mutex)(&kprobe_mutex);
633
634	scoped_guard(cpus_read_lock) {
635	guard(mutex)(&text_mutex);
636
637	/*
638	* Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
639	* kprobes before waiting for quiesence period.
640	*/
641	do_unoptimize_kprobes();
642
643	/*
644	* Step 2: Wait for quiesence period to ensure all potentially
645	* preempted tasks to have normally scheduled. Because optprobe
646	* may modify multiple instructions, there is a chance that Nth
647	* instruction is preempted. In that case, such tasks can return
648	* to 2nd-Nth byte of jump instruction. This wait is for avoiding it.
649	* Note that on non-preemptive kernel, this is transparently converted
650	* to synchronoze_sched() to wait for all interrupts to have completed.
651	*/
652	synchronize_rcu_tasks();
653
654	/* Step 3: Optimize kprobes after quiesence period */
655	do_optimize_kprobes();
656
657	/* Step 4: Free cleaned kprobes after quiesence period */
658	do_free_cleaned_kprobes();
659	}
660
661	/* Step 5: Wake up flushers, and kick optimizer again if needed. */
662	optimizer_passes++;
663	wake_up_var_locked(&optimizer_passes, &kprobe_mutex);
664
665	if (!list_empty(&optimizing_list) \|\| !list_empty(&unoptimizing_list))
666	kick_kprobe_optimizer(); /normal kick/
667	}
668
669	static int kprobe_optimizer_thread(void *data)
670	{
671	while (!kthread_should_stop()) {
672	/* To avoid hung_task, wait in interruptible state. */
673	wait_event_interruptible(kprobe_optimizer_wait,
674	atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE \|\|
675	kthread_should_stop());
676
677	if (kthread_should_stop())
678	break;
679
680	/*
681	* If it was a normal kick, wait for OPTIMIZE_DELAY.
682	* This wait can be interrupted by a flush request.
683	*/
684	if (atomic_read(&optimizer_state) == 1)
685	wait_event_interruptible_timeout(
686	kprobe_optimizer_wait,
687	atomic_read(&optimizer_state) == OPTIMIZER_ST_FLUSHING \|\|
688	kthread_should_stop(),
689	OPTIMIZE_DELAY);
690
691	if (kthread_should_stop())
692	break;
693
694	atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);
695
696	kprobe_optimizer();
697	}
698	return 0;
699	}
700
701	/* Start optimizer after OPTIMIZE_DELAY passed */
702	static void kick_kprobe_optimizer(void)
703	{
704	lockdep_assert_held(&kprobe_mutex);
705	if (atomic_cmpxchg(&optimizer_state,
706	OPTIMIZER_ST_IDLE, OPTIMIZER_ST_KICKED) == OPTIMIZER_ST_IDLE)
707	wake_up(&kprobe_optimizer_wait);
708	}
709
710	static void wait_for_kprobe_optimizer_locked(void)
711	{
712	lockdep_assert_held(&kprobe_mutex);
713
714	while (!list_empty(&optimizing_list) \|\| !list_empty(&unoptimizing_list)) {
715	unsigned long passes = optimizer_passes;
716
717	/*
718	* Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's
719	* idle. If it's already kicked, it will see the state change.
720	*/
721	if (atomic_xchg_acquire(&optimizer_state,
722	OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING)
723	wake_up(&kprobe_optimizer_wait);
724
725	/*
726	* kprobe_optimizer() holds 'kprobe_mutex' for a whole pass, which
727	* this drops while sleeping, so a new count means a full pass ran.
728	*/
729	wait_var_event_mutex(&optimizer_passes,
730	optimizer_passes != passes, &kprobe_mutex);
731	}
732	}
733
734	/* Wait for completing optimization and unoptimization */
735	void wait_for_kprobe_optimizer(void)
736	{
737	guard(mutex)(&kprobe_mutex);
738
739	wait_for_kprobe_optimizer_locked();
740	}
741
742	bool optprobe_queued_unopt(struct optimized_kprobe *op)
743	{
744	struct optimized_kprobe *_op;
745
746	list_for_each_entry(_op, &unoptimizing_list, list) {
747	if (op == _op)
748	return true;
749	}
750
751	return false;
752	}
753
754	/* Optimize kprobe if p is ready to be optimized */
755	static void optimize_kprobe(struct kprobe *p)
756	{
757	struct optimized_kprobe *op;
758
759	/* Check if the kprobe is disabled or not ready for optimization. */
760	if (!kprobe_optready(p) \|\| !kprobes_allow_optimization \|\|
761	(kprobe_disabled(p) \|\| kprobes_all_disarmed))
762	return;
763
764	/* kprobes with 'post_handler' can not be optimized */
765	if (p->post_handler)
766	return;
767
768	op = container_of(p, struct optimized_kprobe, kp);
769
770	/* Check there is no other kprobes at the optimized instructions */
771	if (arch_check_optimized_kprobe(op) < 0)
772	return;
773
774	/* Check if it is already optimized. */
775	if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) {
776	if (optprobe_queued_unopt(op)) {
777	/* This is under unoptimizing. Just dequeue the probe */
778	list_del_init(&op->list);
779	}
780	return;
781	}
782	op->kp.flags \|= KPROBE_FLAG_OPTIMIZED;
783
784	/*
785	* On the 'unoptimizing_list' and 'optimizing_list',
786	* 'op' must have OPTIMIZED flag
787	*/
788	if (WARN_ON_ONCE(!list_empty(&op->list)))
789	return;
790
791	list_add(&op->list, &optimizing_list);
792	kick_kprobe_optimizer();
793	}
794
795	/* Short cut to direct unoptimizing */
796	static void force_unoptimize_kprobe(struct optimized_kprobe *op)
797	{
798	lockdep_assert_cpus_held();
799	arch_unoptimize_kprobe(op);
800	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
801	}
802
803	/* Unoptimize a kprobe if p is optimized */
804	static void unoptimize_kprobe(struct kprobe *p, bool force)
805	{
806	struct optimized_kprobe *op;
807
808	if (!kprobe_aggrprobe(p) \|\| kprobe_disarmed(p))
809	return; /* This is not an optprobe nor optimized */
810
811	op = container_of(p, struct optimized_kprobe, kp);
812	if (!kprobe_optimized(p))
813	return;
814
815	if (!list_empty(&op->list)) {
816	if (optprobe_queued_unopt(op)) {
817	/* Queued in unoptimizing queue */
818	if (force) {
819	/*
820	* Forcibly unoptimize the kprobe here, and queue it
821	* in the freeing list for release afterwards.
822	*/
823	force_unoptimize_kprobe(op);
824	list_move(&op->list, &freeing_list);
825	}
826	} else {
827	/* Dequeue from the optimizing queue */
828	list_del_init(&op->list);
829	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
830	}
831	return;
832	}
833
834	/* Optimized kprobe case */
835	if (force) {
836	/* Forcibly update the code: this is a special case */
837	force_unoptimize_kprobe(op);
838	} else {
839	list_add(&op->list, &unoptimizing_list);
840	kick_kprobe_optimizer();
841	}
842	}
843
844	/* Cancel unoptimizing for reusing */
845	static int reuse_unused_kprobe(struct kprobe *ap)
846	{
847	struct optimized_kprobe *op;
848
849	/*
850	* Unused kprobe MUST be on the way of delayed unoptimizing (means
851	* there is still a relative jump) and disabled.
852	*/
853	op = container_of(ap, struct optimized_kprobe, kp);
854	WARN_ON_ONCE(list_empty(&op->list));
855	/* Enable the probe again */
856	ap->flags &= ~KPROBE_FLAG_DISABLED;
857	/* Optimize it again. (remove from 'op->list') */
858	if (!kprobe_optready(ap))
859	return -EINVAL;
860
861	optimize_kprobe(ap);
862	return 0;
863	}
864
865	/* Remove optimized instructions */
866	static void kill_optimized_kprobe(struct kprobe *p)
867	{
868	struct optimized_kprobe *op;
869
870	op = container_of(p, struct optimized_kprobe, kp);
871	if (!list_empty(&op->list))
872	/* Dequeue from the (un)optimization queue */
873	list_del_init(&op->list);
874	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
875
876	if (kprobe_unused(p)) {
877	/*
878	* Unused kprobe is on unoptimizing or freeing list. We move it
879	* to freeing_list and let the kprobe_optimizer() remove it from
880	* the kprobe hash list and free it.
881	*/
882	if (optprobe_queued_unopt(op))
883	list_move(&op->list, &freeing_list);
884	}
885
886	/* Don't touch the code, because it is already freed. */
887	arch_remove_optimized_kprobe(op);
888	}
889
890	static inline
891	void __prepare_optimized_kprobe(struct optimized_kprobe op, struct kprobe p)
892	{
893	if (!kprobe_ftrace(p))
894	arch_prepare_optimized_kprobe(op, p);
895	}
896
897	/* Try to prepare optimized instructions */
898	static void prepare_optimized_kprobe(struct kprobe *p)
899	{
900	struct optimized_kprobe *op;
901
902	op = container_of(p, struct optimized_kprobe, kp);
903	__prepare_optimized_kprobe(op, p);
904	}
905
906	/* Allocate new optimized_kprobe and try to prepare optimized instructions. */
907	static struct kprobe alloc_aggr_kprobe(struct kprobe p)
908	{
909	struct optimized_kprobe *op;
910
911	op = kzalloc_obj(struct optimized_kprobe);
912	if (!op)
913	return NULL;
914
915	INIT_LIST_HEAD(&op->list);
916	op->kp.addr = p->addr;
917	__prepare_optimized_kprobe(op, p);
918
919	return &op->kp;
920	}
921
922	static void init_aggr_kprobe(struct kprobe ap, struct kprobe p);
923
924	/*
925	* Prepare an optimized_kprobe and optimize it.
926	* NOTE: 'p' must be a normal registered kprobe.
927	*/
928	static void try_to_optimize_kprobe(struct kprobe *p)
929	{
930	struct kprobe *ap;
931	struct optimized_kprobe *op;
932
933	/* Impossible to optimize ftrace-based kprobe. */
934	if (kprobe_ftrace(p))
935	return;
936
937	/* For preparing optimization, jump_label_text_reserved() is called. */
938	guard(cpus_read_lock)();
939	guard(jump_label_lock)();
940	guard(mutex)(&text_mutex);
941
942	ap = alloc_aggr_kprobe(p);
943	if (!ap)
944	return;
945
946	op = container_of(ap, struct optimized_kprobe, kp);
947	if (!arch_prepared_optinsn(&op->optinsn)) {
948	/* If failed to setup optimizing, fallback to kprobe. */
949	arch_remove_optimized_kprobe(op);
950	kfree(op);
951	return;
952	}
953
954	init_aggr_kprobe(ap, p);
955	optimize_kprobe(ap); /* This just kicks optimizer thread. */
956	}
957
958	static void optimize_all_kprobes(void)
959	{
960	struct hlist_head *head;
961	struct kprobe *p;
962	unsigned int i;
963
964	guard(mutex)(&kprobe_mutex);
965	/* If optimization is already allowed, just return. */
966	if (kprobes_allow_optimization)
967	return;
968
969	cpus_read_lock();
970	kprobes_allow_optimization = true;
971	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
972	head = &kprobe_table[i];
973	hlist_for_each_entry(p, head, hlist)
974	if (!kprobe_disabled(p))
975	optimize_kprobe(p);
976	}
977	cpus_read_unlock();
978	pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n");
979	}
980
981	#ifdef CONFIG_SYSCTL
982	static void unoptimize_all_kprobes(void)
983	{
984	struct hlist_head *head;
985	struct kprobe *p;
986	unsigned int i;
987
988	guard(mutex)(&kprobe_mutex);
989	/* If optimization is already prohibited, just return. */
990	if (!kprobes_allow_optimization)
991	return;
992
993	cpus_read_lock();
994	kprobes_allow_optimization = false;
995	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
996	head = &kprobe_table[i];
997	hlist_for_each_entry(p, head, hlist) {
998	if (!kprobe_disabled(p))
999	unoptimize_kprobe(p, false);
1000	}
1001	}
1002	cpus_read_unlock();
1003	/* Wait for unoptimizing completion. */
1004	wait_for_kprobe_optimizer_locked();
1005	pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n");
1006	}
1007
1008	static DEFINE_MUTEX(kprobe_sysctl_mutex);
1009	static int sysctl_kprobes_optimization;
1010	static int proc_kprobes_optimization_handler(const struct ctl_table *table,
1011	int write, void *buffer,
1012	size_t length, loff_t ppos)
1013	{
1014	int ret;
1015
1016	guard(mutex)(&kprobe_sysctl_mutex);
1017	sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
1018	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
1019
1020	if (sysctl_kprobes_optimization)
1021	optimize_all_kprobes();
1022	else
1023	unoptimize_all_kprobes();
1024
1025	return ret;
1026	}
1027
1028	static const struct ctl_table kprobe_sysctls[] = {
1029	{
1030	.procname = "kprobes-optimization",
1031	.data = &sysctl_kprobes_optimization,
1032	.maxlen = sizeof(int),
1033	.mode = 0644,
1034	.proc_handler = proc_kprobes_optimization_handler,
1035	.extra1 = SYSCTL_ZERO,
1036	.extra2 = SYSCTL_ONE,
1037	},
1038	};
1039
1040	static void __init kprobe_sysctls_init(void)
1041	{
1042	register_sysctl_init("debug", kprobe_sysctls);
1043	}
1044	#endif /* CONFIG_SYSCTL */
1045
1046	/* Put a breakpoint for a probe. */
1047	static void __arm_kprobe(struct kprobe *p)
1048	{
1049	struct kprobe *_p;
1050
1051	lockdep_assert_held(&text_mutex);
1052
1053	/* Find the overlapping optimized kprobes. */
1054	_p = get_optimized_kprobe(p->addr);
1055	if (unlikely(_p))
1056	/* Fallback to unoptimized kprobe */
1057	unoptimize_kprobe(_p, true);
1058
1059	arch_arm_kprobe(p);
1060	optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
1061	}
1062
1063	/* Remove the breakpoint of a probe. */
1064	static void __disarm_kprobe(struct kprobe *p, bool reopt)
1065	{
1066	struct kprobe *_p;
1067
1068	lockdep_assert_held(&text_mutex);
1069
1070	/* Try to unoptimize */
1071	unoptimize_kprobe(p, kprobes_all_disarmed);
1072
1073	if (!kprobe_queued(p)) {
1074	arch_disarm_kprobe(p);
1075	/* If another kprobe was blocked, re-optimize it. */
1076	_p = get_optimized_kprobe(p->addr);
1077	if (unlikely(_p) && reopt)
1078	optimize_kprobe(_p);
1079	}
1080	}
1081
1082	static void __init init_optprobe(void)
1083	{
1084	#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
1085	/* Init 'kprobe_optinsn_slots' for allocation */
1086	kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1087	#endif
1088
1089	init_waitqueue_head(&kprobe_optimizer_wait);
1090	atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);
1091	kprobe_optimizer_task = kthread_run(kprobe_optimizer_thread, NULL,
1092	"kprobe-optimizer");
1093	}
1094	#else /* !CONFIG_OPTPROBES */
1095
1096	#define init_optprobe() do {} while (0)
1097	#define optimize_kprobe(p) do {} while (0)
1098	#define unoptimize_kprobe(p, f) do {} while (0)
1099	#define kill_optimized_kprobe(p) do {} while (0)
1100	#define prepare_optimized_kprobe(p) do {} while (0)
1101	#define try_to_optimize_kprobe(p) do {} while (0)
1102	#define __arm_kprobe(p) arch_arm_kprobe(p)
1103	#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
1104	#define kprobe_disarmed(p) kprobe_disabled(p)
1105	#define wait_for_kprobe_optimizer_locked() \
1106	lockdep_assert_held(&kprobe_mutex)
1107
1108	static int reuse_unused_kprobe(struct kprobe *ap)
1109	{
1110	/*
1111	* If the optimized kprobe is NOT supported, the aggr kprobe is
1112	* released at the same time that the last aggregated kprobe is
1113	* unregistered.
1114	* Thus there should be no chance to reuse unused kprobe.
1115	*/
1116	WARN_ON_ONCE(1);
1117	return -EINVAL;
1118	}
1119
1120	static void free_aggr_kprobe(struct kprobe *p)
1121	{
1122	arch_remove_kprobe(p);
1123	kfree(p);
1124	}
1125
1126	static struct kprobe alloc_aggr_kprobe(struct kprobe p)
1127	{
1128	return kzalloc_obj(struct kprobe);
1129	}
1130	#endif /* CONFIG_OPTPROBES */
1131
1132	#ifdef CONFIG_KPROBES_ON_FTRACE
1133	static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
1134	.func = kprobe_ftrace_handler,
1135	.flags = FTRACE_OPS_FL_SAVE_REGS,
1136	};
1137
1138	static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = {
1139	.func = kprobe_ftrace_handler,
1140	.flags = FTRACE_OPS_FL_SAVE_REGS \| FTRACE_OPS_FL_IPMODIFY,
1141	};
1142
1143	static int kprobe_ipmodify_enabled;
1144	static int kprobe_ftrace_enabled;
1145	bool kprobe_ftrace_disabled;
1146
1147	static int __arm_kprobe_ftrace(struct kprobe p, struct ftrace_ops ops,
1148	int *cnt)
1149	{
1150	int ret;
1151
1152	lockdep_assert_held(&kprobe_mutex);
1153
1154	ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
1155	if (ret < 0)
1156	return ret;
1157
1158	if (*cnt == 0) {
1159	ret = register_ftrace_function(ops);
1160	if (ret < 0) {
1161	/*
1162	* At this point, sinec ops is not registered, we should be sefe from
1163	* registering empty filter.
1164	*/
1165	ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1166	return ret;
1167	}
1168	}
1169
1170	(*cnt)++;
1171	return ret;
1172	}
1173
1174	static int arm_kprobe_ftrace(struct kprobe *p)
1175	{
1176	bool ipmodify = (p->post_handler != NULL);
1177
1178	return __arm_kprobe_ftrace(p,
1179	ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1180	ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1181	}
1182
1183	static int __disarm_kprobe_ftrace(struct kprobe p, struct ftrace_ops ops,
1184	int *cnt)
1185	{
1186	int ret;
1187
1188	lockdep_assert_held(&kprobe_mutex);
1189	if (unlikely(kprobe_ftrace_disabled)) {
1190	/* Now ftrace is disabled forever, disarm is already done. */
1191	return 0;
1192	}
1193
1194	if (*cnt == 1) {
1195	ret = unregister_ftrace_function(ops);
1196	if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret))
1197	return ret;
1198	}
1199
1200	(*cnt)--;
1201
1202	ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1203	WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n",
1204	p->addr, ret);
1205	return ret;
1206	}
1207
1208	static int disarm_kprobe_ftrace(struct kprobe *p)
1209	{
1210	bool ipmodify = (p->post_handler != NULL);
1211
1212	return __disarm_kprobe_ftrace(p,
1213	ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1214	ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1215	}
1216
1217	void kprobe_ftrace_kill(void)
1218	{
1219	kprobe_ftrace_disabled = true;
1220	}
1221	#else /* !CONFIG_KPROBES_ON_FTRACE */
1222	static inline int arm_kprobe_ftrace(struct kprobe *p)
1223	{
1224	return -ENODEV;
1225	}
1226
1227	static inline int disarm_kprobe_ftrace(struct kprobe *p)
1228	{
1229	return -ENODEV;
1230	}
1231	#endif
1232
1233	static int prepare_kprobe(struct kprobe *p)
1234	{
1235	/* Must ensure p->addr is really on ftrace */
1236	if (kprobe_ftrace(p))
1237	return arch_prepare_kprobe_ftrace(p);
1238
1239	return arch_prepare_kprobe(p);
1240	}
1241
1242	static int arm_kprobe(struct kprobe *kp)
1243	{
1244	if (unlikely(kprobe_ftrace(kp)))
1245	return arm_kprobe_ftrace(kp);
1246
1247	guard(cpus_read_lock)();
1248	guard(mutex)(&text_mutex);
1249	__arm_kprobe(kp);
1250	return 0;
1251	}
1252
1253	static int disarm_kprobe(struct kprobe *kp, bool reopt)
1254	{
1255	if (unlikely(kprobe_ftrace(kp)))
1256	return disarm_kprobe_ftrace(kp);
1257
1258	guard(cpus_read_lock)();
1259	guard(mutex)(&text_mutex);
1260	__disarm_kprobe(kp, reopt);
1261	return 0;
1262	}
1263
1264	/*
1265	* Aggregate handlers for multiple kprobes support - these handlers
1266	* take care of invoking the individual kprobe handlers on p->list
1267	*/
1268	static int aggr_pre_handler(struct kprobe p, struct pt_regs regs)
1269	{
1270	struct kprobe *kp;
1271
1272	list_for_each_entry_rcu(kp, &p->list, list) {
1273	if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
1274	set_kprobe_instance(kp);
1275	if (kp->pre_handler(kp, regs))
1276	return 1;
1277	}
1278	reset_kprobe_instance();
1279	}
1280	return 0;
1281	}
1282	NOKPROBE_SYMBOL(aggr_pre_handler);
1283
1284	static void aggr_post_handler(struct kprobe p, struct pt_regs regs,
1285	unsigned long flags)
1286	{
1287	struct kprobe *kp;
1288
1289	list_for_each_entry_rcu(kp, &p->list, list) {
1290	if (kp->post_handler && likely(!kprobe_disabled(kp))) {
1291	set_kprobe_instance(kp);
1292	kp->post_handler(kp, regs, flags);
1293	reset_kprobe_instance();
1294	}
1295	}
1296	}
1297	NOKPROBE_SYMBOL(aggr_post_handler);
1298
1299	/* Walks the list and increments 'nmissed' if 'p' has child probes. */
1300	void kprobes_inc_nmissed_count(struct kprobe *p)
1301	{
1302	struct kprobe *kp;
1303
1304	if (!kprobe_aggrprobe(p)) {
1305	p->nmissed++;
1306	} else {
1307	list_for_each_entry_rcu(kp, &p->list, list)
1308	kp->nmissed++;
1309	}
1310	}
1311	NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1312
1313	static struct kprobe kprobe_busy = {
1314	.addr = (void *) get_kprobe,
1315	};
1316
1317	void kprobe_busy_begin(void)
1318	{
1319	struct kprobe_ctlblk *kcb;
1320
1321	preempt_disable();
1322	__this_cpu_write(current_kprobe, &kprobe_busy);
1323	kcb = get_kprobe_ctlblk();
1324	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1325	}
1326
1327	void kprobe_busy_end(void)
1328	{
1329	__this_cpu_write(current_kprobe, NULL);
1330	preempt_enable();
1331	}
1332
1333	/* Add the new probe to 'ap->list'. */
1334	static int add_new_kprobe(struct kprobe ap, struct kprobe p)
1335	{
1336	if (p->post_handler)
1337	unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */
1338
1339	list_add_rcu(&p->list, &ap->list);
1340	if (p->post_handler && !ap->post_handler)
1341	ap->post_handler = aggr_post_handler;
1342
1343	return 0;
1344	}
1345
1346	/*
1347	* Fill in the required fields of the aggregator kprobe. Replace the
1348	* earlier kprobe in the hlist with the aggregator kprobe.
1349	*/
1350	static void init_aggr_kprobe(struct kprobe ap, struct kprobe p)
1351	{
1352	/* Copy the insn slot of 'p' to 'ap'. */
1353	copy_kprobe(p, ap);
1354	flush_insn_slot(ap);
1355	ap->addr = p->addr;
1356	ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1357	ap->pre_handler = aggr_pre_handler;
1358	/* We don't care the kprobe which has gone. */
1359	if (p->post_handler && !kprobe_gone(p))
1360	ap->post_handler = aggr_post_handler;
1361
1362	INIT_LIST_HEAD(&ap->list);
1363	INIT_HLIST_NODE(&ap->hlist);
1364
1365	list_add_rcu(&p->list, &ap->list);
1366	hlist_replace_rcu(&p->hlist, &ap->hlist);
1367	}
1368
1369	/*
1370	* This registers the second or subsequent kprobe at the same address.
1371	*/
1372	static int register_aggr_kprobe(struct kprobe orig_p, struct kprobe p)
1373	{
1374	int ret = 0;
1375	struct kprobe *ap = orig_p;
1376
1377	scoped_guard(cpus_read_lock) {
1378	/* For preparing optimization, jump_label_text_reserved() is called */
1379	guard(jump_label_lock)();
1380	guard(mutex)(&text_mutex);
1381
1382	if (!kprobe_aggrprobe(orig_p)) {
1383	/* If 'orig_p' is not an 'aggr_kprobe', create new one. */
1384	ap = alloc_aggr_kprobe(orig_p);
1385	if (!ap)
1386	return -ENOMEM;
1387	init_aggr_kprobe(ap, orig_p);
1388	} else if (kprobe_unused(ap)) {
1389	/* This probe is going to die. Rescue it */
1390	ret = reuse_unused_kprobe(ap);
1391	if (ret)
1392	return ret;
1393	}
1394
1395	if (kprobe_gone(ap)) {
1396	/*
1397	* Attempting to insert new probe at the same location that
1398	* had a probe in the module vaddr area which already
1399	* freed. So, the instruction slot has already been
1400	* released. We need a new slot for the new probe.
1401	*/
1402	ret = arch_prepare_kprobe(ap);
1403	if (ret)
1404	/*
1405	* Even if fail to allocate new slot, don't need to
1406	* free the 'ap'. It will be used next time, or
1407	* freed by unregister_kprobe().
1408	*/
1409	return ret;
1410
1411	/* Prepare optimized instructions if possible. */
1412	prepare_optimized_kprobe(ap);
1413
1414	/*
1415	* Clear gone flag to prevent allocating new slot again, and
1416	* set disabled flag because it is not armed yet.
1417	*/
1418	ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1419	\| KPROBE_FLAG_DISABLED;
1420	}
1421
1422	/* Copy the insn slot of 'p' to 'ap'. */
1423	copy_kprobe(ap, p);
1424	ret = add_new_kprobe(ap, p);
1425	}
1426
1427	if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1428	ap->flags &= ~KPROBE_FLAG_DISABLED;
1429	if (!kprobes_all_disarmed) {
1430	/* Arm the breakpoint again. */
1431	ret = arm_kprobe(ap);
1432	if (ret) {
1433	ap->flags \|= KPROBE_FLAG_DISABLED;
1434	list_del_rcu(&p->list);
1435	synchronize_rcu();
1436	}
1437	}
1438	}
1439	return ret;
1440	}
1441
1442	bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1443	{
1444	/* The '__kprobes' functions and entry code must not be probed. */
1445	return addr >= (unsigned long)__kprobes_text_start &&
1446	addr < (unsigned long)__kprobes_text_end;
1447	}
1448
1449	static bool __within_kprobe_blacklist(unsigned long addr)
1450	{
1451	struct kprobe_blacklist_entry *ent;
1452
1453	if (arch_within_kprobe_blacklist(addr))
1454	return true;
1455	/*
1456	* If 'kprobe_blacklist' is defined, check the address and
1457	* reject any probe registration in the prohibited area.
1458	* Note: this can return true during transition period where
1459	* (start_addr, end_addr) in the black list is shrinking
1460	* but old entry has not been removed yet. This is acceptable
1461	* because the worst case is that we reject more probes than
1462	* we should.
1463	*/
1464	guard(rcu)();
1465	list_for_each_entry_rcu(ent, &kprobe_blacklist, list) {
1466	if (addr >= ent->start_addr && addr < ent->end_addr)
1467	return true;
1468	}
1469	return false;
1470	}
1471
1472	bool within_kprobe_blacklist(unsigned long addr)
1473	{
1474	char symname[KSYM_NAME_LEN], *p;
1475
1476	if (__within_kprobe_blacklist(addr))
1477	return true;
1478
1479	/* Check if the address is on a suffixed-symbol */
1480	if (!lookup_symbol_name(addr, symname)) {
1481	p = strchr(symname, '.');
1482	if (!p)
1483	return false;
1484	*p = '\0';
1485	addr = (unsigned long)kprobe_lookup_name(symname, 0);
1486	if (addr)
1487	return __within_kprobe_blacklist(addr);
1488	}
1489	return false;
1490	}
1491
1492	/*
1493	* arch_adjust_kprobe_addr - adjust the address
1494	* @addr: symbol base address
1495	* @offset: offset within the symbol
1496	* @on_func_entry: was this @addr+@offset on the function entry
1497	*
1498	* Typically returns @addr + @offset, except for special cases where the
1499	* function might be prefixed by a CFI landing pad, in that case any offset
1500	* inside the landing pad is mapped to the first 'real' instruction of the
1501	* symbol.
1502	*
1503	* Specifically, for things like IBT/BTI, skip the resp. ENDBR/BTI.C
1504	* instruction at +0.
1505	*/
1506	kprobe_opcode_t *__weak arch_adjust_kprobe_addr(unsigned long addr,
1507	unsigned long offset,
1508	bool *on_func_entry)
1509	{
1510	*on_func_entry = !offset;
1511	return (kprobe_opcode_t *)(addr + offset);
1512	}
1513
1514	/*
1515	* If 'symbol_name' is specified, look it up and add the 'offset'
1516	* to it. This way, we can specify a relative address to a symbol.
1517	* This returns encoded errors if it fails to look up symbol or invalid
1518	* combination of parameters.
1519	*/
1520	static kprobe_opcode_t *
1521	_kprobe_addr(kprobe_opcode_t addr, const char symbol_name,
1522	unsigned long offset, bool *on_func_entry)
1523	{
1524	if ((symbol_name && addr) \|\| (!symbol_name && !addr))
1525	return ERR_PTR(-EINVAL);
1526
1527	if (symbol_name) {
1528	/*
1529	* Input: @sym + @offset
1530	* Output: @addr + @offset
1531	*
1532	* NOTE: kprobe_lookup_name() does NOT fold the offset
1533	* argument into it's output!
1534	*/
1535	addr = kprobe_lookup_name(symbol_name, offset);
1536	if (!addr)
1537	return ERR_PTR(-ENOENT);
1538	}
1539
1540	/*
1541	* So here we have @addr + @offset, displace it into a new
1542	* @addr' + @offset' where @addr' is the symbol start address.
1543	*/
1544	addr = (void *)addr + offset;
1545	if (!kallsyms_lookup_size_offset((unsigned long)addr, NULL, &offset))
1546	return ERR_PTR(-ENOENT);
1547	addr = (void *)addr - offset;
1548
1549	/*
1550	* Then ask the architecture to re-combine them, taking care of
1551	* magical function entry details while telling us if this was indeed
1552	* at the start of the function.
1553	*/
1554	addr = arch_adjust_kprobe_addr((unsigned long)addr, offset, on_func_entry);
1555	if (!addr)
1556	return ERR_PTR(-EINVAL);
1557
1558	return addr;
1559	}
1560
1561	static kprobe_opcode_t kprobe_addr(struct kprobe p)
1562	{
1563	bool on_func_entry;
1564
1565	return _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
1566	}
1567
1568	/*
1569	* Check the 'p' is valid and return the aggregator kprobe
1570	* at the same address.
1571	*/
1572	static struct kprobe __get_valid_kprobe(struct kprobe p)
1573	{
1574	struct kprobe ap, list_p;
1575
1576	lockdep_assert_held(&kprobe_mutex);
1577
1578	ap = get_kprobe(p->addr);
1579	if (unlikely(!ap))
1580	return NULL;
1581
1582	if (p == ap)
1583	return ap;
1584
1585	list_for_each_entry(list_p, &ap->list, list)
1586	if (list_p == p)
1587	/* kprobe p is a valid probe */
1588	return ap;
1589
1590	return NULL;
1591	}
1592
1593	/*
1594	* Warn and return error if the kprobe is being re-registered since
1595	* there must be a software bug.
1596	*/
1597	static inline int warn_kprobe_rereg(struct kprobe *p)
1598	{
1599	guard(mutex)(&kprobe_mutex);
1600
1601	if (WARN_ON_ONCE(__get_valid_kprobe(p)))
1602	return -EINVAL;
1603
1604	return 0;
1605	}
1606
1607	static int check_ftrace_location(struct kprobe *p)
1608	{
1609	unsigned long addr = (unsigned long)p->addr;
1610
1611	if (ftrace_location(addr) == addr) {
1612	#ifdef CONFIG_KPROBES_ON_FTRACE
1613	p->flags \|= KPROBE_FLAG_FTRACE;
1614	#else
1615	return -EINVAL;
1616	#endif
1617	}
1618	return 0;
1619	}
1620
1621	static bool is_cfi_preamble_symbol(unsigned long addr)
1622	{
1623	char symbuf[KSYM_NAME_LEN];
1624
1625	if (lookup_symbol_name(addr, symbuf))
1626	return false;
1627
1628	return str_has_prefix(symbuf, "__cfi_") \|\|
1629	str_has_prefix(symbuf, "__pfx_");
1630	}
1631
1632	static int check_kprobe_address_safe(struct kprobe *p,
1633	struct module **probed_mod)
1634	{
1635	int ret;
1636
1637	ret = check_ftrace_location(p);
1638	if (ret)
1639	return ret;
1640
1641	guard(jump_label_lock)();
1642
1643	/* Ensure the address is in a text area, and find a module if exists. */
1644	*probed_mod = NULL;
1645	if (!core_kernel_text((unsigned long) p->addr)) {
1646	guard(rcu)();
1647	*probed_mod = __module_text_address((unsigned long) p->addr);
1648	if (!(*probed_mod))
1649	return -EINVAL;
1650
1651	/*
1652	* We must hold a refcount of the probed module while updating
1653	* its code to prohibit unexpected unloading.
1654	*/
1655	if (unlikely(!try_module_get(*probed_mod)))
1656	return -ENOENT;
1657	}
1658	/* Ensure it is not in reserved area. */
1659	if (in_gate_area_no_mm((unsigned long) p->addr) \|\|
1660	within_kprobe_blacklist((unsigned long) p->addr) \|\|
1661	jump_label_text_reserved(p->addr, p->addr) \|\|
1662	static_call_text_reserved(p->addr, p->addr) \|\|
1663	find_bug((unsigned long)p->addr) \|\|
1664	is_cfi_preamble_symbol((unsigned long)p->addr)) {
1665	module_put(*probed_mod);
1666	return -EINVAL;
1667	}
1668
1669	/* Get module refcount and reject __init functions for loaded modules. */
1670	if (IS_ENABLED(CONFIG_MODULES) && *probed_mod) {
1671	/*
1672	* If the module freed '.init.text', we couldn't insert
1673	* kprobes in there.
1674	*/
1675	if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1676	!module_is_coming(*probed_mod)) {
1677	module_put(*probed_mod);
1678	return -ENOENT;
1679	}
1680	}
1681
1682	return 0;
1683	}
1684
1685	static int __register_kprobe(struct kprobe *p)
1686	{
1687	int ret;
1688	struct kprobe *old_p;
1689
1690	guard(mutex)(&kprobe_mutex);
1691
1692	old_p = get_kprobe(p->addr);
1693	if (old_p)
1694	/* Since this may unoptimize 'old_p', locking 'text_mutex'. */
1695	return register_aggr_kprobe(old_p, p);
1696
1697	scoped_guard(cpus_read_lock) {
1698	/* Prevent text modification */
1699	guard(mutex)(&text_mutex);
1700	ret = prepare_kprobe(p);
1701	if (ret)
1702	return ret;
1703	}
1704
1705	INIT_HLIST_NODE(&p->hlist);
1706	hlist_add_head_rcu(&p->hlist,
1707	&kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1708
1709	if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
1710	ret = arm_kprobe(p);
1711	if (ret) {
1712	hlist_del_rcu(&p->hlist);
1713	synchronize_rcu();
1714	}
1715	}
1716
1717	/* Try to optimize kprobe */
1718	try_to_optimize_kprobe(p);
1719	return 0;
1720	}
1721
1722	int register_kprobe(struct kprobe *p)
1723	{
1724	int ret;
1725	struct module *probed_mod;
1726	kprobe_opcode_t *addr;
1727	bool on_func_entry;
1728
1729	/* Canonicalize probe address from symbol */
1730	addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
1731	if (IS_ERR(addr))
1732	return PTR_ERR(addr);
1733	p->addr = addr;
1734
1735	ret = warn_kprobe_rereg(p);
1736	if (ret)
1737	return ret;
1738
1739	/* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1740	p->flags &= KPROBE_FLAG_DISABLED;
1741	if (on_func_entry)
1742	p->flags \|= KPROBE_FLAG_ON_FUNC_ENTRY;
1743	p->nmissed = 0;
1744	INIT_LIST_HEAD(&p->list);
1745
1746	ret = check_kprobe_address_safe(p, &probed_mod);
1747	if (ret)
1748	return ret;
1749
1750	ret = __register_kprobe(p);
1751
1752	if (probed_mod)
1753	module_put(probed_mod);
1754
1755	return ret;
1756	}
1757	EXPORT_SYMBOL_GPL(register_kprobe);
1758
1759	/* Check if all probes on the 'ap' are disabled. */
1760	static bool aggr_kprobe_disabled(struct kprobe *ap)
1761	{
1762	struct kprobe *kp;
1763
1764	lockdep_assert_held(&kprobe_mutex);
1765
1766	list_for_each_entry(kp, &ap->list, list)
1767	if (!kprobe_disabled(kp))
1768	/*
1769	* Since there is an active probe on the list,
1770	* we can't disable this 'ap'.
1771	*/
1772	return false;
1773
1774	return true;
1775	}
1776
1777	static struct kprobe __disable_kprobe(struct kprobe p)
1778	{
1779	struct kprobe *orig_p;
1780	int ret;
1781
1782	lockdep_assert_held(&kprobe_mutex);
1783
1784	/* Get an original kprobe for return */
1785	orig_p = __get_valid_kprobe(p);
1786	if (unlikely(orig_p == NULL))
1787	return ERR_PTR(-EINVAL);
1788
1789	if (kprobe_disabled(p))
1790	return orig_p;
1791
1792	/* Disable probe if it is a child probe */
1793	if (p != orig_p)
1794	p->flags \|= KPROBE_FLAG_DISABLED;
1795
1796	/* Try to disarm and disable this/parent probe */
1797	if (p == orig_p \|\| aggr_kprobe_disabled(orig_p)) {
1798	/*
1799	* Don't be lazy here. Even if 'kprobes_all_disarmed'
1800	* is false, 'orig_p' might not have been armed yet.
1801	* Note arm_all_kprobes() __tries__ to arm all kprobes
1802	* on the best effort basis.
1803	*/
1804	if (!kprobes_all_disarmed && !kprobe_disabled(orig_p)) {
1805	ret = disarm_kprobe(orig_p, true);
1806	if (ret) {
1807	p->flags &= ~KPROBE_FLAG_DISABLED;
1808	return ERR_PTR(ret);
1809	}
1810	}
1811	orig_p->flags \|= KPROBE_FLAG_DISABLED;
1812	}
1813
1814	return orig_p;
1815	}
1816
1817	/*
1818	* Unregister a kprobe without a scheduler synchronization.
1819	*/
1820	static int __unregister_kprobe_top(struct kprobe *p)
1821	{
1822	struct kprobe ap, list_p;
1823
1824	/* Disable kprobe. This will disarm it if needed. */
1825	ap = __disable_kprobe(p);
1826	if (IS_ERR(ap))
1827	return PTR_ERR(ap);
1828
1829	WARN_ON(ap != p && !kprobe_aggrprobe(ap));
1830
1831	/*
1832	* If the probe is an independent(and non-optimized) kprobe
1833	* (not an aggrprobe), the last kprobe on the aggrprobe, or
1834	* kprobe is already disarmed, just remove from the hash list.
1835	*/
1836	if (ap == p \|\|
1837	(list_is_singular(&ap->list) && kprobe_disarmed(ap))) {
1838	/*
1839	* !disarmed could be happen if the probe is under delayed
1840	* unoptimizing.
1841	*/
1842	hlist_del_rcu(&ap->hlist);
1843	return 0;
1844	}
1845
1846	/* If disabling probe has special handlers, update aggrprobe */
1847	if (p->post_handler && !kprobe_gone(p)) {
1848	list_for_each_entry(list_p, &ap->list, list) {
1849	if ((list_p != p) && (list_p->post_handler))
1850	break;
1851	}
1852	/* No other probe has post_handler */
1853	if (list_entry_is_head(list_p, &ap->list, list)) {
1854	/*
1855	* For the kprobe-on-ftrace case, we keep the
1856	* post_handler setting to identify this aggrprobe
1857	* armed with kprobe_ipmodify_ops.
1858	*/
1859	if (!kprobe_ftrace(ap))
1860	ap->post_handler = NULL;
1861	}
1862	}
1863
1864	/*
1865	* Remove from the aggrprobe: this path will do nothing in
1866	* __unregister_kprobe_bottom().
1867	*/
1868	list_del_rcu(&p->list);
1869	if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1870	/*
1871	* Try to optimize this probe again, because post
1872	* handler may have been changed.
1873	*/
1874	optimize_kprobe(ap);
1875	return 0;
1876
1877	}
1878
1879	static void __unregister_kprobe_bottom(struct kprobe *p)
1880	{
1881	struct kprobe *ap;
1882
1883	if (list_empty(&p->list))
1884	/* This is an independent kprobe */
1885	arch_remove_kprobe(p);
1886	else if (list_is_singular(&p->list)) {
1887	/* This is the last child of an aggrprobe */
1888	ap = list_entry(p->list.next, struct kprobe, list);
1889	list_del(&p->list);
1890	free_aggr_kprobe(ap);
1891	}
1892	/* Otherwise, do nothing. */
1893	}
1894
1895	int register_kprobes(struct kprobe **kps, int num)
1896	{
1897	int i, ret = 0;
1898
1899	if (num <= 0)
1900	return -EINVAL;
1901	for (i = 0; i < num; i++) {
1902	ret = register_kprobe(kps[i]);
1903	if (ret < 0) {
1904	if (i > 0)
1905	unregister_kprobes(kps, i);
1906	break;
1907	}
1908	}
1909	return ret;
1910	}
1911	EXPORT_SYMBOL_GPL(register_kprobes);
1912
1913	void unregister_kprobe(struct kprobe *p)
1914	{
1915	unregister_kprobes(&p, 1);
1916	}
1917	EXPORT_SYMBOL_GPL(unregister_kprobe);
1918
1919	void unregister_kprobes(struct kprobe **kps, int num)
1920	{
1921	int i;
1922
1923	if (num <= 0)
1924	return;
1925	scoped_guard(mutex, &kprobe_mutex) {
1926	for (i = 0; i < num; i++)
1927	if (__unregister_kprobe_top(kps[i]) < 0)
1928	kps[i]->addr = NULL;
1929	}
1930	synchronize_rcu();
1931	for (i = 0; i < num; i++)
1932	if (kps[i]->addr)
1933	__unregister_kprobe_bottom(kps[i]);
1934	}
1935	EXPORT_SYMBOL_GPL(unregister_kprobes);
1936
1937	int __weak kprobe_exceptions_notify(struct notifier_block *self,
1938	unsigned long val, void *data)
1939	{
1940	return NOTIFY_DONE;
1941	}
1942	NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1943
1944	static struct notifier_block kprobe_exceptions_nb = {
1945	.notifier_call = kprobe_exceptions_notify,
1946	.priority = 0x7fffffff /* we need to be notified first */
1947	};
1948
1949	#ifdef CONFIG_KRETPROBES
1950
1951	#if !defined(CONFIG_KRETPROBE_ON_RETHOOK)
1952
1953	/* callbacks for objpool of kretprobe instances */
1954	static int kretprobe_init_inst(void nod, void context)
1955	{
1956	struct kretprobe_instance *ri = nod;
1957
1958	ri->rph = context;
1959	return 0;
1960	}
1961	static int kretprobe_fini_pool(struct objpool_head head, void context)
1962	{
1963	kfree(context);
1964	return 0;
1965	}
1966
1967	static void free_rp_inst_rcu(struct rcu_head *head)
1968	{
1969	struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu);
1970	struct kretprobe_holder *rph = ri->rph;
1971
1972	objpool_drop(ri, &rph->pool);
1973	}
1974	NOKPROBE_SYMBOL(free_rp_inst_rcu);
1975
1976	static void recycle_rp_inst(struct kretprobe_instance *ri)
1977	{
1978	struct kretprobe *rp = get_kretprobe(ri);
1979
1980	if (likely(rp))
1981	objpool_push(ri, &rp->rph->pool);
1982	else
1983	call_rcu(&ri->rcu, free_rp_inst_rcu);
1984	}
1985	NOKPROBE_SYMBOL(recycle_rp_inst);
1986
1987	/*
1988	* This function is called from delayed_put_task_struct() when a task is
1989	* dead and cleaned up to recycle any kretprobe instances associated with
1990	* this task. These left over instances represent probed functions that
1991	* have been called but will never return.
1992	*/
1993	void kprobe_flush_task(struct task_struct *tk)
1994	{
1995	struct kretprobe_instance *ri;
1996	struct llist_node *node;
1997
1998	/* Early boot, not yet initialized. */
1999	if (unlikely(!kprobes_initialized))
2000	return;
2001
2002	kprobe_busy_begin();
2003
2004	node = __llist_del_all(&tk->kretprobe_instances);
2005	while (node) {
2006	ri = container_of(node, struct kretprobe_instance, llist);
2007	node = node->next;
2008
2009	recycle_rp_inst(ri);
2010	}
2011
2012	kprobe_busy_end();
2013	}
2014	NOKPROBE_SYMBOL(kprobe_flush_task);
2015
2016	static inline void free_rp_inst(struct kretprobe *rp)
2017	{
2018	struct kretprobe_holder *rph = rp->rph;
2019
2020	if (!rph)
2021	return;
2022	rp->rph = NULL;
2023	objpool_fini(&rph->pool);
2024	}
2025
2026	/* This assumes the 'tsk' is the current task or the is not running. */
2027	static kprobe_opcode_t __kretprobe_find_ret_addr(struct task_struct tsk,
2028	struct llist_node **cur)
2029	{
2030	struct kretprobe_instance *ri = NULL;
2031	struct llist_node node = cur;
2032
2033	if (!node)
2034	node = tsk->kretprobe_instances.first;
2035	else
2036	node = node->next;
2037
2038	while (node) {
2039	ri = container_of(node, struct kretprobe_instance, llist);
2040	if (ri->ret_addr != kretprobe_trampoline_addr()) {
2041	*cur = node;
2042	return ri->ret_addr;
2043	}
2044	node = node->next;
2045	}
2046	return NULL;
2047	}
2048	NOKPROBE_SYMBOL(__kretprobe_find_ret_addr);
2049
2050	/**
2051	* kretprobe_find_ret_addr -- Find correct return address modified by kretprobe
2052	* @tsk: Target task
2053	* @fp: A frame pointer
2054	* @cur: a storage of the loop cursor llist_node pointer for next call
2055	*
2056	* Find the correct return address modified by a kretprobe on @tsk in unsigned
2057	* long type. If it finds the return address, this returns that address value,
2058	* or this returns 0.
2059	* The @tsk must be 'current' or a task which is not running. @fp is a hint
2060	* to get the currect return address - which is compared with the
2061	* kretprobe_instance::fp field. The @cur is a loop cursor for searching the
2062	* kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
2063	* first call, but '@cur' itself must NOT NULL.
2064	*/
2065	unsigned long kretprobe_find_ret_addr(struct task_struct tsk, void fp,
2066	struct llist_node **cur)
2067	{
2068	struct kretprobe_instance *ri;
2069	kprobe_opcode_t *ret;
2070
2071	if (WARN_ON_ONCE(!cur))
2072	return 0;
2073
2074	do {
2075	ret = __kretprobe_find_ret_addr(tsk, cur);
2076	if (!ret)
2077	break;
2078	ri = container_of(*cur, struct kretprobe_instance, llist);
2079	} while (ri->fp != fp);
2080
2081	return (unsigned long)ret;
2082	}
2083	NOKPROBE_SYMBOL(kretprobe_find_ret_addr);
2084
2085	void __weak arch_kretprobe_fixup_return(struct pt_regs *regs,
2086	kprobe_opcode_t *correct_ret_addr)
2087	{
2088	/*
2089	* Do nothing by default. Please fill this to update the fake return
2090	* address on the stack with the correct one on each arch if possible.
2091	*/
2092	}
2093
2094	unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
2095	void *frame_pointer)
2096	{
2097	struct kretprobe_instance *ri = NULL;
2098	struct llist_node first, node = NULL;
2099	kprobe_opcode_t *correct_ret_addr;
2100	struct kretprobe *rp;
2101
2102	/* Find correct address and all nodes for this frame. */
2103	correct_ret_addr = __kretprobe_find_ret_addr(current, &node);
2104	if (!correct_ret_addr) {
2105	pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n");
2106	BUG_ON(1);
2107	}
2108
2109	/*
2110	* Set the return address as the instruction pointer, because if the
2111	* user handler calls stack_trace_save_regs() with this 'regs',
2112	* the stack trace will start from the instruction pointer.
2113	*/
2114	instruction_pointer_set(regs, (unsigned long)correct_ret_addr);
2115
2116	/* Run the user handler of the nodes. */
2117	first = current->kretprobe_instances.first;
2118	while (first) {
2119	ri = container_of(first, struct kretprobe_instance, llist);
2120
2121	if (WARN_ON_ONCE(ri->fp != frame_pointer))
2122	break;
2123
2124	rp = get_kretprobe(ri);
2125	if (rp && rp->handler) {
2126	struct kprobe *prev = kprobe_running();
2127
2128	__this_cpu_write(current_kprobe, &rp->kp);
2129	ri->ret_addr = correct_ret_addr;
2130	rp->handler(ri, regs);
2131	__this_cpu_write(current_kprobe, prev);
2132	}
2133	if (first == node)
2134	break;
2135
2136	first = first->next;
2137	}
2138
2139	arch_kretprobe_fixup_return(regs, correct_ret_addr);
2140
2141	/* Unlink all nodes for this frame. */
2142	first = current->kretprobe_instances.first;
2143	current->kretprobe_instances.first = node->next;
2144	node->next = NULL;
2145
2146	/* Recycle free instances. */
2147	while (first) {
2148	ri = container_of(first, struct kretprobe_instance, llist);
2149	first = first->next;
2150
2151	recycle_rp_inst(ri);
2152	}
2153
2154	return (unsigned long)correct_ret_addr;
2155	}
2156	NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
2157
2158	/*
2159	* This kprobe pre_handler is registered with every kretprobe. When probe
2160	* hits it will set up the return probe.
2161	*/
2162	static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
2163	{
2164	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2165	struct kretprobe_holder *rph = rp->rph;
2166	struct kretprobe_instance *ri;
2167
2168	ri = objpool_pop(&rph->pool);
2169	if (!ri) {
2170	rp->nmissed++;
2171	return 0;
2172	}
2173
2174	if (rp->entry_handler && rp->entry_handler(ri, regs)) {
2175	objpool_push(ri, &rph->pool);
2176	return 0;
2177	}
2178
2179	arch_prepare_kretprobe(ri, regs);
2180
2181	__llist_add(&ri->llist, &current->kretprobe_instances);
2182
2183	return 0;
2184	}
2185	NOKPROBE_SYMBOL(pre_handler_kretprobe);
2186	#else /* CONFIG_KRETPROBE_ON_RETHOOK */
2187	/*
2188	* This kprobe pre_handler is registered with every kretprobe. When probe
2189	* hits it will set up the return probe.
2190	*/
2191	static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
2192	{
2193	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2194	struct kretprobe_instance *ri;
2195	struct rethook_node *rhn;
2196
2197	rhn = rethook_try_get(rp->rh);
2198	if (!rhn) {
2199	rp->nmissed++;
2200	return 0;
2201	}
2202
2203	ri = container_of(rhn, struct kretprobe_instance, node);
2204
2205	if (rp->entry_handler && rp->entry_handler(ri, regs))
2206	rethook_recycle(rhn);
2207	else
2208	rethook_hook(rhn, regs, kprobe_ftrace(p));
2209
2210	return 0;
2211	}
2212	NOKPROBE_SYMBOL(pre_handler_kretprobe);
2213
2214	static void kretprobe_rethook_handler(struct rethook_node rh, void data,
2215	unsigned long ret_addr,
2216	struct pt_regs *regs)
2217	{
2218	struct kretprobe rp = (struct kretprobe )data;
2219	struct kretprobe_instance *ri;
2220	struct kprobe_ctlblk *kcb;
2221
2222	/* The data must NOT be null. This means rethook data structure is broken. */
2223	if (WARN_ON_ONCE(!data) \|\| !rp->handler)
2224	return;
2225
2226	__this_cpu_write(current_kprobe, &rp->kp);
2227	kcb = get_kprobe_ctlblk();
2228	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
2229
2230	ri = container_of(rh, struct kretprobe_instance, node);
2231	rp->handler(ri, regs);
2232
2233	__this_cpu_write(current_kprobe, NULL);
2234	}
2235	NOKPROBE_SYMBOL(kretprobe_rethook_handler);
2236
2237	#endif /* !CONFIG_KRETPROBE_ON_RETHOOK */
2238
2239	/**
2240	* kprobe_on_func_entry() -- check whether given address is function entry
2241	* @addr: Target address
2242	* @sym: Target symbol name
2243	* @offset: The offset from the symbol or the address
2244	*
2245	* This checks whether the given @addr+@offset or @sym+@offset is on the
2246	* function entry address or not.
2247	* This returns 0 if it is the function entry, or -EINVAL if it is not.
2248	* And also it returns -ENOENT if it fails the symbol or address lookup.
2249	* Caller must pass @addr or @sym (either one must be NULL), or this
2250	* returns -EINVAL.
2251	*/
2252	int kprobe_on_func_entry(kprobe_opcode_t addr, const char sym, unsigned long offset)
2253	{
2254	bool on_func_entry;
2255	kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset, &on_func_entry);
2256
2257	if (IS_ERR(kp_addr))
2258	return PTR_ERR(kp_addr);
2259
2260	if (!on_func_entry)
2261	return -EINVAL;
2262
2263	return 0;
2264	}
2265
2266	int register_kretprobe(struct kretprobe *rp)
2267	{
2268	int ret;
2269	int i;
2270	void *addr;
2271
2272	ret = kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset);
2273	if (ret)
2274	return ret;
2275
2276	/* If only 'rp->kp.addr' is specified, check reregistering kprobes */
2277	if (rp->kp.addr && warn_kprobe_rereg(&rp->kp))
2278	return -EINVAL;
2279
2280	if (kretprobe_blacklist_size) {
2281	addr = kprobe_addr(&rp->kp);
2282	if (IS_ERR(addr))
2283	return PTR_ERR(addr);
2284
2285	for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2286	if (kretprobe_blacklist[i].addr == addr)
2287	return -EINVAL;
2288	}
2289	}
2290
2291	if (rp->data_size > KRETPROBE_MAX_DATA_SIZE)
2292	return -E2BIG;
2293
2294	rp->kp.pre_handler = pre_handler_kretprobe;
2295	rp->kp.post_handler = NULL;
2296
2297	/* Pre-allocate memory for max kretprobe instances */
2298	if (rp->maxactive <= 0)
2299	rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
2300
2301	#ifdef CONFIG_KRETPROBE_ON_RETHOOK
2302	rp->rh = rethook_alloc((void *)rp, kretprobe_rethook_handler,
2303	sizeof(struct kretprobe_instance) +
2304	rp->data_size, rp->maxactive);
2305	if (IS_ERR(rp->rh))
2306	return PTR_ERR(rp->rh);
2307
2308	rp->nmissed = 0;
2309	/* Establish function entry probe point */
2310	ret = register_kprobe(&rp->kp);
2311	if (ret != 0) {
2312	rethook_free(rp->rh);
2313	rp->rh = NULL;
2314	}
2315	#else /* !CONFIG_KRETPROBE_ON_RETHOOK */
2316	rp->rph = kzalloc_obj(struct kretprobe_holder);
2317	if (!rp->rph)
2318	return -ENOMEM;
2319
2320	if (objpool_init(&rp->rph->pool, rp->maxactive, rp->data_size +
2321	sizeof(struct kretprobe_instance), GFP_KERNEL,
2322	rp->rph, kretprobe_init_inst, kretprobe_fini_pool)) {
2323	kfree(rp->rph);
2324	rp->rph = NULL;
2325	return -ENOMEM;
2326	}
2327	rcu_assign_pointer(rp->rph->rp, rp);
2328	rp->nmissed = 0;
2329	/* Establish function entry probe point */
2330	ret = register_kprobe(&rp->kp);
2331	if (ret != 0)
2332	free_rp_inst(rp);
2333	#endif
2334	return ret;
2335	}
2336	EXPORT_SYMBOL_GPL(register_kretprobe);
2337
2338	int register_kretprobes(struct kretprobe **rps, int num)
2339	{
2340	int ret = 0, i;
2341
2342	if (num <= 0)
2343	return -EINVAL;
2344	for (i = 0; i < num; i++) {
2345	ret = register_kretprobe(rps[i]);
2346	if (ret < 0) {
2347	if (i > 0)
2348	unregister_kretprobes(rps, i);
2349	break;
2350	}
2351	}
2352	return ret;
2353	}
2354	EXPORT_SYMBOL_GPL(register_kretprobes);
2355
2356	void unregister_kretprobe(struct kretprobe *rp)
2357	{
2358	unregister_kretprobes(&rp, 1);
2359	}
2360	EXPORT_SYMBOL_GPL(unregister_kretprobe);
2361
2362	void unregister_kretprobes(struct kretprobe **rps, int num)
2363	{
2364	int i;
2365
2366	if (num <= 0)
2367	return;
2368	for (i = 0; i < num; i++) {
2369	guard(mutex)(&kprobe_mutex);
2370
2371	if (__unregister_kprobe_top(&rps[i]->kp) < 0)
2372	rps[i]->kp.addr = NULL;
2373	#ifdef CONFIG_KRETPROBE_ON_RETHOOK
2374	rethook_free(rps[i]->rh);
2375	#else
2376	rcu_assign_pointer(rps[i]->rph->rp, NULL);
2377	#endif
2378	}
2379
2380	synchronize_rcu();
2381	for (i = 0; i < num; i++) {
2382	if (rps[i]->kp.addr) {
2383	__unregister_kprobe_bottom(&rps[i]->kp);
2384	#ifndef CONFIG_KRETPROBE_ON_RETHOOK
2385	free_rp_inst(rps[i]);
2386	#endif
2387	}
2388	}
2389	}
2390	EXPORT_SYMBOL_GPL(unregister_kretprobes);
2391
2392	#else /* CONFIG_KRETPROBES */
2393	int register_kretprobe(struct kretprobe *rp)
2394	{
2395	return -EOPNOTSUPP;
2396	}
2397	EXPORT_SYMBOL_GPL(register_kretprobe);
2398
2399	int register_kretprobes(struct kretprobe **rps, int num)
2400	{
2401	return -EOPNOTSUPP;
2402	}
2403	EXPORT_SYMBOL_GPL(register_kretprobes);
2404
2405	void unregister_kretprobe(struct kretprobe *rp)
2406	{
2407	}
2408	EXPORT_SYMBOL_GPL(unregister_kretprobe);
2409
2410	void unregister_kretprobes(struct kretprobe **rps, int num)
2411	{
2412	}
2413	EXPORT_SYMBOL_GPL(unregister_kretprobes);
2414
2415	static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
2416	{
2417	return 0;
2418	}
2419	NOKPROBE_SYMBOL(pre_handler_kretprobe);
2420
2421	#endif /* CONFIG_KRETPROBES */
2422
2423	/* Set the kprobe gone and remove its instruction buffer. */
2424	static void kill_kprobe(struct kprobe *p)
2425	{
2426	struct kprobe *kp;
2427
2428	lockdep_assert_held(&kprobe_mutex);
2429
2430	/*
2431	* The module is going away. We should disarm the kprobe which
2432	* is using ftrace, because ftrace framework is still available at
2433	* 'MODULE_STATE_GOING' notification.
2434	*/
2435	if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed)
2436	disarm_kprobe_ftrace(p);
2437
2438	p->flags \|= KPROBE_FLAG_GONE;
2439	if (kprobe_aggrprobe(p)) {
2440	/*
2441	* If this is an aggr_kprobe, we have to list all the
2442	* chained probes and mark them GONE.
2443	*/
2444	list_for_each_entry(kp, &p->list, list)
2445	kp->flags \|= KPROBE_FLAG_GONE;
2446	p->post_handler = NULL;
2447	kill_optimized_kprobe(p);
2448	}
2449	/*
2450	* Here, we can remove insn_slot safely, because no thread calls
2451	* the original probed function (which will be freed soon) any more.
2452	*/
2453	arch_remove_kprobe(p);
2454	}
2455
2456	/* Disable one kprobe */
2457	int disable_kprobe(struct kprobe *kp)
2458	{
2459	struct kprobe *p;
2460
2461	guard(mutex)(&kprobe_mutex);
2462
2463	/* Disable this kprobe */
2464	p = __disable_kprobe(kp);
2465
2466	return IS_ERR(p) ? PTR_ERR(p) : 0;
2467	}
2468	EXPORT_SYMBOL_GPL(disable_kprobe);
2469
2470	/* Enable one kprobe */
2471	int enable_kprobe(struct kprobe *kp)
2472	{
2473	int ret = 0;
2474	struct kprobe *p;
2475
2476	guard(mutex)(&kprobe_mutex);
2477
2478	/* Check whether specified probe is valid. */
2479	p = __get_valid_kprobe(kp);
2480	if (unlikely(p == NULL))
2481	return -EINVAL;
2482
2483	if (kprobe_gone(kp))
2484	/* This kprobe has gone, we couldn't enable it. */
2485	return -EINVAL;
2486
2487	if (p != kp)
2488	kp->flags &= ~KPROBE_FLAG_DISABLED;
2489
2490	if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2491	p->flags &= ~KPROBE_FLAG_DISABLED;
2492	ret = arm_kprobe(p);
2493	if (ret) {
2494	p->flags \|= KPROBE_FLAG_DISABLED;
2495	if (p != kp)
2496	kp->flags \|= KPROBE_FLAG_DISABLED;
2497	}
2498	}
2499	return ret;
2500	}
2501	EXPORT_SYMBOL_GPL(enable_kprobe);
2502
2503	/* Caller must NOT call this in usual path. This is only for critical case */
2504	void dump_kprobe(struct kprobe *kp)
2505	{
2506	pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n",
2507	kp->symbol_name, kp->offset, kp->addr);
2508	}
2509	NOKPROBE_SYMBOL(dump_kprobe);
2510
2511	int kprobe_add_ksym_blacklist(unsigned long entry)
2512	{
2513	struct kprobe_blacklist_entry *ent;
2514	unsigned long offset = 0, size = 0;
2515
2516	if (!kernel_text_address(entry) \|\|
2517	!kallsyms_lookup_size_offset(entry, &size, &offset))
2518	return -EINVAL;
2519
2520	ent = kmalloc_obj(*ent);
2521	if (!ent)
2522	return -ENOMEM;
2523	ent->start_addr = entry;
2524	ent->end_addr = entry + size;
2525	INIT_LIST_HEAD(&ent->list);
2526	list_add_tail_rcu(&ent->list, &kprobe_blacklist);
2527
2528	return (int)size;
2529	}
2530
2531	/* Add all symbols in given area into kprobe blacklist */
2532	int kprobe_add_area_blacklist(unsigned long start, unsigned long end)
2533	{
2534	unsigned long entry;
2535	int ret = 0;
2536
2537	for (entry = start; entry < end; entry += ret) {
2538	ret = kprobe_add_ksym_blacklist(entry);
2539	if (ret < 0)
2540	return ret;
2541	if (ret == 0) /* In case of alias symbol */
2542	ret = 1;
2543	}
2544	return 0;
2545	}
2546
2547	int __weak arch_kprobe_get_kallsym(unsigned int symnum, unsigned long value,
2548	char type, char sym)
2549	{
2550	return -ERANGE;
2551	}
2552
2553	int kprobe_get_kallsym(unsigned int symnum, unsigned long value, char type,
2554	char *sym)
2555	{
2556	#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
2557	if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym))
2558	return 0;
2559	#ifdef CONFIG_OPTPROBES
2560	if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym))
2561	return 0;
2562	#endif
2563	#endif
2564	if (!arch_kprobe_get_kallsym(&symnum, value, type, sym))
2565	return 0;
2566	return -ERANGE;
2567	}
2568
2569	int __init __weak arch_populate_kprobe_blacklist(void)
2570	{
2571	return 0;
2572	}
2573
2574	/*
2575	* Lookup and populate the kprobe_blacklist.
2576	*
2577	* Unlike the kretprobe blacklist, we'll need to determine
2578	* the range of addresses that belong to the said functions,
2579	* since a kprobe need not necessarily be at the beginning
2580	* of a function.
2581	*/
2582	static int __init populate_kprobe_blacklist(unsigned long *start,
2583	unsigned long *end)
2584	{
2585	unsigned long entry;
2586	unsigned long *iter;
2587	int ret;
2588
2589	for (iter = start; iter < end; iter++) {
2590	entry = (unsigned long)dereference_symbol_descriptor((void )iter);
2591	ret = kprobe_add_ksym_blacklist(entry);
2592	if (ret == -EINVAL)
2593	continue;
2594	if (ret < 0)
2595	return ret;
2596	}
2597
2598	/* Symbols in '__kprobes_text' are blacklisted */
2599	ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start,
2600	(unsigned long)__kprobes_text_end);
2601	if (ret)
2602	return ret;
2603
2604	/* Symbols in 'noinstr' section are blacklisted */
2605	ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start,
2606	(unsigned long)__noinstr_text_end);
2607
2608	return ret ? : arch_populate_kprobe_blacklist();
2609	}
2610
2611	#ifdef CONFIG_MODULES
2612	/* Remove all symbols in given area from kprobe blacklist */
2613	static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end)
2614	{
2615	struct kprobe_blacklist_entry ent, n;
2616
2617	list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) {
2618	if (ent->start_addr < start \|\| ent->start_addr >= end)
2619	continue;
2620	list_del_rcu(&ent->list);
2621	kfree_rcu(ent, rcu);
2622	}
2623	}
2624
2625	static void kprobe_remove_ksym_blacklist(unsigned long entry)
2626	{
2627	kprobe_remove_area_blacklist(entry, entry + 1);
2628	}
2629
2630	static void add_module_kprobe_blacklist(struct module *mod)
2631	{
2632	unsigned long start, end;
2633	int i;
2634
2635	if (mod->kprobe_blacklist) {
2636	for (i = 0; i < mod->num_kprobe_blacklist; i++)
2637	kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]);
2638	}
2639
2640	start = (unsigned long)mod->kprobes_text_start;
2641	if (start) {
2642	end = start + mod->kprobes_text_size;
2643	kprobe_add_area_blacklist(start, end);
2644	}
2645
2646	start = (unsigned long)mod->noinstr_text_start;
2647	if (start) {
2648	end = start + mod->noinstr_text_size;
2649	kprobe_add_area_blacklist(start, end);
2650	}
2651	}
2652
2653	static void remove_module_kprobe_blacklist(struct module *mod)
2654	{
2655	unsigned long start, end;
2656	int i;
2657
2658	if (mod->kprobe_blacklist) {
2659	for (i = 0; i < mod->num_kprobe_blacklist; i++)
2660	kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]);
2661	}
2662
2663	start = (unsigned long)mod->kprobes_text_start;
2664	if (start) {
2665	end = start + mod->kprobes_text_size;
2666	kprobe_remove_area_blacklist(start, end);
2667	}
2668
2669	start = (unsigned long)mod->noinstr_text_start;
2670	if (start) {
2671	end = start + mod->noinstr_text_size;
2672	kprobe_remove_area_blacklist(start, end);
2673	}
2674	}
2675
2676	/* Module notifier call back, checking kprobes on the module */
2677	static int kprobes_module_callback(struct notifier_block *nb,
2678	unsigned long val, void *data)
2679	{
2680	struct module *mod = data;
2681	struct hlist_head *head;
2682	struct kprobe *p;
2683	unsigned int i;
2684	int checkcore = (val == MODULE_STATE_GOING);
2685
2686	guard(mutex)(&kprobe_mutex);
2687
2688	if (val == MODULE_STATE_COMING)
2689	add_module_kprobe_blacklist(mod);
2690
2691	if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
2692	return NOTIFY_DONE;
2693
2694	/*
2695	* When 'MODULE_STATE_GOING' was notified, both of module '.text' and
2696	* '.init.text' sections would be freed. When 'MODULE_STATE_LIVE' was
2697	* notified, only '.init.text' section would be freed. We need to
2698	* disable kprobes which have been inserted in the sections.
2699	*/
2700	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2701	head = &kprobe_table[i];
2702	hlist_for_each_entry(p, head, hlist)
2703	if (within_module_init((unsigned long)p->addr, mod) \|\|
2704	(checkcore &&
2705	within_module_core((unsigned long)p->addr, mod))) {
2706	/*
2707	* The vaddr this probe is installed will soon
2708	* be vfreed buy not synced to disk. Hence,
2709	* disarming the breakpoint isn't needed.
2710	*
2711	* Note, this will also move any optimized probes
2712	* that are pending to be removed from their
2713	* corresponding lists to the 'freeing_list' and
2714	* will not be touched by the delayed
2715	* kprobe_optimizer() work handler.
2716	*/
2717	kill_kprobe(p);
2718	}
2719	}
2720	if (val == MODULE_STATE_GOING)
2721	remove_module_kprobe_blacklist(mod);
2722	return NOTIFY_DONE;
2723	}
2724
2725	static struct notifier_block kprobe_module_nb = {
2726	.notifier_call = kprobes_module_callback,
2727	.priority = 0
2728	};
2729
2730	static int kprobe_register_module_notifier(void)
2731	{
2732	return register_module_notifier(&kprobe_module_nb);
2733	}
2734	#else
2735	static int kprobe_register_module_notifier(void)
2736	{
2737	return 0;
2738	}
2739	#endif /* CONFIG_MODULES */
2740
2741	void kprobe_free_init_mem(void)
2742	{
2743	void start = (void )(&__init_begin);
2744	void end = (void )(&__init_end);
2745	struct hlist_head *head;
2746	struct kprobe *p;
2747	int i;
2748
2749	guard(mutex)(&kprobe_mutex);
2750
2751	/* Kill all kprobes on initmem because the target code has been freed. */
2752	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2753	head = &kprobe_table[i];
2754	hlist_for_each_entry(p, head, hlist) {
2755	if (start <= (void )p->addr && (void )p->addr < end)
2756	kill_kprobe(p);
2757	}
2758	}
2759	}
2760
2761	static int __init init_kprobes(void)
2762	{
2763	int i, err;
2764
2765	/* FIXME allocate the probe table, currently defined statically */
2766	/* initialize all list heads */
2767	for (i = 0; i < KPROBE_TABLE_SIZE; i++)
2768	INIT_HLIST_HEAD(&kprobe_table[i]);
2769
2770	err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2771	__stop_kprobe_blacklist);
2772	if (err)
2773	pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err);
2774
2775	if (kretprobe_blacklist_size) {
2776	/* lookup the function address from its name */
2777	for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2778	kretprobe_blacklist[i].addr =
2779	kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
2780	if (!kretprobe_blacklist[i].addr)
2781	pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n",
2782	kretprobe_blacklist[i].name);
2783	}
2784	}
2785
2786	/* By default, kprobes are armed */
2787	kprobes_all_disarmed = false;
2788
2789	/* Initialize the optimization infrastructure */
2790	init_optprobe();
2791
2792	err = arch_init_kprobes();
2793	if (!err)
2794	err = register_die_notifier(&kprobe_exceptions_nb);
2795	if (!err)
2796	err = kprobe_register_module_notifier();
2797
2798	kprobes_initialized = (err == 0);
2799	kprobe_sysctls_init();
2800	return err;
2801	}
2802	early_initcall(init_kprobes);
2803
2804	#if defined(CONFIG_OPTPROBES)
2805	static int __init init_optprobes(void)
2806	{
2807	/*
2808	* Enable kprobe optimization - this kicks the optimizer which
2809	* depends on synchronize_rcu_tasks() and ksoftirqd, that is
2810	* not spawned in early initcall. So delay the optimization.
2811	*/
2812	optimize_all_kprobes();
2813
2814	return 0;
2815	}
2816	subsys_initcall(init_optprobes);
2817	#endif
2818
2819	#ifdef CONFIG_DEBUG_FS
2820	static void report_probe(struct seq_file pi, struct kprobe p,
2821	const char sym, int offset, char modname, struct kprobe *pp)
2822	{
2823	char *kprobe_type;
2824	void *addr = p->addr;
2825
2826	if (p->pre_handler == pre_handler_kretprobe)
2827	kprobe_type = "r";
2828	else
2829	kprobe_type = "k";
2830
2831	if (!kallsyms_show_value(pi->file->f_cred))
2832	addr = NULL;
2833
2834	if (sym)
2835	seq_printf(pi, "%px %s %s+0x%x %s ",
2836	addr, kprobe_type, sym, offset,
2837	(modname ? modname : " "));
2838	else /* try to use %pS */
2839	seq_printf(pi, "%px %s %pS ",
2840	addr, kprobe_type, p->addr);
2841
2842	if (!pp)
2843	pp = p;
2844	seq_printf(pi, "%s%s%s%s\n",
2845	(kprobe_gone(p) ? "[GONE]" : ""),
2846	((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
2847	(kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2848	(kprobe_ftrace(pp) ? "[FTRACE]" : ""));
2849	}
2850
2851	static void kprobe_seq_start(struct seq_file f, loff_t *pos)
2852	{
2853	return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2854	}
2855
2856	static void kprobe_seq_next(struct seq_file f, void v, loff_t pos)
2857	{
2858	(*pos)++;
2859	if (*pos >= KPROBE_TABLE_SIZE)
2860	return NULL;
2861	return pos;
2862	}
2863
2864	static void kprobe_seq_stop(struct seq_file f, void v)
2865	{
2866	/* Nothing to do */
2867	}
2868
2869	static int show_kprobe_addr(struct seq_file pi, void v)
2870	{
2871	struct hlist_head *head;
2872	struct kprobe p, kp;
2873	const char *sym;
2874	unsigned int i = (loff_t ) v;
2875	unsigned long offset = 0;
2876	char *modname, namebuf[KSYM_NAME_LEN];
2877
2878	head = &kprobe_table[i];
2879	preempt_disable();
2880	hlist_for_each_entry_rcu(p, head, hlist) {
2881	sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2882	&offset, &modname, namebuf);
2883	if (kprobe_aggrprobe(p)) {
2884	list_for_each_entry_rcu(kp, &p->list, list)
2885	report_probe(pi, kp, sym, offset, modname, p);
2886	} else
2887	report_probe(pi, p, sym, offset, modname, NULL);
2888	}
2889	preempt_enable();
2890	return 0;
2891	}
2892
2893	static const struct seq_operations kprobes_sops = {
2894	.start = kprobe_seq_start,
2895	.next = kprobe_seq_next,
2896	.stop = kprobe_seq_stop,
2897	.show = show_kprobe_addr
2898	};
2899
2900	DEFINE_SEQ_ATTRIBUTE(kprobes);
2901
2902	/* kprobes/blacklist -- shows which functions can not be probed */
2903	static void kprobe_blacklist_seq_start(struct seq_file m, loff_t *pos)
2904	{
2905	mutex_lock(&kprobe_mutex);
2906	return seq_list_start(&kprobe_blacklist, *pos);
2907	}
2908
2909	static void kprobe_blacklist_seq_next(struct seq_file m, void v, loff_t pos)
2910	{
2911	return seq_list_next(v, &kprobe_blacklist, pos);
2912	}
2913
2914	static int kprobe_blacklist_seq_show(struct seq_file m, void v)
2915	{
2916	struct kprobe_blacklist_entry *ent =
2917	list_entry(v, struct kprobe_blacklist_entry, list);
2918
2919	/*
2920	* If '/proc/kallsyms' is not showing kernel address, we won't
2921	* show them here either.
2922	*/
2923	if (!kallsyms_show_value(m->file->f_cred))
2924	seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL,
2925	(void *)ent->start_addr);
2926	else
2927	seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
2928	(void )ent->end_addr, (void )ent->start_addr);
2929	return 0;
2930	}
2931
2932	static void kprobe_blacklist_seq_stop(struct seq_file f, void v)
2933	{
2934	mutex_unlock(&kprobe_mutex);
2935	}
2936
2937	static const struct seq_operations kprobe_blacklist_sops = {
2938	.start = kprobe_blacklist_seq_start,
2939	.next = kprobe_blacklist_seq_next,
2940	.stop = kprobe_blacklist_seq_stop,
2941	.show = kprobe_blacklist_seq_show,
2942	};
2943	DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist);
2944
2945	static int arm_all_kprobes(void)
2946	{
2947	struct hlist_head *head;
2948	struct kprobe *p;
2949	unsigned int i, total = 0, errors = 0;
2950	int err, ret = 0;
2951
2952	guard(mutex)(&kprobe_mutex);
2953
2954	/* If kprobes are armed, just return */
2955	if (!kprobes_all_disarmed)
2956	return 0;
2957
2958	/*
2959	* optimize_kprobe() called by arm_kprobe() checks
2960	* kprobes_all_disarmed, so set kprobes_all_disarmed before
2961	* arm_kprobe.
2962	*/
2963	kprobes_all_disarmed = false;
2964	/* Arming kprobes doesn't optimize kprobe itself */
2965	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2966	head = &kprobe_table[i];
2967	/* Arm all kprobes on a best-effort basis */
2968	hlist_for_each_entry(p, head, hlist) {
2969	if (!kprobe_disabled(p)) {
2970	err = arm_kprobe(p);
2971	if (err) {
2972	errors++;
2973	ret = err;
2974	}
2975	total++;
2976	}
2977	}
2978	}
2979
2980	if (errors)
2981	pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n",
2982	errors, total);
2983	else
2984	pr_info("Kprobes globally enabled\n");
2985
2986	return ret;
2987	}
2988
2989	static int disarm_all_kprobes(void)
2990	{
2991	struct hlist_head *head;
2992	struct kprobe *p;
2993	unsigned int i, total = 0, errors = 0;
2994	int err, ret = 0;
2995
2996	guard(mutex)(&kprobe_mutex);
2997
2998	/* If kprobes are already disarmed, just return */
2999	if (kprobes_all_disarmed)
3000	return 0;
3001
3002	kprobes_all_disarmed = true;
3003
3004	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
3005	head = &kprobe_table[i];
3006	/* Disarm all kprobes on a best-effort basis */
3007	hlist_for_each_entry(p, head, hlist) {
3008	if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
3009	err = disarm_kprobe(p, false);
3010	if (err) {
3011	errors++;
3012	ret = err;
3013	}
3014	total++;
3015	}
3016	}
3017	}
3018
3019	if (errors)
3020	pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n",
3021	errors, total);
3022	else
3023	pr_info("Kprobes globally disabled\n");
3024
3025	/* Wait for disarming all kprobes by optimizer */
3026	wait_for_kprobe_optimizer_locked();
3027	return ret;
3028	}
3029
3030	/*
3031	* XXX: The debugfs bool file interface doesn't allow for callbacks
3032	* when the bool state is switched. We can reuse that facility when
3033	* available
3034	*/
3035	static ssize_t read_enabled_file_bool(struct file *file,
3036	char __user user_buf, size_t count, loff_t ppos)
3037	{
3038	char buf[3];
3039
3040	if (!kprobes_all_disarmed)
3041	buf[0] = '1';
3042	else
3043	buf[0] = '0';
3044	buf[1] = '\n';
3045	buf[2] = 0x00;
3046	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
3047	}
3048
3049	static ssize_t write_enabled_file_bool(struct file *file,
3050	const char __user user_buf, size_t count, loff_t ppos)
3051	{
3052	bool enable;
3053	int ret;
3054
3055	ret = kstrtobool_from_user(user_buf, count, &enable);
3056	if (ret)
3057	return ret;
3058
3059	ret = enable ? arm_all_kprobes() : disarm_all_kprobes();
3060	if (ret)
3061	return ret;
3062
3063	return count;
3064	}
3065
3066	static const struct file_operations fops_kp = {
3067	.read = read_enabled_file_bool,
3068	.write = write_enabled_file_bool,
3069	.llseek = default_llseek,
3070	};
3071
3072	static int __init debugfs_kprobe_init(void)
3073	{
3074	struct dentry *dir;
3075
3076	dir = debugfs_create_dir("kprobes", NULL);
3077
3078	debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops);
3079
3080	debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp);
3081
3082	debugfs_create_file("blacklist", 0400, dir, NULL,
3083	&kprobe_blacklist_fops);
3084
3085	return 0;
3086	}
3087
3088	late_initcall(debugfs_kprobe_init);
3089	#endif /* CONFIG_DEBUG_FS */