linux - kernel/kprobes.c

3043 lines · 74.2 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Kernel Probes (KProbes)
4	*
5	* Copyright (C) IBM Corporation, 2002, 2004
6	*
7	* 2002-Oct Created by Vamsi Krishna S <[email protected]> Kernel
8	* Probes initial implementation (includes suggestions from
9	* Rusty Russell).
10	* 2004-Aug Updated by Prasanna S Panchamukhi <[email protected]> with
11	* hlists and exceptions notifier as suggested by Andi Kleen.
12	* 2004-July Suparna Bhattacharya <[email protected]> added jumper probes
13	* interface to access function arguments.
14	* 2004-Sep Prasanna S Panchamukhi <[email protected]> Changed Kprobes
15	* exceptions notifier to be first on the priority list.
16	* 2005-May Hien Nguyen <[email protected]>, Jim Keniston
17	* <[email protected]> and Prasanna S Panchamukhi
18	* <[email protected]> added function-return probes.
19	*/
20
21	#define pr_fmt(fmt) "kprobes: " fmt
22
23	#include <linux/kprobes.h>
24	#include <linux/hash.h>
25	#include <linux/init.h>
26	#include <linux/slab.h>
27	#include <linux/stddef.h>
28	#include <linux/export.h>
29	#include <linux/moduleloader.h>
30	#include <linux/kallsyms.h>
31	#include <linux/freezer.h>
32	#include <linux/seq_file.h>
33	#include <linux/debugfs.h>
34	#include <linux/sysctl.h>
35	#include <linux/kdebug.h>
36	#include <linux/memory.h>
37	#include <linux/ftrace.h>
38	#include <linux/cpu.h>
39	#include <linux/jump_label.h>
40	#include <linux/static_call.h>
41	#include <linux/perf_event.h>
42
43	#include <asm/sections.h>
44	#include <asm/cacheflush.h>
45	#include <asm/errno.h>
46	#include <linux/uaccess.h>
47
48	#define KPROBE_HASH_BITS 6
49	#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
50
51	#if !defined(CONFIG_OPTPROBES) \|\| !defined(CONFIG_SYSCTL)
52	#define kprobe_sysctls_init() do { } while (0)
53	#endif
54
55	static int kprobes_initialized;
56	/* kprobe_table can be accessed by
57	* - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held.
58	* Or
59	* - RCU hlist traversal under disabling preempt (breakpoint handlers)
60	*/
61	static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
62
63	/* NOTE: change this value only with 'kprobe_mutex' held */
64	static bool kprobes_all_disarmed;
65
66	/* This protects 'kprobe_table' and 'optimizing_list' */
67	static DEFINE_MUTEX(kprobe_mutex);
68	static DEFINE_PER_CPU(struct kprobe *, kprobe_instance);
69
70	kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
71	unsigned int __unused)
72	{
73	return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
74	}
75
76	/*
77	* Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where
78	* kprobes can not probe.
79	*/
80	static LIST_HEAD(kprobe_blacklist);
81
82	#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
83	/*
84	* 'kprobe::ainsn.insn' points to the copy of the instruction to be
85	* single-stepped. x86_64, POWER4 and above have no-exec support and
86	* stepping on the instruction on a vmalloced/kmalloced/data page
87	* is a recipe for disaster
88	*/
89	struct kprobe_insn_page {
90	struct list_head list;
91	kprobe_opcode_t insns; / Page of instruction slots */
92	struct kprobe_insn_cache *cache;
93	int nused;
94	int ngarbage;
95	char slot_used[];
96	};
97
98	#define KPROBE_INSN_PAGE_SIZE(slots) \
99	(offsetof(struct kprobe_insn_page, slot_used) + \
100	(sizeof(char) * (slots)))
101
102	static int slots_per_page(struct kprobe_insn_cache *c)
103	{
104	return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
105	}
106
107	enum kprobe_slot_state {
108	SLOT_CLEAN = 0,
109	SLOT_DIRTY = 1,
110	SLOT_USED = 2,
111	};
112
113	void __weak *alloc_insn_page(void)
114	{
115	/*
116	* Use module_alloc() so this page is within +/- 2GB of where the
117	* kernel image and loaded module images reside. This is required
118	* for most of the architectures.
119	* (e.g. x86-64 needs this to handle the %rip-relative fixups.)
120	*/
121	return module_alloc(PAGE_SIZE);
122	}
123
124	static void free_insn_page(void *page)
125	{
126	module_memfree(page);
127	}
128
129	struct kprobe_insn_cache kprobe_insn_slots = {
130	.mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
131	.alloc = alloc_insn_page,
132	.free = free_insn_page,
133	.sym = KPROBE_INSN_PAGE_SYM,
134	.pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
135	.insn_size = MAX_INSN_SIZE,
136	.nr_garbage = 0,
137	};
138	static int collect_garbage_slots(struct kprobe_insn_cache *c);
139
140	/**
141	* __get_insn_slot() - Find a slot on an executable page for an instruction.
142	* We allocate an executable page if there's no room on existing ones.
143	*/
144	kprobe_opcode_t __get_insn_slot(struct kprobe_insn_cache c)
145	{
146	struct kprobe_insn_page *kip;
147	kprobe_opcode_t *slot = NULL;
148
149	/* Since the slot array is not protected by rcu, we need a mutex */
150	mutex_lock(&c->mutex);
151	retry:
152	rcu_read_lock();
153	list_for_each_entry_rcu(kip, &c->pages, list) {
154	if (kip->nused < slots_per_page(c)) {
155	int i;
156
157	for (i = 0; i < slots_per_page(c); i++) {
158	if (kip->slot_used[i] == SLOT_CLEAN) {
159	kip->slot_used[i] = SLOT_USED;
160	kip->nused++;
161	slot = kip->insns + (i * c->insn_size);
162	rcu_read_unlock();
163	goto out;
164	}
165	}
166	/* kip->nused is broken. Fix it. */
167	kip->nused = slots_per_page(c);
168	WARN_ON(1);
169	}
170	}
171	rcu_read_unlock();
172
173	/* If there are any garbage slots, collect it and try again. */
174	if (c->nr_garbage && collect_garbage_slots(c) == 0)
175	goto retry;
176
177	/* All out of space. Need to allocate a new page. */
178	kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
179	if (!kip)
180	goto out;
181
182	kip->insns = c->alloc();
183	if (!kip->insns) {
184	kfree(kip);
185	goto out;
186	}
187	INIT_LIST_HEAD(&kip->list);
188	memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
189	kip->slot_used[0] = SLOT_USED;
190	kip->nused = 1;
191	kip->ngarbage = 0;
192	kip->cache = c;
193	list_add_rcu(&kip->list, &c->pages);
194	slot = kip->insns;
195
196	/* Record the perf ksymbol register event after adding the page */
197	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns,
198	PAGE_SIZE, false, c->sym);
199	out:
200	mutex_unlock(&c->mutex);
201	return slot;
202	}
203
204	/* Return true if all garbages are collected, otherwise false. */
205	static bool collect_one_slot(struct kprobe_insn_page *kip, int idx)
206	{
207	kip->slot_used[idx] = SLOT_CLEAN;
208	kip->nused--;
209	if (kip->nused == 0) {
210	/*
211	* Page is no longer in use. Free it unless
212	* it's the last one. We keep the last one
213	* so as not to have to set it up again the
214	* next time somebody inserts a probe.
215	*/
216	if (!list_is_singular(&kip->list)) {
217	/*
218	* Record perf ksymbol unregister event before removing
219	* the page.
220	*/
221	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
222	(unsigned long)kip->insns, PAGE_SIZE, true,
223	kip->cache->sym);
224	list_del_rcu(&kip->list);
225	synchronize_rcu();
226	kip->cache->free(kip->insns);
227	kfree(kip);
228	}
229	return true;
230	}
231	return false;
232	}
233
234	static int collect_garbage_slots(struct kprobe_insn_cache *c)
235	{
236	struct kprobe_insn_page kip, next;
237
238	/* Ensure no-one is interrupted on the garbages */
239	synchronize_rcu();
240
241	list_for_each_entry_safe(kip, next, &c->pages, list) {
242	int i;
243
244	if (kip->ngarbage == 0)
245	continue;
246	kip->ngarbage = 0; /* we will collect all garbages */
247	for (i = 0; i < slots_per_page(c); i++) {
248	if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
249	break;
250	}
251	}
252	c->nr_garbage = 0;
253	return 0;
254	}
255
256	void __free_insn_slot(struct kprobe_insn_cache *c,
257	kprobe_opcode_t *slot, int dirty)
258	{
259	struct kprobe_insn_page *kip;
260	long idx;
261
262	mutex_lock(&c->mutex);
263	rcu_read_lock();
264	list_for_each_entry_rcu(kip, &c->pages, list) {
265	idx = ((long)slot - (long)kip->insns) /
266	(c->insn_size * sizeof(kprobe_opcode_t));
267	if (idx >= 0 && idx < slots_per_page(c))
268	goto out;
269	}
270	/* Could not find this slot. */
271	WARN_ON(1);
272	kip = NULL;
273	out:
274	rcu_read_unlock();
275	/* Mark and sweep: this may sleep */
276	if (kip) {
277	/* Check double free */
278	WARN_ON(kip->slot_used[idx] != SLOT_USED);
279	if (dirty) {
280	kip->slot_used[idx] = SLOT_DIRTY;
281	kip->ngarbage++;
282	if (++c->nr_garbage > slots_per_page(c))
283	collect_garbage_slots(c);
284	} else {
285	collect_one_slot(kip, idx);
286	}
287	}
288	mutex_unlock(&c->mutex);
289	}
290
291	/*
292	* Check given address is on the page of kprobe instruction slots.
293	* This will be used for checking whether the address on a stack
294	* is on a text area or not.
295	*/
296	bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
297	{
298	struct kprobe_insn_page *kip;
299	bool ret = false;
300
301	rcu_read_lock();
302	list_for_each_entry_rcu(kip, &c->pages, list) {
303	if (addr >= (unsigned long)kip->insns &&
304	addr < (unsigned long)kip->insns + PAGE_SIZE) {
305	ret = true;
306	break;
307	}
308	}
309	rcu_read_unlock();
310
311	return ret;
312	}
313
314	int kprobe_cache_get_kallsym(struct kprobe_insn_cache c, unsigned int symnum,
315	unsigned long value, char type, char *sym)
316	{
317	struct kprobe_insn_page *kip;
318	int ret = -ERANGE;
319
320	rcu_read_lock();
321	list_for_each_entry_rcu(kip, &c->pages, list) {
322	if ((*symnum)--)
323	continue;
324	strscpy(sym, c->sym, KSYM_NAME_LEN);
325	*type = 't';
326	*value = (unsigned long)kip->insns;
327	ret = 0;
328	break;
329	}
330	rcu_read_unlock();
331
332	return ret;
333	}
334
335	#ifdef CONFIG_OPTPROBES
336	void __weak *alloc_optinsn_page(void)
337	{
338	return alloc_insn_page();
339	}
340
341	void __weak free_optinsn_page(void *page)
342	{
343	free_insn_page(page);
344	}
345
346	/* For optimized_kprobe buffer */
347	struct kprobe_insn_cache kprobe_optinsn_slots = {
348	.mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
349	.alloc = alloc_optinsn_page,
350	.free = free_optinsn_page,
351	.sym = KPROBE_OPTINSN_PAGE_SYM,
352	.pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
353	/* .insn_size is initialized later */
354	.nr_garbage = 0,
355	};
356	#endif
357	#endif
358
359	/* We have preemption disabled.. so it is safe to use __ versions */
360	static inline void set_kprobe_instance(struct kprobe *kp)
361	{
362	__this_cpu_write(kprobe_instance, kp);
363	}
364
365	static inline void reset_kprobe_instance(void)
366	{
367	__this_cpu_write(kprobe_instance, NULL);
368	}
369
370	/*
371	* This routine is called either:
372	* - under the 'kprobe_mutex' - during kprobe_[un]register().
373	* OR
374	* - with preemption disabled - from architecture specific code.
375	*/
376	struct kprobe get_kprobe(void addr)
377	{
378	struct hlist_head *head;
379	struct kprobe *p;
380
381	head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
382	hlist_for_each_entry_rcu(p, head, hlist,
383	lockdep_is_held(&kprobe_mutex)) {
384	if (p->addr == addr)
385	return p;
386	}
387
388	return NULL;
389	}
390	NOKPROBE_SYMBOL(get_kprobe);
391
392	static int aggr_pre_handler(struct kprobe p, struct pt_regs regs);
393
394	/* Return true if 'p' is an aggregator */
395	static inline bool kprobe_aggrprobe(struct kprobe *p)
396	{
397	return p->pre_handler == aggr_pre_handler;
398	}
399
400	/* Return true if 'p' is unused */
401	static inline bool kprobe_unused(struct kprobe *p)
402	{
403	return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
404	list_empty(&p->list);
405	}
406
407	/* Keep all fields in the kprobe consistent. */
408	static inline void copy_kprobe(struct kprobe ap, struct kprobe p)
409	{
410	memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
411	memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
412	}
413
414	#ifdef CONFIG_OPTPROBES
415	/* NOTE: This is protected by 'kprobe_mutex'. */
416	static bool kprobes_allow_optimization;
417
418	/*
419	* Call all 'kprobe::pre_handler' on the list, but ignores its return value.
420	* This must be called from arch-dep optimized caller.
421	*/
422	void opt_pre_handler(struct kprobe p, struct pt_regs regs)
423	{
424	struct kprobe *kp;
425
426	list_for_each_entry_rcu(kp, &p->list, list) {
427	if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
428	set_kprobe_instance(kp);
429	kp->pre_handler(kp, regs);
430	}
431	reset_kprobe_instance();
432	}
433	}
434	NOKPROBE_SYMBOL(opt_pre_handler);
435
436	/* Free optimized instructions and optimized_kprobe */
437	static void free_aggr_kprobe(struct kprobe *p)
438	{
439	struct optimized_kprobe *op;
440
441	op = container_of(p, struct optimized_kprobe, kp);
442	arch_remove_optimized_kprobe(op);
443	arch_remove_kprobe(p);
444	kfree(op);
445	}
446
447	/* Return true if the kprobe is ready for optimization. */
448	static inline int kprobe_optready(struct kprobe *p)
449	{
450	struct optimized_kprobe *op;
451
452	if (kprobe_aggrprobe(p)) {
453	op = container_of(p, struct optimized_kprobe, kp);
454	return arch_prepared_optinsn(&op->optinsn);
455	}
456
457	return 0;
458	}
459
460	/* Return true if the kprobe is disarmed. Note: p must be on hash list */
461	bool kprobe_disarmed(struct kprobe *p)
462	{
463	struct optimized_kprobe *op;
464
465	/* If kprobe is not aggr/opt probe, just return kprobe is disabled */
466	if (!kprobe_aggrprobe(p))
467	return kprobe_disabled(p);
468
469	op = container_of(p, struct optimized_kprobe, kp);
470
471	return kprobe_disabled(p) && list_empty(&op->list);
472	}
473
474	/* Return true if the probe is queued on (un)optimizing lists */
475	static bool kprobe_queued(struct kprobe *p)
476	{
477	struct optimized_kprobe *op;
478
479	if (kprobe_aggrprobe(p)) {
480	op = container_of(p, struct optimized_kprobe, kp);
481	if (!list_empty(&op->list))
482	return true;
483	}
484	return false;
485	}
486
487	/*
488	* Return an optimized kprobe whose optimizing code replaces
489	* instructions including 'addr' (exclude breakpoint).
490	*/
491	static struct kprobe get_optimized_kprobe(kprobe_opcode_t addr)
492	{
493	int i;
494	struct kprobe *p = NULL;
495	struct optimized_kprobe *op;
496
497	/* Don't check i == 0, since that is a breakpoint case. */
498	for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++)
499	p = get_kprobe(addr - i);
500
501	if (p && kprobe_optready(p)) {
502	op = container_of(p, struct optimized_kprobe, kp);
503	if (arch_within_optimized_kprobe(op, addr))
504	return p;
505	}
506
507	return NULL;
508	}
509
510	/* Optimization staging list, protected by 'kprobe_mutex' */
511	static LIST_HEAD(optimizing_list);
512	static LIST_HEAD(unoptimizing_list);
513	static LIST_HEAD(freeing_list);
514
515	static void kprobe_optimizer(struct work_struct *work);
516	static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
517	#define OPTIMIZE_DELAY 5
518
519	/*
520	* Optimize (replace a breakpoint with a jump) kprobes listed on
521	* 'optimizing_list'.
522	*/
523	static void do_optimize_kprobes(void)
524	{
525	lockdep_assert_held(&text_mutex);
526	/*
527	* The optimization/unoptimization refers 'online_cpus' via
528	* stop_machine() and cpu-hotplug modifies the 'online_cpus'.
529	* And same time, 'text_mutex' will be held in cpu-hotplug and here.
530	* This combination can cause a deadlock (cpu-hotplug tries to lock
531	* 'text_mutex' but stop_machine() can not be done because
532	* the 'online_cpus' has been changed)
533	* To avoid this deadlock, caller must have locked cpu-hotplug
534	* for preventing cpu-hotplug outside of 'text_mutex' locking.
535	*/
536	lockdep_assert_cpus_held();
537
538	/* Optimization never be done when disarmed */
539	if (kprobes_all_disarmed \|\| !kprobes_allow_optimization \|\|
540	list_empty(&optimizing_list))
541	return;
542
543	arch_optimize_kprobes(&optimizing_list);
544	}
545
546	/*
547	* Unoptimize (replace a jump with a breakpoint and remove the breakpoint
548	* if need) kprobes listed on 'unoptimizing_list'.
549	*/
550	static void do_unoptimize_kprobes(void)
551	{
552	struct optimized_kprobe op, tmp;
553
554	lockdep_assert_held(&text_mutex);
555	/* See comment in do_optimize_kprobes() */
556	lockdep_assert_cpus_held();
557
558	if (!list_empty(&unoptimizing_list))
559	arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
560
561	/* Loop on 'freeing_list' for disarming and removing from kprobe hash list */
562	list_for_each_entry_safe(op, tmp, &freeing_list, list) {
563	/* Switching from detour code to origin */
564	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
565	/* Disarm probes if marked disabled and not gone */
566	if (kprobe_disabled(&op->kp) && !kprobe_gone(&op->kp))
567	arch_disarm_kprobe(&op->kp);
568	if (kprobe_unused(&op->kp)) {
569	/*
570	* Remove unused probes from hash list. After waiting
571	* for synchronization, these probes are reclaimed.
572	* (reclaiming is done by do_free_cleaned_kprobes().)
573	*/
574	hlist_del_rcu(&op->kp.hlist);
575	} else
576	list_del_init(&op->list);
577	}
578	}
579
580	/* Reclaim all kprobes on the 'freeing_list' */
581	static void do_free_cleaned_kprobes(void)
582	{
583	struct optimized_kprobe op, tmp;
584
585	list_for_each_entry_safe(op, tmp, &freeing_list, list) {
586	list_del_init(&op->list);
587	if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) {
588	/*
589	* This must not happen, but if there is a kprobe
590	* still in use, keep it on kprobes hash list.
591	*/
592	continue;
593	}
594	free_aggr_kprobe(&op->kp);
595	}
596	}
597
598	/* Start optimizer after OPTIMIZE_DELAY passed */
599	static void kick_kprobe_optimizer(void)
600	{
601	schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
602	}
603
604	/* Kprobe jump optimizer */
605	static void kprobe_optimizer(struct work_struct *work)
606	{
607	mutex_lock(&kprobe_mutex);
608	cpus_read_lock();
609	mutex_lock(&text_mutex);
610
611	/*
612	* Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
613	* kprobes before waiting for quiesence period.
614	*/
615	do_unoptimize_kprobes();
616
617	/*
618	* Step 2: Wait for quiesence period to ensure all potentially
619	* preempted tasks to have normally scheduled. Because optprobe
620	* may modify multiple instructions, there is a chance that Nth
621	* instruction is preempted. In that case, such tasks can return
622	* to 2nd-Nth byte of jump instruction. This wait is for avoiding it.
623	* Note that on non-preemptive kernel, this is transparently converted
624	* to synchronoze_sched() to wait for all interrupts to have completed.
625	*/
626	synchronize_rcu_tasks();
627
628	/* Step 3: Optimize kprobes after quiesence period */
629	do_optimize_kprobes();
630
631	/* Step 4: Free cleaned kprobes after quiesence period */
632	do_free_cleaned_kprobes();
633
634	mutex_unlock(&text_mutex);
635	cpus_read_unlock();
636
637	/* Step 5: Kick optimizer again if needed */
638	if (!list_empty(&optimizing_list) \|\| !list_empty(&unoptimizing_list))
639	kick_kprobe_optimizer();
640
641	mutex_unlock(&kprobe_mutex);
642	}
643
644	/* Wait for completing optimization and unoptimization */
645	void wait_for_kprobe_optimizer(void)
646	{
647	mutex_lock(&kprobe_mutex);
648
649	while (!list_empty(&optimizing_list) \|\| !list_empty(&unoptimizing_list)) {
650	mutex_unlock(&kprobe_mutex);
651
652	/* This will also make 'optimizing_work' execute immmediately */
653	flush_delayed_work(&optimizing_work);
654	/* 'optimizing_work' might not have been queued yet, relax */
655	cpu_relax();
656
657	mutex_lock(&kprobe_mutex);
658	}
659
660	mutex_unlock(&kprobe_mutex);
661	}
662
663	bool optprobe_queued_unopt(struct optimized_kprobe *op)
664	{
665	struct optimized_kprobe *_op;
666
667	list_for_each_entry(_op, &unoptimizing_list, list) {
668	if (op == _op)
669	return true;
670	}
671
672	return false;
673	}
674
675	/* Optimize kprobe if p is ready to be optimized */
676	static void optimize_kprobe(struct kprobe *p)
677	{
678	struct optimized_kprobe *op;
679
680	/* Check if the kprobe is disabled or not ready for optimization. */
681	if (!kprobe_optready(p) \|\| !kprobes_allow_optimization \|\|
682	(kprobe_disabled(p) \|\| kprobes_all_disarmed))
683	return;
684
685	/* kprobes with 'post_handler' can not be optimized */
686	if (p->post_handler)
687	return;
688
689	op = container_of(p, struct optimized_kprobe, kp);
690
691	/* Check there is no other kprobes at the optimized instructions */
692	if (arch_check_optimized_kprobe(op) < 0)
693	return;
694
695	/* Check if it is already optimized. */
696	if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) {
697	if (optprobe_queued_unopt(op)) {
698	/* This is under unoptimizing. Just dequeue the probe */
699	list_del_init(&op->list);
700	}
701	return;
702	}
703	op->kp.flags \|= KPROBE_FLAG_OPTIMIZED;
704
705	/*
706	* On the 'unoptimizing_list' and 'optimizing_list',
707	* 'op' must have OPTIMIZED flag
708	*/
709	if (WARN_ON_ONCE(!list_empty(&op->list)))
710	return;
711
712	list_add(&op->list, &optimizing_list);
713	kick_kprobe_optimizer();
714	}
715
716	/* Short cut to direct unoptimizing */
717	static void force_unoptimize_kprobe(struct optimized_kprobe *op)
718	{
719	lockdep_assert_cpus_held();
720	arch_unoptimize_kprobe(op);
721	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
722	}
723
724	/* Unoptimize a kprobe if p is optimized */
725	static void unoptimize_kprobe(struct kprobe *p, bool force)
726	{
727	struct optimized_kprobe *op;
728
729	if (!kprobe_aggrprobe(p) \|\| kprobe_disarmed(p))
730	return; /* This is not an optprobe nor optimized */
731
732	op = container_of(p, struct optimized_kprobe, kp);
733	if (!kprobe_optimized(p))
734	return;
735
736	if (!list_empty(&op->list)) {
737	if (optprobe_queued_unopt(op)) {
738	/* Queued in unoptimizing queue */
739	if (force) {
740	/*
741	* Forcibly unoptimize the kprobe here, and queue it
742	* in the freeing list for release afterwards.
743	*/
744	force_unoptimize_kprobe(op);
745	list_move(&op->list, &freeing_list);
746	}
747	} else {
748	/* Dequeue from the optimizing queue */
749	list_del_init(&op->list);
750	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
751	}
752	return;
753	}
754
755	/* Optimized kprobe case */
756	if (force) {
757	/* Forcibly update the code: this is a special case */
758	force_unoptimize_kprobe(op);
759	} else {
760	list_add(&op->list, &unoptimizing_list);
761	kick_kprobe_optimizer();
762	}
763	}
764
765	/* Cancel unoptimizing for reusing */
766	static int reuse_unused_kprobe(struct kprobe *ap)
767	{
768	struct optimized_kprobe *op;
769
770	/*
771	* Unused kprobe MUST be on the way of delayed unoptimizing (means
772	* there is still a relative jump) and disabled.
773	*/
774	op = container_of(ap, struct optimized_kprobe, kp);
775	WARN_ON_ONCE(list_empty(&op->list));
776	/* Enable the probe again */
777	ap->flags &= ~KPROBE_FLAG_DISABLED;
778	/* Optimize it again. (remove from 'op->list') */
779	if (!kprobe_optready(ap))
780	return -EINVAL;
781
782	optimize_kprobe(ap);
783	return 0;
784	}
785
786	/* Remove optimized instructions */
787	static void kill_optimized_kprobe(struct kprobe *p)
788	{
789	struct optimized_kprobe *op;
790
791	op = container_of(p, struct optimized_kprobe, kp);
792	if (!list_empty(&op->list))
793	/* Dequeue from the (un)optimization queue */
794	list_del_init(&op->list);
795	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
796
797	if (kprobe_unused(p)) {
798	/*
799	* Unused kprobe is on unoptimizing or freeing list. We move it
800	* to freeing_list and let the kprobe_optimizer() remove it from
801	* the kprobe hash list and free it.
802	*/
803	if (optprobe_queued_unopt(op))
804	list_move(&op->list, &freeing_list);
805	}
806
807	/* Don't touch the code, because it is already freed. */
808	arch_remove_optimized_kprobe(op);
809	}
810
811	static inline
812	void __prepare_optimized_kprobe(struct optimized_kprobe op, struct kprobe p)
813	{
814	if (!kprobe_ftrace(p))
815	arch_prepare_optimized_kprobe(op, p);
816	}
817
818	/* Try to prepare optimized instructions */
819	static void prepare_optimized_kprobe(struct kprobe *p)
820	{
821	struct optimized_kprobe *op;
822
823	op = container_of(p, struct optimized_kprobe, kp);
824	__prepare_optimized_kprobe(op, p);
825	}
826
827	/* Allocate new optimized_kprobe and try to prepare optimized instructions. */
828	static struct kprobe alloc_aggr_kprobe(struct kprobe p)
829	{
830	struct optimized_kprobe *op;
831
832	op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
833	if (!op)
834	return NULL;
835
836	INIT_LIST_HEAD(&op->list);
837	op->kp.addr = p->addr;
838	__prepare_optimized_kprobe(op, p);
839
840	return &op->kp;
841	}
842
843	static void init_aggr_kprobe(struct kprobe ap, struct kprobe p);
844
845	/*
846	* Prepare an optimized_kprobe and optimize it.
847	* NOTE: 'p' must be a normal registered kprobe.
848	*/
849	static void try_to_optimize_kprobe(struct kprobe *p)
850	{
851	struct kprobe *ap;
852	struct optimized_kprobe *op;
853
854	/* Impossible to optimize ftrace-based kprobe. */
855	if (kprobe_ftrace(p))
856	return;
857
858	/* For preparing optimization, jump_label_text_reserved() is called. */
859	cpus_read_lock();
860	jump_label_lock();
861	mutex_lock(&text_mutex);
862
863	ap = alloc_aggr_kprobe(p);
864	if (!ap)
865	goto out;
866
867	op = container_of(ap, struct optimized_kprobe, kp);
868	if (!arch_prepared_optinsn(&op->optinsn)) {
869	/* If failed to setup optimizing, fallback to kprobe. */
870	arch_remove_optimized_kprobe(op);
871	kfree(op);
872	goto out;
873	}
874
875	init_aggr_kprobe(ap, p);
876	optimize_kprobe(ap); /* This just kicks optimizer thread. */
877
878	out:
879	mutex_unlock(&text_mutex);
880	jump_label_unlock();
881	cpus_read_unlock();
882	}
883
884	static void optimize_all_kprobes(void)
885	{
886	struct hlist_head *head;
887	struct kprobe *p;
888	unsigned int i;
889
890	mutex_lock(&kprobe_mutex);
891	/* If optimization is already allowed, just return. */
892	if (kprobes_allow_optimization)
893	goto out;
894
895	cpus_read_lock();
896	kprobes_allow_optimization = true;
897	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
898	head = &kprobe_table[i];
899	hlist_for_each_entry(p, head, hlist)
900	if (!kprobe_disabled(p))
901	optimize_kprobe(p);
902	}
903	cpus_read_unlock();
904	pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n");
905	out:
906	mutex_unlock(&kprobe_mutex);
907	}
908
909	#ifdef CONFIG_SYSCTL
910	static void unoptimize_all_kprobes(void)
911	{
912	struct hlist_head *head;
913	struct kprobe *p;
914	unsigned int i;
915
916	mutex_lock(&kprobe_mutex);
917	/* If optimization is already prohibited, just return. */
918	if (!kprobes_allow_optimization) {
919	mutex_unlock(&kprobe_mutex);
920	return;
921	}
922
923	cpus_read_lock();
924	kprobes_allow_optimization = false;
925	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
926	head = &kprobe_table[i];
927	hlist_for_each_entry(p, head, hlist) {
928	if (!kprobe_disabled(p))
929	unoptimize_kprobe(p, false);
930	}
931	}
932	cpus_read_unlock();
933	mutex_unlock(&kprobe_mutex);
934
935	/* Wait for unoptimizing completion. */
936	wait_for_kprobe_optimizer();
937	pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n");
938	}
939
940	static DEFINE_MUTEX(kprobe_sysctl_mutex);
941	static int sysctl_kprobes_optimization;
942	static int proc_kprobes_optimization_handler(struct ctl_table *table,
943	int write, void *buffer,
944	size_t length, loff_t ppos)
945	{
946	int ret;
947
948	mutex_lock(&kprobe_sysctl_mutex);
949	sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
950	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
951
952	if (sysctl_kprobes_optimization)
953	optimize_all_kprobes();
954	else
955	unoptimize_all_kprobes();
956	mutex_unlock(&kprobe_sysctl_mutex);
957
958	return ret;
959	}
960
961	static struct ctl_table kprobe_sysctls[] = {
962	{
963	.procname = "kprobes-optimization",
964	.data = &sysctl_kprobes_optimization,
965	.maxlen = sizeof(int),
966	.mode = 0644,
967	.proc_handler = proc_kprobes_optimization_handler,
968	.extra1 = SYSCTL_ZERO,
969	.extra2 = SYSCTL_ONE,
970	},
971	{}
972	};
973
974	static void __init kprobe_sysctls_init(void)
975	{
976	register_sysctl_init("debug", kprobe_sysctls);
977	}
978	#endif /* CONFIG_SYSCTL */
979
980	/* Put a breakpoint for a probe. */
981	static void __arm_kprobe(struct kprobe *p)
982	{
983	struct kprobe *_p;
984
985	lockdep_assert_held(&text_mutex);
986
987	/* Find the overlapping optimized kprobes. */
988	_p = get_optimized_kprobe(p->addr);
989	if (unlikely(_p))
990	/* Fallback to unoptimized kprobe */
991	unoptimize_kprobe(_p, true);
992
993	arch_arm_kprobe(p);
994	optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
995	}
996
997	/* Remove the breakpoint of a probe. */
998	static void __disarm_kprobe(struct kprobe *p, bool reopt)
999	{
1000	struct kprobe *_p;
1001
1002	lockdep_assert_held(&text_mutex);
1003
1004	/* Try to unoptimize */
1005	unoptimize_kprobe(p, kprobes_all_disarmed);
1006
1007	if (!kprobe_queued(p)) {
1008	arch_disarm_kprobe(p);
1009	/* If another kprobe was blocked, re-optimize it. */
1010	_p = get_optimized_kprobe(p->addr);
1011	if (unlikely(_p) && reopt)
1012	optimize_kprobe(_p);
1013	}
1014	/*
1015	* TODO: Since unoptimization and real disarming will be done by
1016	* the worker thread, we can not check whether another probe are
1017	* unoptimized because of this probe here. It should be re-optimized
1018	* by the worker thread.
1019	*/
1020	}
1021
1022	#else /* !CONFIG_OPTPROBES */
1023
1024	#define optimize_kprobe(p) do {} while (0)
1025	#define unoptimize_kprobe(p, f) do {} while (0)
1026	#define kill_optimized_kprobe(p) do {} while (0)
1027	#define prepare_optimized_kprobe(p) do {} while (0)
1028	#define try_to_optimize_kprobe(p) do {} while (0)
1029	#define __arm_kprobe(p) arch_arm_kprobe(p)
1030	#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
1031	#define kprobe_disarmed(p) kprobe_disabled(p)
1032	#define wait_for_kprobe_optimizer() do {} while (0)
1033
1034	static int reuse_unused_kprobe(struct kprobe *ap)
1035	{
1036	/*
1037	* If the optimized kprobe is NOT supported, the aggr kprobe is
1038	* released at the same time that the last aggregated kprobe is
1039	* unregistered.
1040	* Thus there should be no chance to reuse unused kprobe.
1041	*/
1042	WARN_ON_ONCE(1);
1043	return -EINVAL;
1044	}
1045
1046	static void free_aggr_kprobe(struct kprobe *p)
1047	{
1048	arch_remove_kprobe(p);
1049	kfree(p);
1050	}
1051
1052	static struct kprobe alloc_aggr_kprobe(struct kprobe p)
1053	{
1054	return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
1055	}
1056	#endif /* CONFIG_OPTPROBES */
1057
1058	#ifdef CONFIG_KPROBES_ON_FTRACE
1059	static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
1060	.func = kprobe_ftrace_handler,
1061	.flags = FTRACE_OPS_FL_SAVE_REGS,
1062	};
1063
1064	static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = {
1065	.func = kprobe_ftrace_handler,
1066	.flags = FTRACE_OPS_FL_SAVE_REGS \| FTRACE_OPS_FL_IPMODIFY,
1067	};
1068
1069	static int kprobe_ipmodify_enabled;
1070	static int kprobe_ftrace_enabled;
1071
1072	static int __arm_kprobe_ftrace(struct kprobe p, struct ftrace_ops ops,
1073	int *cnt)
1074	{
1075	int ret;
1076
1077	lockdep_assert_held(&kprobe_mutex);
1078
1079	ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
1080	if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret))
1081	return ret;
1082
1083	if (*cnt == 0) {
1084	ret = register_ftrace_function(ops);
1085	if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret))
1086	goto err_ftrace;
1087	}
1088
1089	(*cnt)++;
1090	return ret;
1091
1092	err_ftrace:
1093	/*
1094	* At this point, sinec ops is not registered, we should be sefe from
1095	* registering empty filter.
1096	*/
1097	ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1098	return ret;
1099	}
1100
1101	static int arm_kprobe_ftrace(struct kprobe *p)
1102	{
1103	bool ipmodify = (p->post_handler != NULL);
1104
1105	return __arm_kprobe_ftrace(p,
1106	ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1107	ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1108	}
1109
1110	static int __disarm_kprobe_ftrace(struct kprobe p, struct ftrace_ops ops,
1111	int *cnt)
1112	{
1113	int ret;
1114
1115	lockdep_assert_held(&kprobe_mutex);
1116
1117	if (*cnt == 1) {
1118	ret = unregister_ftrace_function(ops);
1119	if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret))
1120	return ret;
1121	}
1122
1123	(*cnt)--;
1124
1125	ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1126	WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n",
1127	p->addr, ret);
1128	return ret;
1129	}
1130
1131	static int disarm_kprobe_ftrace(struct kprobe *p)
1132	{
1133	bool ipmodify = (p->post_handler != NULL);
1134
1135	return __disarm_kprobe_ftrace(p,
1136	ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1137	ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1138	}
1139	#else /* !CONFIG_KPROBES_ON_FTRACE */
1140	static inline int arm_kprobe_ftrace(struct kprobe *p)
1141	{
1142	return -ENODEV;
1143	}
1144
1145	static inline int disarm_kprobe_ftrace(struct kprobe *p)
1146	{
1147	return -ENODEV;
1148	}
1149	#endif
1150
1151	static int prepare_kprobe(struct kprobe *p)
1152	{
1153	/* Must ensure p->addr is really on ftrace */
1154	if (kprobe_ftrace(p))
1155	return arch_prepare_kprobe_ftrace(p);
1156
1157	return arch_prepare_kprobe(p);
1158	}
1159
1160	static int arm_kprobe(struct kprobe *kp)
1161	{
1162	if (unlikely(kprobe_ftrace(kp)))
1163	return arm_kprobe_ftrace(kp);
1164
1165	cpus_read_lock();
1166	mutex_lock(&text_mutex);
1167	__arm_kprobe(kp);
1168	mutex_unlock(&text_mutex);
1169	cpus_read_unlock();
1170
1171	return 0;
1172	}
1173
1174	static int disarm_kprobe(struct kprobe *kp, bool reopt)
1175	{
1176	if (unlikely(kprobe_ftrace(kp)))
1177	return disarm_kprobe_ftrace(kp);
1178
1179	cpus_read_lock();
1180	mutex_lock(&text_mutex);
1181	__disarm_kprobe(kp, reopt);
1182	mutex_unlock(&text_mutex);
1183	cpus_read_unlock();
1184
1185	return 0;
1186	}
1187
1188	/*
1189	* Aggregate handlers for multiple kprobes support - these handlers
1190	* take care of invoking the individual kprobe handlers on p->list
1191	*/
1192	static int aggr_pre_handler(struct kprobe p, struct pt_regs regs)
1193	{
1194	struct kprobe *kp;
1195
1196	list_for_each_entry_rcu(kp, &p->list, list) {
1197	if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
1198	set_kprobe_instance(kp);
1199	if (kp->pre_handler(kp, regs))
1200	return 1;
1201	}
1202	reset_kprobe_instance();
1203	}
1204	return 0;
1205	}
1206	NOKPROBE_SYMBOL(aggr_pre_handler);
1207
1208	static void aggr_post_handler(struct kprobe p, struct pt_regs regs,
1209	unsigned long flags)
1210	{
1211	struct kprobe *kp;
1212
1213	list_for_each_entry_rcu(kp, &p->list, list) {
1214	if (kp->post_handler && likely(!kprobe_disabled(kp))) {
1215	set_kprobe_instance(kp);
1216	kp->post_handler(kp, regs, flags);
1217	reset_kprobe_instance();
1218	}
1219	}
1220	}
1221	NOKPROBE_SYMBOL(aggr_post_handler);
1222
1223	/* Walks the list and increments 'nmissed' if 'p' has child probes. */
1224	void kprobes_inc_nmissed_count(struct kprobe *p)
1225	{
1226	struct kprobe *kp;
1227
1228	if (!kprobe_aggrprobe(p)) {
1229	p->nmissed++;
1230	} else {
1231	list_for_each_entry_rcu(kp, &p->list, list)
1232	kp->nmissed++;
1233	}
1234	}
1235	NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1236
1237	static struct kprobe kprobe_busy = {
1238	.addr = (void *) get_kprobe,
1239	};
1240
1241	void kprobe_busy_begin(void)
1242	{
1243	struct kprobe_ctlblk *kcb;
1244
1245	preempt_disable();
1246	__this_cpu_write(current_kprobe, &kprobe_busy);
1247	kcb = get_kprobe_ctlblk();
1248	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1249	}
1250
1251	void kprobe_busy_end(void)
1252	{
1253	__this_cpu_write(current_kprobe, NULL);
1254	preempt_enable();
1255	}
1256
1257	/* Add the new probe to 'ap->list'. */
1258	static int add_new_kprobe(struct kprobe ap, struct kprobe p)
1259	{
1260	if (p->post_handler)
1261	unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */
1262
1263	list_add_rcu(&p->list, &ap->list);
1264	if (p->post_handler && !ap->post_handler)
1265	ap->post_handler = aggr_post_handler;
1266
1267	return 0;
1268	}
1269
1270	/*
1271	* Fill in the required fields of the aggregator kprobe. Replace the
1272	* earlier kprobe in the hlist with the aggregator kprobe.
1273	*/
1274	static void init_aggr_kprobe(struct kprobe ap, struct kprobe p)
1275	{
1276	/* Copy the insn slot of 'p' to 'ap'. */
1277	copy_kprobe(p, ap);
1278	flush_insn_slot(ap);
1279	ap->addr = p->addr;
1280	ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1281	ap->pre_handler = aggr_pre_handler;
1282	/* We don't care the kprobe which has gone. */
1283	if (p->post_handler && !kprobe_gone(p))
1284	ap->post_handler = aggr_post_handler;
1285
1286	INIT_LIST_HEAD(&ap->list);
1287	INIT_HLIST_NODE(&ap->hlist);
1288
1289	list_add_rcu(&p->list, &ap->list);
1290	hlist_replace_rcu(&p->hlist, &ap->hlist);
1291	}
1292
1293	/*
1294	* This registers the second or subsequent kprobe at the same address.
1295	*/
1296	static int register_aggr_kprobe(struct kprobe orig_p, struct kprobe p)
1297	{
1298	int ret = 0;
1299	struct kprobe *ap = orig_p;
1300
1301	cpus_read_lock();
1302
1303	/* For preparing optimization, jump_label_text_reserved() is called */
1304	jump_label_lock();
1305	mutex_lock(&text_mutex);
1306
1307	if (!kprobe_aggrprobe(orig_p)) {
1308	/* If 'orig_p' is not an 'aggr_kprobe', create new one. */
1309	ap = alloc_aggr_kprobe(orig_p);
1310	if (!ap) {
1311	ret = -ENOMEM;
1312	goto out;
1313	}
1314	init_aggr_kprobe(ap, orig_p);
1315	} else if (kprobe_unused(ap)) {
1316	/* This probe is going to die. Rescue it */
1317	ret = reuse_unused_kprobe(ap);
1318	if (ret)
1319	goto out;
1320	}
1321
1322	if (kprobe_gone(ap)) {
1323	/*
1324	* Attempting to insert new probe at the same location that
1325	* had a probe in the module vaddr area which already
1326	* freed. So, the instruction slot has already been
1327	* released. We need a new slot for the new probe.
1328	*/
1329	ret = arch_prepare_kprobe(ap);
1330	if (ret)
1331	/*
1332	* Even if fail to allocate new slot, don't need to
1333	* free the 'ap'. It will be used next time, or
1334	* freed by unregister_kprobe().
1335	*/
1336	goto out;
1337
1338	/* Prepare optimized instructions if possible. */
1339	prepare_optimized_kprobe(ap);
1340
1341	/*
1342	* Clear gone flag to prevent allocating new slot again, and
1343	* set disabled flag because it is not armed yet.
1344	*/
1345	ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1346	\| KPROBE_FLAG_DISABLED;
1347	}
1348
1349	/* Copy the insn slot of 'p' to 'ap'. */
1350	copy_kprobe(ap, p);
1351	ret = add_new_kprobe(ap, p);
1352
1353	out:
1354	mutex_unlock(&text_mutex);
1355	jump_label_unlock();
1356	cpus_read_unlock();
1357
1358	if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1359	ap->flags &= ~KPROBE_FLAG_DISABLED;
1360	if (!kprobes_all_disarmed) {
1361	/* Arm the breakpoint again. */
1362	ret = arm_kprobe(ap);
1363	if (ret) {
1364	ap->flags \|= KPROBE_FLAG_DISABLED;
1365	list_del_rcu(&p->list);
1366	synchronize_rcu();
1367	}
1368	}
1369	}
1370	return ret;
1371	}
1372
1373	bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1374	{
1375	/* The '__kprobes' functions and entry code must not be probed. */
1376	return addr >= (unsigned long)__kprobes_text_start &&
1377	addr < (unsigned long)__kprobes_text_end;
1378	}
1379
1380	static bool __within_kprobe_blacklist(unsigned long addr)
1381	{
1382	struct kprobe_blacklist_entry *ent;
1383
1384	if (arch_within_kprobe_blacklist(addr))
1385	return true;
1386	/*
1387	* If 'kprobe_blacklist' is defined, check the address and
1388	* reject any probe registration in the prohibited area.
1389	*/
1390	list_for_each_entry(ent, &kprobe_blacklist, list) {
1391	if (addr >= ent->start_addr && addr < ent->end_addr)
1392	return true;
1393	}
1394	return false;
1395	}
1396
1397	bool within_kprobe_blacklist(unsigned long addr)
1398	{
1399	char symname[KSYM_NAME_LEN], *p;
1400
1401	if (__within_kprobe_blacklist(addr))
1402	return true;
1403
1404	/* Check if the address is on a suffixed-symbol */
1405	if (!lookup_symbol_name(addr, symname)) {
1406	p = strchr(symname, '.');
1407	if (!p)
1408	return false;
1409	*p = '\0';
1410	addr = (unsigned long)kprobe_lookup_name(symname, 0);
1411	if (addr)
1412	return __within_kprobe_blacklist(addr);
1413	}
1414	return false;
1415	}
1416
1417	/*
1418	* arch_adjust_kprobe_addr - adjust the address
1419	* @addr: symbol base address
1420	* @offset: offset within the symbol
1421	* @on_func_entry: was this @addr+@offset on the function entry
1422	*
1423	* Typically returns @addr + @offset, except for special cases where the
1424	* function might be prefixed by a CFI landing pad, in that case any offset
1425	* inside the landing pad is mapped to the first 'real' instruction of the
1426	* symbol.
1427	*
1428	* Specifically, for things like IBT/BTI, skip the resp. ENDBR/BTI.C
1429	* instruction at +0.
1430	*/
1431	kprobe_opcode_t *__weak arch_adjust_kprobe_addr(unsigned long addr,
1432	unsigned long offset,
1433	bool *on_func_entry)
1434	{
1435	*on_func_entry = !offset;
1436	return (kprobe_opcode_t *)(addr + offset);
1437	}
1438
1439	/*
1440	* If 'symbol_name' is specified, look it up and add the 'offset'
1441	* to it. This way, we can specify a relative address to a symbol.
1442	* This returns encoded errors if it fails to look up symbol or invalid
1443	* combination of parameters.
1444	*/
1445	static kprobe_opcode_t *
1446	_kprobe_addr(kprobe_opcode_t addr, const char symbol_name,
1447	unsigned long offset, bool *on_func_entry)
1448	{
1449	if ((symbol_name && addr) \|\| (!symbol_name && !addr))
1450	goto invalid;
1451
1452	if (symbol_name) {
1453	/*
1454	* Input: @sym + @offset
1455	* Output: @addr + @offset
1456	*
1457	* NOTE: kprobe_lookup_name() does NOT fold the offset
1458	* argument into it's output!
1459	*/
1460	addr = kprobe_lookup_name(symbol_name, offset);
1461	if (!addr)
1462	return ERR_PTR(-ENOENT);
1463	}
1464
1465	/*
1466	* So here we have @addr + @offset, displace it into a new
1467	* @addr' + @offset' where @addr' is the symbol start address.
1468	*/
1469	addr = (void *)addr + offset;
1470	if (!kallsyms_lookup_size_offset((unsigned long)addr, NULL, &offset))
1471	return ERR_PTR(-ENOENT);
1472	addr = (void *)addr - offset;
1473
1474	/*
1475	* Then ask the architecture to re-combine them, taking care of
1476	* magical function entry details while telling us if this was indeed
1477	* at the start of the function.
1478	*/
1479	addr = arch_adjust_kprobe_addr((unsigned long)addr, offset, on_func_entry);
1480	if (addr)
1481	return addr;
1482
1483	invalid:
1484	return ERR_PTR(-EINVAL);
1485	}
1486
1487	static kprobe_opcode_t kprobe_addr(struct kprobe p)
1488	{
1489	bool on_func_entry;
1490	return _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
1491	}
1492
1493	/*
1494	* Check the 'p' is valid and return the aggregator kprobe
1495	* at the same address.
1496	*/
1497	static struct kprobe __get_valid_kprobe(struct kprobe p)
1498	{
1499	struct kprobe ap, list_p;
1500
1501	lockdep_assert_held(&kprobe_mutex);
1502
1503	ap = get_kprobe(p->addr);
1504	if (unlikely(!ap))
1505	return NULL;
1506
1507	if (p != ap) {
1508	list_for_each_entry(list_p, &ap->list, list)
1509	if (list_p == p)
1510	/* kprobe p is a valid probe */
1511	goto valid;
1512	return NULL;
1513	}
1514	valid:
1515	return ap;
1516	}
1517
1518	/*
1519	* Warn and return error if the kprobe is being re-registered since
1520	* there must be a software bug.
1521	*/
1522	static inline int warn_kprobe_rereg(struct kprobe *p)
1523	{
1524	int ret = 0;
1525
1526	mutex_lock(&kprobe_mutex);
1527	if (WARN_ON_ONCE(__get_valid_kprobe(p)))
1528	ret = -EINVAL;
1529	mutex_unlock(&kprobe_mutex);
1530
1531	return ret;
1532	}
1533
1534	static int check_ftrace_location(struct kprobe *p)
1535	{
1536	unsigned long addr = (unsigned long)p->addr;
1537
1538	if (ftrace_location(addr) == addr) {
1539	#ifdef CONFIG_KPROBES_ON_FTRACE
1540	p->flags \|= KPROBE_FLAG_FTRACE;
1541	#else /* !CONFIG_KPROBES_ON_FTRACE */
1542	return -EINVAL;
1543	#endif
1544	}
1545	return 0;
1546	}
1547
1548	static bool is_cfi_preamble_symbol(unsigned long addr)
1549	{
1550	char symbuf[KSYM_NAME_LEN];
1551
1552	if (lookup_symbol_name(addr, symbuf))
1553	return false;
1554
1555	return str_has_prefix("__cfi_", symbuf) \|\|
1556	str_has_prefix("__pfx_", symbuf);
1557	}
1558
1559	static int check_kprobe_address_safe(struct kprobe *p,
1560	struct module **probed_mod)
1561	{
1562	int ret;
1563
1564	ret = check_ftrace_location(p);
1565	if (ret)
1566	return ret;
1567	jump_label_lock();
1568	preempt_disable();
1569
1570	/* Ensure it is not in reserved area nor out of text */
1571	if (!(core_kernel_text((unsigned long) p->addr) \|\|
1572	is_module_text_address((unsigned long) p->addr)) \|\|
1573	in_gate_area_no_mm((unsigned long) p->addr) \|\|
1574	within_kprobe_blacklist((unsigned long) p->addr) \|\|
1575	jump_label_text_reserved(p->addr, p->addr) \|\|
1576	static_call_text_reserved(p->addr, p->addr) \|\|
1577	find_bug((unsigned long)p->addr) \|\|
1578	is_cfi_preamble_symbol((unsigned long)p->addr)) {
1579	ret = -EINVAL;
1580	goto out;
1581	}
1582
1583	/* Check if 'p' is probing a module. */
1584	*probed_mod = __module_text_address((unsigned long) p->addr);
1585	if (*probed_mod) {
1586	/*
1587	* We must hold a refcount of the probed module while updating
1588	* its code to prohibit unexpected unloading.
1589	*/
1590	if (unlikely(!try_module_get(*probed_mod))) {
1591	ret = -ENOENT;
1592	goto out;
1593	}
1594
1595	/*
1596	* If the module freed '.init.text', we couldn't insert
1597	* kprobes in there.
1598	*/
1599	if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1600	(*probed_mod)->state != MODULE_STATE_COMING) {
1601	module_put(*probed_mod);
1602	*probed_mod = NULL;
1603	ret = -ENOENT;
1604	}
1605	}
1606	out:
1607	preempt_enable();
1608	jump_label_unlock();
1609
1610	return ret;
1611	}
1612
1613	int register_kprobe(struct kprobe *p)
1614	{
1615	int ret;
1616	struct kprobe *old_p;
1617	struct module *probed_mod;
1618	kprobe_opcode_t *addr;
1619	bool on_func_entry;
1620
1621	/* Adjust probe address from symbol */
1622	addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
1623	if (IS_ERR(addr))
1624	return PTR_ERR(addr);
1625	p->addr = addr;
1626
1627	ret = warn_kprobe_rereg(p);
1628	if (ret)
1629	return ret;
1630
1631	/* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1632	p->flags &= KPROBE_FLAG_DISABLED;
1633	p->nmissed = 0;
1634	INIT_LIST_HEAD(&p->list);
1635
1636	ret = check_kprobe_address_safe(p, &probed_mod);
1637	if (ret)
1638	return ret;
1639
1640	mutex_lock(&kprobe_mutex);
1641
1642	if (on_func_entry)
1643	p->flags \|= KPROBE_FLAG_ON_FUNC_ENTRY;
1644
1645	old_p = get_kprobe(p->addr);
1646	if (old_p) {
1647	/* Since this may unoptimize 'old_p', locking 'text_mutex'. */
1648	ret = register_aggr_kprobe(old_p, p);
1649	goto out;
1650	}
1651
1652	cpus_read_lock();
1653	/* Prevent text modification */
1654	mutex_lock(&text_mutex);
1655	ret = prepare_kprobe(p);
1656	mutex_unlock(&text_mutex);
1657	cpus_read_unlock();
1658	if (ret)
1659	goto out;
1660
1661	INIT_HLIST_NODE(&p->hlist);
1662	hlist_add_head_rcu(&p->hlist,
1663	&kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1664
1665	if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
1666	ret = arm_kprobe(p);
1667	if (ret) {
1668	hlist_del_rcu(&p->hlist);
1669	synchronize_rcu();
1670	goto out;
1671	}
1672	}
1673
1674	/* Try to optimize kprobe */
1675	try_to_optimize_kprobe(p);
1676	out:
1677	mutex_unlock(&kprobe_mutex);
1678
1679	if (probed_mod)
1680	module_put(probed_mod);
1681
1682	return ret;
1683	}
1684	EXPORT_SYMBOL_GPL(register_kprobe);
1685
1686	/* Check if all probes on the 'ap' are disabled. */
1687	static bool aggr_kprobe_disabled(struct kprobe *ap)
1688	{
1689	struct kprobe *kp;
1690
1691	lockdep_assert_held(&kprobe_mutex);
1692
1693	list_for_each_entry(kp, &ap->list, list)
1694	if (!kprobe_disabled(kp))
1695	/*
1696	* Since there is an active probe on the list,
1697	* we can't disable this 'ap'.
1698	*/
1699	return false;
1700
1701	return true;
1702	}
1703
1704	static struct kprobe __disable_kprobe(struct kprobe p)
1705	{
1706	struct kprobe *orig_p;
1707	int ret;
1708
1709	lockdep_assert_held(&kprobe_mutex);
1710
1711	/* Get an original kprobe for return */
1712	orig_p = __get_valid_kprobe(p);
1713	if (unlikely(orig_p == NULL))
1714	return ERR_PTR(-EINVAL);
1715
1716	if (!kprobe_disabled(p)) {
1717	/* Disable probe if it is a child probe */
1718	if (p != orig_p)
1719	p->flags \|= KPROBE_FLAG_DISABLED;
1720
1721	/* Try to disarm and disable this/parent probe */
1722	if (p == orig_p \|\| aggr_kprobe_disabled(orig_p)) {
1723	/*
1724	* Don't be lazy here. Even if 'kprobes_all_disarmed'
1725	* is false, 'orig_p' might not have been armed yet.
1726	* Note arm_all_kprobes() __tries__ to arm all kprobes
1727	* on the best effort basis.
1728	*/
1729	if (!kprobes_all_disarmed && !kprobe_disabled(orig_p)) {
1730	ret = disarm_kprobe(orig_p, true);
1731	if (ret) {
1732	p->flags &= ~KPROBE_FLAG_DISABLED;
1733	return ERR_PTR(ret);
1734	}
1735	}
1736	orig_p->flags \|= KPROBE_FLAG_DISABLED;
1737	}
1738	}
1739
1740	return orig_p;
1741	}
1742
1743	/*
1744	* Unregister a kprobe without a scheduler synchronization.
1745	*/
1746	static int __unregister_kprobe_top(struct kprobe *p)
1747	{
1748	struct kprobe ap, list_p;
1749
1750	/* Disable kprobe. This will disarm it if needed. */
1751	ap = __disable_kprobe(p);
1752	if (IS_ERR(ap))
1753	return PTR_ERR(ap);
1754
1755	if (ap == p)
1756	/*
1757	* This probe is an independent(and non-optimized) kprobe
1758	* (not an aggrprobe). Remove from the hash list.
1759	*/
1760	goto disarmed;
1761
1762	/* Following process expects this probe is an aggrprobe */
1763	WARN_ON(!kprobe_aggrprobe(ap));
1764
1765	if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1766	/*
1767	* !disarmed could be happen if the probe is under delayed
1768	* unoptimizing.
1769	*/
1770	goto disarmed;
1771	else {
1772	/* If disabling probe has special handlers, update aggrprobe */
1773	if (p->post_handler && !kprobe_gone(p)) {
1774	list_for_each_entry(list_p, &ap->list, list) {
1775	if ((list_p != p) && (list_p->post_handler))
1776	goto noclean;
1777	}
1778	/*
1779	* For the kprobe-on-ftrace case, we keep the
1780	* post_handler setting to identify this aggrprobe
1781	* armed with kprobe_ipmodify_ops.
1782	*/
1783	if (!kprobe_ftrace(ap))
1784	ap->post_handler = NULL;
1785	}
1786	noclean:
1787	/*
1788	* Remove from the aggrprobe: this path will do nothing in
1789	* __unregister_kprobe_bottom().
1790	*/
1791	list_del_rcu(&p->list);
1792	if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1793	/*
1794	* Try to optimize this probe again, because post
1795	* handler may have been changed.
1796	*/
1797	optimize_kprobe(ap);
1798	}
1799	return 0;
1800
1801	disarmed:
1802	hlist_del_rcu(&ap->hlist);
1803	return 0;
1804	}
1805
1806	static void __unregister_kprobe_bottom(struct kprobe *p)
1807	{
1808	struct kprobe *ap;
1809
1810	if (list_empty(&p->list))
1811	/* This is an independent kprobe */
1812	arch_remove_kprobe(p);
1813	else if (list_is_singular(&p->list)) {
1814	/* This is the last child of an aggrprobe */
1815	ap = list_entry(p->list.next, struct kprobe, list);
1816	list_del(&p->list);
1817	free_aggr_kprobe(ap);
1818	}
1819	/* Otherwise, do nothing. */
1820	}
1821
1822	int register_kprobes(struct kprobe **kps, int num)
1823	{
1824	int i, ret = 0;
1825
1826	if (num <= 0)
1827	return -EINVAL;
1828	for (i = 0; i < num; i++) {
1829	ret = register_kprobe(kps[i]);
1830	if (ret < 0) {
1831	if (i > 0)
1832	unregister_kprobes(kps, i);
1833	break;
1834	}
1835	}
1836	return ret;
1837	}
1838	EXPORT_SYMBOL_GPL(register_kprobes);
1839
1840	void unregister_kprobe(struct kprobe *p)
1841	{
1842	unregister_kprobes(&p, 1);
1843	}
1844	EXPORT_SYMBOL_GPL(unregister_kprobe);
1845
1846	void unregister_kprobes(struct kprobe **kps, int num)
1847	{
1848	int i;
1849
1850	if (num <= 0)
1851	return;
1852	mutex_lock(&kprobe_mutex);
1853	for (i = 0; i < num; i++)
1854	if (__unregister_kprobe_top(kps[i]) < 0)
1855	kps[i]->addr = NULL;
1856	mutex_unlock(&kprobe_mutex);
1857
1858	synchronize_rcu();
1859	for (i = 0; i < num; i++)
1860	if (kps[i]->addr)
1861	__unregister_kprobe_bottom(kps[i]);
1862	}
1863	EXPORT_SYMBOL_GPL(unregister_kprobes);
1864
1865	int __weak kprobe_exceptions_notify(struct notifier_block *self,
1866	unsigned long val, void *data)
1867	{
1868	return NOTIFY_DONE;
1869	}
1870	NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1871
1872	static struct notifier_block kprobe_exceptions_nb = {
1873	.notifier_call = kprobe_exceptions_notify,
1874	.priority = 0x7fffffff /* we need to be notified first */
1875	};
1876
1877	#ifdef CONFIG_KRETPROBES
1878
1879	#if !defined(CONFIG_KRETPROBE_ON_RETHOOK)
1880	static void free_rp_inst_rcu(struct rcu_head *head)
1881	{
1882	struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu);
1883
1884	if (refcount_dec_and_test(&ri->rph->ref))
1885	kfree(ri->rph);
1886	kfree(ri);
1887	}
1888	NOKPROBE_SYMBOL(free_rp_inst_rcu);
1889
1890	static void recycle_rp_inst(struct kretprobe_instance *ri)
1891	{
1892	struct kretprobe *rp = get_kretprobe(ri);
1893
1894	if (likely(rp))
1895	freelist_add(&ri->freelist, &rp->freelist);
1896	else
1897	call_rcu(&ri->rcu, free_rp_inst_rcu);
1898	}
1899	NOKPROBE_SYMBOL(recycle_rp_inst);
1900
1901	/*
1902	* This function is called from delayed_put_task_struct() when a task is
1903	* dead and cleaned up to recycle any kretprobe instances associated with
1904	* this task. These left over instances represent probed functions that
1905	* have been called but will never return.
1906	*/
1907	void kprobe_flush_task(struct task_struct *tk)
1908	{
1909	struct kretprobe_instance *ri;
1910	struct llist_node *node;
1911
1912	/* Early boot, not yet initialized. */
1913	if (unlikely(!kprobes_initialized))
1914	return;
1915
1916	kprobe_busy_begin();
1917
1918	node = __llist_del_all(&tk->kretprobe_instances);
1919	while (node) {
1920	ri = container_of(node, struct kretprobe_instance, llist);
1921	node = node->next;
1922
1923	recycle_rp_inst(ri);
1924	}
1925
1926	kprobe_busy_end();
1927	}
1928	NOKPROBE_SYMBOL(kprobe_flush_task);
1929
1930	static inline void free_rp_inst(struct kretprobe *rp)
1931	{
1932	struct kretprobe_instance *ri;
1933	struct freelist_node *node;
1934	int count = 0;
1935
1936	node = rp->freelist.head;
1937	while (node) {
1938	ri = container_of(node, struct kretprobe_instance, freelist);
1939	node = node->next;
1940
1941	kfree(ri);
1942	count++;
1943	}
1944
1945	if (refcount_sub_and_test(count, &rp->rph->ref)) {
1946	kfree(rp->rph);
1947	rp->rph = NULL;
1948	}
1949	}
1950
1951	/* This assumes the 'tsk' is the current task or the is not running. */
1952	static kprobe_opcode_t __kretprobe_find_ret_addr(struct task_struct tsk,
1953	struct llist_node **cur)
1954	{
1955	struct kretprobe_instance *ri = NULL;
1956	struct llist_node node = cur;
1957
1958	if (!node)
1959	node = tsk->kretprobe_instances.first;
1960	else
1961	node = node->next;
1962
1963	while (node) {
1964	ri = container_of(node, struct kretprobe_instance, llist);
1965	if (ri->ret_addr != kretprobe_trampoline_addr()) {
1966	*cur = node;
1967	return ri->ret_addr;
1968	}
1969	node = node->next;
1970	}
1971	return NULL;
1972	}
1973	NOKPROBE_SYMBOL(__kretprobe_find_ret_addr);
1974
1975	/**
1976	* kretprobe_find_ret_addr -- Find correct return address modified by kretprobe
1977	* @tsk: Target task
1978	* @fp: A frame pointer
1979	* @cur: a storage of the loop cursor llist_node pointer for next call
1980	*
1981	* Find the correct return address modified by a kretprobe on @tsk in unsigned
1982	* long type. If it finds the return address, this returns that address value,
1983	* or this returns 0.
1984	* The @tsk must be 'current' or a task which is not running. @fp is a hint
1985	* to get the currect return address - which is compared with the
1986	* kretprobe_instance::fp field. The @cur is a loop cursor for searching the
1987	* kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
1988	* first call, but '@cur' itself must NOT NULL.
1989	*/
1990	unsigned long kretprobe_find_ret_addr(struct task_struct tsk, void fp,
1991	struct llist_node **cur)
1992	{
1993	struct kretprobe_instance *ri = NULL;
1994	kprobe_opcode_t *ret;
1995
1996	if (WARN_ON_ONCE(!cur))
1997	return 0;
1998
1999	do {
2000	ret = __kretprobe_find_ret_addr(tsk, cur);
2001	if (!ret)
2002	break;
2003	ri = container_of(*cur, struct kretprobe_instance, llist);
2004	} while (ri->fp != fp);
2005
2006	return (unsigned long)ret;
2007	}
2008	NOKPROBE_SYMBOL(kretprobe_find_ret_addr);
2009
2010	void __weak arch_kretprobe_fixup_return(struct pt_regs *regs,
2011	kprobe_opcode_t *correct_ret_addr)
2012	{
2013	/*
2014	* Do nothing by default. Please fill this to update the fake return
2015	* address on the stack with the correct one on each arch if possible.
2016	*/
2017	}
2018
2019	unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
2020	void *frame_pointer)
2021	{
2022	struct kretprobe_instance *ri = NULL;
2023	struct llist_node first, node = NULL;
2024	kprobe_opcode_t *correct_ret_addr;
2025	struct kretprobe *rp;
2026
2027	/* Find correct address and all nodes for this frame. */
2028	correct_ret_addr = __kretprobe_find_ret_addr(current, &node);
2029	if (!correct_ret_addr) {
2030	pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n");
2031	BUG_ON(1);
2032	}
2033
2034	/*
2035	* Set the return address as the instruction pointer, because if the
2036	* user handler calls stack_trace_save_regs() with this 'regs',
2037	* the stack trace will start from the instruction pointer.
2038	*/
2039	instruction_pointer_set(regs, (unsigned long)correct_ret_addr);
2040
2041	/* Run the user handler of the nodes. */
2042	first = current->kretprobe_instances.first;
2043	while (first) {
2044	ri = container_of(first, struct kretprobe_instance, llist);
2045
2046	if (WARN_ON_ONCE(ri->fp != frame_pointer))
2047	break;
2048
2049	rp = get_kretprobe(ri);
2050	if (rp && rp->handler) {
2051	struct kprobe *prev = kprobe_running();
2052
2053	__this_cpu_write(current_kprobe, &rp->kp);
2054	ri->ret_addr = correct_ret_addr;
2055	rp->handler(ri, regs);
2056	__this_cpu_write(current_kprobe, prev);
2057	}
2058	if (first == node)
2059	break;
2060
2061	first = first->next;
2062	}
2063
2064	arch_kretprobe_fixup_return(regs, correct_ret_addr);
2065
2066	/* Unlink all nodes for this frame. */
2067	first = current->kretprobe_instances.first;
2068	current->kretprobe_instances.first = node->next;
2069	node->next = NULL;
2070
2071	/* Recycle free instances. */
2072	while (first) {
2073	ri = container_of(first, struct kretprobe_instance, llist);
2074	first = first->next;
2075
2076	recycle_rp_inst(ri);
2077	}
2078
2079	return (unsigned long)correct_ret_addr;
2080	}
2081	NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
2082
2083	/*
2084	* This kprobe pre_handler is registered with every kretprobe. When probe
2085	* hits it will set up the return probe.
2086	*/
2087	static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
2088	{
2089	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2090	struct kretprobe_instance *ri;
2091	struct freelist_node *fn;
2092
2093	fn = freelist_try_get(&rp->freelist);
2094	if (!fn) {
2095	rp->nmissed++;
2096	return 0;
2097	}
2098
2099	ri = container_of(fn, struct kretprobe_instance, freelist);
2100
2101	if (rp->entry_handler && rp->entry_handler(ri, regs)) {
2102	freelist_add(&ri->freelist, &rp->freelist);
2103	return 0;
2104	}
2105
2106	arch_prepare_kretprobe(ri, regs);
2107
2108	__llist_add(&ri->llist, &current->kretprobe_instances);
2109
2110	return 0;
2111	}
2112	NOKPROBE_SYMBOL(pre_handler_kretprobe);
2113	#else /* CONFIG_KRETPROBE_ON_RETHOOK */
2114	/*
2115	* This kprobe pre_handler is registered with every kretprobe. When probe
2116	* hits it will set up the return probe.
2117	*/
2118	static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
2119	{
2120	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2121	struct kretprobe_instance *ri;
2122	struct rethook_node *rhn;
2123
2124	rhn = rethook_try_get(rp->rh);
2125	if (!rhn) {
2126	rp->nmissed++;
2127	return 0;
2128	}
2129
2130	ri = container_of(rhn, struct kretprobe_instance, node);
2131
2132	if (rp->entry_handler && rp->entry_handler(ri, regs))
2133	rethook_recycle(rhn);
2134	else
2135	rethook_hook(rhn, regs, kprobe_ftrace(p));
2136
2137	return 0;
2138	}
2139	NOKPROBE_SYMBOL(pre_handler_kretprobe);
2140
2141	static void kretprobe_rethook_handler(struct rethook_node rh, void data,
2142	unsigned long ret_addr,
2143	struct pt_regs *regs)
2144	{
2145	struct kretprobe rp = (struct kretprobe )data;
2146	struct kretprobe_instance *ri;
2147	struct kprobe_ctlblk *kcb;
2148
2149	/* The data must NOT be null. This means rethook data structure is broken. */
2150	if (WARN_ON_ONCE(!data) \|\| !rp->handler)
2151	return;
2152
2153	__this_cpu_write(current_kprobe, &rp->kp);
2154	kcb = get_kprobe_ctlblk();
2155	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
2156
2157	ri = container_of(rh, struct kretprobe_instance, node);
2158	rp->handler(ri, regs);
2159
2160	__this_cpu_write(current_kprobe, NULL);
2161	}
2162	NOKPROBE_SYMBOL(kretprobe_rethook_handler);
2163
2164	#endif /* !CONFIG_KRETPROBE_ON_RETHOOK */
2165
2166	/**
2167	* kprobe_on_func_entry() -- check whether given address is function entry
2168	* @addr: Target address
2169	* @sym: Target symbol name
2170	* @offset: The offset from the symbol or the address
2171	*
2172	* This checks whether the given @addr+@offset or @sym+@offset is on the
2173	* function entry address or not.
2174	* This returns 0 if it is the function entry, or -EINVAL if it is not.
2175	* And also it returns -ENOENT if it fails the symbol or address lookup.
2176	* Caller must pass @addr or @sym (either one must be NULL), or this
2177	* returns -EINVAL.
2178	*/
2179	int kprobe_on_func_entry(kprobe_opcode_t addr, const char sym, unsigned long offset)
2180	{
2181	bool on_func_entry;
2182	kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset, &on_func_entry);
2183
2184	if (IS_ERR(kp_addr))
2185	return PTR_ERR(kp_addr);
2186
2187	if (!on_func_entry)
2188	return -EINVAL;
2189
2190	return 0;
2191	}
2192
2193	int register_kretprobe(struct kretprobe *rp)
2194	{
2195	int ret;
2196	struct kretprobe_instance *inst;
2197	int i;
2198	void *addr;
2199
2200	ret = kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset);
2201	if (ret)
2202	return ret;
2203
2204	/* If only 'rp->kp.addr' is specified, check reregistering kprobes */
2205	if (rp->kp.addr && warn_kprobe_rereg(&rp->kp))
2206	return -EINVAL;
2207
2208	if (kretprobe_blacklist_size) {
2209	addr = kprobe_addr(&rp->kp);
2210	if (IS_ERR(addr))
2211	return PTR_ERR(addr);
2212
2213	for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2214	if (kretprobe_blacklist[i].addr == addr)
2215	return -EINVAL;
2216	}
2217	}
2218
2219	if (rp->data_size > KRETPROBE_MAX_DATA_SIZE)
2220	return -E2BIG;
2221
2222	rp->kp.pre_handler = pre_handler_kretprobe;
2223	rp->kp.post_handler = NULL;
2224
2225	/* Pre-allocate memory for max kretprobe instances */
2226	if (rp->maxactive <= 0)
2227	rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
2228
2229	#ifdef CONFIG_KRETPROBE_ON_RETHOOK
2230	rp->rh = rethook_alloc((void *)rp, kretprobe_rethook_handler);
2231	if (!rp->rh)
2232	return -ENOMEM;
2233
2234	for (i = 0; i < rp->maxactive; i++) {
2235	inst = kzalloc(sizeof(struct kretprobe_instance) +
2236	rp->data_size, GFP_KERNEL);
2237	if (inst == NULL) {
2238	rethook_free(rp->rh);
2239	rp->rh = NULL;
2240	return -ENOMEM;
2241	}
2242	rethook_add_node(rp->rh, &inst->node);
2243	}
2244	rp->nmissed = 0;
2245	/* Establish function entry probe point */
2246	ret = register_kprobe(&rp->kp);
2247	if (ret != 0) {
2248	rethook_free(rp->rh);
2249	rp->rh = NULL;
2250	}
2251	#else /* !CONFIG_KRETPROBE_ON_RETHOOK */
2252	rp->freelist.head = NULL;
2253	rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL);
2254	if (!rp->rph)
2255	return -ENOMEM;
2256
2257	rp->rph->rp = rp;
2258	for (i = 0; i < rp->maxactive; i++) {
2259	inst = kzalloc(sizeof(struct kretprobe_instance) +
2260	rp->data_size, GFP_KERNEL);
2261	if (inst == NULL) {
2262	refcount_set(&rp->rph->ref, i);
2263	free_rp_inst(rp);
2264	return -ENOMEM;
2265	}
2266	inst->rph = rp->rph;
2267	freelist_add(&inst->freelist, &rp->freelist);
2268	}
2269	refcount_set(&rp->rph->ref, i);
2270
2271	rp->nmissed = 0;
2272	/* Establish function entry probe point */
2273	ret = register_kprobe(&rp->kp);
2274	if (ret != 0)
2275	free_rp_inst(rp);
2276	#endif
2277	return ret;
2278	}
2279	EXPORT_SYMBOL_GPL(register_kretprobe);
2280
2281	int register_kretprobes(struct kretprobe **rps, int num)
2282	{
2283	int ret = 0, i;
2284
2285	if (num <= 0)
2286	return -EINVAL;
2287	for (i = 0; i < num; i++) {
2288	ret = register_kretprobe(rps[i]);
2289	if (ret < 0) {
2290	if (i > 0)
2291	unregister_kretprobes(rps, i);
2292	break;
2293	}
2294	}
2295	return ret;
2296	}
2297	EXPORT_SYMBOL_GPL(register_kretprobes);
2298
2299	void unregister_kretprobe(struct kretprobe *rp)
2300	{
2301	unregister_kretprobes(&rp, 1);
2302	}
2303	EXPORT_SYMBOL_GPL(unregister_kretprobe);
2304
2305	void unregister_kretprobes(struct kretprobe **rps, int num)
2306	{
2307	int i;
2308
2309	if (num <= 0)
2310	return;
2311	mutex_lock(&kprobe_mutex);
2312	for (i = 0; i < num; i++) {
2313	if (__unregister_kprobe_top(&rps[i]->kp) < 0)
2314	rps[i]->kp.addr = NULL;
2315	#ifdef CONFIG_KRETPROBE_ON_RETHOOK
2316	rethook_free(rps[i]->rh);
2317	#else
2318	rps[i]->rph->rp = NULL;
2319	#endif
2320	}
2321	mutex_unlock(&kprobe_mutex);
2322
2323	synchronize_rcu();
2324	for (i = 0; i < num; i++) {
2325	if (rps[i]->kp.addr) {
2326	__unregister_kprobe_bottom(&rps[i]->kp);
2327	#ifndef CONFIG_KRETPROBE_ON_RETHOOK
2328	free_rp_inst(rps[i]);
2329	#endif
2330	}
2331	}
2332	}
2333	EXPORT_SYMBOL_GPL(unregister_kretprobes);
2334
2335	#else /* CONFIG_KRETPROBES */
2336	int register_kretprobe(struct kretprobe *rp)
2337	{
2338	return -EOPNOTSUPP;
2339	}
2340	EXPORT_SYMBOL_GPL(register_kretprobe);
2341
2342	int register_kretprobes(struct kretprobe **rps, int num)
2343	{
2344	return -EOPNOTSUPP;
2345	}
2346	EXPORT_SYMBOL_GPL(register_kretprobes);
2347
2348	void unregister_kretprobe(struct kretprobe *rp)
2349	{
2350	}
2351	EXPORT_SYMBOL_GPL(unregister_kretprobe);
2352
2353	void unregister_kretprobes(struct kretprobe **rps, int num)
2354	{
2355	}
2356	EXPORT_SYMBOL_GPL(unregister_kretprobes);
2357
2358	static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
2359	{
2360	return 0;
2361	}
2362	NOKPROBE_SYMBOL(pre_handler_kretprobe);
2363
2364	#endif /* CONFIG_KRETPROBES */
2365
2366	/* Set the kprobe gone and remove its instruction buffer. */
2367	static void kill_kprobe(struct kprobe *p)
2368	{
2369	struct kprobe *kp;
2370
2371	lockdep_assert_held(&kprobe_mutex);
2372
2373	/*
2374	* The module is going away. We should disarm the kprobe which
2375	* is using ftrace, because ftrace framework is still available at
2376	* 'MODULE_STATE_GOING' notification.
2377	*/
2378	if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed)
2379	disarm_kprobe_ftrace(p);
2380
2381	p->flags \|= KPROBE_FLAG_GONE;
2382	if (kprobe_aggrprobe(p)) {
2383	/*
2384	* If this is an aggr_kprobe, we have to list all the
2385	* chained probes and mark them GONE.
2386	*/
2387	list_for_each_entry(kp, &p->list, list)
2388	kp->flags \|= KPROBE_FLAG_GONE;
2389	p->post_handler = NULL;
2390	kill_optimized_kprobe(p);
2391	}
2392	/*
2393	* Here, we can remove insn_slot safely, because no thread calls
2394	* the original probed function (which will be freed soon) any more.
2395	*/
2396	arch_remove_kprobe(p);
2397	}
2398
2399	/* Disable one kprobe */
2400	int disable_kprobe(struct kprobe *kp)
2401	{
2402	int ret = 0;
2403	struct kprobe *p;
2404
2405	mutex_lock(&kprobe_mutex);
2406
2407	/* Disable this kprobe */
2408	p = __disable_kprobe(kp);
2409	if (IS_ERR(p))
2410	ret = PTR_ERR(p);
2411
2412	mutex_unlock(&kprobe_mutex);
2413	return ret;
2414	}
2415	EXPORT_SYMBOL_GPL(disable_kprobe);
2416
2417	/* Enable one kprobe */
2418	int enable_kprobe(struct kprobe *kp)
2419	{
2420	int ret = 0;
2421	struct kprobe *p;
2422
2423	mutex_lock(&kprobe_mutex);
2424
2425	/* Check whether specified probe is valid. */
2426	p = __get_valid_kprobe(kp);
2427	if (unlikely(p == NULL)) {
2428	ret = -EINVAL;
2429	goto out;
2430	}
2431
2432	if (kprobe_gone(kp)) {
2433	/* This kprobe has gone, we couldn't enable it. */
2434	ret = -EINVAL;
2435	goto out;
2436	}
2437
2438	if (p != kp)
2439	kp->flags &= ~KPROBE_FLAG_DISABLED;
2440
2441	if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2442	p->flags &= ~KPROBE_FLAG_DISABLED;
2443	ret = arm_kprobe(p);
2444	if (ret) {
2445	p->flags \|= KPROBE_FLAG_DISABLED;
2446	if (p != kp)
2447	kp->flags \|= KPROBE_FLAG_DISABLED;
2448	}
2449	}
2450	out:
2451	mutex_unlock(&kprobe_mutex);
2452	return ret;
2453	}
2454	EXPORT_SYMBOL_GPL(enable_kprobe);
2455
2456	/* Caller must NOT call this in usual path. This is only for critical case */
2457	void dump_kprobe(struct kprobe *kp)
2458	{
2459	pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n",
2460	kp->symbol_name, kp->offset, kp->addr);
2461	}
2462	NOKPROBE_SYMBOL(dump_kprobe);
2463
2464	int kprobe_add_ksym_blacklist(unsigned long entry)
2465	{
2466	struct kprobe_blacklist_entry *ent;
2467	unsigned long offset = 0, size = 0;
2468
2469	if (!kernel_text_address(entry) \|\|
2470	!kallsyms_lookup_size_offset(entry, &size, &offset))
2471	return -EINVAL;
2472
2473	ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2474	if (!ent)
2475	return -ENOMEM;
2476	ent->start_addr = entry;
2477	ent->end_addr = entry + size;
2478	INIT_LIST_HEAD(&ent->list);
2479	list_add_tail(&ent->list, &kprobe_blacklist);
2480
2481	return (int)size;
2482	}
2483
2484	/* Add all symbols in given area into kprobe blacklist */
2485	int kprobe_add_area_blacklist(unsigned long start, unsigned long end)
2486	{
2487	unsigned long entry;
2488	int ret = 0;
2489
2490	for (entry = start; entry < end; entry += ret) {
2491	ret = kprobe_add_ksym_blacklist(entry);
2492	if (ret < 0)
2493	return ret;
2494	if (ret == 0) /* In case of alias symbol */
2495	ret = 1;
2496	}
2497	return 0;
2498	}
2499
2500	/* Remove all symbols in given area from kprobe blacklist */
2501	static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end)
2502	{
2503	struct kprobe_blacklist_entry ent, n;
2504
2505	list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) {
2506	if (ent->start_addr < start \|\| ent->start_addr >= end)
2507	continue;
2508	list_del(&ent->list);
2509	kfree(ent);
2510	}
2511	}
2512
2513	static void kprobe_remove_ksym_blacklist(unsigned long entry)
2514	{
2515	kprobe_remove_area_blacklist(entry, entry + 1);
2516	}
2517
2518	int __weak arch_kprobe_get_kallsym(unsigned int symnum, unsigned long value,
2519	char type, char sym)
2520	{
2521	return -ERANGE;
2522	}
2523
2524	int kprobe_get_kallsym(unsigned int symnum, unsigned long value, char type,
2525	char *sym)
2526	{
2527	#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
2528	if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym))
2529	return 0;
2530	#ifdef CONFIG_OPTPROBES
2531	if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym))
2532	return 0;
2533	#endif
2534	#endif
2535	if (!arch_kprobe_get_kallsym(&symnum, value, type, sym))
2536	return 0;
2537	return -ERANGE;
2538	}
2539
2540	int __init __weak arch_populate_kprobe_blacklist(void)
2541	{
2542	return 0;
2543	}
2544
2545	/*
2546	* Lookup and populate the kprobe_blacklist.
2547	*
2548	* Unlike the kretprobe blacklist, we'll need to determine
2549	* the range of addresses that belong to the said functions,
2550	* since a kprobe need not necessarily be at the beginning
2551	* of a function.
2552	*/
2553	static int __init populate_kprobe_blacklist(unsigned long *start,
2554	unsigned long *end)
2555	{
2556	unsigned long entry;
2557	unsigned long *iter;
2558	int ret;
2559
2560	for (iter = start; iter < end; iter++) {
2561	entry = (unsigned long)dereference_symbol_descriptor((void )iter);
2562	ret = kprobe_add_ksym_blacklist(entry);
2563	if (ret == -EINVAL)
2564	continue;
2565	if (ret < 0)
2566	return ret;
2567	}
2568
2569	/* Symbols in '__kprobes_text' are blacklisted */
2570	ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start,
2571	(unsigned long)__kprobes_text_end);
2572	if (ret)
2573	return ret;
2574
2575	/* Symbols in 'noinstr' section are blacklisted */
2576	ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start,
2577	(unsigned long)__noinstr_text_end);
2578
2579	return ret ? : arch_populate_kprobe_blacklist();
2580	}
2581
2582	static void add_module_kprobe_blacklist(struct module *mod)
2583	{
2584	unsigned long start, end;
2585	int i;
2586
2587	if (mod->kprobe_blacklist) {
2588	for (i = 0; i < mod->num_kprobe_blacklist; i++)
2589	kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]);
2590	}
2591
2592	start = (unsigned long)mod->kprobes_text_start;
2593	if (start) {
2594	end = start + mod->kprobes_text_size;
2595	kprobe_add_area_blacklist(start, end);
2596	}
2597
2598	start = (unsigned long)mod->noinstr_text_start;
2599	if (start) {
2600	end = start + mod->noinstr_text_size;
2601	kprobe_add_area_blacklist(start, end);
2602	}
2603	}
2604
2605	static void remove_module_kprobe_blacklist(struct module *mod)
2606	{
2607	unsigned long start, end;
2608	int i;
2609
2610	if (mod->kprobe_blacklist) {
2611	for (i = 0; i < mod->num_kprobe_blacklist; i++)
2612	kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]);
2613	}
2614
2615	start = (unsigned long)mod->kprobes_text_start;
2616	if (start) {
2617	end = start + mod->kprobes_text_size;
2618	kprobe_remove_area_blacklist(start, end);
2619	}
2620
2621	start = (unsigned long)mod->noinstr_text_start;
2622	if (start) {
2623	end = start + mod->noinstr_text_size;
2624	kprobe_remove_area_blacklist(start, end);
2625	}
2626	}
2627
2628	/* Module notifier call back, checking kprobes on the module */
2629	static int kprobes_module_callback(struct notifier_block *nb,
2630	unsigned long val, void *data)
2631	{
2632	struct module *mod = data;
2633	struct hlist_head *head;
2634	struct kprobe *p;
2635	unsigned int i;
2636	int checkcore = (val == MODULE_STATE_GOING);
2637
2638	if (val == MODULE_STATE_COMING) {
2639	mutex_lock(&kprobe_mutex);
2640	add_module_kprobe_blacklist(mod);
2641	mutex_unlock(&kprobe_mutex);
2642	}
2643	if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
2644	return NOTIFY_DONE;
2645
2646	/*
2647	* When 'MODULE_STATE_GOING' was notified, both of module '.text' and
2648	* '.init.text' sections would be freed. When 'MODULE_STATE_LIVE' was
2649	* notified, only '.init.text' section would be freed. We need to
2650	* disable kprobes which have been inserted in the sections.
2651	*/
2652	mutex_lock(&kprobe_mutex);
2653	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2654	head = &kprobe_table[i];
2655	hlist_for_each_entry(p, head, hlist)
2656	if (within_module_init((unsigned long)p->addr, mod) \|\|
2657	(checkcore &&
2658	within_module_core((unsigned long)p->addr, mod))) {
2659	/*
2660	* The vaddr this probe is installed will soon
2661	* be vfreed buy not synced to disk. Hence,
2662	* disarming the breakpoint isn't needed.
2663	*
2664	* Note, this will also move any optimized probes
2665	* that are pending to be removed from their
2666	* corresponding lists to the 'freeing_list' and
2667	* will not be touched by the delayed
2668	* kprobe_optimizer() work handler.
2669	*/
2670	kill_kprobe(p);
2671	}
2672	}
2673	if (val == MODULE_STATE_GOING)
2674	remove_module_kprobe_blacklist(mod);
2675	mutex_unlock(&kprobe_mutex);
2676	return NOTIFY_DONE;
2677	}
2678
2679	static struct notifier_block kprobe_module_nb = {
2680	.notifier_call = kprobes_module_callback,
2681	.priority = 0
2682	};
2683
2684	void kprobe_free_init_mem(void)
2685	{
2686	void start = (void )(&__init_begin);
2687	void end = (void )(&__init_end);
2688	struct hlist_head *head;
2689	struct kprobe *p;
2690	int i;
2691
2692	mutex_lock(&kprobe_mutex);
2693
2694	/* Kill all kprobes on initmem because the target code has been freed. */
2695	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2696	head = &kprobe_table[i];
2697	hlist_for_each_entry(p, head, hlist) {
2698	if (start <= (void )p->addr && (void )p->addr < end)
2699	kill_kprobe(p);
2700	}
2701	}
2702
2703	mutex_unlock(&kprobe_mutex);
2704	}
2705
2706	static int __init init_kprobes(void)
2707	{
2708	int i, err;
2709
2710	/* FIXME allocate the probe table, currently defined statically */
2711	/* initialize all list heads */
2712	for (i = 0; i < KPROBE_TABLE_SIZE; i++)
2713	INIT_HLIST_HEAD(&kprobe_table[i]);
2714
2715	err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2716	__stop_kprobe_blacklist);
2717	if (err)
2718	pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err);
2719
2720	if (kretprobe_blacklist_size) {
2721	/* lookup the function address from its name */
2722	for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2723	kretprobe_blacklist[i].addr =
2724	kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
2725	if (!kretprobe_blacklist[i].addr)
2726	pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n",
2727	kretprobe_blacklist[i].name);
2728	}
2729	}
2730
2731	/* By default, kprobes are armed */
2732	kprobes_all_disarmed = false;
2733
2734	#if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2735	/* Init 'kprobe_optinsn_slots' for allocation */
2736	kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2737	#endif
2738
2739	err = arch_init_kprobes();
2740	if (!err)
2741	err = register_die_notifier(&kprobe_exceptions_nb);
2742	if (!err)
2743	err = register_module_notifier(&kprobe_module_nb);
2744
2745	kprobes_initialized = (err == 0);
2746	kprobe_sysctls_init();
2747	return err;
2748	}
2749	early_initcall(init_kprobes);
2750
2751	#if defined(CONFIG_OPTPROBES)
2752	static int __init init_optprobes(void)
2753	{
2754	/*
2755	* Enable kprobe optimization - this kicks the optimizer which
2756	* depends on synchronize_rcu_tasks() and ksoftirqd, that is
2757	* not spawned in early initcall. So delay the optimization.
2758	*/
2759	optimize_all_kprobes();
2760
2761	return 0;
2762	}
2763	subsys_initcall(init_optprobes);
2764	#endif
2765
2766	#ifdef CONFIG_DEBUG_FS
2767	static void report_probe(struct seq_file pi, struct kprobe p,
2768	const char sym, int offset, char modname, struct kprobe *pp)
2769	{
2770	char *kprobe_type;
2771	void *addr = p->addr;
2772
2773	if (p->pre_handler == pre_handler_kretprobe)
2774	kprobe_type = "r";
2775	else
2776	kprobe_type = "k";
2777
2778	if (!kallsyms_show_value(pi->file->f_cred))
2779	addr = NULL;
2780
2781	if (sym)
2782	seq_printf(pi, "%px %s %s+0x%x %s ",
2783	addr, kprobe_type, sym, offset,
2784	(modname ? modname : " "));
2785	else /* try to use %pS */
2786	seq_printf(pi, "%px %s %pS ",
2787	addr, kprobe_type, p->addr);
2788
2789	if (!pp)
2790	pp = p;
2791	seq_printf(pi, "%s%s%s%s\n",
2792	(kprobe_gone(p) ? "[GONE]" : ""),
2793	((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
2794	(kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2795	(kprobe_ftrace(pp) ? "[FTRACE]" : ""));
2796	}
2797
2798	static void kprobe_seq_start(struct seq_file f, loff_t *pos)
2799	{
2800	return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2801	}
2802
2803	static void kprobe_seq_next(struct seq_file f, void v, loff_t pos)
2804	{
2805	(*pos)++;
2806	if (*pos >= KPROBE_TABLE_SIZE)
2807	return NULL;
2808	return pos;
2809	}
2810
2811	static void kprobe_seq_stop(struct seq_file f, void v)
2812	{
2813	/* Nothing to do */
2814	}
2815
2816	static int show_kprobe_addr(struct seq_file pi, void v)
2817	{
2818	struct hlist_head *head;
2819	struct kprobe p, kp;
2820	const char *sym = NULL;
2821	unsigned int i = (loff_t ) v;
2822	unsigned long offset = 0;
2823	char *modname, namebuf[KSYM_NAME_LEN];
2824
2825	head = &kprobe_table[i];
2826	preempt_disable();
2827	hlist_for_each_entry_rcu(p, head, hlist) {
2828	sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2829	&offset, &modname, namebuf);
2830	if (kprobe_aggrprobe(p)) {
2831	list_for_each_entry_rcu(kp, &p->list, list)
2832	report_probe(pi, kp, sym, offset, modname, p);
2833	} else
2834	report_probe(pi, p, sym, offset, modname, NULL);
2835	}
2836	preempt_enable();
2837	return 0;
2838	}
2839
2840	static const struct seq_operations kprobes_sops = {
2841	.start = kprobe_seq_start,
2842	.next = kprobe_seq_next,
2843	.stop = kprobe_seq_stop,
2844	.show = show_kprobe_addr
2845	};
2846
2847	DEFINE_SEQ_ATTRIBUTE(kprobes);
2848
2849	/* kprobes/blacklist -- shows which functions can not be probed */
2850	static void kprobe_blacklist_seq_start(struct seq_file m, loff_t *pos)
2851	{
2852	mutex_lock(&kprobe_mutex);
2853	return seq_list_start(&kprobe_blacklist, *pos);
2854	}
2855
2856	static void kprobe_blacklist_seq_next(struct seq_file m, void v, loff_t pos)
2857	{
2858	return seq_list_next(v, &kprobe_blacklist, pos);
2859	}
2860
2861	static int kprobe_blacklist_seq_show(struct seq_file m, void v)
2862	{
2863	struct kprobe_blacklist_entry *ent =
2864	list_entry(v, struct kprobe_blacklist_entry, list);
2865
2866	/*
2867	* If '/proc/kallsyms' is not showing kernel address, we won't
2868	* show them here either.
2869	*/
2870	if (!kallsyms_show_value(m->file->f_cred))
2871	seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL,
2872	(void *)ent->start_addr);
2873	else
2874	seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
2875	(void )ent->end_addr, (void )ent->start_addr);
2876	return 0;
2877	}
2878
2879	static void kprobe_blacklist_seq_stop(struct seq_file f, void v)
2880	{
2881	mutex_unlock(&kprobe_mutex);
2882	}
2883
2884	static const struct seq_operations kprobe_blacklist_sops = {
2885	.start = kprobe_blacklist_seq_start,
2886	.next = kprobe_blacklist_seq_next,
2887	.stop = kprobe_blacklist_seq_stop,
2888	.show = kprobe_blacklist_seq_show,
2889	};
2890	DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist);
2891
2892	static int arm_all_kprobes(void)
2893	{
2894	struct hlist_head *head;
2895	struct kprobe *p;
2896	unsigned int i, total = 0, errors = 0;
2897	int err, ret = 0;
2898
2899	mutex_lock(&kprobe_mutex);
2900
2901	/* If kprobes are armed, just return */
2902	if (!kprobes_all_disarmed)
2903	goto already_enabled;
2904
2905	/*
2906	* optimize_kprobe() called by arm_kprobe() checks
2907	* kprobes_all_disarmed, so set kprobes_all_disarmed before
2908	* arm_kprobe.
2909	*/
2910	kprobes_all_disarmed = false;
2911	/* Arming kprobes doesn't optimize kprobe itself */
2912	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2913	head = &kprobe_table[i];
2914	/* Arm all kprobes on a best-effort basis */
2915	hlist_for_each_entry(p, head, hlist) {
2916	if (!kprobe_disabled(p)) {
2917	err = arm_kprobe(p);
2918	if (err) {
2919	errors++;
2920	ret = err;
2921	}
2922	total++;
2923	}
2924	}
2925	}
2926
2927	if (errors)
2928	pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n",
2929	errors, total);
2930	else
2931	pr_info("Kprobes globally enabled\n");
2932
2933	already_enabled:
2934	mutex_unlock(&kprobe_mutex);
2935	return ret;
2936	}
2937
2938	static int disarm_all_kprobes(void)
2939	{
2940	struct hlist_head *head;
2941	struct kprobe *p;
2942	unsigned int i, total = 0, errors = 0;
2943	int err, ret = 0;
2944
2945	mutex_lock(&kprobe_mutex);
2946
2947	/* If kprobes are already disarmed, just return */
2948	if (kprobes_all_disarmed) {
2949	mutex_unlock(&kprobe_mutex);
2950	return 0;
2951	}
2952
2953	kprobes_all_disarmed = true;
2954
2955	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2956	head = &kprobe_table[i];
2957	/* Disarm all kprobes on a best-effort basis */
2958	hlist_for_each_entry(p, head, hlist) {
2959	if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
2960	err = disarm_kprobe(p, false);
2961	if (err) {
2962	errors++;
2963	ret = err;
2964	}
2965	total++;
2966	}
2967	}
2968	}
2969
2970	if (errors)
2971	pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n",
2972	errors, total);
2973	else
2974	pr_info("Kprobes globally disabled\n");
2975
2976	mutex_unlock(&kprobe_mutex);
2977
2978	/* Wait for disarming all kprobes by optimizer */
2979	wait_for_kprobe_optimizer();
2980
2981	return ret;
2982	}
2983
2984	/*
2985	* XXX: The debugfs bool file interface doesn't allow for callbacks
2986	* when the bool state is switched. We can reuse that facility when
2987	* available
2988	*/
2989	static ssize_t read_enabled_file_bool(struct file *file,
2990	char __user user_buf, size_t count, loff_t ppos)
2991	{
2992	char buf[3];
2993
2994	if (!kprobes_all_disarmed)
2995	buf[0] = '1';
2996	else
2997	buf[0] = '0';
2998	buf[1] = '\n';
2999	buf[2] = 0x00;
3000	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
3001	}
3002
3003	static ssize_t write_enabled_file_bool(struct file *file,
3004	const char __user user_buf, size_t count, loff_t ppos)
3005	{
3006	bool enable;
3007	int ret;
3008
3009	ret = kstrtobool_from_user(user_buf, count, &enable);
3010	if (ret)
3011	return ret;
3012
3013	ret = enable ? arm_all_kprobes() : disarm_all_kprobes();
3014	if (ret)
3015	return ret;
3016
3017	return count;
3018	}
3019
3020	static const struct file_operations fops_kp = {
3021	.read = read_enabled_file_bool,
3022	.write = write_enabled_file_bool,
3023	.llseek = default_llseek,
3024	};
3025
3026	static int __init debugfs_kprobe_init(void)
3027	{
3028	struct dentry *dir;
3029
3030	dir = debugfs_create_dir("kprobes", NULL);
3031
3032	debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops);
3033
3034	debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp);
3035
3036	debugfs_create_file("blacklist", 0400, dir, NULL,
3037	&kprobe_blacklist_fops);
3038
3039	return 0;
3040	}
3041
3042	late_initcall(debugfs_kprobe_init);
3043	#endif /* CONFIG_DEBUG_FS */