linux - kernel/module/main.c

3391 lines · 85.8 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2002 Richard Henderson
4	* Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5	* Copyright (C) 2023 Luis Chamberlain <[email protected]>
6	*/
7
8	#define INCLUDE_VERMAGIC
9
10	#include <linux/export.h>
11	#include <linux/extable.h>
12	#include <linux/moduleloader.h>
13	#include <linux/module_signature.h>
14	#include <linux/trace_events.h>
15	#include <linux/init.h>
16	#include <linux/kallsyms.h>
17	#include <linux/buildid.h>
18	#include <linux/fs.h>
19	#include <linux/kernel.h>
20	#include <linux/kernel_read_file.h>
21	#include <linux/kstrtox.h>
22	#include <linux/slab.h>
23	#include <linux/vmalloc.h>
24	#include <linux/elf.h>
25	#include <linux/seq_file.h>
26	#include <linux/syscalls.h>
27	#include <linux/fcntl.h>
28	#include <linux/rcupdate.h>
29	#include <linux/capability.h>
30	#include <linux/cpu.h>
31	#include <linux/moduleparam.h>
32	#include <linux/errno.h>
33	#include <linux/err.h>
34	#include <linux/vermagic.h>
35	#include <linux/notifier.h>
36	#include <linux/sched.h>
37	#include <linux/device.h>
38	#include <linux/string.h>
39	#include <linux/mutex.h>
40	#include <linux/rculist.h>
41	#include <linux/uaccess.h>
42	#include <asm/cacheflush.h>
43	#include <linux/set_memory.h>
44	#include <asm/mmu_context.h>
45	#include <linux/license.h>
46	#include <asm/sections.h>
47	#include <linux/tracepoint.h>
48	#include <linux/ftrace.h>
49	#include <linux/livepatch.h>
50	#include <linux/async.h>
51	#include <linux/percpu.h>
52	#include <linux/kmemleak.h>
53	#include <linux/jump_label.h>
54	#include <linux/pfn.h>
55	#include <linux/bsearch.h>
56	#include <linux/dynamic_debug.h>
57	#include <linux/audit.h>
58	#include <linux/cfi.h>
59	#include <linux/codetag.h>
60	#include <linux/debugfs.h>
61	#include <linux/execmem.h>
62	#include <uapi/linux/module.h>
63	#include "internal.h"
64
65	#define CREATE_TRACE_POINTS
66	#include <trace/events/module.h>
67
68	/*
69	* Mutex protects:
70	* 1) List of modules (also safely readable with preempt_disable),
71	* 2) module_use links,
72	* 3) mod_tree.addr_min/mod_tree.addr_max.
73	* (delete and add uses RCU list operations).
74	*/
75	DEFINE_MUTEX(module_mutex);
76	LIST_HEAD(modules);
77
78	/* Work queue for freeing init sections in success case */
79	static void do_free_init(struct work_struct *w);
80	static DECLARE_WORK(init_free_wq, do_free_init);
81	static LLIST_HEAD(init_free_list);
82
83	struct mod_tree_root mod_tree __cacheline_aligned = {
84	.addr_min = -1UL,
85	};
86
87	struct symsearch {
88	const struct kernel_symbol start, stop;
89	const s32 *crcs;
90	enum mod_license license;
91	};
92
93	/*
94	* Bounds of module memory, for speeding up __module_address.
95	* Protected by module_mutex.
96	*/
97	static void __mod_update_bounds(enum mod_mem_type type __maybe_unused, void *base,
98	unsigned int size, struct mod_tree_root *tree)
99	{
100	unsigned long min = (unsigned long)base;
101	unsigned long max = min + size;
102
103	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
104	if (mod_mem_type_is_core_data(type)) {
105	if (min < tree->data_addr_min)
106	tree->data_addr_min = min;
107	if (max > tree->data_addr_max)
108	tree->data_addr_max = max;
109	return;
110	}
111	#endif
112	if (min < tree->addr_min)
113	tree->addr_min = min;
114	if (max > tree->addr_max)
115	tree->addr_max = max;
116	}
117
118	static void mod_update_bounds(struct module *mod)
119	{
120	for_each_mod_mem_type(type) {
121	struct module_memory *mod_mem = &mod->mem[type];
122
123	if (mod_mem->size)
124	__mod_update_bounds(type, mod_mem->base, mod_mem->size, &mod_tree);
125	}
126	}
127
128	/* Block module loading/unloading? */
129	int modules_disabled;
130	core_param(nomodule, modules_disabled, bint, 0);
131
132	/* Waiting for a module to finish initializing? */
133	static DECLARE_WAIT_QUEUE_HEAD(module_wq);
134
135	static BLOCKING_NOTIFIER_HEAD(module_notify_list);
136
137	int register_module_notifier(struct notifier_block *nb)
138	{
139	return blocking_notifier_chain_register(&module_notify_list, nb);
140	}
141	EXPORT_SYMBOL(register_module_notifier);
142
143	int unregister_module_notifier(struct notifier_block *nb)
144	{
145	return blocking_notifier_chain_unregister(&module_notify_list, nb);
146	}
147	EXPORT_SYMBOL(unregister_module_notifier);
148
149	/*
150	* We require a truly strong try_module_get(): 0 means success.
151	* Otherwise an error is returned due to ongoing or failed
152	* initialization etc.
153	*/
154	static inline int strong_try_module_get(struct module *mod)
155	{
156	BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
157	if (mod && mod->state == MODULE_STATE_COMING)
158	return -EBUSY;
159	if (try_module_get(mod))
160	return 0;
161	else
162	return -ENOENT;
163	}
164
165	static inline void add_taint_module(struct module *mod, unsigned flag,
166	enum lockdep_ok lockdep_ok)
167	{
168	add_taint(flag, lockdep_ok);
169	set_bit(flag, &mod->taints);
170	}
171
172	/*
173	* A thread that wants to hold a reference to a module only while it
174	* is running can call this to safely exit.
175	*/
176	void __noreturn __module_put_and_kthread_exit(struct module *mod, long code)
177	{
178	module_put(mod);
179	kthread_exit(code);
180	}
181	EXPORT_SYMBOL(__module_put_and_kthread_exit);
182
183	/* Find a module section: 0 means not found. */
184	static unsigned int find_sec(const struct load_info info, const char name)
185	{
186	unsigned int i;
187
188	for (i = 1; i < info->hdr->e_shnum; i++) {
189	Elf_Shdr *shdr = &info->sechdrs[i];
190	/* Alloc bit cleared means "ignore it." */
191	if ((shdr->sh_flags & SHF_ALLOC)
192	&& strcmp(info->secstrings + shdr->sh_name, name) == 0)
193	return i;
194	}
195	return 0;
196	}
197
198	/* Find a module section, or NULL. */
199	static void section_addr(const struct load_info info, const char *name)
200	{
201	/* Section 0 has sh_addr 0. */
202	return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
203	}
204
205	/* Find a module section, or NULL. Fill in number of "objects" in section. */
206	static void section_objs(const struct load_info info,
207	const char *name,
208	size_t object_size,
209	unsigned int *num)
210	{
211	unsigned int sec = find_sec(info, name);
212
213	/* Section 0 has sh_addr 0 and sh_size 0. */
214	*num = info->sechdrs[sec].sh_size / object_size;
215	return (void *)info->sechdrs[sec].sh_addr;
216	}
217
218	/* Find a module section: 0 means not found. Ignores SHF_ALLOC flag. */
219	static unsigned int find_any_sec(const struct load_info info, const char name)
220	{
221	unsigned int i;
222
223	for (i = 1; i < info->hdr->e_shnum; i++) {
224	Elf_Shdr *shdr = &info->sechdrs[i];
225	if (strcmp(info->secstrings + shdr->sh_name, name) == 0)
226	return i;
227	}
228	return 0;
229	}
230
231	/*
232	* Find a module section, or NULL. Fill in number of "objects" in section.
233	* Ignores SHF_ALLOC flag.
234	*/
235	static __maybe_unused void any_section_objs(const struct load_info info,
236	const char *name,
237	size_t object_size,
238	unsigned int *num)
239	{
240	unsigned int sec = find_any_sec(info, name);
241
242	/* Section 0 has sh_addr 0 and sh_size 0. */
243	*num = info->sechdrs[sec].sh_size / object_size;
244	return (void *)info->sechdrs[sec].sh_addr;
245	}
246
247	#ifndef CONFIG_MODVERSIONS
248	#define symversion(base, idx) NULL
249	#else
250	#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
251	#endif
252
253	static const char kernel_symbol_name(const struct kernel_symbol sym)
254	{
255	#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
256	return offset_to_ptr(&sym->name_offset);
257	#else
258	return sym->name;
259	#endif
260	}
261
262	static const char kernel_symbol_namespace(const struct kernel_symbol sym)
263	{
264	#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
265	if (!sym->namespace_offset)
266	return NULL;
267	return offset_to_ptr(&sym->namespace_offset);
268	#else
269	return sym->namespace;
270	#endif
271	}
272
273	int cmp_name(const void name, const void sym)
274	{
275	return strcmp(name, kernel_symbol_name(sym));
276	}
277
278	static bool find_exported_symbol_in_section(const struct symsearch *syms,
279	struct module *owner,
280	struct find_symbol_arg *fsa)
281	{
282	struct kernel_symbol *sym;
283
284	if (!fsa->gplok && syms->license == GPL_ONLY)
285	return false;
286
287	sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
288	sizeof(struct kernel_symbol), cmp_name);
289	if (!sym)
290	return false;
291
292	fsa->owner = owner;
293	fsa->crc = symversion(syms->crcs, sym - syms->start);
294	fsa->sym = sym;
295	fsa->license = syms->license;
296
297	return true;
298	}
299
300	/*
301	* Find an exported symbol and return it, along with, (optional) crc and
302	* (optional) module which owns it. Needs preempt disabled or module_mutex.
303	*/
304	bool find_symbol(struct find_symbol_arg *fsa)
305	{
306	static const struct symsearch arr[] = {
307	{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,
308	NOT_GPL_ONLY },
309	{ __start___ksymtab_gpl, __stop___ksymtab_gpl,
310	__start___kcrctab_gpl,
311	GPL_ONLY },
312	};
313	struct module *mod;
314	unsigned int i;
315
316	module_assert_mutex_or_preempt();
317
318	for (i = 0; i < ARRAY_SIZE(arr); i++)
319	if (find_exported_symbol_in_section(&arr[i], NULL, fsa))
320	return true;
321
322	list_for_each_entry_rcu(mod, &modules, list,
323	lockdep_is_held(&module_mutex)) {
324	struct symsearch arr[] = {
325	{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
326	NOT_GPL_ONLY },
327	{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
328	mod->gpl_crcs,
329	GPL_ONLY },
330	};
331
332	if (mod->state == MODULE_STATE_UNFORMED)
333	continue;
334
335	for (i = 0; i < ARRAY_SIZE(arr); i++)
336	if (find_exported_symbol_in_section(&arr[i], mod, fsa))
337	return true;
338	}
339
340	pr_debug("Failed to find symbol %s\n", fsa->name);
341	return false;
342	}
343
344	/*
345	* Search for module by name: must hold module_mutex (or preempt disabled
346	* for read-only access).
347	*/
348	struct module find_module_all(const char name, size_t len,
349	bool even_unformed)
350	{
351	struct module *mod;
352
353	module_assert_mutex_or_preempt();
354
355	list_for_each_entry_rcu(mod, &modules, list,
356	lockdep_is_held(&module_mutex)) {
357	if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
358	continue;
359	if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
360	return mod;
361	}
362	return NULL;
363	}
364
365	struct module find_module(const char name)
366	{
367	return find_module_all(name, strlen(name), false);
368	}
369
370	#ifdef CONFIG_SMP
371
372	static inline void __percpu mod_percpu(struct module mod)
373	{
374	return mod->percpu;
375	}
376
377	static int percpu_modalloc(struct module mod, struct load_info info)
378	{
379	Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
380	unsigned long align = pcpusec->sh_addralign;
381
382	if (!pcpusec->sh_size)
383	return 0;
384
385	if (align > PAGE_SIZE) {
386	pr_warn("%s: per-cpu alignment %li > %li\n",
387	mod->name, align, PAGE_SIZE);
388	align = PAGE_SIZE;
389	}
390
391	mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
392	if (!mod->percpu) {
393	pr_warn("%s: Could not allocate %lu bytes percpu data\n",
394	mod->name, (unsigned long)pcpusec->sh_size);
395	return -ENOMEM;
396	}
397	mod->percpu_size = pcpusec->sh_size;
398	return 0;
399	}
400
401	static void percpu_modfree(struct module *mod)
402	{
403	free_percpu(mod->percpu);
404	}
405
406	static unsigned int find_pcpusec(struct load_info *info)
407	{
408	return find_sec(info, ".data..percpu");
409	}
410
411	static void percpu_modcopy(struct module *mod,
412	const void *from, unsigned long size)
413	{
414	int cpu;
415
416	for_each_possible_cpu(cpu)
417	memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
418	}
419
420	bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
421	{
422	struct module *mod;
423	unsigned int cpu;
424
425	preempt_disable();
426
427	list_for_each_entry_rcu(mod, &modules, list) {
428	if (mod->state == MODULE_STATE_UNFORMED)
429	continue;
430	if (!mod->percpu_size)
431	continue;
432	for_each_possible_cpu(cpu) {
433	void *start = per_cpu_ptr(mod->percpu, cpu);
434	void va = (void )addr;
435
436	if (va >= start && va < start + mod->percpu_size) {
437	if (can_addr) {
438	*can_addr = (unsigned long) (va - start);
439	*can_addr += (unsigned long)
440	per_cpu_ptr(mod->percpu,
441	get_boot_cpu_id());
442	}
443	preempt_enable();
444	return true;
445	}
446	}
447	}
448
449	preempt_enable();
450	return false;
451	}
452
453	/**
454	* is_module_percpu_address() - test whether address is from module static percpu
455	* @addr: address to test
456	*
457	* Test whether @addr belongs to module static percpu area.
458	*
459	* Return: %true if @addr is from module static percpu area
460	*/
461	bool is_module_percpu_address(unsigned long addr)
462	{
463	return __is_module_percpu_address(addr, NULL);
464	}
465
466	#else /* ... !CONFIG_SMP */
467
468	static inline void __percpu mod_percpu(struct module mod)
469	{
470	return NULL;
471	}
472	static int percpu_modalloc(struct module mod, struct load_info info)
473	{
474	/* UP modules shouldn't have this section: ENOMEM isn't quite right */
475	if (info->sechdrs[info->index.pcpu].sh_size != 0)
476	return -ENOMEM;
477	return 0;
478	}
479	static inline void percpu_modfree(struct module *mod)
480	{
481	}
482	static unsigned int find_pcpusec(struct load_info *info)
483	{
484	return 0;
485	}
486	static inline void percpu_modcopy(struct module *mod,
487	const void *from, unsigned long size)
488	{
489	/* pcpusec should be 0, and size of that section should be 0. */
490	BUG_ON(size != 0);
491	}
492	bool is_module_percpu_address(unsigned long addr)
493	{
494	return false;
495	}
496
497	bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
498	{
499	return false;
500	}
501
502	#endif /* CONFIG_SMP */
503
504	#define MODINFO_ATTR(field) \
505	static void setup_modinfo_##field(struct module mod, const char s) \
506	{ \
507	mod->field = kstrdup(s, GFP_KERNEL); \
508	} \
509	static ssize_t show_modinfo_##field(struct module_attribute *mattr, \
510	struct module_kobject mk, char buffer) \
511	{ \
512	return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
513	} \
514	static int modinfo_##field##_exists(struct module *mod) \
515	{ \
516	return mod->field != NULL; \
517	} \
518	static void free_modinfo_##field(struct module *mod) \
519	{ \
520	kfree(mod->field); \
521	mod->field = NULL; \
522	} \
523	static struct module_attribute modinfo_##field = { \
524	.attr = { .name = __stringify(field), .mode = 0444 }, \
525	.show = show_modinfo_##field, \
526	.setup = setup_modinfo_##field, \
527	.test = modinfo_##field##_exists, \
528	.free = free_modinfo_##field, \
529	};
530
531	MODINFO_ATTR(version);
532	MODINFO_ATTR(srcversion);
533
534	static struct {
535	char name[MODULE_NAME_LEN + 1];
536	char taints[MODULE_FLAGS_BUF_SIZE];
537	} last_unloaded_module;
538
539	#ifdef CONFIG_MODULE_UNLOAD
540
541	EXPORT_TRACEPOINT_SYMBOL(module_get);
542
543	/* MODULE_REF_BASE is the base reference count by kmodule loader. */
544	#define MODULE_REF_BASE 1
545
546	/* Init the unload section of the module. */
547	static int module_unload_init(struct module *mod)
548	{
549	/*
550	* Initialize reference counter to MODULE_REF_BASE.
551	* refcnt == 0 means module is going.
552	*/
553	atomic_set(&mod->refcnt, MODULE_REF_BASE);
554
555	INIT_LIST_HEAD(&mod->source_list);
556	INIT_LIST_HEAD(&mod->target_list);
557
558	/* Hold reference count during initialization. */
559	atomic_inc(&mod->refcnt);
560
561	return 0;
562	}
563
564	/* Does a already use b? */
565	static int already_uses(struct module a, struct module b)
566	{
567	struct module_use *use;
568
569	list_for_each_entry(use, &b->source_list, source_list) {
570	if (use->source == a)
571	return 1;
572	}
573	pr_debug("%s does not use %s!\n", a->name, b->name);
574	return 0;
575	}
576
577	/*
578	* Module a uses b
579	* - we add 'a' as a "source", 'b' as a "target" of module use
580	* - the module_use is added to the list of 'b' sources (so
581	* 'b' can walk the list to see who sourced them), and of 'a'
582	* targets (so 'a' can see what modules it targets).
583	*/
584	static int add_module_usage(struct module a, struct module b)
585	{
586	struct module_use *use;
587
588	pr_debug("Allocating new usage for %s.\n", a->name);
589	use = kmalloc(sizeof(*use), GFP_ATOMIC);
590	if (!use)
591	return -ENOMEM;
592
593	use->source = a;
594	use->target = b;
595	list_add(&use->source_list, &b->source_list);
596	list_add(&use->target_list, &a->target_list);
597	return 0;
598	}
599
600	/* Module a uses b: caller needs module_mutex() */
601	static int ref_module(struct module a, struct module b)
602	{
603	int err;
604
605	if (b == NULL \|\| already_uses(a, b))
606	return 0;
607
608	/* If module isn't available, we fail. */
609	err = strong_try_module_get(b);
610	if (err)
611	return err;
612
613	err = add_module_usage(a, b);
614	if (err) {
615	module_put(b);
616	return err;
617	}
618	return 0;
619	}
620
621	/* Clear the unload stuff of the module. */
622	static void module_unload_free(struct module *mod)
623	{
624	struct module_use use, tmp;
625
626	mutex_lock(&module_mutex);
627	list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
628	struct module *i = use->target;
629	pr_debug("%s unusing %s\n", mod->name, i->name);
630	module_put(i);
631	list_del(&use->source_list);
632	list_del(&use->target_list);
633	kfree(use);
634	}
635	mutex_unlock(&module_mutex);
636	}
637
638	#ifdef CONFIG_MODULE_FORCE_UNLOAD
639	static inline int try_force_unload(unsigned int flags)
640	{
641	int ret = (flags & O_TRUNC);
642	if (ret)
643	add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
644	return ret;
645	}
646	#else
647	static inline int try_force_unload(unsigned int flags)
648	{
649	return 0;
650	}
651	#endif /* CONFIG_MODULE_FORCE_UNLOAD */
652
653	/* Try to release refcount of module, 0 means success. */
654	static int try_release_module_ref(struct module *mod)
655	{
656	int ret;
657
658	/* Try to decrement refcnt which we set at loading */
659	ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
660	BUG_ON(ret < 0);
661	if (ret)
662	/* Someone can put this right now, recover with checking */
663	ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
664
665	return ret;
666	}
667
668	static int try_stop_module(struct module mod, int flags, int forced)
669	{
670	/* If it's not unused, quit unless we're forcing. */
671	if (try_release_module_ref(mod) != 0) {
672	*forced = try_force_unload(flags);
673	if (!(*forced))
674	return -EWOULDBLOCK;
675	}
676
677	/* Mark it as dying. */
678	mod->state = MODULE_STATE_GOING;
679
680	return 0;
681	}
682
683	/**
684	* module_refcount() - return the refcount or -1 if unloading
685	* @mod: the module we're checking
686	*
687	* Return:
688	* -1 if the module is in the process of unloading
689	* otherwise the number of references in the kernel to the module
690	*/
691	int module_refcount(struct module *mod)
692	{
693	return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
694	}
695	EXPORT_SYMBOL(module_refcount);
696
697	/* This exists whether we can unload or not */
698	static void free_module(struct module *mod);
699
700	SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
701	unsigned int, flags)
702	{
703	struct module *mod;
704	char name[MODULE_NAME_LEN];
705	char buf[MODULE_FLAGS_BUF_SIZE];
706	int ret, forced = 0;
707
708	if (!capable(CAP_SYS_MODULE) \|\| modules_disabled)
709	return -EPERM;
710
711	if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
712	return -EFAULT;
713	name[MODULE_NAME_LEN-1] = '\0';
714
715	audit_log_kern_module(name);
716
717	if (mutex_lock_interruptible(&module_mutex) != 0)
718	return -EINTR;
719
720	mod = find_module(name);
721	if (!mod) {
722	ret = -ENOENT;
723	goto out;
724	}
725
726	if (!list_empty(&mod->source_list)) {
727	/* Other modules depend on us: get rid of them first. */
728	ret = -EWOULDBLOCK;
729	goto out;
730	}
731
732	/* Doing init or already dying? */
733	if (mod->state != MODULE_STATE_LIVE) {
734	/* FIXME: if (force), slam module count damn the torpedoes */
735	pr_debug("%s already dying\n", mod->name);
736	ret = -EBUSY;
737	goto out;
738	}
739
740	/* If it has an init func, it must have an exit func to unload */
741	if (mod->init && !mod->exit) {
742	forced = try_force_unload(flags);
743	if (!forced) {
744	/* This module can't be removed */
745	ret = -EBUSY;
746	goto out;
747	}
748	}
749
750	ret = try_stop_module(mod, flags, &forced);
751	if (ret != 0)
752	goto out;
753
754	mutex_unlock(&module_mutex);
755	/* Final destruction now no one is using it. */
756	if (mod->exit != NULL)
757	mod->exit();
758	blocking_notifier_call_chain(&module_notify_list,
759	MODULE_STATE_GOING, mod);
760	klp_module_going(mod);
761	ftrace_release_mod(mod);
762
763	async_synchronize_full();
764
765	/* Store the name and taints of the last unloaded module for diagnostic purposes */
766	strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name));
767	strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.taints));
768
769	free_module(mod);
770	/* someone could wait for the module in add_unformed_module() */
771	wake_up_all(&module_wq);
772	return 0;
773	out:
774	mutex_unlock(&module_mutex);
775	return ret;
776	}
777
778	void __symbol_put(const char *symbol)
779	{
780	struct find_symbol_arg fsa = {
781	.name = symbol,
782	.gplok = true,
783	};
784
785	preempt_disable();
786	BUG_ON(!find_symbol(&fsa));
787	module_put(fsa.owner);
788	preempt_enable();
789	}
790	EXPORT_SYMBOL(__symbol_put);
791
792	/* Note this assumes addr is a function, which it currently always is. */
793	void symbol_put_addr(void *addr)
794	{
795	struct module *modaddr;
796	unsigned long a = (unsigned long)dereference_function_descriptor(addr);
797
798	if (core_kernel_text(a))
799	return;
800
801	/*
802	* Even though we hold a reference on the module; we still need to
803	* disable preemption in order to safely traverse the data structure.
804	*/
805	preempt_disable();
806	modaddr = __module_text_address(a);
807	BUG_ON(!modaddr);
808	module_put(modaddr);
809	preempt_enable();
810	}
811	EXPORT_SYMBOL_GPL(symbol_put_addr);
812
813	static ssize_t show_refcnt(struct module_attribute *mattr,
814	struct module_kobject mk, char buffer)
815	{
816	return sprintf(buffer, "%i\n", module_refcount(mk->mod));
817	}
818
819	static struct module_attribute modinfo_refcnt =
820	__ATTR(refcnt, 0444, show_refcnt, NULL);
821
822	void __module_get(struct module *module)
823	{
824	if (module) {
825	atomic_inc(&module->refcnt);
826	trace_module_get(module, _RET_IP_);
827	}
828	}
829	EXPORT_SYMBOL(__module_get);
830
831	bool try_module_get(struct module *module)
832	{
833	bool ret = true;
834
835	if (module) {
836	/* Note: here, we can fail to get a reference */
837	if (likely(module_is_live(module) &&
838	atomic_inc_not_zero(&module->refcnt) != 0))
839	trace_module_get(module, _RET_IP_);
840	else
841	ret = false;
842	}
843	return ret;
844	}
845	EXPORT_SYMBOL(try_module_get);
846
847	void module_put(struct module *module)
848	{
849	int ret;
850
851	if (module) {
852	ret = atomic_dec_if_positive(&module->refcnt);
853	WARN_ON(ret < 0); /* Failed to put refcount */
854	trace_module_put(module, _RET_IP_);
855	}
856	}
857	EXPORT_SYMBOL(module_put);
858
859	#else /* !CONFIG_MODULE_UNLOAD */
860	static inline void module_unload_free(struct module *mod)
861	{
862	}
863
864	static int ref_module(struct module a, struct module b)
865	{
866	return strong_try_module_get(b);
867	}
868
869	static inline int module_unload_init(struct module *mod)
870	{
871	return 0;
872	}
873	#endif /* CONFIG_MODULE_UNLOAD */
874
875	size_t module_flags_taint(unsigned long taints, char *buf)
876	{
877	size_t l = 0;
878	int i;
879
880	for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
881	if (taint_flags[i].module && test_bit(i, &taints))
882	buf[l++] = taint_flags[i].c_true;
883	}
884
885	return l;
886	}
887
888	static ssize_t show_initstate(struct module_attribute *mattr,
889	struct module_kobject mk, char buffer)
890	{
891	const char *state = "unknown";
892
893	switch (mk->mod->state) {
894	case MODULE_STATE_LIVE:
895	state = "live";
896	break;
897	case MODULE_STATE_COMING:
898	state = "coming";
899	break;
900	case MODULE_STATE_GOING:
901	state = "going";
902	break;
903	default:
904	BUG();
905	}
906	return sprintf(buffer, "%s\n", state);
907	}
908
909	static struct module_attribute modinfo_initstate =
910	__ATTR(initstate, 0444, show_initstate, NULL);
911
912	static ssize_t store_uevent(struct module_attribute *mattr,
913	struct module_kobject *mk,
914	const char *buffer, size_t count)
915	{
916	int rc;
917
918	rc = kobject_synth_uevent(&mk->kobj, buffer, count);
919	return rc ? rc : count;
920	}
921
922	struct module_attribute module_uevent =
923	__ATTR(uevent, 0200, NULL, store_uevent);
924
925	static ssize_t show_coresize(struct module_attribute *mattr,
926	struct module_kobject mk, char buffer)
927	{
928	unsigned int size = mk->mod->mem[MOD_TEXT].size;
929
930	if (!IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC)) {
931	for_class_mod_mem_type(type, core_data)
932	size += mk->mod->mem[type].size;
933	}
934	return sprintf(buffer, "%u\n", size);
935	}
936
937	static struct module_attribute modinfo_coresize =
938	__ATTR(coresize, 0444, show_coresize, NULL);
939
940	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
941	static ssize_t show_datasize(struct module_attribute *mattr,
942	struct module_kobject mk, char buffer)
943	{
944	unsigned int size = 0;
945
946	for_class_mod_mem_type(type, core_data)
947	size += mk->mod->mem[type].size;
948	return sprintf(buffer, "%u\n", size);
949	}
950
951	static struct module_attribute modinfo_datasize =
952	__ATTR(datasize, 0444, show_datasize, NULL);
953	#endif
954
955	static ssize_t show_initsize(struct module_attribute *mattr,
956	struct module_kobject mk, char buffer)
957	{
958	unsigned int size = 0;
959
960	for_class_mod_mem_type(type, init)
961	size += mk->mod->mem[type].size;
962	return sprintf(buffer, "%u\n", size);
963	}
964
965	static struct module_attribute modinfo_initsize =
966	__ATTR(initsize, 0444, show_initsize, NULL);
967
968	static ssize_t show_taint(struct module_attribute *mattr,
969	struct module_kobject mk, char buffer)
970	{
971	size_t l;
972
973	l = module_flags_taint(mk->mod->taints, buffer);
974	buffer[l++] = '\n';
975	return l;
976	}
977
978	static struct module_attribute modinfo_taint =
979	__ATTR(taint, 0444, show_taint, NULL);
980
981	struct module_attribute *modinfo_attrs[] = {
982	&module_uevent,
983	&modinfo_version,
984	&modinfo_srcversion,
985	&modinfo_initstate,
986	&modinfo_coresize,
987	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
988	&modinfo_datasize,
989	#endif
990	&modinfo_initsize,
991	&modinfo_taint,
992	#ifdef CONFIG_MODULE_UNLOAD
993	&modinfo_refcnt,
994	#endif
995	NULL,
996	};
997
998	size_t modinfo_attrs_count = ARRAY_SIZE(modinfo_attrs);
999
1000	static const char vermagic[] = VERMAGIC_STRING;
1001
1002	int try_to_force_load(struct module mod, const char reason)
1003	{
1004	#ifdef CONFIG_MODULE_FORCE_LOAD
1005	if (!test_taint(TAINT_FORCED_MODULE))
1006	pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
1007	add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
1008	return 0;
1009	#else
1010	return -ENOEXEC;
1011	#endif
1012	}
1013
1014	/* Parse tag=value strings from .modinfo section */
1015	char module_next_tag_pair(char string, unsigned long *secsize)
1016	{
1017	/* Skip non-zero chars */
1018	while (string[0]) {
1019	string++;
1020	if ((*secsize)-- <= 1)
1021	return NULL;
1022	}
1023
1024	/* Skip any zero padding. */
1025	while (!string[0]) {
1026	string++;
1027	if ((*secsize)-- <= 1)
1028	return NULL;
1029	}
1030	return string;
1031	}
1032
1033	static char get_next_modinfo(const struct load_info info, const char *tag,
1034	char *prev)
1035	{
1036	char *p;
1037	unsigned int taglen = strlen(tag);
1038	Elf_Shdr *infosec = &info->sechdrs[info->index.info];
1039	unsigned long size = infosec->sh_size;
1040
1041	/*
1042	* get_modinfo() calls made before rewrite_section_headers()
1043	* must use sh_offset, as sh_addr isn't set!
1044	*/
1045	char modinfo = (char )info->hdr + infosec->sh_offset;
1046
1047	if (prev) {
1048	size -= prev - modinfo;
1049	modinfo = module_next_tag_pair(prev, &size);
1050	}
1051
1052	for (p = modinfo; p; p = module_next_tag_pair(p, &size)) {
1053	if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
1054	return p + taglen + 1;
1055	}
1056	return NULL;
1057	}
1058
1059	static char get_modinfo(const struct load_info info, const char *tag)
1060	{
1061	return get_next_modinfo(info, tag, NULL);
1062	}
1063
1064	static int verify_namespace_is_imported(const struct load_info *info,
1065	const struct kernel_symbol *sym,
1066	struct module *mod)
1067	{
1068	const char *namespace;
1069	char *imported_namespace;
1070
1071	namespace = kernel_symbol_namespace(sym);
1072	if (namespace && namespace[0]) {
1073	for_each_modinfo_entry(imported_namespace, info, "import_ns") {
1074	if (strcmp(namespace, imported_namespace) == 0)
1075	return 0;
1076	}
1077	#ifdef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
1078	pr_warn(
1079	#else
1080	pr_err(
1081	#endif
1082	"%s: module uses symbol (%s) from namespace %s, but does not import it.\n",
1083	mod->name, kernel_symbol_name(sym), namespace);
1084	#ifndef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
1085	return -EINVAL;
1086	#endif
1087	}
1088	return 0;
1089	}
1090
1091	static bool inherit_taint(struct module mod, struct module owner, const char *name)
1092	{
1093	if (!owner \|\| !test_bit(TAINT_PROPRIETARY_MODULE, &owner->taints))
1094	return true;
1095
1096	if (mod->using_gplonly_symbols) {
1097	pr_err("%s: module using GPL-only symbols uses symbols %s from proprietary module %s.\n",
1098	mod->name, name, owner->name);
1099	return false;
1100	}
1101
1102	if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) {
1103	pr_warn("%s: module uses symbols %s from proprietary module %s, inheriting taint.\n",
1104	mod->name, name, owner->name);
1105	set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints);
1106	}
1107	return true;
1108	}
1109
1110	/* Resolve a symbol for this module. I.e. if we find one, record usage. */
1111	static const struct kernel_symbol resolve_symbol(struct module mod,
1112	const struct load_info *info,
1113	const char *name,
1114	char ownername[])
1115	{
1116	struct find_symbol_arg fsa = {
1117	.name = name,
1118	.gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)),
1119	.warn = true,
1120	};
1121	int err;
1122
1123	/*
1124	* The module_mutex should not be a heavily contended lock;
1125	* if we get the occasional sleep here, we'll go an extra iteration
1126	* in the wait_event_interruptible(), which is harmless.
1127	*/
1128	sched_annotate_sleep();
1129	mutex_lock(&module_mutex);
1130	if (!find_symbol(&fsa))
1131	goto unlock;
1132
1133	if (fsa.license == GPL_ONLY)
1134	mod->using_gplonly_symbols = true;
1135
1136	if (!inherit_taint(mod, fsa.owner, name)) {
1137	fsa.sym = NULL;
1138	goto getname;
1139	}
1140
1141	if (!check_version(info, name, mod, fsa.crc)) {
1142	fsa.sym = ERR_PTR(-EINVAL);
1143	goto getname;
1144	}
1145
1146	err = verify_namespace_is_imported(info, fsa.sym, mod);
1147	if (err) {
1148	fsa.sym = ERR_PTR(err);
1149	goto getname;
1150	}
1151
1152	err = ref_module(mod, fsa.owner);
1153	if (err) {
1154	fsa.sym = ERR_PTR(err);
1155	goto getname;
1156	}
1157
1158	getname:
1159	/* We must make copy under the lock if we failed to get ref. */
1160	strncpy(ownername, module_name(fsa.owner), MODULE_NAME_LEN);
1161	unlock:
1162	mutex_unlock(&module_mutex);
1163	return fsa.sym;
1164	}
1165
1166	static const struct kernel_symbol *
1167	resolve_symbol_wait(struct module *mod,
1168	const struct load_info *info,
1169	const char *name)
1170	{
1171	const struct kernel_symbol *ksym;
1172	char owner[MODULE_NAME_LEN];
1173
1174	if (wait_event_interruptible_timeout(module_wq,
1175	!IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
1176	\|\| PTR_ERR(ksym) != -EBUSY,
1177	30 * HZ) <= 0) {
1178	pr_warn("%s: gave up waiting for init of module %s.\n",
1179	mod->name, owner);
1180	}
1181	return ksym;
1182	}
1183
1184	void __weak module_arch_cleanup(struct module *mod)
1185	{
1186	}
1187
1188	void __weak module_arch_freeing_init(struct module *mod)
1189	{
1190	}
1191
1192	static int module_memory_alloc(struct module *mod, enum mod_mem_type type)
1193	{
1194	unsigned int size = PAGE_ALIGN(mod->mem[type].size);
1195	enum execmem_type execmem_type;
1196	void *ptr;
1197
1198	mod->mem[type].size = size;
1199
1200	if (mod_mem_type_is_data(type))
1201	execmem_type = EXECMEM_MODULE_DATA;
1202	else
1203	execmem_type = EXECMEM_MODULE_TEXT;
1204
1205	ptr = execmem_alloc(execmem_type, size);
1206	if (!ptr)
1207	return -ENOMEM;
1208
1209	/*
1210	* The pointer to these blocks of memory are stored on the module
1211	* structure and we keep that around so long as the module is
1212	* around. We only free that memory when we unload the module.
1213	* Just mark them as not being a leak then. The .init* ELF
1214	* sections do get freed after boot so we could treat them
1215	* slightly differently with kmemleak_ignore() and only grey
1216	* them out as they work as typical memory allocations which
1217	* do eventually get freed, but let's just keep things simple
1218	* and avoid any false positives.
1219	*/
1220	kmemleak_not_leak(ptr);
1221
1222	memset(ptr, 0, size);
1223	mod->mem[type].base = ptr;
1224
1225	return 0;
1226	}
1227
1228	static void module_memory_free(struct module *mod, enum mod_mem_type type,
1229	bool unload_codetags)
1230	{
1231	void *ptr = mod->mem[type].base;
1232
1233	if (!unload_codetags && mod_mem_type_is_core_data(type))
1234	return;
1235
1236	execmem_free(ptr);
1237	}
1238
1239	static void free_mod_mem(struct module *mod, bool unload_codetags)
1240	{
1241	for_each_mod_mem_type(type) {
1242	struct module_memory *mod_mem = &mod->mem[type];
1243
1244	if (type == MOD_DATA)
1245	continue;
1246
1247	/* Free lock-classes; relies on the preceding sync_rcu(). */
1248	lockdep_free_key_range(mod_mem->base, mod_mem->size);
1249	if (mod_mem->size)
1250	module_memory_free(mod, type, unload_codetags);
1251	}
1252
1253	/* MOD_DATA hosts mod, so free it at last */
1254	lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size);
1255	module_memory_free(mod, MOD_DATA, unload_codetags);
1256	}
1257
1258	/* Free a module, remove from lists, etc. */
1259	static void free_module(struct module *mod)
1260	{
1261	bool unload_codetags;
1262
1263	trace_module_free(mod);
1264
1265	unload_codetags = codetag_unload_module(mod);
1266	if (!unload_codetags)
1267	pr_warn("%s: memory allocation(s) from the module still alive, cannot unload cleanly\n",
1268	mod->name);
1269
1270	mod_sysfs_teardown(mod);
1271
1272	/*
1273	* We leave it in list to prevent duplicate loads, but make sure
1274	* that noone uses it while it's being deconstructed.
1275	*/
1276	mutex_lock(&module_mutex);
1277	mod->state = MODULE_STATE_UNFORMED;
1278	mutex_unlock(&module_mutex);
1279
1280	/* Arch-specific cleanup. */
1281	module_arch_cleanup(mod);
1282
1283	/* Module unload stuff */
1284	module_unload_free(mod);
1285
1286	/* Free any allocated parameters. */
1287	destroy_params(mod->kp, mod->num_kp);
1288
1289	if (is_livepatch_module(mod))
1290	free_module_elf(mod);
1291
1292	/* Now we can delete it from the lists */
1293	mutex_lock(&module_mutex);
1294	/* Unlink carefully: kallsyms could be walking list. */
1295	list_del_rcu(&mod->list);
1296	mod_tree_remove(mod);
1297	/* Remove this module from bug list, this uses list_del_rcu */
1298	module_bug_cleanup(mod);
1299	/* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
1300	synchronize_rcu();
1301	if (try_add_tainted_module(mod))
1302	pr_err("%s: adding tainted module to the unloaded tainted modules list failed.\n",
1303	mod->name);
1304	mutex_unlock(&module_mutex);
1305
1306	/* This may be empty, but that's OK */
1307	module_arch_freeing_init(mod);
1308	kfree(mod->args);
1309	percpu_modfree(mod);
1310
1311	free_mod_mem(mod, unload_codetags);
1312	}
1313
1314	void __symbol_get(const char symbol)
1315	{
1316	struct find_symbol_arg fsa = {
1317	.name = symbol,
1318	.gplok = true,
1319	.warn = true,
1320	};
1321
1322	preempt_disable();
1323	if (!find_symbol(&fsa))
1324	goto fail;
1325	if (fsa.license != GPL_ONLY) {
1326	pr_warn("failing symbol_get of non-GPLONLY symbol %s.\n",
1327	symbol);
1328	goto fail;
1329	}
1330	if (strong_try_module_get(fsa.owner))
1331	goto fail;
1332	preempt_enable();
1333	return (void *)kernel_symbol_value(fsa.sym);
1334	fail:
1335	preempt_enable();
1336	return NULL;
1337	}
1338	EXPORT_SYMBOL_GPL(__symbol_get);
1339
1340	/*
1341	* Ensure that an exported symbol [global namespace] does not already exist
1342	* in the kernel or in some other module's exported symbol table.
1343	*
1344	* You must hold the module_mutex.
1345	*/
1346	static int verify_exported_symbols(struct module *mod)
1347	{
1348	unsigned int i;
1349	const struct kernel_symbol *s;
1350	struct {
1351	const struct kernel_symbol *sym;
1352	unsigned int num;
1353	} arr[] = {
1354	{ mod->syms, mod->num_syms },
1355	{ mod->gpl_syms, mod->num_gpl_syms },
1356	};
1357
1358	for (i = 0; i < ARRAY_SIZE(arr); i++) {
1359	for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
1360	struct find_symbol_arg fsa = {
1361	.name = kernel_symbol_name(s),
1362	.gplok = true,
1363	};
1364	if (find_symbol(&fsa)) {
1365	pr_err("%s: exports duplicate symbol %s"
1366	" (owned by %s)\n",
1367	mod->name, kernel_symbol_name(s),
1368	module_name(fsa.owner));
1369	return -ENOEXEC;
1370	}
1371	}
1372	}
1373	return 0;
1374	}
1375
1376	static bool ignore_undef_symbol(Elf_Half emachine, const char *name)
1377	{
1378	/*
1379	* On x86, PIC code and Clang non-PIC code may have call foo@PLT. GNU as
1380	* before 2.37 produces an unreferenced _GLOBAL_OFFSET_TABLE_ on x86-64.
1381	* i386 has a similar problem but may not deserve a fix.
1382	*
1383	* If we ever have to ignore many symbols, consider refactoring the code to
1384	* only warn if referenced by a relocation.
1385	*/
1386	if (emachine == EM_386 \|\| emachine == EM_X86_64)
1387	return !strcmp(name, "_GLOBAL_OFFSET_TABLE_");
1388	return false;
1389	}
1390
1391	/* Change all symbols so that st_value encodes the pointer directly. */
1392	static int simplify_symbols(struct module mod, const struct load_info info)
1393	{
1394	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
1395	Elf_Sym sym = (void )symsec->sh_addr;
1396	unsigned long secbase;
1397	unsigned int i;
1398	int ret = 0;
1399	const struct kernel_symbol *ksym;
1400
1401	for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
1402	const char *name = info->strtab + sym[i].st_name;
1403
1404	switch (sym[i].st_shndx) {
1405	case SHN_COMMON:
1406	/* Ignore common symbols */
1407	if (!strncmp(name, "__gnu_lto", 9))
1408	break;
1409
1410	/*
1411	* We compiled with -fno-common. These are not
1412	* supposed to happen.
1413	*/
1414	pr_debug("Common symbol: %s\n", name);
1415	pr_warn("%s: please compile with -fno-common\n",
1416	mod->name);
1417	ret = -ENOEXEC;
1418	break;
1419
1420	case SHN_ABS:
1421	/* Don't need to do anything */
1422	pr_debug("Absolute symbol: 0x%08lx %s\n",
1423	(long)sym[i].st_value, name);
1424	break;
1425
1426	case SHN_LIVEPATCH:
1427	/* Livepatch symbols are resolved by livepatch */
1428	break;
1429
1430	case SHN_UNDEF:
1431	ksym = resolve_symbol_wait(mod, info, name);
1432	/* Ok if resolved. */
1433	if (ksym && !IS_ERR(ksym)) {
1434	sym[i].st_value = kernel_symbol_value(ksym);
1435	break;
1436	}
1437
1438	/* Ok if weak or ignored. */
1439	if (!ksym &&
1440	(ELF_ST_BIND(sym[i].st_info) == STB_WEAK \|\|
1441	ignore_undef_symbol(info->hdr->e_machine, name)))
1442	break;
1443
1444	ret = PTR_ERR(ksym) ?: -ENOENT;
1445	pr_warn("%s: Unknown symbol %s (err %d)\n",
1446	mod->name, name, ret);
1447	break;
1448
1449	default:
1450	/* Divert to percpu allocation if a percpu var. */
1451	if (sym[i].st_shndx == info->index.pcpu)
1452	secbase = (unsigned long)mod_percpu(mod);
1453	else
1454	secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
1455	sym[i].st_value += secbase;
1456	break;
1457	}
1458	}
1459
1460	return ret;
1461	}
1462
1463	static int apply_relocations(struct module mod, const struct load_info info)
1464	{
1465	unsigned int i;
1466	int err = 0;
1467
1468	/* Now do relocations. */
1469	for (i = 1; i < info->hdr->e_shnum; i++) {
1470	unsigned int infosec = info->sechdrs[i].sh_info;
1471
1472	/* Not a valid relocation section? */
1473	if (infosec >= info->hdr->e_shnum)
1474	continue;
1475
1476	/* Don't bother with non-allocated sections */
1477	if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
1478	continue;
1479
1480	if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH)
1481	err = klp_apply_section_relocs(mod, info->sechdrs,
1482	info->secstrings,
1483	info->strtab,
1484	info->index.sym, i,
1485	NULL);
1486	else if (info->sechdrs[i].sh_type == SHT_REL)
1487	err = apply_relocate(info->sechdrs, info->strtab,
1488	info->index.sym, i, mod);
1489	else if (info->sechdrs[i].sh_type == SHT_RELA)
1490	err = apply_relocate_add(info->sechdrs, info->strtab,
1491	info->index.sym, i, mod);
1492	if (err < 0)
1493	break;
1494	}
1495	return err;
1496	}
1497
1498	/* Additional bytes needed by arch in front of individual sections */
1499	unsigned int __weak arch_mod_section_prepend(struct module *mod,
1500	unsigned int section)
1501	{
1502	/* default implementation just returns zero */
1503	return 0;
1504	}
1505
1506	long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
1507	Elf_Shdr *sechdr, unsigned int section)
1508	{
1509	long offset;
1510	long mask = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK) << SH_ENTSIZE_TYPE_SHIFT;
1511
1512	mod->mem[type].size += arch_mod_section_prepend(mod, section);
1513	offset = ALIGN(mod->mem[type].size, sechdr->sh_addralign ?: 1);
1514	mod->mem[type].size = offset + sechdr->sh_size;
1515
1516	WARN_ON_ONCE(offset & mask);
1517	return offset \| mask;
1518	}
1519
1520	bool module_init_layout_section(const char *sname)
1521	{
1522	#ifndef CONFIG_MODULE_UNLOAD
1523	if (module_exit_section(sname))
1524	return true;
1525	#endif
1526	return module_init_section(sname);
1527	}
1528
1529	static void __layout_sections(struct module mod, struct load_info info, bool is_init)
1530	{
1531	unsigned int m, i;
1532
1533	static const unsigned long masks[][2] = {
1534	/*
1535	* NOTE: all executable code must be the first section
1536	* in this array; otherwise modify the text_size
1537	* finder in the two loops below
1538	*/
1539	{ SHF_EXECINSTR \| SHF_ALLOC, ARCH_SHF_SMALL },
1540	{ SHF_ALLOC, SHF_WRITE \| ARCH_SHF_SMALL },
1541	{ SHF_RO_AFTER_INIT \| SHF_ALLOC, ARCH_SHF_SMALL },
1542	{ SHF_WRITE \| SHF_ALLOC, ARCH_SHF_SMALL },
1543	{ ARCH_SHF_SMALL \| SHF_ALLOC, 0 }
1544	};
1545	static const int core_m_to_mem_type[] = {
1546	MOD_TEXT,
1547	MOD_RODATA,
1548	MOD_RO_AFTER_INIT,
1549	MOD_DATA,
1550	MOD_DATA,
1551	};
1552	static const int init_m_to_mem_type[] = {
1553	MOD_INIT_TEXT,
1554	MOD_INIT_RODATA,
1555	MOD_INVALID,
1556	MOD_INIT_DATA,
1557	MOD_INIT_DATA,
1558	};
1559
1560	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1561	enum mod_mem_type type = is_init ? init_m_to_mem_type[m] : core_m_to_mem_type[m];
1562
1563	for (i = 0; i < info->hdr->e_shnum; ++i) {
1564	Elf_Shdr *s = &info->sechdrs[i];
1565	const char *sname = info->secstrings + s->sh_name;
1566
1567	if ((s->sh_flags & masks[m][0]) != masks[m][0]
1568	\|\| (s->sh_flags & masks[m][1])
1569	\|\| s->sh_entsize != ~0UL
1570	\|\| is_init != module_init_layout_section(sname))
1571	continue;
1572
1573	if (WARN_ON_ONCE(type == MOD_INVALID))
1574	continue;
1575
1576	s->sh_entsize = module_get_offset_and_type(mod, type, s, i);
1577	pr_debug("\t%s\n", sname);
1578	}
1579	}
1580	}
1581
1582	/*
1583	* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
1584	* might -- code, read-only data, read-write data, small data. Tally
1585	* sizes, and place the offsets into sh_entsize fields: high bit means it
1586	* belongs in init.
1587	*/
1588	static void layout_sections(struct module mod, struct load_info info)
1589	{
1590	unsigned int i;
1591
1592	for (i = 0; i < info->hdr->e_shnum; i++)
1593	info->sechdrs[i].sh_entsize = ~0UL;
1594
1595	pr_debug("Core section allocation order for %s:\n", mod->name);
1596	__layout_sections(mod, info, false);
1597
1598	pr_debug("Init section allocation order for %s:\n", mod->name);
1599	__layout_sections(mod, info, true);
1600	}
1601
1602	static void module_license_taint_check(struct module mod, const char license)
1603	{
1604	if (!license)
1605	license = "unspecified";
1606
1607	if (!license_is_gpl_compatible(license)) {
1608	if (!test_taint(TAINT_PROPRIETARY_MODULE))
1609	pr_warn("%s: module license '%s' taints kernel.\n",
1610	mod->name, license);
1611	add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
1612	LOCKDEP_NOW_UNRELIABLE);
1613	}
1614	}
1615
1616	static void setup_modinfo(struct module mod, struct load_info info)
1617	{
1618	struct module_attribute *attr;
1619	int i;
1620
1621	for (i = 0; (attr = modinfo_attrs[i]); i++) {
1622	if (attr->setup)
1623	attr->setup(mod, get_modinfo(info, attr->attr.name));
1624	}
1625	}
1626
1627	static void free_modinfo(struct module *mod)
1628	{
1629	struct module_attribute *attr;
1630	int i;
1631
1632	for (i = 0; (attr = modinfo_attrs[i]); i++) {
1633	if (attr->free)
1634	attr->free(mod);
1635	}
1636	}
1637
1638	bool __weak module_init_section(const char *name)
1639	{
1640	return strstarts(name, ".init");
1641	}
1642
1643	bool __weak module_exit_section(const char *name)
1644	{
1645	return strstarts(name, ".exit");
1646	}
1647
1648	static int validate_section_offset(struct load_info info, Elf_Shdr shdr)
1649	{
1650	#if defined(CONFIG_64BIT)
1651	unsigned long long secend;
1652	#else
1653	unsigned long secend;
1654	#endif
1655
1656	/*
1657	* Check for both overflow and offset/size being
1658	* too large.
1659	*/
1660	secend = shdr->sh_offset + shdr->sh_size;
1661	if (secend < shdr->sh_offset \|\| secend > info->len)
1662	return -ENOEXEC;
1663
1664	return 0;
1665	}
1666
1667	/*
1668	* Check userspace passed ELF module against our expectations, and cache
1669	* useful variables for further processing as we go.
1670	*
1671	* This does basic validity checks against section offsets and sizes, the
1672	* section name string table, and the indices used for it (sh_name).
1673	*
1674	* As a last step, since we're already checking the ELF sections we cache
1675	* useful variables which will be used later for our convenience:
1676	*
1677	* o pointers to section headers
1678	* o cache the modinfo symbol section
1679	* o cache the string symbol section
1680	* o cache the module section
1681	*
1682	* As a last step we set info->mod to the temporary copy of the module in
1683	* info->hdr. The final one will be allocated in move_module(). Any
1684	* modifications we make to our copy of the module will be carried over
1685	* to the final minted module.
1686	*/
1687	static int elf_validity_cache_copy(struct load_info *info, int flags)
1688	{
1689	unsigned int i;
1690	Elf_Shdr shdr, strhdr;
1691	int err;
1692	unsigned int num_mod_secs = 0, mod_idx;
1693	unsigned int num_info_secs = 0, info_idx;
1694	unsigned int num_sym_secs = 0, sym_idx;
1695
1696	if (info->len < sizeof(*(info->hdr))) {
1697	pr_err("Invalid ELF header len %lu\n", info->len);
1698	goto no_exec;
1699	}
1700
1701	if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0) {
1702	pr_err("Invalid ELF header magic: != %s\n", ELFMAG);
1703	goto no_exec;
1704	}
1705	if (info->hdr->e_type != ET_REL) {
1706	pr_err("Invalid ELF header type: %u != %u\n",
1707	info->hdr->e_type, ET_REL);
1708	goto no_exec;
1709	}
1710	if (!elf_check_arch(info->hdr)) {
1711	pr_err("Invalid architecture in ELF header: %u\n",
1712	info->hdr->e_machine);
1713	goto no_exec;
1714	}
1715	if (!module_elf_check_arch(info->hdr)) {
1716	pr_err("Invalid module architecture in ELF header: %u\n",
1717	info->hdr->e_machine);
1718	goto no_exec;
1719	}
1720	if (info->hdr->e_shentsize != sizeof(Elf_Shdr)) {
1721	pr_err("Invalid ELF section header size\n");
1722	goto no_exec;
1723	}
1724
1725	/*
1726	* e_shnum is 16 bits, and sizeof(Elf_Shdr) is
1727	* known and small. So e_shnum * sizeof(Elf_Shdr)
1728	* will not overflow unsigned long on any platform.
1729	*/
1730	if (info->hdr->e_shoff >= info->len
1731	\|\| (info->hdr->e_shnum * sizeof(Elf_Shdr) >
1732	info->len - info->hdr->e_shoff)) {
1733	pr_err("Invalid ELF section header overflow\n");
1734	goto no_exec;
1735	}
1736
1737	info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
1738
1739	/*
1740	* Verify if the section name table index is valid.
1741	*/
1742	if (info->hdr->e_shstrndx == SHN_UNDEF
1743	\|\| info->hdr->e_shstrndx >= info->hdr->e_shnum) {
1744	pr_err("Invalid ELF section name index: %d \|\| e_shstrndx (%d) >= e_shnum (%d)\n",
1745	info->hdr->e_shstrndx, info->hdr->e_shstrndx,
1746	info->hdr->e_shnum);
1747	goto no_exec;
1748	}
1749
1750	strhdr = &info->sechdrs[info->hdr->e_shstrndx];
1751	err = validate_section_offset(info, strhdr);
1752	if (err < 0) {
1753	pr_err("Invalid ELF section hdr(type %u)\n", strhdr->sh_type);
1754	return err;
1755	}
1756
1757	/*
1758	* The section name table must be NUL-terminated, as required
1759	* by the spec. This makes strcmp and pr_* calls that access
1760	* strings in the section safe.
1761	*/
1762	info->secstrings = (void *)info->hdr + strhdr->sh_offset;
1763	if (strhdr->sh_size == 0) {
1764	pr_err("empty section name table\n");
1765	goto no_exec;
1766	}
1767	if (info->secstrings[strhdr->sh_size - 1] != '\0') {
1768	pr_err("ELF Spec violation: section name table isn't null terminated\n");
1769	goto no_exec;
1770	}
1771
1772	/*
1773	* The code assumes that section 0 has a length of zero and
1774	* an addr of zero, so check for it.
1775	*/
1776	if (info->sechdrs[0].sh_type != SHT_NULL
1777	\|\| info->sechdrs[0].sh_size != 0
1778	\|\| info->sechdrs[0].sh_addr != 0) {
1779	pr_err("ELF Spec violation: section 0 type(%d)!=SH_NULL or non-zero len or addr\n",
1780	info->sechdrs[0].sh_type);
1781	goto no_exec;
1782	}
1783
1784	for (i = 1; i < info->hdr->e_shnum; i++) {
1785	shdr = &info->sechdrs[i];
1786	switch (shdr->sh_type) {
1787	case SHT_NULL:
1788	case SHT_NOBITS:
1789	continue;
1790	case SHT_SYMTAB:
1791	if (shdr->sh_link == SHN_UNDEF
1792	\|\| shdr->sh_link >= info->hdr->e_shnum) {
1793	pr_err("Invalid ELF sh_link!=SHN_UNDEF(%d) or (sh_link(%d) >= hdr->e_shnum(%d)\n",
1794	shdr->sh_link, shdr->sh_link,
1795	info->hdr->e_shnum);
1796	goto no_exec;
1797	}
1798	num_sym_secs++;
1799	sym_idx = i;
1800	fallthrough;
1801	default:
1802	err = validate_section_offset(info, shdr);
1803	if (err < 0) {
1804	pr_err("Invalid ELF section in module (section %u type %u)\n",
1805	i, shdr->sh_type);
1806	return err;
1807	}
1808	if (strcmp(info->secstrings + shdr->sh_name,
1809	".gnu.linkonce.this_module") == 0) {
1810	num_mod_secs++;
1811	mod_idx = i;
1812	} else if (strcmp(info->secstrings + shdr->sh_name,
1813	".modinfo") == 0) {
1814	num_info_secs++;
1815	info_idx = i;
1816	}
1817
1818	if (shdr->sh_flags & SHF_ALLOC) {
1819	if (shdr->sh_name >= strhdr->sh_size) {
1820	pr_err("Invalid ELF section name in module (section %u type %u)\n",
1821	i, shdr->sh_type);
1822	return -ENOEXEC;
1823	}
1824	}
1825	break;
1826	}
1827	}
1828
1829	if (num_info_secs > 1) {
1830	pr_err("Only one .modinfo section must exist.\n");
1831	goto no_exec;
1832	} else if (num_info_secs == 1) {
1833	/* Try to find a name early so we can log errors with a module name */
1834	info->index.info = info_idx;
1835	info->name = get_modinfo(info, "name");
1836	}
1837
1838	if (num_sym_secs != 1) {
1839	pr_warn("%s: module has no symbols (stripped?)\n",
1840	info->name ?: "(missing .modinfo section or name field)");
1841	goto no_exec;
1842	}
1843
1844	/* Sets internal symbols and strings. */
1845	info->index.sym = sym_idx;
1846	shdr = &info->sechdrs[sym_idx];
1847	info->index.str = shdr->sh_link;
1848	info->strtab = (char *)info->hdr + info->sechdrs[info->index.str].sh_offset;
1849
1850	/*
1851	* The ".gnu.linkonce.this_module" ELF section is special. It is
1852	* what modpost uses to refer to __this_module and let's use rely
1853	* on THIS_MODULE to point to &__this_module properly. The kernel's
1854	* modpost declares it on each modules's *.mod.c file. If the struct
1855	* module of the kernel changes a full kernel rebuild is required.
1856	*
1857	* We have a few expectaions for this special section, the following
1858	* code validates all this for us:
1859	*
1860	* o Only one section must exist
1861	* o We expect the kernel to always have to allocate it: SHF_ALLOC
1862	* o The section size must match the kernel's run time's struct module
1863	* size
1864	*/
1865	if (num_mod_secs != 1) {
1866	pr_err("module %s: Only one .gnu.linkonce.this_module section must exist.\n",
1867	info->name ?: "(missing .modinfo section or name field)");
1868	goto no_exec;
1869	}
1870
1871	shdr = &info->sechdrs[mod_idx];
1872
1873	/*
1874	* This is already implied on the switch above, however let's be
1875	* pedantic about it.
1876	*/
1877	if (shdr->sh_type == SHT_NOBITS) {
1878	pr_err("module %s: .gnu.linkonce.this_module section must have a size set\n",
1879	info->name ?: "(missing .modinfo section or name field)");
1880	goto no_exec;
1881	}
1882
1883	if (!(shdr->sh_flags & SHF_ALLOC)) {
1884	pr_err("module %s: .gnu.linkonce.this_module must occupy memory during process execution\n",
1885	info->name ?: "(missing .modinfo section or name field)");
1886	goto no_exec;
1887	}
1888
1889	if (shdr->sh_size != sizeof(struct module)) {
1890	pr_err("module %s: .gnu.linkonce.this_module section size must match the kernel's built struct module size at run time\n",
1891	info->name ?: "(missing .modinfo section or name field)");
1892	goto no_exec;
1893	}
1894
1895	info->index.mod = mod_idx;
1896
1897	/* This is temporary: point mod into copy of data. */
1898	info->mod = (void *)info->hdr + shdr->sh_offset;
1899
1900	/*
1901	* If we didn't load the .modinfo 'name' field earlier, fall back to
1902	* on-disk struct mod 'name' field.
1903	*/
1904	if (!info->name)
1905	info->name = info->mod->name;
1906
1907	if (flags & MODULE_INIT_IGNORE_MODVERSIONS)
1908	info->index.vers = 0; /* Pretend no __versions section! */
1909	else
1910	info->index.vers = find_sec(info, "__versions");
1911
1912	info->index.pcpu = find_pcpusec(info);
1913
1914	return 0;
1915
1916	no_exec:
1917	return -ENOEXEC;
1918	}
1919
1920	#define COPY_CHUNK_SIZE (16*PAGE_SIZE)
1921
1922	static int copy_chunked_from_user(void dst, const void __user usrc, unsigned long len)
1923	{
1924	do {
1925	unsigned long n = min(len, COPY_CHUNK_SIZE);
1926
1927	if (copy_from_user(dst, usrc, n) != 0)
1928	return -EFAULT;
1929	cond_resched();
1930	dst += n;
1931	usrc += n;
1932	len -= n;
1933	} while (len);
1934	return 0;
1935	}
1936
1937	static int check_modinfo_livepatch(struct module mod, struct load_info info)
1938	{
1939	if (!get_modinfo(info, "livepatch"))
1940	/* Nothing more to do */
1941	return 0;
1942
1943	if (set_livepatch_module(mod))
1944	return 0;
1945
1946	pr_err("%s: module is marked as livepatch module, but livepatch support is disabled",
1947	mod->name);
1948	return -ENOEXEC;
1949	}
1950
1951	static void check_modinfo_retpoline(struct module mod, struct load_info info)
1952	{
1953	if (retpoline_module_ok(get_modinfo(info, "retpoline")))
1954	return;
1955
1956	pr_warn("%s: loading module not compiled with retpoline compiler.\n",
1957	mod->name);
1958	}
1959
1960	/* Sets info->hdr and info->len. */
1961	static int copy_module_from_user(const void __user *umod, unsigned long len,
1962	struct load_info *info)
1963	{
1964	int err;
1965
1966	info->len = len;
1967	if (info->len < sizeof(*(info->hdr)))
1968	return -ENOEXEC;
1969
1970	err = security_kernel_load_data(LOADING_MODULE, true);
1971	if (err)
1972	return err;
1973
1974	/* Suck in entire file: we'll want most of it. */
1975	info->hdr = __vmalloc(info->len, GFP_KERNEL \| __GFP_NOWARN);
1976	if (!info->hdr)
1977	return -ENOMEM;
1978
1979	if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
1980	err = -EFAULT;
1981	goto out;
1982	}
1983
1984	err = security_kernel_post_load_data((char *)info->hdr, info->len,
1985	LOADING_MODULE, "init_module");
1986	out:
1987	if (err)
1988	vfree(info->hdr);
1989
1990	return err;
1991	}
1992
1993	static void free_copy(struct load_info *info, int flags)
1994	{
1995	if (flags & MODULE_INIT_COMPRESSED_FILE)
1996	module_decompress_cleanup(info);
1997	else
1998	vfree(info->hdr);
1999	}
2000
2001	static int rewrite_section_headers(struct load_info *info, int flags)
2002	{
2003	unsigned int i;
2004
2005	/* This should always be true, but let's be sure. */
2006	info->sechdrs[0].sh_addr = 0;
2007
2008	for (i = 1; i < info->hdr->e_shnum; i++) {
2009	Elf_Shdr *shdr = &info->sechdrs[i];
2010
2011	/*
2012	* Mark all sections sh_addr with their address in the
2013	* temporary image.
2014	*/
2015	shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
2016
2017	}
2018
2019	/* Track but don't keep modinfo and version sections. */
2020	info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
2021	info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
2022
2023	return 0;
2024	}
2025
2026	/*
2027	* These calls taint the kernel depending certain module circumstances */
2028	static void module_augment_kernel_taints(struct module mod, struct load_info info)
2029	{
2030	int prev_taint = test_taint(TAINT_PROPRIETARY_MODULE);
2031
2032	if (!get_modinfo(info, "intree")) {
2033	if (!test_taint(TAINT_OOT_MODULE))
2034	pr_warn("%s: loading out-of-tree module taints kernel.\n",
2035	mod->name);
2036	add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
2037	}
2038
2039	check_modinfo_retpoline(mod, info);
2040
2041	if (get_modinfo(info, "staging")) {
2042	add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
2043	pr_warn("%s: module is from the staging directory, the quality "
2044	"is unknown, you have been warned.\n", mod->name);
2045	}
2046
2047	if (is_livepatch_module(mod)) {
2048	add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
2049	pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n",
2050	mod->name);
2051	}
2052
2053	module_license_taint_check(mod, get_modinfo(info, "license"));
2054
2055	if (get_modinfo(info, "test")) {
2056	if (!test_taint(TAINT_TEST))
2057	pr_warn("%s: loading test module taints kernel.\n",
2058	mod->name);
2059	add_taint_module(mod, TAINT_TEST, LOCKDEP_STILL_OK);
2060	}
2061	#ifdef CONFIG_MODULE_SIG
2062	mod->sig_ok = info->sig_ok;
2063	if (!mod->sig_ok) {
2064	pr_notice_once("%s: module verification failed: signature "
2065	"and/or required key missing - tainting "
2066	"kernel\n", mod->name);
2067	add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
2068	}
2069	#endif
2070
2071	/*
2072	* ndiswrapper is under GPL by itself, but loads proprietary modules.
2073	* Don't use add_taint_module(), as it would prevent ndiswrapper from
2074	* using GPL-only symbols it needs.
2075	*/
2076	if (strcmp(mod->name, "ndiswrapper") == 0)
2077	add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
2078
2079	/* driverloader was caught wrongly pretending to be under GPL */
2080	if (strcmp(mod->name, "driverloader") == 0)
2081	add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
2082	LOCKDEP_NOW_UNRELIABLE);
2083
2084	/* lve claims to be GPL but upstream won't provide source */
2085	if (strcmp(mod->name, "lve") == 0)
2086	add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
2087	LOCKDEP_NOW_UNRELIABLE);
2088
2089	if (!prev_taint && test_taint(TAINT_PROPRIETARY_MODULE))
2090	pr_warn("%s: module license taints kernel.\n", mod->name);
2091
2092	}
2093
2094	static int check_modinfo(struct module mod, struct load_info info, int flags)
2095	{
2096	const char *modmagic = get_modinfo(info, "vermagic");
2097	int err;
2098
2099	if (flags & MODULE_INIT_IGNORE_VERMAGIC)
2100	modmagic = NULL;
2101
2102	/* This is allowed: modprobe --force will invalidate it. */
2103	if (!modmagic) {
2104	err = try_to_force_load(mod, "bad vermagic");
2105	if (err)
2106	return err;
2107	} else if (!same_magic(modmagic, vermagic, info->index.vers)) {
2108	pr_err("%s: version magic '%s' should be '%s'\n",
2109	info->name, modmagic, vermagic);
2110	return -ENOEXEC;
2111	}
2112
2113	err = check_modinfo_livepatch(mod, info);
2114	if (err)
2115	return err;
2116
2117	return 0;
2118	}
2119
2120	static int find_module_sections(struct module mod, struct load_info info)
2121	{
2122	mod->kp = section_objs(info, "__param",
2123	sizeof(*mod->kp), &mod->num_kp);
2124	mod->syms = section_objs(info, "__ksymtab",
2125	sizeof(*mod->syms), &mod->num_syms);
2126	mod->crcs = section_addr(info, "__kcrctab");
2127	mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
2128	sizeof(*mod->gpl_syms),
2129	&mod->num_gpl_syms);
2130	mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
2131
2132	#ifdef CONFIG_CONSTRUCTORS
2133	mod->ctors = section_objs(info, ".ctors",
2134	sizeof(*mod->ctors), &mod->num_ctors);
2135	if (!mod->ctors)
2136	mod->ctors = section_objs(info, ".init_array",
2137	sizeof(*mod->ctors), &mod->num_ctors);
2138	else if (find_sec(info, ".init_array")) {
2139	/*
2140	* This shouldn't happen with same compiler and binutils
2141	* building all parts of the module.
2142	*/
2143	pr_warn("%s: has both .ctors and .init_array.\n",
2144	mod->name);
2145	return -EINVAL;
2146	}
2147	#endif
2148
2149	mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1,
2150	&mod->noinstr_text_size);
2151
2152	#ifdef CONFIG_TRACEPOINTS
2153	mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
2154	sizeof(*mod->tracepoints_ptrs),
2155	&mod->num_tracepoints);
2156	#endif
2157	#ifdef CONFIG_TREE_SRCU
2158	mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs",
2159	sizeof(*mod->srcu_struct_ptrs),
2160	&mod->num_srcu_structs);
2161	#endif
2162	#ifdef CONFIG_BPF_EVENTS
2163	mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map",
2164	sizeof(*mod->bpf_raw_events),
2165	&mod->num_bpf_raw_events);
2166	#endif
2167	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
2168	mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size);
2169	#endif
2170	#ifdef CONFIG_JUMP_LABEL
2171	mod->jump_entries = section_objs(info, "__jump_table",
2172	sizeof(*mod->jump_entries),
2173	&mod->num_jump_entries);
2174	#endif
2175	#ifdef CONFIG_EVENT_TRACING
2176	mod->trace_events = section_objs(info, "_ftrace_events",
2177	sizeof(*mod->trace_events),
2178	&mod->num_trace_events);
2179	mod->trace_evals = section_objs(info, "_ftrace_eval_map",
2180	sizeof(*mod->trace_evals),
2181	&mod->num_trace_evals);
2182	#endif
2183	#ifdef CONFIG_TRACING
2184	mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
2185	sizeof(*mod->trace_bprintk_fmt_start),
2186	&mod->num_trace_bprintk_fmt);
2187	#endif
2188	#ifdef CONFIG_FTRACE_MCOUNT_RECORD
2189	/* sechdrs[0].sh_size is always zero */
2190	mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION,
2191	sizeof(*mod->ftrace_callsites),
2192	&mod->num_ftrace_callsites);
2193	#endif
2194	#ifdef CONFIG_FUNCTION_ERROR_INJECTION
2195	mod->ei_funcs = section_objs(info, "_error_injection_whitelist",
2196	sizeof(*mod->ei_funcs),
2197	&mod->num_ei_funcs);
2198	#endif
2199	#ifdef CONFIG_KPROBES
2200	mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1,
2201	&mod->kprobes_text_size);
2202	mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist",
2203	sizeof(unsigned long),
2204	&mod->num_kprobe_blacklist);
2205	#endif
2206	#ifdef CONFIG_PRINTK_INDEX
2207	mod->printk_index_start = section_objs(info, ".printk_index",
2208	sizeof(*mod->printk_index_start),
2209	&mod->printk_index_size);
2210	#endif
2211	#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
2212	mod->static_call_sites = section_objs(info, ".static_call_sites",
2213	sizeof(*mod->static_call_sites),
2214	&mod->num_static_call_sites);
2215	#endif
2216	#if IS_ENABLED(CONFIG_KUNIT)
2217	mod->kunit_suites = section_objs(info, ".kunit_test_suites",
2218	sizeof(*mod->kunit_suites),
2219	&mod->num_kunit_suites);
2220	mod->kunit_init_suites = section_objs(info, ".kunit_init_test_suites",
2221	sizeof(*mod->kunit_init_suites),
2222	&mod->num_kunit_init_suites);
2223	#endif
2224
2225	mod->extable = section_objs(info, "__ex_table",
2226	sizeof(*mod->extable), &mod->num_exentries);
2227
2228	if (section_addr(info, "__obsparm"))
2229	pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
2230
2231	#ifdef CONFIG_DYNAMIC_DEBUG_CORE
2232	mod->dyndbg_info.descs = section_objs(info, "__dyndbg",
2233	sizeof(*mod->dyndbg_info.descs),
2234	&mod->dyndbg_info.num_descs);
2235	mod->dyndbg_info.classes = section_objs(info, "__dyndbg_classes",
2236	sizeof(*mod->dyndbg_info.classes),
2237	&mod->dyndbg_info.num_classes);
2238	#endif
2239
2240	return 0;
2241	}
2242
2243	static int move_module(struct module mod, struct load_info info)
2244	{
2245	int i;
2246	enum mod_mem_type t = 0;
2247	int ret = -ENOMEM;
2248
2249	for_each_mod_mem_type(type) {
2250	if (!mod->mem[type].size) {
2251	mod->mem[type].base = NULL;
2252	continue;
2253	}
2254
2255	ret = module_memory_alloc(mod, type);
2256	if (ret) {
2257	t = type;
2258	goto out_enomem;
2259	}
2260	}
2261
2262	/* Transfer each section which specifies SHF_ALLOC */
2263	pr_debug("Final section addresses for %s:\n", mod->name);
2264	for (i = 0; i < info->hdr->e_shnum; i++) {
2265	void *dest;
2266	Elf_Shdr *shdr = &info->sechdrs[i];
2267	enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
2268
2269	if (!(shdr->sh_flags & SHF_ALLOC))
2270	continue;
2271
2272	dest = mod->mem[type].base + (shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK);
2273
2274	if (shdr->sh_type != SHT_NOBITS) {
2275	/*
2276	* Our ELF checker already validated this, but let's
2277	* be pedantic and make the goal clearer. We actually
2278	* end up copying over all modifications made to the
2279	* userspace copy of the entire struct module.
2280	*/
2281	if (i == info->index.mod &&
2282	(WARN_ON_ONCE(shdr->sh_size != sizeof(struct module)))) {
2283	ret = -ENOEXEC;
2284	goto out_enomem;
2285	}
2286	memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
2287	}
2288	/*
2289	* Update the userspace copy's ELF section address to point to
2290	* our newly allocated memory as a pure convenience so that
2291	* users of info can keep taking advantage and using the newly
2292	* minted official memory area.
2293	*/
2294	shdr->sh_addr = (unsigned long)dest;
2295	pr_debug("\t0x%lx 0x%.8lx %s\n", (long)shdr->sh_addr,
2296	(long)shdr->sh_size, info->secstrings + shdr->sh_name);
2297	}
2298
2299	return 0;
2300	out_enomem:
2301	for (t--; t >= 0; t--)
2302	module_memory_free(mod, t, true);
2303	return ret;
2304	}
2305
2306	static int check_export_symbol_versions(struct module *mod)
2307	{
2308	#ifdef CONFIG_MODVERSIONS
2309	if ((mod->num_syms && !mod->crcs) \|\|
2310	(mod->num_gpl_syms && !mod->gpl_crcs)) {
2311	return try_to_force_load(mod,
2312	"no versions for exported symbols");
2313	}
2314	#endif
2315	return 0;
2316	}
2317
2318	static void flush_module_icache(const struct module *mod)
2319	{
2320	/*
2321	* Flush the instruction cache, since we've played with text.
2322	* Do it before processing of module parameters, so the module
2323	* can provide parameter accessor functions of its own.
2324	*/
2325	for_each_mod_mem_type(type) {
2326	const struct module_memory *mod_mem = &mod->mem[type];
2327
2328	if (mod_mem->size) {
2329	flush_icache_range((unsigned long)mod_mem->base,
2330	(unsigned long)mod_mem->base + mod_mem->size);
2331	}
2332	}
2333	}
2334
2335	bool __weak module_elf_check_arch(Elf_Ehdr *hdr)
2336	{
2337	return true;
2338	}
2339
2340	int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
2341	Elf_Shdr *sechdrs,
2342	char *secstrings,
2343	struct module *mod)
2344	{
2345	return 0;
2346	}
2347
2348	/* module_blacklist is a comma-separated list of module names */
2349	static char *module_blacklist;
2350	static bool blacklisted(const char *module_name)
2351	{
2352	const char *p;
2353	size_t len;
2354
2355	if (!module_blacklist)
2356	return false;
2357
2358	for (p = module_blacklist; *p; p += len) {
2359	len = strcspn(p, ",");
2360	if (strlen(module_name) == len && !memcmp(module_name, p, len))
2361	return true;
2362	if (p[len] == ',')
2363	len++;
2364	}
2365	return false;
2366	}
2367	core_param(module_blacklist, module_blacklist, charp, 0400);
2368
2369	static struct module layout_and_allocate(struct load_info info, int flags)
2370	{
2371	struct module *mod;
2372	unsigned int ndx;
2373	int err;
2374
2375	/* Allow arches to frob section contents and sizes. */
2376	err = module_frob_arch_sections(info->hdr, info->sechdrs,
2377	info->secstrings, info->mod);
2378	if (err < 0)
2379	return ERR_PTR(err);
2380
2381	err = module_enforce_rwx_sections(info->hdr, info->sechdrs,
2382	info->secstrings, info->mod);
2383	if (err < 0)
2384	return ERR_PTR(err);
2385
2386	/* We will do a special allocation for per-cpu sections later. */
2387	info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
2388
2389	/*
2390	* Mark ro_after_init section with SHF_RO_AFTER_INIT so that
2391	* layout_sections() can put it in the right place.
2392	* Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set.
2393	*/
2394	ndx = find_sec(info, ".data..ro_after_init");
2395	if (ndx)
2396	info->sechdrs[ndx].sh_flags \|= SHF_RO_AFTER_INIT;
2397	/*
2398	* Mark the __jump_table section as ro_after_init as well: these data
2399	* structures are never modified, with the exception of entries that
2400	* refer to code in the __init section, which are annotated as such
2401	* at module load time.
2402	*/
2403	ndx = find_sec(info, "__jump_table");
2404	if (ndx)
2405	info->sechdrs[ndx].sh_flags \|= SHF_RO_AFTER_INIT;
2406
2407	/*
2408	* Determine total sizes, and put offsets in sh_entsize. For now
2409	* this is done generically; there doesn't appear to be any
2410	* special cases for the architectures.
2411	*/
2412	layout_sections(info->mod, info);
2413	layout_symtab(info->mod, info);
2414
2415	/* Allocate and move to the final place */
2416	err = move_module(info->mod, info);
2417	if (err)
2418	return ERR_PTR(err);
2419
2420	/* Module has been copied to its final place now: return it. */
2421	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2422	kmemleak_load_module(mod, info);
2423	return mod;
2424	}
2425
2426	/* mod is no longer valid after this! */
2427	static void module_deallocate(struct module mod, struct load_info info)
2428	{
2429	percpu_modfree(mod);
2430	module_arch_freeing_init(mod);
2431
2432	free_mod_mem(mod, true);
2433	}
2434
2435	int __weak module_finalize(const Elf_Ehdr *hdr,
2436	const Elf_Shdr *sechdrs,
2437	struct module *me)
2438	{
2439	return 0;
2440	}
2441
2442	static int post_relocation(struct module mod, const struct load_info info)
2443	{
2444	/* Sort exception table now relocations are done. */
2445	sort_extable(mod->extable, mod->extable + mod->num_exentries);
2446
2447	/* Copy relocated percpu area over. */
2448	percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
2449	info->sechdrs[info->index.pcpu].sh_size);
2450
2451	/* Setup kallsyms-specific fields. */
2452	add_kallsyms(mod, info);
2453
2454	/* Arch-specific module finalizing. */
2455	return module_finalize(info->hdr, info->sechdrs, mod);
2456	}
2457
2458	/* Call module constructors. */
2459	static void do_mod_ctors(struct module *mod)
2460	{
2461	#ifdef CONFIG_CONSTRUCTORS
2462	unsigned long i;
2463
2464	for (i = 0; i < mod->num_ctors; i++)
2465	mod->ctors[i]();
2466	#endif
2467	}
2468
2469	/* For freeing module_init on success, in case kallsyms traversing */
2470	struct mod_initfree {
2471	struct llist_node node;
2472	void *init_text;
2473	void *init_data;
2474	void *init_rodata;
2475	};
2476
2477	static void do_free_init(struct work_struct *w)
2478	{
2479	struct llist_node pos, n, *list;
2480	struct mod_initfree *initfree;
2481
2482	list = llist_del_all(&init_free_list);
2483
2484	synchronize_rcu();
2485
2486	llist_for_each_safe(pos, n, list) {
2487	initfree = container_of(pos, struct mod_initfree, node);
2488	execmem_free(initfree->init_text);
2489	execmem_free(initfree->init_data);
2490	execmem_free(initfree->init_rodata);
2491	kfree(initfree);
2492	}
2493	}
2494
2495	void flush_module_init_free_work(void)
2496	{
2497	flush_work(&init_free_wq);
2498	}
2499
2500	#undef MODULE_PARAM_PREFIX
2501	#define MODULE_PARAM_PREFIX "module."
2502	/* Default value for module->async_probe_requested */
2503	static bool async_probe;
2504	module_param(async_probe, bool, 0644);
2505
2506	/*
2507	* This is where the real work happens.
2508	*
2509	* Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
2510	* helper command 'lx-symbols'.
2511	*/
2512	static noinline int do_init_module(struct module *mod)
2513	{
2514	int ret = 0;
2515	struct mod_initfree *freeinit;
2516	#if defined(CONFIG_MODULE_STATS)
2517	unsigned int text_size = 0, total_size = 0;
2518
2519	for_each_mod_mem_type(type) {
2520	const struct module_memory *mod_mem = &mod->mem[type];
2521	if (mod_mem->size) {
2522	total_size += mod_mem->size;
2523	if (type == MOD_TEXT \|\| type == MOD_INIT_TEXT)
2524	text_size += mod_mem->size;
2525	}
2526	}
2527	#endif
2528
2529	freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
2530	if (!freeinit) {
2531	ret = -ENOMEM;
2532	goto fail;
2533	}
2534	freeinit->init_text = mod->mem[MOD_INIT_TEXT].base;
2535	freeinit->init_data = mod->mem[MOD_INIT_DATA].base;
2536	freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base;
2537
2538	do_mod_ctors(mod);
2539	/* Start the module */
2540	if (mod->init != NULL)
2541	ret = do_one_initcall(mod->init);
2542	if (ret < 0) {
2543	goto fail_free_freeinit;
2544	}
2545	if (ret > 0) {
2546	pr_warn("%s: '%s'->init suspiciously returned %d, it should "
2547	"follow 0/-E convention\n"
2548	"%s: loading module anyway...\n",
2549	__func__, mod->name, ret, __func__);
2550	dump_stack();
2551	}
2552
2553	/* Now it's a first class citizen! */
2554	mod->state = MODULE_STATE_LIVE;
2555	blocking_notifier_call_chain(&module_notify_list,
2556	MODULE_STATE_LIVE, mod);
2557
2558	/* Delay uevent until module has finished its init routine */
2559	kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
2560
2561	/*
2562	* We need to finish all async code before the module init sequence
2563	* is done. This has potential to deadlock if synchronous module
2564	* loading is requested from async (which is not allowed!).
2565	*
2566	* See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous
2567	* request_module() from async workers") for more details.
2568	*/
2569	if (!mod->async_probe_requested)
2570	async_synchronize_full();
2571
2572	ftrace_free_mem(mod, mod->mem[MOD_INIT_TEXT].base,
2573	mod->mem[MOD_INIT_TEXT].base + mod->mem[MOD_INIT_TEXT].size);
2574	mutex_lock(&module_mutex);
2575	/* Drop initial reference. */
2576	module_put(mod);
2577	trim_init_extable(mod);
2578	#ifdef CONFIG_KALLSYMS
2579	/* Switch to core kallsyms now init is done: kallsyms may be walking! */
2580	rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
2581	#endif
2582	ret = module_enable_rodata_ro(mod, true);
2583	if (ret)
2584	goto fail_mutex_unlock;
2585	mod_tree_remove_init(mod);
2586	module_arch_freeing_init(mod);
2587	for_class_mod_mem_type(type, init) {
2588	mod->mem[type].base = NULL;
2589	mod->mem[type].size = 0;
2590	}
2591
2592	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
2593	/* .BTF is not SHF_ALLOC and will get removed, so sanitize pointer */
2594	mod->btf_data = NULL;
2595	#endif
2596	/*
2597	* We want to free module_init, but be aware that kallsyms may be
2598	* walking this with preempt disabled. In all the failure paths, we
2599	* call synchronize_rcu(), but we don't want to slow down the success
2600	* path. execmem_free() cannot be called in an interrupt, so do the
2601	* work and call synchronize_rcu() in a work queue.
2602	*
2603	* Note that execmem_alloc() on most architectures creates W+X page
2604	* mappings which won't be cleaned up until do_free_init() runs. Any
2605	* code such as mark_rodata_ro() which depends on those mappings to
2606	* be cleaned up needs to sync with the queued work by invoking
2607	* flush_module_init_free_work().
2608	*/
2609	if (llist_add(&freeinit->node, &init_free_list))
2610	schedule_work(&init_free_wq);
2611
2612	mutex_unlock(&module_mutex);
2613	wake_up_all(&module_wq);
2614
2615	mod_stat_add_long(text_size, &total_text_size);
2616	mod_stat_add_long(total_size, &total_mod_size);
2617
2618	mod_stat_inc(&modcount);
2619
2620	return 0;
2621
2622	fail_mutex_unlock:
2623	mutex_unlock(&module_mutex);
2624	fail_free_freeinit:
2625	kfree(freeinit);
2626	fail:
2627	/* Try to protect us from buggy refcounters. */
2628	mod->state = MODULE_STATE_GOING;
2629	synchronize_rcu();
2630	module_put(mod);
2631	blocking_notifier_call_chain(&module_notify_list,
2632	MODULE_STATE_GOING, mod);
2633	klp_module_going(mod);
2634	ftrace_release_mod(mod);
2635	free_module(mod);
2636	wake_up_all(&module_wq);
2637
2638	return ret;
2639	}
2640
2641	static int may_init_module(void)
2642	{
2643	if (!capable(CAP_SYS_MODULE) \|\| modules_disabled)
2644	return -EPERM;
2645
2646	return 0;
2647	}
2648
2649	/* Is this module of this name done loading? No locks held. */
2650	static bool finished_loading(const char *name)
2651	{
2652	struct module *mod;
2653	bool ret;
2654
2655	/*
2656	* The module_mutex should not be a heavily contended lock;
2657	* if we get the occasional sleep here, we'll go an extra iteration
2658	* in the wait_event_interruptible(), which is harmless.
2659	*/
2660	sched_annotate_sleep();
2661	mutex_lock(&module_mutex);
2662	mod = find_module_all(name, strlen(name), true);
2663	ret = !mod \|\| mod->state == MODULE_STATE_LIVE
2664	\|\| mod->state == MODULE_STATE_GOING;
2665	mutex_unlock(&module_mutex);
2666
2667	return ret;
2668	}
2669
2670	/* Must be called with module_mutex held */
2671	static int module_patient_check_exists(const char *name,
2672	enum fail_dup_mod_reason reason)
2673	{
2674	struct module *old;
2675	int err = 0;
2676
2677	old = find_module_all(name, strlen(name), true);
2678	if (old == NULL)
2679	return 0;
2680
2681	if (old->state == MODULE_STATE_COMING \|\|
2682	old->state == MODULE_STATE_UNFORMED) {
2683	/* Wait in case it fails to load. */
2684	mutex_unlock(&module_mutex);
2685	err = wait_event_interruptible(module_wq,
2686	finished_loading(name));
2687	mutex_lock(&module_mutex);
2688	if (err)
2689	return err;
2690
2691	/* The module might have gone in the meantime. */
2692	old = find_module_all(name, strlen(name), true);
2693	}
2694
2695	if (try_add_failed_module(name, reason))
2696	pr_warn("Could not add fail-tracking for module: %s\n", name);
2697
2698	/*
2699	* We are here only when the same module was being loaded. Do
2700	* not try to load it again right now. It prevents long delays
2701	* caused by serialized module load failures. It might happen
2702	* when more devices of the same type trigger load of
2703	* a particular module.
2704	*/
2705	if (old && old->state == MODULE_STATE_LIVE)
2706	return -EEXIST;
2707	return -EBUSY;
2708	}
2709
2710	/*
2711	* We try to place it in the list now to make sure it's unique before
2712	* we dedicate too many resources. In particular, temporary percpu
2713	* memory exhaustion.
2714	*/
2715	static int add_unformed_module(struct module *mod)
2716	{
2717	int err;
2718
2719	mod->state = MODULE_STATE_UNFORMED;
2720
2721	mutex_lock(&module_mutex);
2722	err = module_patient_check_exists(mod->name, FAIL_DUP_MOD_LOAD);
2723	if (err)
2724	goto out;
2725
2726	mod_update_bounds(mod);
2727	list_add_rcu(&mod->list, &modules);
2728	mod_tree_insert(mod);
2729	err = 0;
2730
2731	out:
2732	mutex_unlock(&module_mutex);
2733	return err;
2734	}
2735
2736	static int complete_formation(struct module mod, struct load_info info)
2737	{
2738	int err;
2739
2740	mutex_lock(&module_mutex);
2741
2742	/* Find duplicate symbols (must be called under lock). */
2743	err = verify_exported_symbols(mod);
2744	if (err < 0)
2745	goto out;
2746
2747	/* These rely on module_mutex for list integrity. */
2748	module_bug_finalize(info->hdr, info->sechdrs, mod);
2749	module_cfi_finalize(info->hdr, info->sechdrs, mod);
2750
2751	err = module_enable_rodata_ro(mod, false);
2752	if (err)
2753	goto out_strict_rwx;
2754	err = module_enable_data_nx(mod);
2755	if (err)
2756	goto out_strict_rwx;
2757	err = module_enable_text_rox(mod);
2758	if (err)
2759	goto out_strict_rwx;
2760
2761	/*
2762	* Mark state as coming so strong_try_module_get() ignores us,
2763	* but kallsyms etc. can see us.
2764	*/
2765	mod->state = MODULE_STATE_COMING;
2766	mutex_unlock(&module_mutex);
2767
2768	return 0;
2769
2770	out_strict_rwx:
2771	module_bug_cleanup(mod);
2772	out:
2773	mutex_unlock(&module_mutex);
2774	return err;
2775	}
2776
2777	static int prepare_coming_module(struct module *mod)
2778	{
2779	int err;
2780
2781	ftrace_module_enable(mod);
2782	err = klp_module_coming(mod);
2783	if (err)
2784	return err;
2785
2786	err = blocking_notifier_call_chain_robust(&module_notify_list,
2787	MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
2788	err = notifier_to_errno(err);
2789	if (err)
2790	klp_module_going(mod);
2791
2792	return err;
2793	}
2794
2795	static int unknown_module_param_cb(char param, char val, const char *modname,
2796	void *arg)
2797	{
2798	struct module *mod = arg;
2799	int ret;
2800
2801	if (strcmp(param, "async_probe") == 0) {
2802	if (kstrtobool(val, &mod->async_probe_requested))
2803	mod->async_probe_requested = true;
2804	return 0;
2805	}
2806
2807	/* Check for magic 'dyndbg' arg */
2808	ret = ddebug_dyndbg_module_param_cb(param, val, modname);
2809	if (ret != 0)
2810	pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
2811	return 0;
2812	}
2813
2814	/* Module within temporary copy, this doesn't do any allocation */
2815	static int early_mod_check(struct load_info *info, int flags)
2816	{
2817	int err;
2818
2819	/*
2820	* Now that we know we have the correct module name, check
2821	* if it's blacklisted.
2822	*/
2823	if (blacklisted(info->name)) {
2824	pr_err("Module %s is blacklisted\n", info->name);
2825	return -EPERM;
2826	}
2827
2828	err = rewrite_section_headers(info, flags);
2829	if (err)
2830	return err;
2831
2832	/* Check module struct version now, before we try to use module. */
2833	if (!check_modstruct_version(info, info->mod))
2834	return -ENOEXEC;
2835
2836	err = check_modinfo(info->mod, info, flags);
2837	if (err)
2838	return err;
2839
2840	mutex_lock(&module_mutex);
2841	err = module_patient_check_exists(info->mod->name, FAIL_DUP_MOD_BECOMING);
2842	mutex_unlock(&module_mutex);
2843
2844	return err;
2845	}
2846
2847	/*
2848	* Allocate and load the module: note that size of section 0 is always
2849	* zero, and we rely on this for optional sections.
2850	*/
2851	static int load_module(struct load_info info, const char __user uargs,
2852	int flags)
2853	{
2854	struct module *mod;
2855	bool module_allocated = false;
2856	long err = 0;
2857	char *after_dashes;
2858
2859	/*
2860	* Do the signature check (if any) first. All that
2861	* the signature check needs is info->len, it does
2862	* not need any of the section info. That can be
2863	* set up later. This will minimize the chances
2864	* of a corrupt module causing problems before
2865	* we even get to the signature check.
2866	*
2867	* The check will also adjust info->len by stripping
2868	* off the sig length at the end of the module, making
2869	* checks against info->len more correct.
2870	*/
2871	err = module_sig_check(info, flags);
2872	if (err)
2873	goto free_copy;
2874
2875	/*
2876	* Do basic sanity checks against the ELF header and
2877	* sections. Cache useful sections and set the
2878	* info->mod to the userspace passed struct module.
2879	*/
2880	err = elf_validity_cache_copy(info, flags);
2881	if (err)
2882	goto free_copy;
2883
2884	err = early_mod_check(info, flags);
2885	if (err)
2886	goto free_copy;
2887
2888	/* Figure out module layout, and allocate all the memory. */
2889	mod = layout_and_allocate(info, flags);
2890	if (IS_ERR(mod)) {
2891	err = PTR_ERR(mod);
2892	goto free_copy;
2893	}
2894
2895	module_allocated = true;
2896
2897	audit_log_kern_module(mod->name);
2898
2899	/* Reserve our place in the list. */
2900	err = add_unformed_module(mod);
2901	if (err)
2902	goto free_module;
2903
2904	/*
2905	* We are tainting your kernel if your module gets into
2906	* the modules linked list somehow.
2907	*/
2908	module_augment_kernel_taints(mod, info);
2909
2910	/* To avoid stressing percpu allocator, do this once we're unique. */
2911	err = percpu_modalloc(mod, info);
2912	if (err)
2913	goto unlink_mod;
2914
2915	/* Now module is in final location, initialize linked lists, etc. */
2916	err = module_unload_init(mod);
2917	if (err)
2918	goto unlink_mod;
2919
2920	init_param_lock(mod);
2921
2922	/*
2923	* Now we've got everything in the final locations, we can
2924	* find optional sections.
2925	*/
2926	err = find_module_sections(mod, info);
2927	if (err)
2928	goto free_unload;
2929
2930	err = check_export_symbol_versions(mod);
2931	if (err)
2932	goto free_unload;
2933
2934	/* Set up MODINFO_ATTR fields */
2935	setup_modinfo(mod, info);
2936
2937	/* Fix up syms, so that st_value is a pointer to location. */
2938	err = simplify_symbols(mod, info);
2939	if (err < 0)
2940	goto free_modinfo;
2941
2942	err = apply_relocations(mod, info);
2943	if (err < 0)
2944	goto free_modinfo;
2945
2946	err = post_relocation(mod, info);
2947	if (err < 0)
2948	goto free_modinfo;
2949
2950	flush_module_icache(mod);
2951
2952	/* Now copy in args */
2953	mod->args = strndup_user(uargs, ~0UL >> 1);
2954	if (IS_ERR(mod->args)) {
2955	err = PTR_ERR(mod->args);
2956	goto free_arch_cleanup;
2957	}
2958
2959	init_build_id(mod, info);
2960
2961	/* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
2962	ftrace_module_init(mod);
2963
2964	/* Finally it's fully formed, ready to start executing. */
2965	err = complete_formation(mod, info);
2966	if (err)
2967	goto ddebug_cleanup;
2968
2969	err = prepare_coming_module(mod);
2970	if (err)
2971	goto bug_cleanup;
2972
2973	mod->async_probe_requested = async_probe;
2974
2975	/* Module is ready to execute: parsing args may do that. */
2976	after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
2977	-32768, 32767, mod,
2978	unknown_module_param_cb);
2979	if (IS_ERR(after_dashes)) {
2980	err = PTR_ERR(after_dashes);
2981	goto coming_cleanup;
2982	} else if (after_dashes) {
2983	pr_warn("%s: parameters '%s' after `--' ignored\n",
2984	mod->name, after_dashes);
2985	}
2986
2987	/* Link in to sysfs. */
2988	err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
2989	if (err < 0)
2990	goto coming_cleanup;
2991
2992	if (is_livepatch_module(mod)) {
2993	err = copy_module_elf(mod, info);
2994	if (err < 0)
2995	goto sysfs_cleanup;
2996	}
2997
2998	/* Get rid of temporary copy. */
2999	free_copy(info, flags);
3000
3001	codetag_load_module(mod);
3002
3003	/* Done! */
3004	trace_module_load(mod);
3005
3006	return do_init_module(mod);
3007
3008	sysfs_cleanup:
3009	mod_sysfs_teardown(mod);
3010	coming_cleanup:
3011	mod->state = MODULE_STATE_GOING;
3012	destroy_params(mod->kp, mod->num_kp);
3013	blocking_notifier_call_chain(&module_notify_list,
3014	MODULE_STATE_GOING, mod);
3015	klp_module_going(mod);
3016	bug_cleanup:
3017	mod->state = MODULE_STATE_GOING;
3018	/* module_bug_cleanup needs module_mutex protection */
3019	mutex_lock(&module_mutex);
3020	module_bug_cleanup(mod);
3021	mutex_unlock(&module_mutex);
3022
3023	ddebug_cleanup:
3024	ftrace_release_mod(mod);
3025	synchronize_rcu();
3026	kfree(mod->args);
3027	free_arch_cleanup:
3028	module_arch_cleanup(mod);
3029	free_modinfo:
3030	free_modinfo(mod);
3031	free_unload:
3032	module_unload_free(mod);
3033	unlink_mod:
3034	mutex_lock(&module_mutex);
3035	/* Unlink carefully: kallsyms could be walking list. */
3036	list_del_rcu(&mod->list);
3037	mod_tree_remove(mod);
3038	wake_up_all(&module_wq);
3039	/* Wait for RCU-sched synchronizing before releasing mod->list. */
3040	synchronize_rcu();
3041	mutex_unlock(&module_mutex);
3042	free_module:
3043	mod_stat_bump_invalid(info, flags);
3044	/* Free lock-classes; relies on the preceding sync_rcu() */
3045	for_class_mod_mem_type(type, core_data) {
3046	lockdep_free_key_range(mod->mem[type].base,
3047	mod->mem[type].size);
3048	}
3049
3050	module_deallocate(mod, info);
3051	free_copy:
3052	/*
3053	* The info->len is always set. We distinguish between
3054	* failures once the proper module was allocated and
3055	* before that.
3056	*/
3057	if (!module_allocated)
3058	mod_stat_bump_becoming(info, flags);
3059	free_copy(info, flags);
3060	return err;
3061	}
3062
3063	SYSCALL_DEFINE3(init_module, void __user *, umod,
3064	unsigned long, len, const char __user *, uargs)
3065	{
3066	int err;
3067	struct load_info info = { };
3068
3069	err = may_init_module();
3070	if (err)
3071	return err;
3072
3073	pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
3074	umod, len, uargs);
3075
3076	err = copy_module_from_user(umod, len, &info);
3077	if (err) {
3078	mod_stat_inc(&failed_kreads);
3079	mod_stat_add_long(len, &invalid_kread_bytes);
3080	return err;
3081	}
3082
3083	return load_module(&info, uargs, 0);
3084	}
3085
3086	struct idempotent {
3087	const void *cookie;
3088	struct hlist_node entry;
3089	struct completion complete;
3090	int ret;
3091	};
3092
3093	#define IDEM_HASH_BITS 8
3094	static struct hlist_head idem_hash[1 << IDEM_HASH_BITS];
3095	static DEFINE_SPINLOCK(idem_lock);
3096
3097	static bool idempotent(struct idempotent u, const void cookie)
3098	{
3099	int hash = hash_ptr(cookie, IDEM_HASH_BITS);
3100	struct hlist_head *head = idem_hash + hash;
3101	struct idempotent *existing;
3102	bool first;
3103
3104	u->ret = 0;
3105	u->cookie = cookie;
3106	init_completion(&u->complete);
3107
3108	spin_lock(&idem_lock);
3109	first = true;
3110	hlist_for_each_entry(existing, head, entry) {
3111	if (existing->cookie != cookie)
3112	continue;
3113	first = false;
3114	break;
3115	}
3116	hlist_add_head(&u->entry, idem_hash + hash);
3117	spin_unlock(&idem_lock);
3118
3119	return !first;
3120	}
3121
3122	/*
3123	* We were the first one with 'cookie' on the list, and we ended
3124	* up completing the operation. We now need to walk the list,
3125	* remove everybody - which includes ourselves - fill in the return
3126	* value, and then complete the operation.
3127	*/
3128	static int idempotent_complete(struct idempotent *u, int ret)
3129	{
3130	const void *cookie = u->cookie;
3131	int hash = hash_ptr(cookie, IDEM_HASH_BITS);
3132	struct hlist_head *head = idem_hash + hash;
3133	struct hlist_node *next;
3134	struct idempotent *pos;
3135
3136	spin_lock(&idem_lock);
3137	hlist_for_each_entry_safe(pos, next, head, entry) {
3138	if (pos->cookie != cookie)
3139	continue;
3140	hlist_del(&pos->entry);
3141	pos->ret = ret;
3142	complete(&pos->complete);
3143	}
3144	spin_unlock(&idem_lock);
3145	return ret;
3146	}
3147
3148	static int init_module_from_file(struct file f, const char __user uargs, int flags)
3149	{
3150	struct load_info info = { };
3151	void *buf = NULL;
3152	int len;
3153
3154	len = kernel_read_file(f, 0, &buf, INT_MAX, NULL, READING_MODULE);
3155	if (len < 0) {
3156	mod_stat_inc(&failed_kreads);
3157	return len;
3158	}
3159
3160	if (flags & MODULE_INIT_COMPRESSED_FILE) {
3161	int err = module_decompress(&info, buf, len);
3162	vfree(buf); /* compressed data is no longer needed */
3163	if (err) {
3164	mod_stat_inc(&failed_decompress);
3165	mod_stat_add_long(len, &invalid_decompress_bytes);
3166	return err;
3167	}
3168	} else {
3169	info.hdr = buf;
3170	info.len = len;
3171	}
3172
3173	return load_module(&info, uargs, flags);
3174	}
3175
3176	static int idempotent_init_module(struct file f, const char __user uargs, int flags)
3177	{
3178	struct idempotent idem;
3179
3180	if (!f \|\| !(f->f_mode & FMODE_READ))
3181	return -EBADF;
3182
3183	/* See if somebody else is doing the operation? */
3184	if (idempotent(&idem, file_inode(f))) {
3185	wait_for_completion(&idem.complete);
3186	return idem.ret;
3187	}
3188
3189	/* Otherwise, we'll do it and complete others */
3190	return idempotent_complete(&idem,
3191	init_module_from_file(f, uargs, flags));
3192	}
3193
3194	SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
3195	{
3196	int err;
3197	struct fd f;
3198
3199	err = may_init_module();
3200	if (err)
3201	return err;
3202
3203	pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
3204
3205	if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
3206	\|MODULE_INIT_IGNORE_VERMAGIC
3207	\|MODULE_INIT_COMPRESSED_FILE))
3208	return -EINVAL;
3209
3210	f = fdget(fd);
3211	err = idempotent_init_module(f.file, uargs, flags);
3212	fdput(f);
3213	return err;
3214	}
3215
3216	/* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
3217	char module_flags(struct module mod, char *buf, bool show_state)
3218	{
3219	int bx = 0;
3220
3221	BUG_ON(mod->state == MODULE_STATE_UNFORMED);
3222	if (!mod->taints && !show_state)
3223	goto out;
3224	if (mod->taints \|\|
3225	mod->state == MODULE_STATE_GOING \|\|
3226	mod->state == MODULE_STATE_COMING) {
3227	buf[bx++] = '(';
3228	bx += module_flags_taint(mod->taints, buf + bx);
3229	/* Show a - for module-is-being-unloaded */
3230	if (mod->state == MODULE_STATE_GOING && show_state)
3231	buf[bx++] = '-';
3232	/* Show a + for module-is-being-loaded */
3233	if (mod->state == MODULE_STATE_COMING && show_state)
3234	buf[bx++] = '+';
3235	buf[bx++] = ')';
3236	}
3237	out:
3238	buf[bx] = '\0';
3239
3240	return buf;
3241	}
3242
3243	/* Given an address, look for it in the module exception tables. */
3244	const struct exception_table_entry *search_module_extables(unsigned long addr)
3245	{
3246	const struct exception_table_entry *e = NULL;
3247	struct module *mod;
3248
3249	preempt_disable();
3250	mod = __module_address(addr);
3251	if (!mod)
3252	goto out;
3253
3254	if (!mod->num_exentries)
3255	goto out;
3256
3257	e = search_extable(mod->extable,
3258	mod->num_exentries,
3259	addr);
3260	out:
3261	preempt_enable();
3262
3263	/*
3264	* Now, if we found one, we are running inside it now, hence
3265	* we cannot unload the module, hence no refcnt needed.
3266	*/
3267	return e;
3268	}
3269
3270	/**
3271	* is_module_address() - is this address inside a module?
3272	* @addr: the address to check.
3273	*
3274	* See is_module_text_address() if you simply want to see if the address
3275	* is code (not data).
3276	*/
3277	bool is_module_address(unsigned long addr)
3278	{
3279	bool ret;
3280
3281	preempt_disable();
3282	ret = __module_address(addr) != NULL;
3283	preempt_enable();
3284
3285	return ret;
3286	}
3287
3288	/**
3289	* __module_address() - get the module which contains an address.
3290	* @addr: the address.
3291	*
3292	* Must be called with preempt disabled or module mutex held so that
3293	* module doesn't get freed during this.
3294	*/
3295	struct module *__module_address(unsigned long addr)
3296	{
3297	struct module *mod;
3298
3299	if (addr >= mod_tree.addr_min && addr <= mod_tree.addr_max)
3300	goto lookup;
3301
3302	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
3303	if (addr >= mod_tree.data_addr_min && addr <= mod_tree.data_addr_max)
3304	goto lookup;
3305	#endif
3306
3307	return NULL;
3308
3309	lookup:
3310	module_assert_mutex_or_preempt();
3311
3312	mod = mod_find(addr, &mod_tree);
3313	if (mod) {
3314	BUG_ON(!within_module(addr, mod));
3315	if (mod->state == MODULE_STATE_UNFORMED)
3316	mod = NULL;
3317	}
3318	return mod;
3319	}
3320
3321	/**
3322	* is_module_text_address() - is this address inside module code?
3323	* @addr: the address to check.
3324	*
3325	* See is_module_address() if you simply want to see if the address is
3326	* anywhere in a module. See kernel_text_address() for testing if an
3327	* address corresponds to kernel or module code.
3328	*/
3329	bool is_module_text_address(unsigned long addr)
3330	{
3331	bool ret;
3332
3333	preempt_disable();
3334	ret = __module_text_address(addr) != NULL;
3335	preempt_enable();
3336
3337	return ret;
3338	}
3339
3340	/**
3341	* __module_text_address() - get the module whose code contains an address.
3342	* @addr: the address.
3343	*
3344	* Must be called with preempt disabled or module mutex held so that
3345	* module doesn't get freed during this.
3346	*/
3347	struct module *__module_text_address(unsigned long addr)
3348	{
3349	struct module *mod = __module_address(addr);
3350	if (mod) {
3351	/* Make sure it's within the text section. */
3352	if (!within_module_mem_type(addr, mod, MOD_TEXT) &&
3353	!within_module_mem_type(addr, mod, MOD_INIT_TEXT))
3354	mod = NULL;
3355	}
3356	return mod;
3357	}
3358
3359	/* Don't grab lock, we're oopsing. */
3360	void print_modules(void)
3361	{
3362	struct module *mod;
3363	char buf[MODULE_FLAGS_BUF_SIZE];
3364
3365	printk(KERN_DEFAULT "Modules linked in:");
3366	/* Most callers should already have preempt disabled, but make sure */
3367	preempt_disable();
3368	list_for_each_entry_rcu(mod, &modules, list) {
3369	if (mod->state == MODULE_STATE_UNFORMED)
3370	continue;
3371	pr_cont(" %s%s", mod->name, module_flags(mod, buf, true));
3372	}
3373
3374	print_unloaded_tainted_modules();
3375	preempt_enable();
3376	if (last_unloaded_module.name[0])
3377	pr_cont(" [last unloaded: %s%s]", last_unloaded_module.name,
3378	last_unloaded_module.taints);
3379	pr_cont("\n");
3380	}
3381
3382	#ifdef CONFIG_MODULE_DEBUGFS
3383	struct dentry *mod_debugfs_root;
3384
3385	static int module_debugfs_init(void)
3386	{
3387	mod_debugfs_root = debugfs_create_dir("modules", NULL);
3388	return 0;
3389	}
3390	module_init(module_debugfs_init);
3391	#endif