linux - kernel/module/main.c

3889 lines · 99.1 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2002 Richard Henderson
4	* Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5	* Copyright (C) 2023 Luis Chamberlain <[email protected]>
6	*/
7
8	#define INCLUDE_VERMAGIC
9
10	#include <linux/export.h>
11	#include <linux/extable.h>
12	#include <linux/moduleloader.h>
13	#include <linux/module_signature.h>
14	#include <linux/trace_events.h>
15	#include <linux/init.h>
16	#include <linux/kallsyms.h>
17	#include <linux/buildid.h>
18	#include <linux/fs.h>
19	#include <linux/kernel.h>
20	#include <linux/kernel_read_file.h>
21	#include <linux/kstrtox.h>
22	#include <linux/slab.h>
23	#include <linux/vmalloc.h>
24	#include <linux/elf.h>
25	#include <linux/seq_file.h>
26	#include <linux/syscalls.h>
27	#include <linux/fcntl.h>
28	#include <linux/rcupdate.h>
29	#include <linux/capability.h>
30	#include <linux/cpu.h>
31	#include <linux/moduleparam.h>
32	#include <linux/errno.h>
33	#include <linux/err.h>
34	#include <linux/vermagic.h>
35	#include <linux/notifier.h>
36	#include <linux/sched.h>
37	#include <linux/device.h>
38	#include <linux/string.h>
39	#include <linux/mutex.h>
40	#include <linux/rculist.h>
41	#include <linux/uaccess.h>
42	#include <asm/cacheflush.h>
43	#include <linux/set_memory.h>
44	#include <asm/mmu_context.h>
45	#include <linux/license.h>
46	#include <asm/sections.h>
47	#include <linux/tracepoint.h>
48	#include <linux/ftrace.h>
49	#include <linux/livepatch.h>
50	#include <linux/async.h>
51	#include <linux/percpu.h>
52	#include <linux/kmemleak.h>
53	#include <linux/jump_label.h>
54	#include <linux/pfn.h>
55	#include <linux/bsearch.h>
56	#include <linux/dynamic_debug.h>
57	#include <linux/audit.h>
58	#include <linux/cfi.h>
59	#include <linux/codetag.h>
60	#include <linux/debugfs.h>
61	#include <linux/execmem.h>
62	#include <uapi/linux/module.h>
63	#include "internal.h"
64
65	#define CREATE_TRACE_POINTS
66	#include <trace/events/module.h>
67
68	/*
69	* Mutex protects:
70	* 1) List of modules (also safely readable within RCU read section),
71	* 2) module_use links,
72	* 3) mod_tree.addr_min/mod_tree.addr_max.
73	* (delete and add uses RCU list operations).
74	*/
75	DEFINE_MUTEX(module_mutex);
76	LIST_HEAD(modules);
77
78	/* Work queue for freeing init sections in success case */
79	static void do_free_init(struct work_struct *w);
80	static DECLARE_WORK(init_free_wq, do_free_init);
81	static LLIST_HEAD(init_free_list);
82
83	struct mod_tree_root mod_tree __cacheline_aligned = {
84	.addr_min = -1UL,
85	};
86
87	struct symsearch {
88	const struct kernel_symbol start, stop;
89	const u32 *crcs;
90	enum mod_license license;
91	};
92
93	/*
94	* Bounds of module memory, for speeding up __module_address.
95	* Protected by module_mutex.
96	*/
97	static void __mod_update_bounds(enum mod_mem_type type __maybe_unused, void *base,
98	unsigned int size, struct mod_tree_root *tree)
99	{
100	unsigned long min = (unsigned long)base;
101	unsigned long max = min + size;
102
103	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
104	if (mod_mem_type_is_core_data(type)) {
105	if (min < tree->data_addr_min)
106	tree->data_addr_min = min;
107	if (max > tree->data_addr_max)
108	tree->data_addr_max = max;
109	return;
110	}
111	#endif
112	if (min < tree->addr_min)
113	tree->addr_min = min;
114	if (max > tree->addr_max)
115	tree->addr_max = max;
116	}
117
118	static void mod_update_bounds(struct module *mod)
119	{
120	for_each_mod_mem_type(type) {
121	struct module_memory *mod_mem = &mod->mem[type];
122
123	if (mod_mem->size)
124	__mod_update_bounds(type, mod_mem->base, mod_mem->size, &mod_tree);
125	}
126	}
127
128	/* Block module loading/unloading? */
129	int modules_disabled;
130	core_param(nomodule, modules_disabled, bint, 0);
131
132	/* Waiting for a module to finish initializing? */
133	static DECLARE_WAIT_QUEUE_HEAD(module_wq);
134
135	static BLOCKING_NOTIFIER_HEAD(module_notify_list);
136
137	int register_module_notifier(struct notifier_block *nb)
138	{
139	return blocking_notifier_chain_register(&module_notify_list, nb);
140	}
141	EXPORT_SYMBOL(register_module_notifier);
142
143	int unregister_module_notifier(struct notifier_block *nb)
144	{
145	return blocking_notifier_chain_unregister(&module_notify_list, nb);
146	}
147	EXPORT_SYMBOL(unregister_module_notifier);
148
149	/*
150	* We require a truly strong try_module_get(): 0 means success.
151	* Otherwise an error is returned due to ongoing or failed
152	* initialization etc.
153	*/
154	static inline int strong_try_module_get(struct module *mod)
155	{
156	BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
157	if (mod && mod->state == MODULE_STATE_COMING)
158	return -EBUSY;
159	if (try_module_get(mod))
160	return 0;
161	else
162	return -ENOENT;
163	}
164
165	static inline void add_taint_module(struct module *mod, unsigned flag,
166	enum lockdep_ok lockdep_ok)
167	{
168	add_taint(flag, lockdep_ok);
169	set_bit(flag, &mod->taints);
170	}
171
172	/*
173	* Like strncmp(), except s/-/_/g as per scripts/Makefile.lib:name-fix-token rule.
174	*/
175	static int mod_strncmp(const char str_a, const char str_b, size_t n)
176	{
177	for (int i = 0; i < n; i++) {
178	char a = str_a[i];
179	char b = str_b[i];
180	int d;
181
182	if (a == '-') a = '_';
183	if (b == '-') b = '_';
184
185	d = a - b;
186	if (d)
187	return d;
188
189	if (!a)
190	break;
191	}
192
193	return 0;
194	}
195
196	/*
197	* A thread that wants to hold a reference to a module only while it
198	* is running can call this to safely exit.
199	*/
200	void __noreturn __module_put_and_kthread_exit(struct module *mod, long code)
201	{
202	module_put(mod);
203	kthread_exit(code);
204	}
205	EXPORT_SYMBOL(__module_put_and_kthread_exit);
206
207	/* Find a module section: 0 means not found. */
208	static unsigned int find_sec(const struct load_info info, const char name)
209	{
210	unsigned int i;
211
212	for (i = 1; i < info->hdr->e_shnum; i++) {
213	Elf_Shdr *shdr = &info->sechdrs[i];
214	/* Alloc bit cleared means "ignore it." */
215	if ((shdr->sh_flags & SHF_ALLOC)
216	&& strcmp(info->secstrings + shdr->sh_name, name) == 0)
217	return i;
218	}
219	return 0;
220	}
221
222	/**
223	* find_any_unique_sec() - Find a unique section index by name
224	* @info: Load info for the module to scan
225	* @name: Name of the section we're looking for
226	*
227	* Locates a unique section by name. Ignores SHF_ALLOC.
228	*
229	* Return: Section index if found uniquely, zero if absent, negative count
230	* of total instances if multiple were found.
231	*/
232	static int find_any_unique_sec(const struct load_info info, const char name)
233	{
234	unsigned int idx;
235	unsigned int count = 0;
236	int i;
237
238	for (i = 1; i < info->hdr->e_shnum; i++) {
239	if (strcmp(info->secstrings + info->sechdrs[i].sh_name,
240	name) == 0) {
241	count++;
242	idx = i;
243	}
244	}
245	if (count == 1) {
246	return idx;
247	} else if (count == 0) {
248	return 0;
249	} else {
250	return -count;
251	}
252	}
253
254	/* Find a module section, or NULL. */
255	static void section_addr(const struct load_info info, const char *name)
256	{
257	/* Section 0 has sh_addr 0. */
258	return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
259	}
260
261	/* Find a module section, or NULL. Fill in number of "objects" in section. */
262	static void section_objs(const struct load_info info,
263	const char *name,
264	size_t object_size,
265	unsigned int *num)
266	{
267	unsigned int sec = find_sec(info, name);
268
269	/* Section 0 has sh_addr 0 and sh_size 0. */
270	*num = info->sechdrs[sec].sh_size / object_size;
271	return (void *)info->sechdrs[sec].sh_addr;
272	}
273
274	/* Find a module section: 0 means not found. Ignores SHF_ALLOC flag. */
275	static unsigned int find_any_sec(const struct load_info info, const char name)
276	{
277	unsigned int i;
278
279	for (i = 1; i < info->hdr->e_shnum; i++) {
280	Elf_Shdr *shdr = &info->sechdrs[i];
281	if (strcmp(info->secstrings + shdr->sh_name, name) == 0)
282	return i;
283	}
284	return 0;
285	}
286
287	/*
288	* Find a module section, or NULL. Fill in number of "objects" in section.
289	* Ignores SHF_ALLOC flag.
290	*/
291	static __maybe_unused void any_section_objs(const struct load_info info,
292	const char *name,
293	size_t object_size,
294	unsigned int *num)
295	{
296	unsigned int sec = find_any_sec(info, name);
297
298	/* Section 0 has sh_addr 0 and sh_size 0. */
299	*num = info->sechdrs[sec].sh_size / object_size;
300	return (void *)info->sechdrs[sec].sh_addr;
301	}
302
303	#ifndef CONFIG_MODVERSIONS
304	#define symversion(base, idx) NULL
305	#else
306	#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
307	#endif
308
309	static const char kernel_symbol_name(const struct kernel_symbol sym)
310	{
311	#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
312	return offset_to_ptr(&sym->name_offset);
313	#else
314	return sym->name;
315	#endif
316	}
317
318	static const char kernel_symbol_namespace(const struct kernel_symbol sym)
319	{
320	#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
321	if (!sym->namespace_offset)
322	return NULL;
323	return offset_to_ptr(&sym->namespace_offset);
324	#else
325	return sym->namespace;
326	#endif
327	}
328
329	int cmp_name(const void name, const void sym)
330	{
331	return strcmp(name, kernel_symbol_name(sym));
332	}
333
334	static bool find_exported_symbol_in_section(const struct symsearch *syms,
335	struct module *owner,
336	struct find_symbol_arg *fsa)
337	{
338	struct kernel_symbol *sym;
339
340	if (!fsa->gplok && syms->license == GPL_ONLY)
341	return false;
342
343	sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
344	sizeof(struct kernel_symbol), cmp_name);
345	if (!sym)
346	return false;
347
348	fsa->owner = owner;
349	fsa->crc = symversion(syms->crcs, sym - syms->start);
350	fsa->sym = sym;
351	fsa->license = syms->license;
352
353	return true;
354	}
355
356	/*
357	* Find an exported symbol and return it, along with, (optional) crc and
358	* (optional) module which owns it. Needs RCU or module_mutex.
359	*/
360	bool find_symbol(struct find_symbol_arg *fsa)
361	{
362	static const struct symsearch arr[] = {
363	{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,
364	NOT_GPL_ONLY },
365	{ __start___ksymtab_gpl, __stop___ksymtab_gpl,
366	__start___kcrctab_gpl,
367	GPL_ONLY },
368	};
369	struct module *mod;
370	unsigned int i;
371
372	for (i = 0; i < ARRAY_SIZE(arr); i++)
373	if (find_exported_symbol_in_section(&arr[i], NULL, fsa))
374	return true;
375
376	list_for_each_entry_rcu(mod, &modules, list,
377	lockdep_is_held(&module_mutex)) {
378	struct symsearch arr[] = {
379	{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
380	NOT_GPL_ONLY },
381	{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
382	mod->gpl_crcs,
383	GPL_ONLY },
384	};
385
386	if (mod->state == MODULE_STATE_UNFORMED)
387	continue;
388
389	for (i = 0; i < ARRAY_SIZE(arr); i++)
390	if (find_exported_symbol_in_section(&arr[i], mod, fsa))
391	return true;
392	}
393
394	pr_debug("Failed to find symbol %s\n", fsa->name);
395	return false;
396	}
397
398	/*
399	* Search for module by name: must hold module_mutex (or RCU for read-only
400	* access).
401	*/
402	struct module find_module_all(const char name, size_t len,
403	bool even_unformed)
404	{
405	struct module *mod;
406
407	list_for_each_entry_rcu(mod, &modules, list,
408	lockdep_is_held(&module_mutex)) {
409	if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
410	continue;
411	if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
412	return mod;
413	}
414	return NULL;
415	}
416
417	struct module find_module(const char name)
418	{
419	return find_module_all(name, strlen(name), false);
420	}
421
422	#ifdef CONFIG_SMP
423
424	static inline void __percpu mod_percpu(struct module mod)
425	{
426	return mod->percpu;
427	}
428
429	static int percpu_modalloc(struct module mod, struct load_info info)
430	{
431	Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
432	unsigned long align = pcpusec->sh_addralign;
433
434	if (!pcpusec->sh_size)
435	return 0;
436
437	if (align > PAGE_SIZE) {
438	pr_warn("%s: per-cpu alignment %li > %li\n",
439	mod->name, align, PAGE_SIZE);
440	align = PAGE_SIZE;
441	}
442
443	mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
444	if (!mod->percpu) {
445	pr_warn("%s: Could not allocate %lu bytes percpu data\n",
446	mod->name, (unsigned long)pcpusec->sh_size);
447	return -ENOMEM;
448	}
449	mod->percpu_size = pcpusec->sh_size;
450	return 0;
451	}
452
453	static void percpu_modfree(struct module *mod)
454	{
455	free_percpu(mod->percpu);
456	}
457
458	static unsigned int find_pcpusec(struct load_info *info)
459	{
460	return find_sec(info, ".data..percpu");
461	}
462
463	static void percpu_modcopy(struct module *mod,
464	const void *from, unsigned long size)
465	{
466	int cpu;
467
468	for_each_possible_cpu(cpu)
469	memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
470	}
471
472	bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
473	{
474	struct module *mod;
475	unsigned int cpu;
476
477	guard(rcu)();
478	list_for_each_entry_rcu(mod, &modules, list) {
479	if (mod->state == MODULE_STATE_UNFORMED)
480	continue;
481	if (!mod->percpu_size)
482	continue;
483	for_each_possible_cpu(cpu) {
484	void *start = per_cpu_ptr(mod->percpu, cpu);
485	void va = (void )addr;
486
487	if (va >= start && va < start + mod->percpu_size) {
488	if (can_addr) {
489	*can_addr = (unsigned long) (va - start);
490	*can_addr += (unsigned long)
491	per_cpu_ptr(mod->percpu,
492	get_boot_cpu_id());
493	}
494	return true;
495	}
496	}
497	}
498	return false;
499	}
500
501	/**
502	* is_module_percpu_address() - test whether address is from module static percpu
503	* @addr: address to test
504	*
505	* Test whether @addr belongs to module static percpu area.
506	*
507	* Return: %true if @addr is from module static percpu area
508	*/
509	bool is_module_percpu_address(unsigned long addr)
510	{
511	return __is_module_percpu_address(addr, NULL);
512	}
513
514	#else /* ... !CONFIG_SMP */
515
516	static inline void __percpu mod_percpu(struct module mod)
517	{
518	return NULL;
519	}
520	static int percpu_modalloc(struct module mod, struct load_info info)
521	{
522	/* UP modules shouldn't have this section: ENOMEM isn't quite right */
523	if (info->sechdrs[info->index.pcpu].sh_size != 0)
524	return -ENOMEM;
525	return 0;
526	}
527	static inline void percpu_modfree(struct module *mod)
528	{
529	}
530	static unsigned int find_pcpusec(struct load_info *info)
531	{
532	return 0;
533	}
534	static inline void percpu_modcopy(struct module *mod,
535	const void *from, unsigned long size)
536	{
537	/* pcpusec should be 0, and size of that section should be 0. */
538	BUG_ON(size != 0);
539	}
540	bool is_module_percpu_address(unsigned long addr)
541	{
542	return false;
543	}
544
545	bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
546	{
547	return false;
548	}
549
550	#endif /* CONFIG_SMP */
551
552	#define MODINFO_ATTR(field) \
553	static void setup_modinfo_##field(struct module mod, const char s) \
554	{ \
555	mod->field = kstrdup(s, GFP_KERNEL); \
556	} \
557	static ssize_t show_modinfo_##field(const struct module_attribute *mattr, \
558	struct module_kobject mk, char buffer) \
559	{ \
560	return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
561	} \
562	static int modinfo_##field##_exists(struct module *mod) \
563	{ \
564	return mod->field != NULL; \
565	} \
566	static void free_modinfo_##field(struct module *mod) \
567	{ \
568	kfree(mod->field); \
569	mod->field = NULL; \
570	} \
571	static const struct module_attribute modinfo_##field = { \
572	.attr = { .name = __stringify(field), .mode = 0444 }, \
573	.show = show_modinfo_##field, \
574	.setup = setup_modinfo_##field, \
575	.test = modinfo_##field##_exists, \
576	.free = free_modinfo_##field, \
577	};
578
579	MODINFO_ATTR(version);
580	MODINFO_ATTR(srcversion);
581
582	static struct {
583	char name[MODULE_NAME_LEN + 1];
584	char taints[MODULE_FLAGS_BUF_SIZE];
585	} last_unloaded_module;
586
587	#ifdef CONFIG_MODULE_UNLOAD
588
589	EXPORT_TRACEPOINT_SYMBOL(module_get);
590
591	/* MODULE_REF_BASE is the base reference count by kmodule loader. */
592	#define MODULE_REF_BASE 1
593
594	/* Init the unload section of the module. */
595	static int module_unload_init(struct module *mod)
596	{
597	/*
598	* Initialize reference counter to MODULE_REF_BASE.
599	* refcnt == 0 means module is going.
600	*/
601	atomic_set(&mod->refcnt, MODULE_REF_BASE);
602
603	INIT_LIST_HEAD(&mod->source_list);
604	INIT_LIST_HEAD(&mod->target_list);
605
606	/* Hold reference count during initialization. */
607	atomic_inc(&mod->refcnt);
608
609	return 0;
610	}
611
612	/* Does a already use b? */
613	static int already_uses(struct module a, struct module b)
614	{
615	struct module_use *use;
616
617	list_for_each_entry(use, &b->source_list, source_list) {
618	if (use->source == a)
619	return 1;
620	}
621	pr_debug("%s does not use %s!\n", a->name, b->name);
622	return 0;
623	}
624
625	/*
626	* Module a uses b
627	* - we add 'a' as a "source", 'b' as a "target" of module use
628	* - the module_use is added to the list of 'b' sources (so
629	* 'b' can walk the list to see who sourced them), and of 'a'
630	* targets (so 'a' can see what modules it targets).
631	*/
632	static int add_module_usage(struct module a, struct module b)
633	{
634	struct module_use *use;
635
636	pr_debug("Allocating new usage for %s.\n", a->name);
637	use = kmalloc(sizeof(*use), GFP_ATOMIC);
638	if (!use)
639	return -ENOMEM;
640
641	use->source = a;
642	use->target = b;
643	list_add(&use->source_list, &b->source_list);
644	list_add(&use->target_list, &a->target_list);
645	return 0;
646	}
647
648	/* Module a uses b: caller needs module_mutex() */
649	static int ref_module(struct module a, struct module b)
650	{
651	int err;
652
653	if (b == NULL \|\| already_uses(a, b))
654	return 0;
655
656	/* If module isn't available, we fail. */
657	err = strong_try_module_get(b);
658	if (err)
659	return err;
660
661	err = add_module_usage(a, b);
662	if (err) {
663	module_put(b);
664	return err;
665	}
666	return 0;
667	}
668
669	/* Clear the unload stuff of the module. */
670	static void module_unload_free(struct module *mod)
671	{
672	struct module_use use, tmp;
673
674	mutex_lock(&module_mutex);
675	list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
676	struct module *i = use->target;
677	pr_debug("%s unusing %s\n", mod->name, i->name);
678	module_put(i);
679	list_del(&use->source_list);
680	list_del(&use->target_list);
681	kfree(use);
682	}
683	mutex_unlock(&module_mutex);
684	}
685
686	#ifdef CONFIG_MODULE_FORCE_UNLOAD
687	static inline int try_force_unload(unsigned int flags)
688	{
689	int ret = (flags & O_TRUNC);
690	if (ret)
691	add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
692	return ret;
693	}
694	#else
695	static inline int try_force_unload(unsigned int flags)
696	{
697	return 0;
698	}
699	#endif /* CONFIG_MODULE_FORCE_UNLOAD */
700
701	/* Try to release refcount of module, 0 means success. */
702	static int try_release_module_ref(struct module *mod)
703	{
704	int ret;
705
706	/* Try to decrement refcnt which we set at loading */
707	ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
708	BUG_ON(ret < 0);
709	if (ret)
710	/* Someone can put this right now, recover with checking */
711	ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
712
713	return ret;
714	}
715
716	static int try_stop_module(struct module mod, int flags, int forced)
717	{
718	/* If it's not unused, quit unless we're forcing. */
719	if (try_release_module_ref(mod) != 0) {
720	*forced = try_force_unload(flags);
721	if (!(*forced))
722	return -EWOULDBLOCK;
723	}
724
725	/* Mark it as dying. */
726	mod->state = MODULE_STATE_GOING;
727
728	return 0;
729	}
730
731	/**
732	* module_refcount() - return the refcount or -1 if unloading
733	* @mod: the module we're checking
734	*
735	* Return:
736	* -1 if the module is in the process of unloading
737	* otherwise the number of references in the kernel to the module
738	*/
739	int module_refcount(struct module *mod)
740	{
741	return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
742	}
743	EXPORT_SYMBOL(module_refcount);
744
745	/* This exists whether we can unload or not */
746	static void free_module(struct module *mod);
747
748	SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
749	unsigned int, flags)
750	{
751	struct module *mod;
752	char name[MODULE_NAME_LEN];
753	char buf[MODULE_FLAGS_BUF_SIZE];
754	int ret, forced = 0;
755
756	if (!capable(CAP_SYS_MODULE) \|\| modules_disabled)
757	return -EPERM;
758
759	if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
760	return -EFAULT;
761	name[MODULE_NAME_LEN-1] = '\0';
762
763	audit_log_kern_module(name);
764
765	if (mutex_lock_interruptible(&module_mutex) != 0)
766	return -EINTR;
767
768	mod = find_module(name);
769	if (!mod) {
770	ret = -ENOENT;
771	goto out;
772	}
773
774	if (!list_empty(&mod->source_list)) {
775	/* Other modules depend on us: get rid of them first. */
776	ret = -EWOULDBLOCK;
777	goto out;
778	}
779
780	/* Doing init or already dying? */
781	if (mod->state != MODULE_STATE_LIVE) {
782	/* FIXME: if (force), slam module count damn the torpedoes */
783	pr_debug("%s already dying\n", mod->name);
784	ret = -EBUSY;
785	goto out;
786	}
787
788	/* If it has an init func, it must have an exit func to unload */
789	if (mod->init && !mod->exit) {
790	forced = try_force_unload(flags);
791	if (!forced) {
792	/* This module can't be removed */
793	ret = -EBUSY;
794	goto out;
795	}
796	}
797
798	ret = try_stop_module(mod, flags, &forced);
799	if (ret != 0)
800	goto out;
801
802	mutex_unlock(&module_mutex);
803	/* Final destruction now no one is using it. */
804	if (mod->exit != NULL)
805	mod->exit();
806	blocking_notifier_call_chain(&module_notify_list,
807	MODULE_STATE_GOING, mod);
808	klp_module_going(mod);
809	ftrace_release_mod(mod);
810
811	async_synchronize_full();
812
813	/* Store the name and taints of the last unloaded module for diagnostic purposes */
814	strscpy(last_unloaded_module.name, mod->name);
815	strscpy(last_unloaded_module.taints, module_flags(mod, buf, false));
816
817	free_module(mod);
818	/* someone could wait for the module in add_unformed_module() */
819	wake_up_all(&module_wq);
820	return 0;
821	out:
822	mutex_unlock(&module_mutex);
823	return ret;
824	}
825
826	void __symbol_put(const char *symbol)
827	{
828	struct find_symbol_arg fsa = {
829	.name = symbol,
830	.gplok = true,
831	};
832
833	guard(rcu)();
834	BUG_ON(!find_symbol(&fsa));
835	module_put(fsa.owner);
836	}
837	EXPORT_SYMBOL(__symbol_put);
838
839	/* Note this assumes addr is a function, which it currently always is. */
840	void symbol_put_addr(void *addr)
841	{
842	struct module *modaddr;
843	unsigned long a = (unsigned long)dereference_function_descriptor(addr);
844
845	if (core_kernel_text(a))
846	return;
847
848	/*
849	* Even though we hold a reference on the module; we still need to
850	* RCU read section in order to safely traverse the data structure.
851	*/
852	guard(rcu)();
853	modaddr = __module_text_address(a);
854	BUG_ON(!modaddr);
855	module_put(modaddr);
856	}
857	EXPORT_SYMBOL_GPL(symbol_put_addr);
858
859	static ssize_t show_refcnt(const struct module_attribute *mattr,
860	struct module_kobject mk, char buffer)
861	{
862	return sprintf(buffer, "%i\n", module_refcount(mk->mod));
863	}
864
865	static const struct module_attribute modinfo_refcnt =
866	__ATTR(refcnt, 0444, show_refcnt, NULL);
867
868	void __module_get(struct module *module)
869	{
870	if (module) {
871	atomic_inc(&module->refcnt);
872	trace_module_get(module, _RET_IP_);
873	}
874	}
875	EXPORT_SYMBOL(__module_get);
876
877	bool try_module_get(struct module *module)
878	{
879	bool ret = true;
880
881	if (module) {
882	/* Note: here, we can fail to get a reference */
883	if (likely(module_is_live(module) &&
884	atomic_inc_not_zero(&module->refcnt) != 0))
885	trace_module_get(module, _RET_IP_);
886	else
887	ret = false;
888	}
889	return ret;
890	}
891	EXPORT_SYMBOL(try_module_get);
892
893	void module_put(struct module *module)
894	{
895	int ret;
896
897	if (module) {
898	ret = atomic_dec_if_positive(&module->refcnt);
899	WARN_ON(ret < 0); /* Failed to put refcount */
900	trace_module_put(module, _RET_IP_);
901	}
902	}
903	EXPORT_SYMBOL(module_put);
904
905	#else /* !CONFIG_MODULE_UNLOAD */
906	static inline void module_unload_free(struct module *mod)
907	{
908	}
909
910	static int ref_module(struct module a, struct module b)
911	{
912	return strong_try_module_get(b);
913	}
914
915	static inline int module_unload_init(struct module *mod)
916	{
917	return 0;
918	}
919	#endif /* CONFIG_MODULE_UNLOAD */
920
921	size_t module_flags_taint(unsigned long taints, char *buf)
922	{
923	size_t l = 0;
924	int i;
925
926	for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
927	if (taint_flags[i].module && test_bit(i, &taints))
928	buf[l++] = taint_flags[i].c_true;
929	}
930
931	return l;
932	}
933
934	static ssize_t show_initstate(const struct module_attribute *mattr,
935	struct module_kobject mk, char buffer)
936	{
937	const char *state = "unknown";
938
939	switch (mk->mod->state) {
940	case MODULE_STATE_LIVE:
941	state = "live";
942	break;
943	case MODULE_STATE_COMING:
944	state = "coming";
945	break;
946	case MODULE_STATE_GOING:
947	state = "going";
948	break;
949	default:
950	BUG();
951	}
952	return sprintf(buffer, "%s\n", state);
953	}
954
955	static const struct module_attribute modinfo_initstate =
956	__ATTR(initstate, 0444, show_initstate, NULL);
957
958	static ssize_t store_uevent(const struct module_attribute *mattr,
959	struct module_kobject *mk,
960	const char *buffer, size_t count)
961	{
962	int rc;
963
964	rc = kobject_synth_uevent(&mk->kobj, buffer, count);
965	return rc ? rc : count;
966	}
967
968	const struct module_attribute module_uevent =
969	__ATTR(uevent, 0200, NULL, store_uevent);
970
971	static ssize_t show_coresize(const struct module_attribute *mattr,
972	struct module_kobject mk, char buffer)
973	{
974	unsigned int size = mk->mod->mem[MOD_TEXT].size;
975
976	if (!IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC)) {
977	for_class_mod_mem_type(type, core_data)
978	size += mk->mod->mem[type].size;
979	}
980	return sprintf(buffer, "%u\n", size);
981	}
982
983	static const struct module_attribute modinfo_coresize =
984	__ATTR(coresize, 0444, show_coresize, NULL);
985
986	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
987	static ssize_t show_datasize(const struct module_attribute *mattr,
988	struct module_kobject mk, char buffer)
989	{
990	unsigned int size = 0;
991
992	for_class_mod_mem_type(type, core_data)
993	size += mk->mod->mem[type].size;
994	return sprintf(buffer, "%u\n", size);
995	}
996
997	static const struct module_attribute modinfo_datasize =
998	__ATTR(datasize, 0444, show_datasize, NULL);
999	#endif
1000
1001	static ssize_t show_initsize(const struct module_attribute *mattr,
1002	struct module_kobject mk, char buffer)
1003	{
1004	unsigned int size = 0;
1005
1006	for_class_mod_mem_type(type, init)
1007	size += mk->mod->mem[type].size;
1008	return sprintf(buffer, "%u\n", size);
1009	}
1010
1011	static const struct module_attribute modinfo_initsize =
1012	__ATTR(initsize, 0444, show_initsize, NULL);
1013
1014	static ssize_t show_taint(const struct module_attribute *mattr,
1015	struct module_kobject mk, char buffer)
1016	{
1017	size_t l;
1018
1019	l = module_flags_taint(mk->mod->taints, buffer);
1020	buffer[l++] = '\n';
1021	return l;
1022	}
1023
1024	static const struct module_attribute modinfo_taint =
1025	__ATTR(taint, 0444, show_taint, NULL);
1026
1027	const struct module_attribute *const modinfo_attrs[] = {
1028	&module_uevent,
1029	&modinfo_version,
1030	&modinfo_srcversion,
1031	&modinfo_initstate,
1032	&modinfo_coresize,
1033	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
1034	&modinfo_datasize,
1035	#endif
1036	&modinfo_initsize,
1037	&modinfo_taint,
1038	#ifdef CONFIG_MODULE_UNLOAD
1039	&modinfo_refcnt,
1040	#endif
1041	NULL,
1042	};
1043
1044	const size_t modinfo_attrs_count = ARRAY_SIZE(modinfo_attrs);
1045
1046	static const char vermagic[] = VERMAGIC_STRING;
1047
1048	int try_to_force_load(struct module mod, const char reason)
1049	{
1050	#ifdef CONFIG_MODULE_FORCE_LOAD
1051	if (!test_taint(TAINT_FORCED_MODULE))
1052	pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
1053	add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
1054	return 0;
1055	#else
1056	return -ENOEXEC;
1057	#endif
1058	}
1059
1060	/* Parse tag=value strings from .modinfo section */
1061	char module_next_tag_pair(char string, unsigned long *secsize)
1062	{
1063	/* Skip non-zero chars */
1064	while (string[0]) {
1065	string++;
1066	if ((*secsize)-- <= 1)
1067	return NULL;
1068	}
1069
1070	/* Skip any zero padding. */
1071	while (!string[0]) {
1072	string++;
1073	if ((*secsize)-- <= 1)
1074	return NULL;
1075	}
1076	return string;
1077	}
1078
1079	static char get_next_modinfo(const struct load_info info, const char *tag,
1080	char *prev)
1081	{
1082	char *p;
1083	unsigned int taglen = strlen(tag);
1084	Elf_Shdr *infosec = &info->sechdrs[info->index.info];
1085	unsigned long size = infosec->sh_size;
1086
1087	/*
1088	* get_modinfo() calls made before rewrite_section_headers()
1089	* must use sh_offset, as sh_addr isn't set!
1090	*/
1091	char modinfo = (char )info->hdr + infosec->sh_offset;
1092
1093	if (prev) {
1094	size -= prev - modinfo;
1095	modinfo = module_next_tag_pair(prev, &size);
1096	}
1097
1098	for (p = modinfo; p; p = module_next_tag_pair(p, &size)) {
1099	if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
1100	return p + taglen + 1;
1101	}
1102	return NULL;
1103	}
1104
1105	static char get_modinfo(const struct load_info info, const char *tag)
1106	{
1107	return get_next_modinfo(info, tag, NULL);
1108	}
1109
1110	/**
1111	* verify_module_namespace() - does @modname have access to this symbol's @namespace
1112	* @namespace: export symbol namespace
1113	* @modname: module name
1114	*
1115	* If @namespace is prefixed with "module:" to indicate it is a module namespace
1116	* then test if @modname matches any of the comma separated patterns.
1117	*
1118	* The patterns only support tail-glob.
1119	*/
1120	static bool verify_module_namespace(const char namespace, const char modname)
1121	{
1122	size_t len, modlen = strlen(modname);
1123	const char *prefix = "module:";
1124	const char *sep;
1125	bool glob;
1126
1127	if (!strstarts(namespace, prefix))
1128	return false;
1129
1130	for (namespace += strlen(prefix); *namespace; namespace = sep) {
1131	sep = strchrnul(namespace, ',');
1132	len = sep - namespace;
1133
1134	glob = false;
1135	if (sep[-1] == '*') {
1136	len--;
1137	glob = true;
1138	}
1139
1140	if (*sep)
1141	sep++;
1142
1143	if (mod_strncmp(namespace, modname, len) == 0 && (glob \|\| len == modlen))
1144	return true;
1145	}
1146
1147	return false;
1148	}
1149
1150	static int verify_namespace_is_imported(const struct load_info *info,
1151	const struct kernel_symbol *sym,
1152	struct module *mod)
1153	{
1154	const char *namespace;
1155	char *imported_namespace;
1156
1157	namespace = kernel_symbol_namespace(sym);
1158	if (namespace && namespace[0]) {
1159
1160	if (verify_module_namespace(namespace, mod->name))
1161	return 0;
1162
1163	for_each_modinfo_entry(imported_namespace, info, "import_ns") {
1164	if (strcmp(namespace, imported_namespace) == 0)
1165	return 0;
1166	}
1167	#ifdef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
1168	pr_warn(
1169	#else
1170	pr_err(
1171	#endif
1172	"%s: module uses symbol (%s) from namespace %s, but does not import it.\n",
1173	mod->name, kernel_symbol_name(sym), namespace);
1174	#ifndef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
1175	return -EINVAL;
1176	#endif
1177	}
1178	return 0;
1179	}
1180
1181	static bool inherit_taint(struct module mod, struct module owner, const char *name)
1182	{
1183	if (!owner \|\| !test_bit(TAINT_PROPRIETARY_MODULE, &owner->taints))
1184	return true;
1185
1186	if (mod->using_gplonly_symbols) {
1187	pr_err("%s: module using GPL-only symbols uses symbols %s from proprietary module %s.\n",
1188	mod->name, name, owner->name);
1189	return false;
1190	}
1191
1192	if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) {
1193	pr_warn("%s: module uses symbols %s from proprietary module %s, inheriting taint.\n",
1194	mod->name, name, owner->name);
1195	set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints);
1196	}
1197	return true;
1198	}
1199
1200	/* Resolve a symbol for this module. I.e. if we find one, record usage. */
1201	static const struct kernel_symbol resolve_symbol(struct module mod,
1202	const struct load_info *info,
1203	const char *name,
1204	char ownername[])
1205	{
1206	struct find_symbol_arg fsa = {
1207	.name = name,
1208	.gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)),
1209	.warn = true,
1210	};
1211	int err;
1212
1213	/*
1214	* The module_mutex should not be a heavily contended lock;
1215	* if we get the occasional sleep here, we'll go an extra iteration
1216	* in the wait_event_interruptible(), which is harmless.
1217	*/
1218	sched_annotate_sleep();
1219	mutex_lock(&module_mutex);
1220	if (!find_symbol(&fsa))
1221	goto unlock;
1222
1223	if (fsa.license == GPL_ONLY)
1224	mod->using_gplonly_symbols = true;
1225
1226	if (!inherit_taint(mod, fsa.owner, name)) {
1227	fsa.sym = NULL;
1228	goto getname;
1229	}
1230
1231	if (!check_version(info, name, mod, fsa.crc)) {
1232	fsa.sym = ERR_PTR(-EINVAL);
1233	goto getname;
1234	}
1235
1236	err = verify_namespace_is_imported(info, fsa.sym, mod);
1237	if (err) {
1238	fsa.sym = ERR_PTR(err);
1239	goto getname;
1240	}
1241
1242	err = ref_module(mod, fsa.owner);
1243	if (err) {
1244	fsa.sym = ERR_PTR(err);
1245	goto getname;
1246	}
1247
1248	getname:
1249	/* We must make copy under the lock if we failed to get ref. */
1250	strscpy(ownername, module_name(fsa.owner), MODULE_NAME_LEN);
1251	unlock:
1252	mutex_unlock(&module_mutex);
1253	return fsa.sym;
1254	}
1255
1256	static const struct kernel_symbol *
1257	resolve_symbol_wait(struct module *mod,
1258	const struct load_info *info,
1259	const char *name)
1260	{
1261	const struct kernel_symbol *ksym;
1262	char owner[MODULE_NAME_LEN];
1263
1264	if (wait_event_interruptible_timeout(module_wq,
1265	!IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
1266	\|\| PTR_ERR(ksym) != -EBUSY,
1267	30 * HZ) <= 0) {
1268	pr_warn("%s: gave up waiting for init of module %s.\n",
1269	mod->name, owner);
1270	}
1271	return ksym;
1272	}
1273
1274	void __weak module_arch_cleanup(struct module *mod)
1275	{
1276	}
1277
1278	void __weak module_arch_freeing_init(struct module *mod)
1279	{
1280	}
1281
1282	static int module_memory_alloc(struct module *mod, enum mod_mem_type type)
1283	{
1284	unsigned int size = PAGE_ALIGN(mod->mem[type].size);
1285	enum execmem_type execmem_type;
1286	void *ptr;
1287
1288	mod->mem[type].size = size;
1289
1290	if (mod_mem_type_is_data(type))
1291	execmem_type = EXECMEM_MODULE_DATA;
1292	else
1293	execmem_type = EXECMEM_MODULE_TEXT;
1294
1295	ptr = execmem_alloc(execmem_type, size);
1296	if (!ptr)
1297	return -ENOMEM;
1298
1299	if (execmem_is_rox(execmem_type)) {
1300	int err = execmem_make_temp_rw(ptr, size);
1301
1302	if (err) {
1303	execmem_free(ptr);
1304	return -ENOMEM;
1305	}
1306
1307	mod->mem[type].is_rox = true;
1308	}
1309
1310	/*
1311	* The pointer to these blocks of memory are stored on the module
1312	* structure and we keep that around so long as the module is
1313	* around. We only free that memory when we unload the module.
1314	* Just mark them as not being a leak then. The .init* ELF
1315	* sections do get freed after boot so we could treat them
1316	* slightly differently with kmemleak_ignore() and only grey
1317	* them out as they work as typical memory allocations which
1318	* do eventually get freed, but let's just keep things simple
1319	* and avoid any false positives.
1320	*/
1321	if (!mod->mem[type].is_rox)
1322	kmemleak_not_leak(ptr);
1323
1324	memset(ptr, 0, size);
1325	mod->mem[type].base = ptr;
1326
1327	return 0;
1328	}
1329
1330	static void module_memory_restore_rox(struct module *mod)
1331	{
1332	for_class_mod_mem_type(type, text) {
1333	struct module_memory *mem = &mod->mem[type];
1334
1335	if (mem->is_rox)
1336	execmem_restore_rox(mem->base, mem->size);
1337	}
1338	}
1339
1340	static void module_memory_free(struct module *mod, enum mod_mem_type type)
1341	{
1342	struct module_memory *mem = &mod->mem[type];
1343
1344	execmem_free(mem->base);
1345	}
1346
1347	static void free_mod_mem(struct module *mod)
1348	{
1349	for_each_mod_mem_type(type) {
1350	struct module_memory *mod_mem = &mod->mem[type];
1351
1352	if (type == MOD_DATA)
1353	continue;
1354
1355	/* Free lock-classes; relies on the preceding sync_rcu(). */
1356	lockdep_free_key_range(mod_mem->base, mod_mem->size);
1357	if (mod_mem->size)
1358	module_memory_free(mod, type);
1359	}
1360
1361	/* MOD_DATA hosts mod, so free it at last */
1362	lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size);
1363	module_memory_free(mod, MOD_DATA);
1364	}
1365
1366	/* Free a module, remove from lists, etc. */
1367	static void free_module(struct module *mod)
1368	{
1369	trace_module_free(mod);
1370
1371	codetag_unload_module(mod);
1372
1373	mod_sysfs_teardown(mod);
1374
1375	/*
1376	* We leave it in list to prevent duplicate loads, but make sure
1377	* that noone uses it while it's being deconstructed.
1378	*/
1379	mutex_lock(&module_mutex);
1380	mod->state = MODULE_STATE_UNFORMED;
1381	mutex_unlock(&module_mutex);
1382
1383	/* Arch-specific cleanup. */
1384	module_arch_cleanup(mod);
1385
1386	/* Module unload stuff */
1387	module_unload_free(mod);
1388
1389	/* Free any allocated parameters. */
1390	destroy_params(mod->kp, mod->num_kp);
1391
1392	if (is_livepatch_module(mod))
1393	free_module_elf(mod);
1394
1395	/* Now we can delete it from the lists */
1396	mutex_lock(&module_mutex);
1397	/* Unlink carefully: kallsyms could be walking list. */
1398	list_del_rcu(&mod->list);
1399	mod_tree_remove(mod);
1400	/* Remove this module from bug list, this uses list_del_rcu */
1401	module_bug_cleanup(mod);
1402	/* Wait for RCU synchronizing before releasing mod->list and buglist. */
1403	synchronize_rcu();
1404	if (try_add_tainted_module(mod))
1405	pr_err("%s: adding tainted module to the unloaded tainted modules list failed.\n",
1406	mod->name);
1407	mutex_unlock(&module_mutex);
1408
1409	/* This may be empty, but that's OK */
1410	module_arch_freeing_init(mod);
1411	kfree(mod->args);
1412	percpu_modfree(mod);
1413
1414	free_mod_mem(mod);
1415	}
1416
1417	void __symbol_get(const char symbol)
1418	{
1419	struct find_symbol_arg fsa = {
1420	.name = symbol,
1421	.gplok = true,
1422	.warn = true,
1423	};
1424
1425	scoped_guard(rcu) {
1426	if (!find_symbol(&fsa))
1427	return NULL;
1428	if (fsa.license != GPL_ONLY) {
1429	pr_warn("failing symbol_get of non-GPLONLY symbol %s.\n",
1430	symbol);
1431	return NULL;
1432	}
1433	if (strong_try_module_get(fsa.owner))
1434	return NULL;
1435	}
1436	return (void *)kernel_symbol_value(fsa.sym);
1437	}
1438	EXPORT_SYMBOL_GPL(__symbol_get);
1439
1440	/*
1441	* Ensure that an exported symbol [global namespace] does not already exist
1442	* in the kernel or in some other module's exported symbol table.
1443	*
1444	* You must hold the module_mutex.
1445	*/
1446	static int verify_exported_symbols(struct module *mod)
1447	{
1448	unsigned int i;
1449	const struct kernel_symbol *s;
1450	struct {
1451	const struct kernel_symbol *sym;
1452	unsigned int num;
1453	} arr[] = {
1454	{ mod->syms, mod->num_syms },
1455	{ mod->gpl_syms, mod->num_gpl_syms },
1456	};
1457
1458	for (i = 0; i < ARRAY_SIZE(arr); i++) {
1459	for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
1460	struct find_symbol_arg fsa = {
1461	.name = kernel_symbol_name(s),
1462	.gplok = true,
1463	};
1464	if (find_symbol(&fsa)) {
1465	pr_err("%s: exports duplicate symbol %s"
1466	" (owned by %s)\n",
1467	mod->name, kernel_symbol_name(s),
1468	module_name(fsa.owner));
1469	return -ENOEXEC;
1470	}
1471	}
1472	}
1473	return 0;
1474	}
1475
1476	static bool ignore_undef_symbol(Elf_Half emachine, const char *name)
1477	{
1478	/*
1479	* On x86, PIC code and Clang non-PIC code may have call foo@PLT. GNU as
1480	* before 2.37 produces an unreferenced _GLOBAL_OFFSET_TABLE_ on x86-64.
1481	* i386 has a similar problem but may not deserve a fix.
1482	*
1483	* If we ever have to ignore many symbols, consider refactoring the code to
1484	* only warn if referenced by a relocation.
1485	*/
1486	if (emachine == EM_386 \|\| emachine == EM_X86_64)
1487	return !strcmp(name, "_GLOBAL_OFFSET_TABLE_");
1488	return false;
1489	}
1490
1491	/* Change all symbols so that st_value encodes the pointer directly. */
1492	static int simplify_symbols(struct module mod, const struct load_info info)
1493	{
1494	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
1495	Elf_Sym sym = (void )symsec->sh_addr;
1496	unsigned long secbase;
1497	unsigned int i;
1498	int ret = 0;
1499	const struct kernel_symbol *ksym;
1500
1501	for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
1502	const char *name = info->strtab + sym[i].st_name;
1503
1504	switch (sym[i].st_shndx) {
1505	case SHN_COMMON:
1506	/* Ignore common symbols */
1507	if (!strncmp(name, "__gnu_lto", 9))
1508	break;
1509
1510	/*
1511	* We compiled with -fno-common. These are not
1512	* supposed to happen.
1513	*/
1514	pr_debug("Common symbol: %s\n", name);
1515	pr_warn("%s: please compile with -fno-common\n",
1516	mod->name);
1517	ret = -ENOEXEC;
1518	break;
1519
1520	case SHN_ABS:
1521	/* Don't need to do anything */
1522	pr_debug("Absolute symbol: 0x%08lx %s\n",
1523	(long)sym[i].st_value, name);
1524	break;
1525
1526	case SHN_LIVEPATCH:
1527	/* Livepatch symbols are resolved by livepatch */
1528	break;
1529
1530	case SHN_UNDEF:
1531	ksym = resolve_symbol_wait(mod, info, name);
1532	/* Ok if resolved. */
1533	if (ksym && !IS_ERR(ksym)) {
1534	sym[i].st_value = kernel_symbol_value(ksym);
1535	break;
1536	}
1537
1538	/* Ok if weak or ignored. */
1539	if (!ksym &&
1540	(ELF_ST_BIND(sym[i].st_info) == STB_WEAK \|\|
1541	ignore_undef_symbol(info->hdr->e_machine, name)))
1542	break;
1543
1544	ret = PTR_ERR(ksym) ?: -ENOENT;
1545	pr_warn("%s: Unknown symbol %s (err %d)\n",
1546	mod->name, name, ret);
1547	break;
1548
1549	default:
1550	/* Divert to percpu allocation if a percpu var. */
1551	if (sym[i].st_shndx == info->index.pcpu)
1552	secbase = (unsigned long)mod_percpu(mod);
1553	else
1554	secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
1555	sym[i].st_value += secbase;
1556	break;
1557	}
1558	}
1559
1560	return ret;
1561	}
1562
1563	static int apply_relocations(struct module mod, const struct load_info info)
1564	{
1565	unsigned int i;
1566	int err = 0;
1567
1568	/* Now do relocations. */
1569	for (i = 1; i < info->hdr->e_shnum; i++) {
1570	unsigned int infosec = info->sechdrs[i].sh_info;
1571
1572	/* Not a valid relocation section? */
1573	if (infosec >= info->hdr->e_shnum)
1574	continue;
1575
1576	/*
1577	* Don't bother with non-allocated sections.
1578	* An exception is the percpu section, which has separate allocations
1579	* for individual CPUs. We relocate the percpu section in the initial
1580	* ELF template and subsequently copy it to the per-CPU destinations.
1581	*/
1582	if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC) &&
1583	(!infosec \|\| infosec != info->index.pcpu))
1584	continue;
1585
1586	if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH)
1587	err = klp_apply_section_relocs(mod, info->sechdrs,
1588	info->secstrings,
1589	info->strtab,
1590	info->index.sym, i,
1591	NULL);
1592	else if (info->sechdrs[i].sh_type == SHT_REL)
1593	err = apply_relocate(info->sechdrs, info->strtab,
1594	info->index.sym, i, mod);
1595	else if (info->sechdrs[i].sh_type == SHT_RELA)
1596	err = apply_relocate_add(info->sechdrs, info->strtab,
1597	info->index.sym, i, mod);
1598	if (err < 0)
1599	break;
1600	}
1601	return err;
1602	}
1603
1604	/* Additional bytes needed by arch in front of individual sections */
1605	unsigned int __weak arch_mod_section_prepend(struct module *mod,
1606	unsigned int section)
1607	{
1608	/* default implementation just returns zero */
1609	return 0;
1610	}
1611
1612	long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
1613	Elf_Shdr *sechdr, unsigned int section)
1614	{
1615	long offset;
1616	long mask = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK) << SH_ENTSIZE_TYPE_SHIFT;
1617
1618	mod->mem[type].size += arch_mod_section_prepend(mod, section);
1619	offset = ALIGN(mod->mem[type].size, sechdr->sh_addralign ?: 1);
1620	mod->mem[type].size = offset + sechdr->sh_size;
1621
1622	WARN_ON_ONCE(offset & mask);
1623	return offset \| mask;
1624	}
1625
1626	bool module_init_layout_section(const char *sname)
1627	{
1628	#ifndef CONFIG_MODULE_UNLOAD
1629	if (module_exit_section(sname))
1630	return true;
1631	#endif
1632	return module_init_section(sname);
1633	}
1634
1635	static void __layout_sections(struct module mod, struct load_info info, bool is_init)
1636	{
1637	unsigned int m, i;
1638
1639	/*
1640	* { Mask of required section header flags,
1641	* Mask of excluded section header flags }
1642	*/
1643	static const unsigned long masks[][2] = {
1644	{ SHF_EXECINSTR \| SHF_ALLOC, ARCH_SHF_SMALL },
1645	{ SHF_ALLOC, SHF_WRITE \| ARCH_SHF_SMALL },
1646	{ SHF_RO_AFTER_INIT \| SHF_ALLOC, ARCH_SHF_SMALL },
1647	{ SHF_WRITE \| SHF_ALLOC, ARCH_SHF_SMALL },
1648	{ ARCH_SHF_SMALL \| SHF_ALLOC, 0 }
1649	};
1650	static const int core_m_to_mem_type[] = {
1651	MOD_TEXT,
1652	MOD_RODATA,
1653	MOD_RO_AFTER_INIT,
1654	MOD_DATA,
1655	MOD_DATA,
1656	};
1657	static const int init_m_to_mem_type[] = {
1658	MOD_INIT_TEXT,
1659	MOD_INIT_RODATA,
1660	MOD_INVALID,
1661	MOD_INIT_DATA,
1662	MOD_INIT_DATA,
1663	};
1664
1665	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1666	enum mod_mem_type type = is_init ? init_m_to_mem_type[m] : core_m_to_mem_type[m];
1667
1668	for (i = 0; i < info->hdr->e_shnum; ++i) {
1669	Elf_Shdr *s = &info->sechdrs[i];
1670	const char *sname = info->secstrings + s->sh_name;
1671
1672	if ((s->sh_flags & masks[m][0]) != masks[m][0]
1673	\|\| (s->sh_flags & masks[m][1])
1674	\|\| s->sh_entsize != ~0UL
1675	\|\| is_init != module_init_layout_section(sname))
1676	continue;
1677
1678	if (WARN_ON_ONCE(type == MOD_INVALID))
1679	continue;
1680
1681	/*
1682	* Do not allocate codetag memory as we load it into
1683	* preallocated contiguous memory.
1684	*/
1685	if (codetag_needs_module_section(mod, sname, s->sh_size)) {
1686	/*
1687	* s->sh_entsize won't be used but populate the
1688	* type field to avoid confusion.
1689	*/
1690	s->sh_entsize = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK)
1691	<< SH_ENTSIZE_TYPE_SHIFT;
1692	continue;
1693	}
1694
1695	s->sh_entsize = module_get_offset_and_type(mod, type, s, i);
1696	pr_debug("\t%s\n", sname);
1697	}
1698	}
1699	}
1700
1701	/*
1702	* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
1703	* might -- code, read-only data, read-write data, small data. Tally
1704	* sizes, and place the offsets into sh_entsize fields: high bit means it
1705	* belongs in init.
1706	*/
1707	static void layout_sections(struct module mod, struct load_info info)
1708	{
1709	unsigned int i;
1710
1711	for (i = 0; i < info->hdr->e_shnum; i++)
1712	info->sechdrs[i].sh_entsize = ~0UL;
1713
1714	pr_debug("Core section allocation order for %s:\n", mod->name);
1715	__layout_sections(mod, info, false);
1716
1717	pr_debug("Init section allocation order for %s:\n", mod->name);
1718	__layout_sections(mod, info, true);
1719	}
1720
1721	static void module_license_taint_check(struct module mod, const char license)
1722	{
1723	if (!license)
1724	license = "unspecified";
1725
1726	if (!license_is_gpl_compatible(license)) {
1727	if (!test_taint(TAINT_PROPRIETARY_MODULE))
1728	pr_warn("%s: module license '%s' taints kernel.\n",
1729	mod->name, license);
1730	add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
1731	LOCKDEP_NOW_UNRELIABLE);
1732	}
1733	}
1734
1735	static int setup_modinfo(struct module mod, struct load_info info)
1736	{
1737	const struct module_attribute *attr;
1738	char *imported_namespace;
1739	int i;
1740
1741	for (i = 0; (attr = modinfo_attrs[i]); i++) {
1742	if (attr->setup)
1743	attr->setup(mod, get_modinfo(info, attr->attr.name));
1744	}
1745
1746	for_each_modinfo_entry(imported_namespace, info, "import_ns") {
1747	/*
1748	* 'module:' prefixed namespaces are implicit, disallow
1749	* explicit imports.
1750	*/
1751	if (strstarts(imported_namespace, "module:")) {
1752	pr_err("%s: module tries to import module namespace: %s\n",
1753	mod->name, imported_namespace);
1754	return -EPERM;
1755	}
1756	}
1757
1758	return 0;
1759	}
1760
1761	static void free_modinfo(struct module *mod)
1762	{
1763	const struct module_attribute *attr;
1764	int i;
1765
1766	for (i = 0; (attr = modinfo_attrs[i]); i++) {
1767	if (attr->free)
1768	attr->free(mod);
1769	}
1770	}
1771
1772	bool __weak module_init_section(const char *name)
1773	{
1774	return strstarts(name, ".init");
1775	}
1776
1777	bool __weak module_exit_section(const char *name)
1778	{
1779	return strstarts(name, ".exit");
1780	}
1781
1782	static int validate_section_offset(const struct load_info info, Elf_Shdr shdr)
1783	{
1784	#if defined(CONFIG_64BIT)
1785	unsigned long long secend;
1786	#else
1787	unsigned long secend;
1788	#endif
1789
1790	/*
1791	* Check for both overflow and offset/size being
1792	* too large.
1793	*/
1794	secend = shdr->sh_offset + shdr->sh_size;
1795	if (secend < shdr->sh_offset \|\| secend > info->len)
1796	return -ENOEXEC;
1797
1798	return 0;
1799	}
1800
1801	/**
1802	* elf_validity_ehdr() - Checks an ELF header for module validity
1803	* @info: Load info containing the ELF header to check
1804	*
1805	* Checks whether an ELF header could belong to a valid module. Checks:
1806	*
1807	* * ELF header is within the data the user provided
1808	* * ELF magic is present
1809	* * It is relocatable (not final linked, not core file, etc.)
1810	* * The header's machine type matches what the architecture expects.
1811	* * Optional arch-specific hook for other properties
1812	* - module_elf_check_arch() is currently only used by PPC to check
1813	* ELF ABI version, but may be used by others in the future.
1814	*
1815	* Return: %0 if valid, %-ENOEXEC on failure.
1816	*/
1817	static int elf_validity_ehdr(const struct load_info *info)
1818	{
1819	if (info->len < sizeof(*(info->hdr))) {
1820	pr_err("Invalid ELF header len %lu\n", info->len);
1821	return -ENOEXEC;
1822	}
1823	if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0) {
1824	pr_err("Invalid ELF header magic: != %s\n", ELFMAG);
1825	return -ENOEXEC;
1826	}
1827	if (info->hdr->e_type != ET_REL) {
1828	pr_err("Invalid ELF header type: %u != %u\n",
1829	info->hdr->e_type, ET_REL);
1830	return -ENOEXEC;
1831	}
1832	if (!elf_check_arch(info->hdr)) {
1833	pr_err("Invalid architecture in ELF header: %u\n",
1834	info->hdr->e_machine);
1835	return -ENOEXEC;
1836	}
1837	if (!module_elf_check_arch(info->hdr)) {
1838	pr_err("Invalid module architecture in ELF header: %u\n",
1839	info->hdr->e_machine);
1840	return -ENOEXEC;
1841	}
1842	return 0;
1843	}
1844
1845	/**
1846	* elf_validity_cache_sechdrs() - Cache section headers if valid
1847	* @info: Load info to compute section headers from
1848	*
1849	* Checks:
1850	*
1851	* * ELF header is valid (see elf_validity_ehdr())
1852	* * Section headers are the size we expect
1853	* * Section array fits in the user provided data
1854	* * Section index 0 is NULL
1855	* * Section contents are inbounds
1856	*
1857	* Then updates @info with a &load_info->sechdrs pointer if valid.
1858	*
1859	* Return: %0 if valid, negative error code if validation failed.
1860	*/
1861	static int elf_validity_cache_sechdrs(struct load_info *info)
1862	{
1863	Elf_Shdr *sechdrs;
1864	Elf_Shdr *shdr;
1865	int i;
1866	int err;
1867
1868	err = elf_validity_ehdr(info);
1869	if (err < 0)
1870	return err;
1871
1872	if (info->hdr->e_shentsize != sizeof(Elf_Shdr)) {
1873	pr_err("Invalid ELF section header size\n");
1874	return -ENOEXEC;
1875	}
1876
1877	/*
1878	* e_shnum is 16 bits, and sizeof(Elf_Shdr) is
1879	* known and small. So e_shnum * sizeof(Elf_Shdr)
1880	* will not overflow unsigned long on any platform.
1881	*/
1882	if (info->hdr->e_shoff >= info->len
1883	\|\| (info->hdr->e_shnum * sizeof(Elf_Shdr) >
1884	info->len - info->hdr->e_shoff)) {
1885	pr_err("Invalid ELF section header overflow\n");
1886	return -ENOEXEC;
1887	}
1888
1889	sechdrs = (void *)info->hdr + info->hdr->e_shoff;
1890
1891	/*
1892	* The code assumes that section 0 has a length of zero and
1893	* an addr of zero, so check for it.
1894	*/
1895	if (sechdrs[0].sh_type != SHT_NULL
1896	\|\| sechdrs[0].sh_size != 0
1897	\|\| sechdrs[0].sh_addr != 0) {
1898	pr_err("ELF Spec violation: section 0 type(%d)!=SH_NULL or non-zero len or addr\n",
1899	sechdrs[0].sh_type);
1900	return -ENOEXEC;
1901	}
1902
1903	/* Validate contents are inbounds */
1904	for (i = 1; i < info->hdr->e_shnum; i++) {
1905	shdr = &sechdrs[i];
1906	switch (shdr->sh_type) {
1907	case SHT_NULL:
1908	case SHT_NOBITS:
1909	/* No contents, offset/size don't mean anything */
1910	continue;
1911	default:
1912	err = validate_section_offset(info, shdr);
1913	if (err < 0) {
1914	pr_err("Invalid ELF section in module (section %u type %u)\n",
1915	i, shdr->sh_type);
1916	return err;
1917	}
1918	}
1919	}
1920
1921	info->sechdrs = sechdrs;
1922
1923	return 0;
1924	}
1925
1926	/**
1927	* elf_validity_cache_secstrings() - Caches section names if valid
1928	* @info: Load info to cache section names from. Must have valid sechdrs.
1929	*
1930	* Specifically checks:
1931	*
1932	* * Section name table index is inbounds of section headers
1933	* * Section name table is not empty
1934	* * Section name table is NUL terminated
1935	* * All section name offsets are inbounds of the section
1936	*
1937	* Then updates @info with a &load_info->secstrings pointer if valid.
1938	*
1939	* Return: %0 if valid, negative error code if validation failed.
1940	*/
1941	static int elf_validity_cache_secstrings(struct load_info *info)
1942	{
1943	Elf_Shdr strhdr, shdr;
1944	char *secstrings;
1945	int i;
1946
1947	/*
1948	* Verify if the section name table index is valid.
1949	*/
1950	if (info->hdr->e_shstrndx == SHN_UNDEF
1951	\|\| info->hdr->e_shstrndx >= info->hdr->e_shnum) {
1952	pr_err("Invalid ELF section name index: %d \|\| e_shstrndx (%d) >= e_shnum (%d)\n",
1953	info->hdr->e_shstrndx, info->hdr->e_shstrndx,
1954	info->hdr->e_shnum);
1955	return -ENOEXEC;
1956	}
1957
1958	strhdr = &info->sechdrs[info->hdr->e_shstrndx];
1959
1960	/*
1961	* The section name table must be NUL-terminated, as required
1962	* by the spec. This makes strcmp and pr_* calls that access
1963	* strings in the section safe.
1964	*/
1965	secstrings = (void *)info->hdr + strhdr->sh_offset;
1966	if (strhdr->sh_size == 0) {
1967	pr_err("empty section name table\n");
1968	return -ENOEXEC;
1969	}
1970	if (secstrings[strhdr->sh_size - 1] != '\0') {
1971	pr_err("ELF Spec violation: section name table isn't null terminated\n");
1972	return -ENOEXEC;
1973	}
1974
1975	for (i = 0; i < info->hdr->e_shnum; i++) {
1976	shdr = &info->sechdrs[i];
1977	/* SHT_NULL means sh_name has an undefined value */
1978	if (shdr->sh_type == SHT_NULL)
1979	continue;
1980	if (shdr->sh_name >= strhdr->sh_size) {
1981	pr_err("Invalid ELF section name in module (section %u type %u)\n",
1982	i, shdr->sh_type);
1983	return -ENOEXEC;
1984	}
1985	}
1986
1987	info->secstrings = secstrings;
1988	return 0;
1989	}
1990
1991	/**
1992	* elf_validity_cache_index_info() - Validate and cache modinfo section
1993	* @info: Load info to populate the modinfo index on.
1994	* Must have &load_info->sechdrs and &load_info->secstrings populated
1995	*
1996	* Checks that if there is a .modinfo section, it is unique.
1997	* Then, it caches its index in &load_info->index.info.
1998	* Finally, it tries to populate the name to improve error messages.
1999	*
2000	* Return: %0 if valid, %-ENOEXEC if multiple modinfo sections were found.
2001	*/
2002	static int elf_validity_cache_index_info(struct load_info *info)
2003	{
2004	int info_idx;
2005
2006	info_idx = find_any_unique_sec(info, ".modinfo");
2007
2008	if (info_idx == 0)
2009	/* Early return, no .modinfo */
2010	return 0;
2011
2012	if (info_idx < 0) {
2013	pr_err("Only one .modinfo section must exist.\n");
2014	return -ENOEXEC;
2015	}
2016
2017	info->index.info = info_idx;
2018	/* Try to find a name early so we can log errors with a module name */
2019	info->name = get_modinfo(info, "name");
2020
2021	return 0;
2022	}
2023
2024	/**
2025	* elf_validity_cache_index_mod() - Validates and caches this_module section
2026	* @info: Load info to cache this_module on.
2027	* Must have &load_info->sechdrs and &load_info->secstrings populated
2028	*
2029	* The ".gnu.linkonce.this_module" ELF section is special. It is what modpost
2030	* uses to refer to __this_module and let's use rely on THIS_MODULE to point
2031	* to &__this_module properly. The kernel's modpost declares it on each
2032	* modules's *.mod.c file. If the struct module of the kernel changes a full
2033	* kernel rebuild is required.
2034	*
2035	* We have a few expectations for this special section, this function
2036	* validates all this for us:
2037	*
2038	* * The section has contents
2039	* * The section is unique
2040	* * We expect the kernel to always have to allocate it: SHF_ALLOC
2041	* * The section size must match the kernel's run time's struct module
2042	* size
2043	*
2044	* If all checks pass, the index will be cached in &load_info->index.mod
2045	*
2046	* Return: %0 on validation success, %-ENOEXEC on failure
2047	*/
2048	static int elf_validity_cache_index_mod(struct load_info *info)
2049	{
2050	Elf_Shdr *shdr;
2051	int mod_idx;
2052
2053	mod_idx = find_any_unique_sec(info, ".gnu.linkonce.this_module");
2054	if (mod_idx <= 0) {
2055	pr_err("module %s: Exactly one .gnu.linkonce.this_module section must exist.\n",
2056	info->name ?: "(missing .modinfo section or name field)");
2057	return -ENOEXEC;
2058	}
2059
2060	shdr = &info->sechdrs[mod_idx];
2061
2062	if (shdr->sh_type == SHT_NOBITS) {
2063	pr_err("module %s: .gnu.linkonce.this_module section must have a size set\n",
2064	info->name ?: "(missing .modinfo section or name field)");
2065	return -ENOEXEC;
2066	}
2067
2068	if (!(shdr->sh_flags & SHF_ALLOC)) {
2069	pr_err("module %s: .gnu.linkonce.this_module must occupy memory during process execution\n",
2070	info->name ?: "(missing .modinfo section or name field)");
2071	return -ENOEXEC;
2072	}
2073
2074	if (shdr->sh_size != sizeof(struct module)) {
2075	pr_err("module %s: .gnu.linkonce.this_module section size must match the kernel's built struct module size at run time\n",
2076	info->name ?: "(missing .modinfo section or name field)");
2077	return -ENOEXEC;
2078	}
2079
2080	info->index.mod = mod_idx;
2081
2082	return 0;
2083	}
2084
2085	/**
2086	* elf_validity_cache_index_sym() - Validate and cache symtab index
2087	* @info: Load info to cache symtab index in.
2088	* Must have &load_info->sechdrs and &load_info->secstrings populated.
2089	*
2090	* Checks that there is exactly one symbol table, then caches its index in
2091	* &load_info->index.sym.
2092	*
2093	* Return: %0 if valid, %-ENOEXEC on failure.
2094	*/
2095	static int elf_validity_cache_index_sym(struct load_info *info)
2096	{
2097	unsigned int sym_idx;
2098	unsigned int num_sym_secs = 0;
2099	int i;
2100
2101	for (i = 1; i < info->hdr->e_shnum; i++) {
2102	if (info->sechdrs[i].sh_type == SHT_SYMTAB) {
2103	num_sym_secs++;
2104	sym_idx = i;
2105	}
2106	}
2107
2108	if (num_sym_secs != 1) {
2109	pr_warn("%s: module has no symbols (stripped?)\n",
2110	info->name ?: "(missing .modinfo section or name field)");
2111	return -ENOEXEC;
2112	}
2113
2114	info->index.sym = sym_idx;
2115
2116	return 0;
2117	}
2118
2119	/**
2120	* elf_validity_cache_index_str() - Validate and cache strtab index
2121	* @info: Load info to cache strtab index in.
2122	* Must have &load_info->sechdrs and &load_info->secstrings populated.
2123	* Must have &load_info->index.sym populated.
2124	*
2125	* Looks at the symbol table's associated string table, makes sure it is
2126	* in-bounds, and caches it.
2127	*
2128	* Return: %0 if valid, %-ENOEXEC on failure.
2129	*/
2130	static int elf_validity_cache_index_str(struct load_info *info)
2131	{
2132	unsigned int str_idx = info->sechdrs[info->index.sym].sh_link;
2133
2134	if (str_idx == SHN_UNDEF \|\| str_idx >= info->hdr->e_shnum) {
2135	pr_err("Invalid ELF sh_link!=SHN_UNDEF(%d) or (sh_link(%d) >= hdr->e_shnum(%d)\n",
2136	str_idx, str_idx, info->hdr->e_shnum);
2137	return -ENOEXEC;
2138	}
2139
2140	info->index.str = str_idx;
2141	return 0;
2142	}
2143
2144	/**
2145	* elf_validity_cache_index_versions() - Validate and cache version indices
2146	* @info: Load info to cache version indices in.
2147	* Must have &load_info->sechdrs and &load_info->secstrings populated.
2148	* @flags: Load flags, relevant to suppress version loading, see
2149	* uapi/linux/module.h
2150	*
2151	* If we're ignoring modversions based on @flags, zero all version indices
2152	* and return validity. Othewrise check:
2153	*
2154	* * If "__version_ext_crcs" is present, "__version_ext_names" is present
2155	* * There is a name present for every crc
2156	*
2157	* Then populate:
2158	*
2159	* * &load_info->index.vers
2160	* * &load_info->index.vers_ext_crc
2161	* * &load_info->index.vers_ext_names
2162	*
2163	* if present.
2164	*
2165	* Return: %0 if valid, %-ENOEXEC on failure.
2166	*/
2167	static int elf_validity_cache_index_versions(struct load_info *info, int flags)
2168	{
2169	unsigned int vers_ext_crc;
2170	unsigned int vers_ext_name;
2171	size_t crc_count;
2172	size_t remaining_len;
2173	size_t name_size;
2174	char *name;
2175
2176	/* If modversions were suppressed, pretend we didn't find any */
2177	if (flags & MODULE_INIT_IGNORE_MODVERSIONS) {
2178	info->index.vers = 0;
2179	info->index.vers_ext_crc = 0;
2180	info->index.vers_ext_name = 0;
2181	return 0;
2182	}
2183
2184	vers_ext_crc = find_sec(info, "__version_ext_crcs");
2185	vers_ext_name = find_sec(info, "__version_ext_names");
2186
2187	/* If we have one field, we must have the other */
2188	if (!!vers_ext_crc != !!vers_ext_name) {
2189	pr_err("extended version crc+name presence does not match");
2190	return -ENOEXEC;
2191	}
2192
2193	/*
2194	* If we have extended version information, we should have the same
2195	* number of entries in every section.
2196	*/
2197	if (vers_ext_crc) {
2198	crc_count = info->sechdrs[vers_ext_crc].sh_size / sizeof(u32);
2199	name = (void *)info->hdr +
2200	info->sechdrs[vers_ext_name].sh_offset;
2201	remaining_len = info->sechdrs[vers_ext_name].sh_size;
2202
2203	while (crc_count--) {
2204	name_size = strnlen(name, remaining_len) + 1;
2205	if (name_size > remaining_len) {
2206	pr_err("more extended version crcs than names");
2207	return -ENOEXEC;
2208	}
2209	remaining_len -= name_size;
2210	name += name_size;
2211	}
2212	}
2213
2214	info->index.vers = find_sec(info, "__versions");
2215	info->index.vers_ext_crc = vers_ext_crc;
2216	info->index.vers_ext_name = vers_ext_name;
2217	return 0;
2218	}
2219
2220	/**
2221	* elf_validity_cache_index() - Resolve, validate, cache section indices
2222	* @info: Load info to read from and update.
2223	* &load_info->sechdrs and &load_info->secstrings must be populated.
2224	* @flags: Load flags, relevant to suppress version loading, see
2225	* uapi/linux/module.h
2226	*
2227	* Populates &load_info->index, validating as it goes.
2228	* See child functions for per-field validation:
2229	*
2230	* * elf_validity_cache_index_info()
2231	* * elf_validity_cache_index_mod()
2232	* * elf_validity_cache_index_sym()
2233	* * elf_validity_cache_index_str()
2234	* * elf_validity_cache_index_versions()
2235	*
2236	* If CONFIG_SMP is enabled, load the percpu section by name with no
2237	* validation.
2238	*
2239	* Return: 0 on success, negative error code if an index failed validation.
2240	*/
2241	static int elf_validity_cache_index(struct load_info *info, int flags)
2242	{
2243	int err;
2244
2245	err = elf_validity_cache_index_info(info);
2246	if (err < 0)
2247	return err;
2248	err = elf_validity_cache_index_mod(info);
2249	if (err < 0)
2250	return err;
2251	err = elf_validity_cache_index_sym(info);
2252	if (err < 0)
2253	return err;
2254	err = elf_validity_cache_index_str(info);
2255	if (err < 0)
2256	return err;
2257	err = elf_validity_cache_index_versions(info, flags);
2258	if (err < 0)
2259	return err;
2260
2261	info->index.pcpu = find_pcpusec(info);
2262
2263	return 0;
2264	}
2265
2266	/**
2267	* elf_validity_cache_strtab() - Validate and cache symbol string table
2268	* @info: Load info to read from and update.
2269	* Must have &load_info->sechdrs and &load_info->secstrings populated.
2270	* Must have &load_info->index populated.
2271	*
2272	* Checks:
2273	*
2274	* * The string table is not empty.
2275	* * The string table starts and ends with NUL (required by ELF spec).
2276	* * Every &Elf_Sym->st_name offset in the symbol table is inbounds of the
2277	* string table.
2278	*
2279	* And caches the pointer as &load_info->strtab in @info.
2280	*
2281	* Return: 0 on success, negative error code if a check failed.
2282	*/
2283	static int elf_validity_cache_strtab(struct load_info *info)
2284	{
2285	Elf_Shdr *str_shdr = &info->sechdrs[info->index.str];
2286	Elf_Shdr *sym_shdr = &info->sechdrs[info->index.sym];
2287	char strtab = (char )info->hdr + str_shdr->sh_offset;
2288	Elf_Sym syms = (void )info->hdr + sym_shdr->sh_offset;
2289	int i;
2290
2291	if (str_shdr->sh_size == 0) {
2292	pr_err("empty symbol string table\n");
2293	return -ENOEXEC;
2294	}
2295	if (strtab[0] != '\0') {
2296	pr_err("symbol string table missing leading NUL\n");
2297	return -ENOEXEC;
2298	}
2299	if (strtab[str_shdr->sh_size - 1] != '\0') {
2300	pr_err("symbol string table isn't NUL terminated\n");
2301	return -ENOEXEC;
2302	}
2303
2304	/*
2305	* Now that we know strtab is correctly structured, check symbol
2306	* starts are inbounds before they're used later.
2307	*/
2308	for (i = 0; i < sym_shdr->sh_size / sizeof(*syms); i++) {
2309	if (syms[i].st_name >= str_shdr->sh_size) {
2310	pr_err("symbol name out of bounds in string table");
2311	return -ENOEXEC;
2312	}
2313	}
2314
2315	info->strtab = strtab;
2316	return 0;
2317	}
2318
2319	/*
2320	* Check userspace passed ELF module against our expectations, and cache
2321	* useful variables for further processing as we go.
2322	*
2323	* This does basic validity checks against section offsets and sizes, the
2324	* section name string table, and the indices used for it (sh_name).
2325	*
2326	* As a last step, since we're already checking the ELF sections we cache
2327	* useful variables which will be used later for our convenience:
2328	*
2329	* o pointers to section headers
2330	* o cache the modinfo symbol section
2331	* o cache the string symbol section
2332	* o cache the module section
2333	*
2334	* As a last step we set info->mod to the temporary copy of the module in
2335	* info->hdr. The final one will be allocated in move_module(). Any
2336	* modifications we make to our copy of the module will be carried over
2337	* to the final minted module.
2338	*/
2339	static int elf_validity_cache_copy(struct load_info *info, int flags)
2340	{
2341	int err;
2342
2343	err = elf_validity_cache_sechdrs(info);
2344	if (err < 0)
2345	return err;
2346	err = elf_validity_cache_secstrings(info);
2347	if (err < 0)
2348	return err;
2349	err = elf_validity_cache_index(info, flags);
2350	if (err < 0)
2351	return err;
2352	err = elf_validity_cache_strtab(info);
2353	if (err < 0)
2354	return err;
2355
2356	/* This is temporary: point mod into copy of data. */
2357	info->mod = (void *)info->hdr + info->sechdrs[info->index.mod].sh_offset;
2358
2359	/*
2360	* If we didn't load the .modinfo 'name' field earlier, fall back to
2361	* on-disk struct mod 'name' field.
2362	*/
2363	if (!info->name)
2364	info->name = info->mod->name;
2365
2366	return 0;
2367	}
2368
2369	#define COPY_CHUNK_SIZE (16*PAGE_SIZE)
2370
2371	static int copy_chunked_from_user(void dst, const void __user usrc, unsigned long len)
2372	{
2373	do {
2374	unsigned long n = min(len, COPY_CHUNK_SIZE);
2375
2376	if (copy_from_user(dst, usrc, n) != 0)
2377	return -EFAULT;
2378	cond_resched();
2379	dst += n;
2380	usrc += n;
2381	len -= n;
2382	} while (len);
2383	return 0;
2384	}
2385
2386	static int check_modinfo_livepatch(struct module mod, struct load_info info)
2387	{
2388	if (!get_modinfo(info, "livepatch"))
2389	/* Nothing more to do */
2390	return 0;
2391
2392	if (set_livepatch_module(mod))
2393	return 0;
2394
2395	pr_err("%s: module is marked as livepatch module, but livepatch support is disabled",
2396	mod->name);
2397	return -ENOEXEC;
2398	}
2399
2400	static void check_modinfo_retpoline(struct module mod, struct load_info info)
2401	{
2402	if (retpoline_module_ok(get_modinfo(info, "retpoline")))
2403	return;
2404
2405	pr_warn("%s: loading module not compiled with retpoline compiler.\n",
2406	mod->name);
2407	}
2408
2409	/* Sets info->hdr and info->len. */
2410	static int copy_module_from_user(const void __user *umod, unsigned long len,
2411	struct load_info *info)
2412	{
2413	int err;
2414
2415	info->len = len;
2416	if (info->len < sizeof(*(info->hdr)))
2417	return -ENOEXEC;
2418
2419	err = security_kernel_load_data(LOADING_MODULE, true);
2420	if (err)
2421	return err;
2422
2423	/* Suck in entire file: we'll want most of it. */
2424	info->hdr = __vmalloc(info->len, GFP_KERNEL \| __GFP_NOWARN);
2425	if (!info->hdr)
2426	return -ENOMEM;
2427
2428	if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
2429	err = -EFAULT;
2430	goto out;
2431	}
2432
2433	err = security_kernel_post_load_data((char *)info->hdr, info->len,
2434	LOADING_MODULE, "init_module");
2435	out:
2436	if (err)
2437	vfree(info->hdr);
2438
2439	return err;
2440	}
2441
2442	static void free_copy(struct load_info *info, int flags)
2443	{
2444	if (flags & MODULE_INIT_COMPRESSED_FILE)
2445	module_decompress_cleanup(info);
2446	else
2447	vfree(info->hdr);
2448	}
2449
2450	static int rewrite_section_headers(struct load_info *info, int flags)
2451	{
2452	unsigned int i;
2453
2454	/* This should always be true, but let's be sure. */
2455	info->sechdrs[0].sh_addr = 0;
2456
2457	for (i = 1; i < info->hdr->e_shnum; i++) {
2458	Elf_Shdr *shdr = &info->sechdrs[i];
2459
2460	/*
2461	* Mark all sections sh_addr with their address in the
2462	* temporary image.
2463	*/
2464	shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
2465
2466	}
2467
2468	/* Track but don't keep modinfo and version sections. */
2469	info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
2470	info->sechdrs[info->index.vers_ext_crc].sh_flags &=
2471	~(unsigned long)SHF_ALLOC;
2472	info->sechdrs[info->index.vers_ext_name].sh_flags &=
2473	~(unsigned long)SHF_ALLOC;
2474	info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
2475
2476	return 0;
2477	}
2478
2479	static const char *const module_license_offenders[] = {
2480	/* driverloader was caught wrongly pretending to be under GPL */
2481	"driverloader",
2482
2483	/* lve claims to be GPL but upstream won't provide source */
2484	"lve",
2485	};
2486
2487	/*
2488	* These calls taint the kernel depending certain module circumstances */
2489	static void module_augment_kernel_taints(struct module mod, struct load_info info)
2490	{
2491	int prev_taint = test_taint(TAINT_PROPRIETARY_MODULE);
2492	size_t i;
2493
2494	if (!get_modinfo(info, "intree")) {
2495	if (!test_taint(TAINT_OOT_MODULE))
2496	pr_warn("%s: loading out-of-tree module taints kernel.\n",
2497	mod->name);
2498	add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
2499	}
2500
2501	check_modinfo_retpoline(mod, info);
2502
2503	if (get_modinfo(info, "staging")) {
2504	add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
2505	pr_warn("%s: module is from the staging directory, the quality "
2506	"is unknown, you have been warned.\n", mod->name);
2507	}
2508
2509	if (is_livepatch_module(mod)) {
2510	add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
2511	pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n",
2512	mod->name);
2513	}
2514
2515	module_license_taint_check(mod, get_modinfo(info, "license"));
2516
2517	if (get_modinfo(info, "test")) {
2518	if (!test_taint(TAINT_TEST))
2519	pr_warn("%s: loading test module taints kernel.\n",
2520	mod->name);
2521	add_taint_module(mod, TAINT_TEST, LOCKDEP_STILL_OK);
2522	}
2523	#ifdef CONFIG_MODULE_SIG
2524	mod->sig_ok = info->sig_ok;
2525	if (!mod->sig_ok) {
2526	pr_notice_once("%s: module verification failed: signature "
2527	"and/or required key missing - tainting "
2528	"kernel\n", mod->name);
2529	add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
2530	}
2531	#endif
2532
2533	/*
2534	* ndiswrapper is under GPL by itself, but loads proprietary modules.
2535	* Don't use add_taint_module(), as it would prevent ndiswrapper from
2536	* using GPL-only symbols it needs.
2537	*/
2538	if (strcmp(mod->name, "ndiswrapper") == 0)
2539	add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
2540
2541	for (i = 0; i < ARRAY_SIZE(module_license_offenders); ++i) {
2542	if (strcmp(mod->name, module_license_offenders[i]) == 0)
2543	add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
2544	LOCKDEP_NOW_UNRELIABLE);
2545	}
2546
2547	if (!prev_taint && test_taint(TAINT_PROPRIETARY_MODULE))
2548	pr_warn("%s: module license taints kernel.\n", mod->name);
2549
2550	}
2551
2552	static int check_modinfo(struct module mod, struct load_info info, int flags)
2553	{
2554	const char *modmagic = get_modinfo(info, "vermagic");
2555	int err;
2556
2557	if (flags & MODULE_INIT_IGNORE_VERMAGIC)
2558	modmagic = NULL;
2559
2560	/* This is allowed: modprobe --force will invalidate it. */
2561	if (!modmagic) {
2562	err = try_to_force_load(mod, "bad vermagic");
2563	if (err)
2564	return err;
2565	} else if (!same_magic(modmagic, vermagic, info->index.vers)) {
2566	pr_err("%s: version magic '%s' should be '%s'\n",
2567	info->name, modmagic, vermagic);
2568	return -ENOEXEC;
2569	}
2570
2571	err = check_modinfo_livepatch(mod, info);
2572	if (err)
2573	return err;
2574
2575	return 0;
2576	}
2577
2578	static int find_module_sections(struct module mod, struct load_info info)
2579	{
2580	mod->kp = section_objs(info, "__param",
2581	sizeof(*mod->kp), &mod->num_kp);
2582	mod->syms = section_objs(info, "__ksymtab",
2583	sizeof(*mod->syms), &mod->num_syms);
2584	mod->crcs = section_addr(info, "__kcrctab");
2585	mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
2586	sizeof(*mod->gpl_syms),
2587	&mod->num_gpl_syms);
2588	mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
2589
2590	#ifdef CONFIG_CONSTRUCTORS
2591	mod->ctors = section_objs(info, ".ctors",
2592	sizeof(*mod->ctors), &mod->num_ctors);
2593	if (!mod->ctors)
2594	mod->ctors = section_objs(info, ".init_array",
2595	sizeof(*mod->ctors), &mod->num_ctors);
2596	else if (find_sec(info, ".init_array")) {
2597	/*
2598	* This shouldn't happen with same compiler and binutils
2599	* building all parts of the module.
2600	*/
2601	pr_warn("%s: has both .ctors and .init_array.\n",
2602	mod->name);
2603	return -EINVAL;
2604	}
2605	#endif
2606
2607	mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1,
2608	&mod->noinstr_text_size);
2609
2610	#ifdef CONFIG_TRACEPOINTS
2611	mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
2612	sizeof(*mod->tracepoints_ptrs),
2613	&mod->num_tracepoints);
2614	#endif
2615	#ifdef CONFIG_TREE_SRCU
2616	mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs",
2617	sizeof(*mod->srcu_struct_ptrs),
2618	&mod->num_srcu_structs);
2619	#endif
2620	#ifdef CONFIG_BPF_EVENTS
2621	mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map",
2622	sizeof(*mod->bpf_raw_events),
2623	&mod->num_bpf_raw_events);
2624	#endif
2625	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
2626	mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size);
2627	mod->btf_base_data = any_section_objs(info, ".BTF.base", 1,
2628	&mod->btf_base_data_size);
2629	#endif
2630	#ifdef CONFIG_JUMP_LABEL
2631	mod->jump_entries = section_objs(info, "__jump_table",
2632	sizeof(*mod->jump_entries),
2633	&mod->num_jump_entries);
2634	#endif
2635	#ifdef CONFIG_EVENT_TRACING
2636	mod->trace_events = section_objs(info, "_ftrace_events",
2637	sizeof(*mod->trace_events),
2638	&mod->num_trace_events);
2639	mod->trace_evals = section_objs(info, "_ftrace_eval_map",
2640	sizeof(*mod->trace_evals),
2641	&mod->num_trace_evals);
2642	#endif
2643	#ifdef CONFIG_TRACING
2644	mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
2645	sizeof(*mod->trace_bprintk_fmt_start),
2646	&mod->num_trace_bprintk_fmt);
2647	#endif
2648	#ifdef CONFIG_FTRACE_MCOUNT_RECORD
2649	/* sechdrs[0].sh_size is always zero */
2650	mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION,
2651	sizeof(*mod->ftrace_callsites),
2652	&mod->num_ftrace_callsites);
2653	#endif
2654	#ifdef CONFIG_FUNCTION_ERROR_INJECTION
2655	mod->ei_funcs = section_objs(info, "_error_injection_whitelist",
2656	sizeof(*mod->ei_funcs),
2657	&mod->num_ei_funcs);
2658	#endif
2659	#ifdef CONFIG_KPROBES
2660	mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1,
2661	&mod->kprobes_text_size);
2662	mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist",
2663	sizeof(unsigned long),
2664	&mod->num_kprobe_blacklist);
2665	#endif
2666	#ifdef CONFIG_PRINTK_INDEX
2667	mod->printk_index_start = section_objs(info, ".printk_index",
2668	sizeof(*mod->printk_index_start),
2669	&mod->printk_index_size);
2670	#endif
2671	#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
2672	mod->static_call_sites = section_objs(info, ".static_call_sites",
2673	sizeof(*mod->static_call_sites),
2674	&mod->num_static_call_sites);
2675	#endif
2676	#if IS_ENABLED(CONFIG_KUNIT)
2677	mod->kunit_suites = section_objs(info, ".kunit_test_suites",
2678	sizeof(*mod->kunit_suites),
2679	&mod->num_kunit_suites);
2680	mod->kunit_init_suites = section_objs(info, ".kunit_init_test_suites",
2681	sizeof(*mod->kunit_init_suites),
2682	&mod->num_kunit_init_suites);
2683	#endif
2684
2685	mod->extable = section_objs(info, "__ex_table",
2686	sizeof(*mod->extable), &mod->num_exentries);
2687
2688	if (section_addr(info, "__obsparm"))
2689	pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
2690
2691	#ifdef CONFIG_DYNAMIC_DEBUG_CORE
2692	mod->dyndbg_info.descs = section_objs(info, "__dyndbg",
2693	sizeof(*mod->dyndbg_info.descs),
2694	&mod->dyndbg_info.num_descs);
2695	mod->dyndbg_info.classes = section_objs(info, "__dyndbg_classes",
2696	sizeof(*mod->dyndbg_info.classes),
2697	&mod->dyndbg_info.num_classes);
2698	#endif
2699
2700	return 0;
2701	}
2702
2703	static int move_module(struct module mod, struct load_info info)
2704	{
2705	int i, ret;
2706	enum mod_mem_type t = MOD_MEM_NUM_TYPES;
2707	bool codetag_section_found = false;
2708
2709	for_each_mod_mem_type(type) {
2710	if (!mod->mem[type].size) {
2711	mod->mem[type].base = NULL;
2712	continue;
2713	}
2714
2715	ret = module_memory_alloc(mod, type);
2716	if (ret) {
2717	t = type;
2718	goto out_err;
2719	}
2720	}
2721
2722	/* Transfer each section which specifies SHF_ALLOC */
2723	pr_debug("Final section addresses for %s:\n", mod->name);
2724	for (i = 0; i < info->hdr->e_shnum; i++) {
2725	void *dest;
2726	Elf_Shdr *shdr = &info->sechdrs[i];
2727	const char *sname;
2728
2729	if (!(shdr->sh_flags & SHF_ALLOC))
2730	continue;
2731
2732	sname = info->secstrings + shdr->sh_name;
2733	/*
2734	* Load codetag sections separately as they might still be used
2735	* after module unload.
2736	*/
2737	if (codetag_needs_module_section(mod, sname, shdr->sh_size)) {
2738	dest = codetag_alloc_module_section(mod, sname, shdr->sh_size,
2739	arch_mod_section_prepend(mod, i), shdr->sh_addralign);
2740	if (WARN_ON(!dest)) {
2741	ret = -EINVAL;
2742	goto out_err;
2743	}
2744	if (IS_ERR(dest)) {
2745	ret = PTR_ERR(dest);
2746	goto out_err;
2747	}
2748	codetag_section_found = true;
2749	} else {
2750	enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
2751	unsigned long offset = shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK;
2752
2753	dest = mod->mem[type].base + offset;
2754	}
2755
2756	if (shdr->sh_type != SHT_NOBITS) {
2757	/*
2758	* Our ELF checker already validated this, but let's
2759	* be pedantic and make the goal clearer. We actually
2760	* end up copying over all modifications made to the
2761	* userspace copy of the entire struct module.
2762	*/
2763	if (i == info->index.mod &&
2764	(WARN_ON_ONCE(shdr->sh_size != sizeof(struct module)))) {
2765	ret = -ENOEXEC;
2766	goto out_err;
2767	}
2768	memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
2769	}
2770	/*
2771	* Update the userspace copy's ELF section address to point to
2772	* our newly allocated memory as a pure convenience so that
2773	* users of info can keep taking advantage and using the newly
2774	* minted official memory area.
2775	*/
2776	shdr->sh_addr = (unsigned long)dest;
2777	pr_debug("\t0x%lx 0x%.8lx %s\n", (long)shdr->sh_addr,
2778	(long)shdr->sh_size, info->secstrings + shdr->sh_name);
2779	}
2780
2781	return 0;
2782	out_err:
2783	module_memory_restore_rox(mod);
2784	while (t--)
2785	module_memory_free(mod, t);
2786	if (codetag_section_found)
2787	codetag_free_module_sections(mod);
2788
2789	return ret;
2790	}
2791
2792	static int check_export_symbol_versions(struct module *mod)
2793	{
2794	#ifdef CONFIG_MODVERSIONS
2795	if ((mod->num_syms && !mod->crcs) \|\|
2796	(mod->num_gpl_syms && !mod->gpl_crcs)) {
2797	return try_to_force_load(mod,
2798	"no versions for exported symbols");
2799	}
2800	#endif
2801	return 0;
2802	}
2803
2804	static void flush_module_icache(const struct module *mod)
2805	{
2806	/*
2807	* Flush the instruction cache, since we've played with text.
2808	* Do it before processing of module parameters, so the module
2809	* can provide parameter accessor functions of its own.
2810	*/
2811	for_each_mod_mem_type(type) {
2812	const struct module_memory *mod_mem = &mod->mem[type];
2813
2814	if (mod_mem->size) {
2815	flush_icache_range((unsigned long)mod_mem->base,
2816	(unsigned long)mod_mem->base + mod_mem->size);
2817	}
2818	}
2819	}
2820
2821	bool __weak module_elf_check_arch(Elf_Ehdr *hdr)
2822	{
2823	return true;
2824	}
2825
2826	int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
2827	Elf_Shdr *sechdrs,
2828	char *secstrings,
2829	struct module *mod)
2830	{
2831	return 0;
2832	}
2833
2834	/* module_blacklist is a comma-separated list of module names */
2835	static char *module_blacklist;
2836	static bool blacklisted(const char *module_name)
2837	{
2838	const char *p;
2839	size_t len;
2840
2841	if (!module_blacklist)
2842	return false;
2843
2844	for (p = module_blacklist; *p; p += len) {
2845	len = strcspn(p, ",");
2846	if (strlen(module_name) == len && !memcmp(module_name, p, len))
2847	return true;
2848	if (p[len] == ',')
2849	len++;
2850	}
2851	return false;
2852	}
2853	core_param(module_blacklist, module_blacklist, charp, 0400);
2854
2855	static struct module layout_and_allocate(struct load_info info, int flags)
2856	{
2857	struct module *mod;
2858	int err;
2859
2860	/* Allow arches to frob section contents and sizes. */
2861	err = module_frob_arch_sections(info->hdr, info->sechdrs,
2862	info->secstrings, info->mod);
2863	if (err < 0)
2864	return ERR_PTR(err);
2865
2866	err = module_enforce_rwx_sections(info->hdr, info->sechdrs,
2867	info->secstrings, info->mod);
2868	if (err < 0)
2869	return ERR_PTR(err);
2870
2871	/* We will do a special allocation for per-cpu sections later. */
2872	info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
2873
2874	/*
2875	* Mark relevant sections as SHF_RO_AFTER_INIT so layout_sections() can
2876	* put them in the right place.
2877	* Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set.
2878	*/
2879	module_mark_ro_after_init(info->hdr, info->sechdrs, info->secstrings);
2880
2881	/*
2882	* Determine total sizes, and put offsets in sh_entsize. For now
2883	* this is done generically; there doesn't appear to be any
2884	* special cases for the architectures.
2885	*/
2886	layout_sections(info->mod, info);
2887	layout_symtab(info->mod, info);
2888
2889	/* Allocate and move to the final place */
2890	err = move_module(info->mod, info);
2891	if (err)
2892	return ERR_PTR(err);
2893
2894	/* Module has been copied to its final place now: return it. */
2895	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2896	kmemleak_load_module(mod, info);
2897	codetag_module_replaced(info->mod, mod);
2898
2899	return mod;
2900	}
2901
2902	/* mod is no longer valid after this! */
2903	static void module_deallocate(struct module mod, struct load_info info)
2904	{
2905	percpu_modfree(mod);
2906	module_arch_freeing_init(mod);
2907	codetag_free_module_sections(mod);
2908
2909	free_mod_mem(mod);
2910	}
2911
2912	int __weak module_finalize(const Elf_Ehdr *hdr,
2913	const Elf_Shdr *sechdrs,
2914	struct module *me)
2915	{
2916	return 0;
2917	}
2918
2919	static int post_relocation(struct module mod, const struct load_info info)
2920	{
2921	/* Sort exception table now relocations are done. */
2922	sort_extable(mod->extable, mod->extable + mod->num_exentries);
2923
2924	/* Copy relocated percpu area over. */
2925	percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
2926	info->sechdrs[info->index.pcpu].sh_size);
2927
2928	/* Setup kallsyms-specific fields. */
2929	add_kallsyms(mod, info);
2930
2931	/* Arch-specific module finalizing. */
2932	return module_finalize(info->hdr, info->sechdrs, mod);
2933	}
2934
2935	/* Call module constructors. */
2936	static void do_mod_ctors(struct module *mod)
2937	{
2938	#ifdef CONFIG_CONSTRUCTORS
2939	unsigned long i;
2940
2941	for (i = 0; i < mod->num_ctors; i++)
2942	mod->ctors[i]();
2943	#endif
2944	}
2945
2946	/* For freeing module_init on success, in case kallsyms traversing */
2947	struct mod_initfree {
2948	struct llist_node node;
2949	void *init_text;
2950	void *init_data;
2951	void *init_rodata;
2952	};
2953
2954	static void do_free_init(struct work_struct *w)
2955	{
2956	struct llist_node pos, n, *list;
2957	struct mod_initfree *initfree;
2958
2959	list = llist_del_all(&init_free_list);
2960
2961	synchronize_rcu();
2962
2963	llist_for_each_safe(pos, n, list) {
2964	initfree = container_of(pos, struct mod_initfree, node);
2965	execmem_free(initfree->init_text);
2966	execmem_free(initfree->init_data);
2967	execmem_free(initfree->init_rodata);
2968	kfree(initfree);
2969	}
2970	}
2971
2972	void flush_module_init_free_work(void)
2973	{
2974	flush_work(&init_free_wq);
2975	}
2976
2977	#undef MODULE_PARAM_PREFIX
2978	#define MODULE_PARAM_PREFIX "module."
2979	/* Default value for module->async_probe_requested */
2980	static bool async_probe;
2981	module_param(async_probe, bool, 0644);
2982
2983	/*
2984	* This is where the real work happens.
2985	*
2986	* Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
2987	* helper command 'lx-symbols'.
2988	*/
2989	static noinline int do_init_module(struct module *mod)
2990	{
2991	int ret = 0;
2992	struct mod_initfree *freeinit;
2993	#if defined(CONFIG_MODULE_STATS)
2994	unsigned int text_size = 0, total_size = 0;
2995
2996	for_each_mod_mem_type(type) {
2997	const struct module_memory *mod_mem = &mod->mem[type];
2998	if (mod_mem->size) {
2999	total_size += mod_mem->size;
3000	if (type == MOD_TEXT \|\| type == MOD_INIT_TEXT)
3001	text_size += mod_mem->size;
3002	}
3003	}
3004	#endif
3005
3006	freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
3007	if (!freeinit) {
3008	ret = -ENOMEM;
3009	goto fail;
3010	}
3011	freeinit->init_text = mod->mem[MOD_INIT_TEXT].base;
3012	freeinit->init_data = mod->mem[MOD_INIT_DATA].base;
3013	freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base;
3014
3015	do_mod_ctors(mod);
3016	/* Start the module */
3017	if (mod->init != NULL)
3018	ret = do_one_initcall(mod->init);
3019	if (ret < 0) {
3020	goto fail_free_freeinit;
3021	}
3022	if (ret > 0) {
3023	pr_warn("%s: '%s'->init suspiciously returned %d, it should "
3024	"follow 0/-E convention\n"
3025	"%s: loading module anyway...\n",
3026	__func__, mod->name, ret, __func__);
3027	dump_stack();
3028	}
3029
3030	/* Now it's a first class citizen! */
3031	mod->state = MODULE_STATE_LIVE;
3032	blocking_notifier_call_chain(&module_notify_list,
3033	MODULE_STATE_LIVE, mod);
3034
3035	/* Delay uevent until module has finished its init routine */
3036	kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
3037
3038	/*
3039	* We need to finish all async code before the module init sequence
3040	* is done. This has potential to deadlock if synchronous module
3041	* loading is requested from async (which is not allowed!).
3042	*
3043	* See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous
3044	* request_module() from async workers") for more details.
3045	*/
3046	if (!mod->async_probe_requested)
3047	async_synchronize_full();
3048
3049	ftrace_free_mem(mod, mod->mem[MOD_INIT_TEXT].base,
3050	mod->mem[MOD_INIT_TEXT].base + mod->mem[MOD_INIT_TEXT].size);
3051	mutex_lock(&module_mutex);
3052	/* Drop initial reference. */
3053	module_put(mod);
3054	trim_init_extable(mod);
3055	#ifdef CONFIG_KALLSYMS
3056	/* Switch to core kallsyms now init is done: kallsyms may be walking! */
3057	rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
3058	#endif
3059	ret = module_enable_rodata_ro_after_init(mod);
3060	if (ret)
3061	pr_warn("%s: module_enable_rodata_ro_after_init() returned %d, "
3062	"ro_after_init data might still be writable\n",
3063	mod->name, ret);
3064
3065	mod_tree_remove_init(mod);
3066	module_arch_freeing_init(mod);
3067	for_class_mod_mem_type(type, init) {
3068	mod->mem[type].base = NULL;
3069	mod->mem[type].size = 0;
3070	}
3071
3072	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
3073	/* .BTF is not SHF_ALLOC and will get removed, so sanitize pointers */
3074	mod->btf_data = NULL;
3075	mod->btf_base_data = NULL;
3076	#endif
3077	/*
3078	* We want to free module_init, but be aware that kallsyms may be
3079	* walking this within an RCU read section. In all the failure paths, we
3080	* call synchronize_rcu(), but we don't want to slow down the success
3081	* path. execmem_free() cannot be called in an interrupt, so do the
3082	* work and call synchronize_rcu() in a work queue.
3083	*
3084	* Note that execmem_alloc() on most architectures creates W+X page
3085	* mappings which won't be cleaned up until do_free_init() runs. Any
3086	* code such as mark_rodata_ro() which depends on those mappings to
3087	* be cleaned up needs to sync with the queued work by invoking
3088	* flush_module_init_free_work().
3089	*/
3090	if (llist_add(&freeinit->node, &init_free_list))
3091	schedule_work(&init_free_wq);
3092
3093	mutex_unlock(&module_mutex);
3094	wake_up_all(&module_wq);
3095
3096	mod_stat_add_long(text_size, &total_text_size);
3097	mod_stat_add_long(total_size, &total_mod_size);
3098
3099	mod_stat_inc(&modcount);
3100
3101	return 0;
3102
3103	fail_free_freeinit:
3104	kfree(freeinit);
3105	fail:
3106	/* Try to protect us from buggy refcounters. */
3107	mod->state = MODULE_STATE_GOING;
3108	synchronize_rcu();
3109	module_put(mod);
3110	blocking_notifier_call_chain(&module_notify_list,
3111	MODULE_STATE_GOING, mod);
3112	klp_module_going(mod);
3113	ftrace_release_mod(mod);
3114	free_module(mod);
3115	wake_up_all(&module_wq);
3116
3117	return ret;
3118	}
3119
3120	static int may_init_module(void)
3121	{
3122	if (!capable(CAP_SYS_MODULE) \|\| modules_disabled)
3123	return -EPERM;
3124
3125	return 0;
3126	}
3127
3128	/* Is this module of this name done loading? No locks held. */
3129	static bool finished_loading(const char *name)
3130	{
3131	struct module *mod;
3132	bool ret;
3133
3134	/*
3135	* The module_mutex should not be a heavily contended lock;
3136	* if we get the occasional sleep here, we'll go an extra iteration
3137	* in the wait_event_interruptible(), which is harmless.
3138	*/
3139	sched_annotate_sleep();
3140	mutex_lock(&module_mutex);
3141	mod = find_module_all(name, strlen(name), true);
3142	ret = !mod \|\| mod->state == MODULE_STATE_LIVE
3143	\|\| mod->state == MODULE_STATE_GOING;
3144	mutex_unlock(&module_mutex);
3145
3146	return ret;
3147	}
3148
3149	/* Must be called with module_mutex held */
3150	static int module_patient_check_exists(const char *name,
3151	enum fail_dup_mod_reason reason)
3152	{
3153	struct module *old;
3154	int err = 0;
3155
3156	old = find_module_all(name, strlen(name), true);
3157	if (old == NULL)
3158	return 0;
3159
3160	if (old->state == MODULE_STATE_COMING \|\|
3161	old->state == MODULE_STATE_UNFORMED) {
3162	/* Wait in case it fails to load. */
3163	mutex_unlock(&module_mutex);
3164	err = wait_event_interruptible(module_wq,
3165	finished_loading(name));
3166	mutex_lock(&module_mutex);
3167	if (err)
3168	return err;
3169
3170	/* The module might have gone in the meantime. */
3171	old = find_module_all(name, strlen(name), true);
3172	}
3173
3174	if (try_add_failed_module(name, reason))
3175	pr_warn("Could not add fail-tracking for module: %s\n", name);
3176
3177	/*
3178	* We are here only when the same module was being loaded. Do
3179	* not try to load it again right now. It prevents long delays
3180	* caused by serialized module load failures. It might happen
3181	* when more devices of the same type trigger load of
3182	* a particular module.
3183	*/
3184	if (old && old->state == MODULE_STATE_LIVE)
3185	return -EEXIST;
3186	return -EBUSY;
3187	}
3188
3189	/*
3190	* We try to place it in the list now to make sure it's unique before
3191	* we dedicate too many resources. In particular, temporary percpu
3192	* memory exhaustion.
3193	*/
3194	static int add_unformed_module(struct module *mod)
3195	{
3196	int err;
3197
3198	mod->state = MODULE_STATE_UNFORMED;
3199
3200	mutex_lock(&module_mutex);
3201	err = module_patient_check_exists(mod->name, FAIL_DUP_MOD_LOAD);
3202	if (err)
3203	goto out;
3204
3205	mod_update_bounds(mod);
3206	list_add_rcu(&mod->list, &modules);
3207	mod_tree_insert(mod);
3208	err = 0;
3209
3210	out:
3211	mutex_unlock(&module_mutex);
3212	return err;
3213	}
3214
3215	static int complete_formation(struct module mod, struct load_info info)
3216	{
3217	int err;
3218
3219	mutex_lock(&module_mutex);
3220
3221	/* Find duplicate symbols (must be called under lock). */
3222	err = verify_exported_symbols(mod);
3223	if (err < 0)
3224	goto out;
3225
3226	/* These rely on module_mutex for list integrity. */
3227	module_bug_finalize(info->hdr, info->sechdrs, mod);
3228	module_cfi_finalize(info->hdr, info->sechdrs, mod);
3229
3230	err = module_enable_rodata_ro(mod);
3231	if (err)
3232	goto out_strict_rwx;
3233	err = module_enable_data_nx(mod);
3234	if (err)
3235	goto out_strict_rwx;
3236	err = module_enable_text_rox(mod);
3237	if (err)
3238	goto out_strict_rwx;
3239
3240	/*
3241	* Mark state as coming so strong_try_module_get() ignores us,
3242	* but kallsyms etc. can see us.
3243	*/
3244	mod->state = MODULE_STATE_COMING;
3245	mutex_unlock(&module_mutex);
3246
3247	return 0;
3248
3249	out_strict_rwx:
3250	module_bug_cleanup(mod);
3251	out:
3252	mutex_unlock(&module_mutex);
3253	return err;
3254	}
3255
3256	static int prepare_coming_module(struct module *mod)
3257	{
3258	int err;
3259
3260	ftrace_module_enable(mod);
3261	err = klp_module_coming(mod);
3262	if (err)
3263	return err;
3264
3265	err = blocking_notifier_call_chain_robust(&module_notify_list,
3266	MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
3267	err = notifier_to_errno(err);
3268	if (err)
3269	klp_module_going(mod);
3270
3271	return err;
3272	}
3273
3274	static int unknown_module_param_cb(char param, char val, const char *modname,
3275	void *arg)
3276	{
3277	struct module *mod = arg;
3278	int ret;
3279
3280	if (strcmp(param, "async_probe") == 0) {
3281	if (kstrtobool(val, &mod->async_probe_requested))
3282	mod->async_probe_requested = true;
3283	return 0;
3284	}
3285
3286	/* Check for magic 'dyndbg' arg */
3287	ret = ddebug_dyndbg_module_param_cb(param, val, modname);
3288	if (ret != 0)
3289	pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
3290	return 0;
3291	}
3292
3293	/* Module within temporary copy, this doesn't do any allocation */
3294	static int early_mod_check(struct load_info *info, int flags)
3295	{
3296	int err;
3297
3298	/*
3299	* Now that we know we have the correct module name, check
3300	* if it's blacklisted.
3301	*/
3302	if (blacklisted(info->name)) {
3303	pr_err("Module %s is blacklisted\n", info->name);
3304	return -EPERM;
3305	}
3306
3307	err = rewrite_section_headers(info, flags);
3308	if (err)
3309	return err;
3310
3311	/* Check module struct version now, before we try to use module. */
3312	if (!check_modstruct_version(info, info->mod))
3313	return -ENOEXEC;
3314
3315	err = check_modinfo(info->mod, info, flags);
3316	if (err)
3317	return err;
3318
3319	mutex_lock(&module_mutex);
3320	err = module_patient_check_exists(info->mod->name, FAIL_DUP_MOD_BECOMING);
3321	mutex_unlock(&module_mutex);
3322
3323	return err;
3324	}
3325
3326	/*
3327	* Allocate and load the module: note that size of section 0 is always
3328	* zero, and we rely on this for optional sections.
3329	*/
3330	static int load_module(struct load_info info, const char __user uargs,
3331	int flags)
3332	{
3333	struct module *mod;
3334	bool module_allocated = false;
3335	long err = 0;
3336	char *after_dashes;
3337
3338	/*
3339	* Do the signature check (if any) first. All that
3340	* the signature check needs is info->len, it does
3341	* not need any of the section info. That can be
3342	* set up later. This will minimize the chances
3343	* of a corrupt module causing problems before
3344	* we even get to the signature check.
3345	*
3346	* The check will also adjust info->len by stripping
3347	* off the sig length at the end of the module, making
3348	* checks against info->len more correct.
3349	*/
3350	err = module_sig_check(info, flags);
3351	if (err)
3352	goto free_copy;
3353
3354	/*
3355	* Do basic sanity checks against the ELF header and
3356	* sections. Cache useful sections and set the
3357	* info->mod to the userspace passed struct module.
3358	*/
3359	err = elf_validity_cache_copy(info, flags);
3360	if (err)
3361	goto free_copy;
3362
3363	err = early_mod_check(info, flags);
3364	if (err)
3365	goto free_copy;
3366
3367	/* Figure out module layout, and allocate all the memory. */
3368	mod = layout_and_allocate(info, flags);
3369	if (IS_ERR(mod)) {
3370	err = PTR_ERR(mod);
3371	goto free_copy;
3372	}
3373
3374	module_allocated = true;
3375
3376	audit_log_kern_module(mod->name);
3377
3378	/* Reserve our place in the list. */
3379	err = add_unformed_module(mod);
3380	if (err)
3381	goto free_module;
3382
3383	/*
3384	* We are tainting your kernel if your module gets into
3385	* the modules linked list somehow.
3386	*/
3387	module_augment_kernel_taints(mod, info);
3388
3389	/* To avoid stressing percpu allocator, do this once we're unique. */
3390	err = percpu_modalloc(mod, info);
3391	if (err)
3392	goto unlink_mod;
3393
3394	/* Now module is in final location, initialize linked lists, etc. */
3395	err = module_unload_init(mod);
3396	if (err)
3397	goto unlink_mod;
3398
3399	init_param_lock(mod);
3400
3401	/*
3402	* Now we've got everything in the final locations, we can
3403	* find optional sections.
3404	*/
3405	err = find_module_sections(mod, info);
3406	if (err)
3407	goto free_unload;
3408
3409	err = check_export_symbol_versions(mod);
3410	if (err)
3411	goto free_unload;
3412
3413	/* Set up MODINFO_ATTR fields */
3414	err = setup_modinfo(mod, info);
3415	if (err)
3416	goto free_modinfo;
3417
3418	/* Fix up syms, so that st_value is a pointer to location. */
3419	err = simplify_symbols(mod, info);
3420	if (err < 0)
3421	goto free_modinfo;
3422
3423	err = apply_relocations(mod, info);
3424	if (err < 0)
3425	goto free_modinfo;
3426
3427	err = post_relocation(mod, info);
3428	if (err < 0)
3429	goto free_modinfo;
3430
3431	flush_module_icache(mod);
3432
3433	/* Now copy in args */
3434	mod->args = strndup_user(uargs, ~0UL >> 1);
3435	if (IS_ERR(mod->args)) {
3436	err = PTR_ERR(mod->args);
3437	goto free_arch_cleanup;
3438	}
3439
3440	init_build_id(mod, info);
3441
3442	/* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
3443	ftrace_module_init(mod);
3444
3445	/* Finally it's fully formed, ready to start executing. */
3446	err = complete_formation(mod, info);
3447	if (err)
3448	goto ddebug_cleanup;
3449
3450	err = prepare_coming_module(mod);
3451	if (err)
3452	goto bug_cleanup;
3453
3454	mod->async_probe_requested = async_probe;
3455
3456	/* Module is ready to execute: parsing args may do that. */
3457	after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
3458	-32768, 32767, mod,
3459	unknown_module_param_cb);
3460	if (IS_ERR(after_dashes)) {
3461	err = PTR_ERR(after_dashes);
3462	goto coming_cleanup;
3463	} else if (after_dashes) {
3464	pr_warn("%s: parameters '%s' after `--' ignored\n",
3465	mod->name, after_dashes);
3466	}
3467
3468	/* Link in to sysfs. */
3469	err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
3470	if (err < 0)
3471	goto coming_cleanup;
3472
3473	if (is_livepatch_module(mod)) {
3474	err = copy_module_elf(mod, info);
3475	if (err < 0)
3476	goto sysfs_cleanup;
3477	}
3478
3479	if (codetag_load_module(mod))
3480	goto sysfs_cleanup;
3481
3482	/* Get rid of temporary copy. */
3483	free_copy(info, flags);
3484
3485	/* Done! */
3486	trace_module_load(mod);
3487
3488	return do_init_module(mod);
3489
3490	sysfs_cleanup:
3491	mod_sysfs_teardown(mod);
3492	coming_cleanup:
3493	mod->state = MODULE_STATE_GOING;
3494	destroy_params(mod->kp, mod->num_kp);
3495	blocking_notifier_call_chain(&module_notify_list,
3496	MODULE_STATE_GOING, mod);
3497	klp_module_going(mod);
3498	bug_cleanup:
3499	mod->state = MODULE_STATE_GOING;
3500	/* module_bug_cleanup needs module_mutex protection */
3501	mutex_lock(&module_mutex);
3502	module_bug_cleanup(mod);
3503	mutex_unlock(&module_mutex);
3504
3505	ddebug_cleanup:
3506	ftrace_release_mod(mod);
3507	synchronize_rcu();
3508	kfree(mod->args);
3509	free_arch_cleanup:
3510	module_arch_cleanup(mod);
3511	free_modinfo:
3512	free_modinfo(mod);
3513	free_unload:
3514	module_unload_free(mod);
3515	unlink_mod:
3516	mutex_lock(&module_mutex);
3517	/* Unlink carefully: kallsyms could be walking list. */
3518	list_del_rcu(&mod->list);
3519	mod_tree_remove(mod);
3520	wake_up_all(&module_wq);
3521	/* Wait for RCU-sched synchronizing before releasing mod->list. */
3522	synchronize_rcu();
3523	mutex_unlock(&module_mutex);
3524	free_module:
3525	mod_stat_bump_invalid(info, flags);
3526	/* Free lock-classes; relies on the preceding sync_rcu() */
3527	for_class_mod_mem_type(type, core_data) {
3528	lockdep_free_key_range(mod->mem[type].base,
3529	mod->mem[type].size);
3530	}
3531
3532	module_memory_restore_rox(mod);
3533	module_deallocate(mod, info);
3534	free_copy:
3535	/*
3536	* The info->len is always set. We distinguish between
3537	* failures once the proper module was allocated and
3538	* before that.
3539	*/
3540	if (!module_allocated)
3541	mod_stat_bump_becoming(info, flags);
3542	free_copy(info, flags);
3543	return err;
3544	}
3545
3546	SYSCALL_DEFINE3(init_module, void __user *, umod,
3547	unsigned long, len, const char __user *, uargs)
3548	{
3549	int err;
3550	struct load_info info = { };
3551
3552	err = may_init_module();
3553	if (err)
3554	return err;
3555
3556	pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
3557	umod, len, uargs);
3558
3559	err = copy_module_from_user(umod, len, &info);
3560	if (err) {
3561	mod_stat_inc(&failed_kreads);
3562	mod_stat_add_long(len, &invalid_kread_bytes);
3563	return err;
3564	}
3565
3566	return load_module(&info, uargs, 0);
3567	}
3568
3569	struct idempotent {
3570	const void *cookie;
3571	struct hlist_node entry;
3572	struct completion complete;
3573	int ret;
3574	};
3575
3576	#define IDEM_HASH_BITS 8
3577	static struct hlist_head idem_hash[1 << IDEM_HASH_BITS];
3578	static DEFINE_SPINLOCK(idem_lock);
3579
3580	static bool idempotent(struct idempotent u, const void cookie)
3581	{
3582	int hash = hash_ptr(cookie, IDEM_HASH_BITS);
3583	struct hlist_head *head = idem_hash + hash;
3584	struct idempotent *existing;
3585	bool first;
3586
3587	u->ret = -EINTR;
3588	u->cookie = cookie;
3589	init_completion(&u->complete);
3590
3591	spin_lock(&idem_lock);
3592	first = true;
3593	hlist_for_each_entry(existing, head, entry) {
3594	if (existing->cookie != cookie)
3595	continue;
3596	first = false;
3597	break;
3598	}
3599	hlist_add_head(&u->entry, idem_hash + hash);
3600	spin_unlock(&idem_lock);
3601
3602	return !first;
3603	}
3604
3605	/*
3606	* We were the first one with 'cookie' on the list, and we ended
3607	* up completing the operation. We now need to walk the list,
3608	* remove everybody - which includes ourselves - fill in the return
3609	* value, and then complete the operation.
3610	*/
3611	static int idempotent_complete(struct idempotent *u, int ret)
3612	{
3613	const void *cookie = u->cookie;
3614	int hash = hash_ptr(cookie, IDEM_HASH_BITS);
3615	struct hlist_head *head = idem_hash + hash;
3616	struct hlist_node *next;
3617	struct idempotent *pos;
3618
3619	spin_lock(&idem_lock);
3620	hlist_for_each_entry_safe(pos, next, head, entry) {
3621	if (pos->cookie != cookie)
3622	continue;
3623	hlist_del_init(&pos->entry);
3624	pos->ret = ret;
3625	complete(&pos->complete);
3626	}
3627	spin_unlock(&idem_lock);
3628	return ret;
3629	}
3630
3631	/*
3632	* Wait for the idempotent worker.
3633	*
3634	* If we get interrupted, we need to remove ourselves from the
3635	* the idempotent list, and the completion may still come in.
3636	*
3637	* The 'idem_lock' protects against the race, and 'idem.ret' was
3638	* initialized to -EINTR and is thus always the right return
3639	* value even if the idempotent work then completes between
3640	* the wait_for_completion and the cleanup.
3641	*/
3642	static int idempotent_wait_for_completion(struct idempotent *u)
3643	{
3644	if (wait_for_completion_interruptible(&u->complete)) {
3645	spin_lock(&idem_lock);
3646	if (!hlist_unhashed(&u->entry))
3647	hlist_del(&u->entry);
3648	spin_unlock(&idem_lock);
3649	}
3650	return u->ret;
3651	}
3652
3653	static int init_module_from_file(struct file f, const char __user uargs, int flags)
3654	{
3655	struct load_info info = { };
3656	void *buf = NULL;
3657	int len;
3658
3659	len = kernel_read_file(f, 0, &buf, INT_MAX, NULL, READING_MODULE);
3660	if (len < 0) {
3661	mod_stat_inc(&failed_kreads);
3662	return len;
3663	}
3664
3665	if (flags & MODULE_INIT_COMPRESSED_FILE) {
3666	int err = module_decompress(&info, buf, len);
3667	vfree(buf); /* compressed data is no longer needed */
3668	if (err) {
3669	mod_stat_inc(&failed_decompress);
3670	mod_stat_add_long(len, &invalid_decompress_bytes);
3671	return err;
3672	}
3673	} else {
3674	info.hdr = buf;
3675	info.len = len;
3676	}
3677
3678	return load_module(&info, uargs, flags);
3679	}
3680
3681	static int idempotent_init_module(struct file f, const char __user uargs, int flags)
3682	{
3683	struct idempotent idem;
3684
3685	if (!(f->f_mode & FMODE_READ))
3686	return -EBADF;
3687
3688	/* Are we the winners of the race and get to do this? */
3689	if (!idempotent(&idem, file_inode(f))) {
3690	int ret = init_module_from_file(f, uargs, flags);
3691	return idempotent_complete(&idem, ret);
3692	}
3693
3694	/*
3695	* Somebody else won the race and is loading the module.
3696	*/
3697	return idempotent_wait_for_completion(&idem);
3698	}
3699
3700	SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
3701	{
3702	int err = may_init_module();
3703	if (err)
3704	return err;
3705
3706	pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
3707
3708	if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
3709	\|MODULE_INIT_IGNORE_VERMAGIC
3710	\|MODULE_INIT_COMPRESSED_FILE))
3711	return -EINVAL;
3712
3713	CLASS(fd, f)(fd);
3714	if (fd_empty(f))
3715	return -EBADF;
3716	return idempotent_init_module(fd_file(f), uargs, flags);
3717	}
3718
3719	/* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
3720	char module_flags(struct module mod, char *buf, bool show_state)
3721	{
3722	int bx = 0;
3723
3724	BUG_ON(mod->state == MODULE_STATE_UNFORMED);
3725	if (!mod->taints && !show_state)
3726	goto out;
3727	if (mod->taints \|\|
3728	mod->state == MODULE_STATE_GOING \|\|
3729	mod->state == MODULE_STATE_COMING) {
3730	buf[bx++] = '(';
3731	bx += module_flags_taint(mod->taints, buf + bx);
3732	/* Show a - for module-is-being-unloaded */
3733	if (mod->state == MODULE_STATE_GOING && show_state)
3734	buf[bx++] = '-';
3735	/* Show a + for module-is-being-loaded */
3736	if (mod->state == MODULE_STATE_COMING && show_state)
3737	buf[bx++] = '+';
3738	buf[bx++] = ')';
3739	}
3740	out:
3741	buf[bx] = '\0';
3742
3743	return buf;
3744	}
3745
3746	/* Given an address, look for it in the module exception tables. */
3747	const struct exception_table_entry *search_module_extables(unsigned long addr)
3748	{
3749	struct module *mod;
3750
3751	guard(rcu)();
3752	mod = __module_address(addr);
3753	if (!mod)
3754	return NULL;
3755
3756	if (!mod->num_exentries)
3757	return NULL;
3758	/*
3759	* The address passed here belongs to a module that is currently
3760	* invoked (we are running inside it). Therefore its module::refcnt
3761	* needs already be >0 to ensure that it is not removed at this stage.
3762	* All other user need to invoke this function within a RCU read
3763	* section.
3764	*/
3765	return search_extable(mod->extable, mod->num_exentries, addr);
3766	}
3767
3768	/**
3769	* is_module_address() - is this address inside a module?
3770	* @addr: the address to check.
3771	*
3772	* See is_module_text_address() if you simply want to see if the address
3773	* is code (not data).
3774	*/
3775	bool is_module_address(unsigned long addr)
3776	{
3777	guard(rcu)();
3778	return __module_address(addr) != NULL;
3779	}
3780
3781	/**
3782	* __module_address() - get the module which contains an address.
3783	* @addr: the address.
3784	*
3785	* Must be called within RCU read section or module mutex held so that
3786	* module doesn't get freed during this.
3787	*/
3788	struct module *__module_address(unsigned long addr)
3789	{
3790	struct module *mod;
3791
3792	if (addr >= mod_tree.addr_min && addr <= mod_tree.addr_max)
3793	goto lookup;
3794
3795	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
3796	if (addr >= mod_tree.data_addr_min && addr <= mod_tree.data_addr_max)
3797	goto lookup;
3798	#endif
3799
3800	return NULL;
3801
3802	lookup:
3803	mod = mod_find(addr, &mod_tree);
3804	if (mod) {
3805	BUG_ON(!within_module(addr, mod));
3806	if (mod->state == MODULE_STATE_UNFORMED)
3807	mod = NULL;
3808	}
3809	return mod;
3810	}
3811
3812	/**
3813	* is_module_text_address() - is this address inside module code?
3814	* @addr: the address to check.
3815	*
3816	* See is_module_address() if you simply want to see if the address is
3817	* anywhere in a module. See kernel_text_address() for testing if an
3818	* address corresponds to kernel or module code.
3819	*/
3820	bool is_module_text_address(unsigned long addr)
3821	{
3822	guard(rcu)();
3823	return __module_text_address(addr) != NULL;
3824	}
3825
3826	void module_for_each_mod(int(func)(struct module mod, void data), void data)
3827	{
3828	struct module *mod;
3829
3830	guard(rcu)();
3831	list_for_each_entry_rcu(mod, &modules, list) {
3832	if (mod->state == MODULE_STATE_UNFORMED)
3833	continue;
3834	if (func(mod, data))
3835	break;
3836	}
3837	}
3838
3839	/**
3840	* __module_text_address() - get the module whose code contains an address.
3841	* @addr: the address.
3842	*
3843	* Must be called within RCU read section or module mutex held so that
3844	* module doesn't get freed during this.
3845	*/
3846	struct module *__module_text_address(unsigned long addr)
3847	{
3848	struct module *mod = __module_address(addr);
3849	if (mod) {
3850	/* Make sure it's within the text section. */
3851	if (!within_module_mem_type(addr, mod, MOD_TEXT) &&
3852	!within_module_mem_type(addr, mod, MOD_INIT_TEXT))
3853	mod = NULL;
3854	}
3855	return mod;
3856	}
3857
3858	/* Don't grab lock, we're oopsing. */
3859	void print_modules(void)
3860	{
3861	struct module *mod;
3862	char buf[MODULE_FLAGS_BUF_SIZE];
3863
3864	printk(KERN_DEFAULT "Modules linked in:");
3865	/* Most callers should already have preempt disabled, but make sure */
3866	guard(rcu)();
3867	list_for_each_entry_rcu(mod, &modules, list) {
3868	if (mod->state == MODULE_STATE_UNFORMED)
3869	continue;
3870	pr_cont(" %s%s", mod->name, module_flags(mod, buf, true));
3871	}
3872
3873	print_unloaded_tainted_modules();
3874	if (last_unloaded_module.name[0])
3875	pr_cont(" [last unloaded: %s%s]", last_unloaded_module.name,
3876	last_unloaded_module.taints);
3877	pr_cont("\n");
3878	}
3879
3880	#ifdef CONFIG_MODULE_DEBUGFS
3881	struct dentry *mod_debugfs_root;
3882
3883	static int module_debugfs_init(void)
3884	{
3885	mod_debugfs_root = debugfs_create_dir("modules", NULL);
3886	return 0;
3887	}
3888	module_init(module_debugfs_init);
3889	#endif