linux - kernel/module/main.c

3417 lines · 86.6 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2002 Richard Henderson
4	* Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5	* Copyright (C) 2023 Luis Chamberlain <[email protected]>
6	*/
7
8	#define INCLUDE_VERMAGIC
9
10	#include <linux/export.h>
11	#include <linux/extable.h>
12	#include <linux/moduleloader.h>
13	#include <linux/module_signature.h>
14	#include <linux/trace_events.h>
15	#include <linux/init.h>
16	#include <linux/kallsyms.h>
17	#include <linux/buildid.h>
18	#include <linux/fs.h>
19	#include <linux/kernel.h>
20	#include <linux/kernel_read_file.h>
21	#include <linux/kstrtox.h>
22	#include <linux/slab.h>
23	#include <linux/vmalloc.h>
24	#include <linux/elf.h>
25	#include <linux/seq_file.h>
26	#include <linux/syscalls.h>
27	#include <linux/fcntl.h>
28	#include <linux/rcupdate.h>
29	#include <linux/capability.h>
30	#include <linux/cpu.h>
31	#include <linux/moduleparam.h>
32	#include <linux/errno.h>
33	#include <linux/err.h>
34	#include <linux/vermagic.h>
35	#include <linux/notifier.h>
36	#include <linux/sched.h>
37	#include <linux/device.h>
38	#include <linux/string.h>
39	#include <linux/mutex.h>
40	#include <linux/rculist.h>
41	#include <linux/uaccess.h>
42	#include <asm/cacheflush.h>
43	#include <linux/set_memory.h>
44	#include <asm/mmu_context.h>
45	#include <linux/license.h>
46	#include <asm/sections.h>
47	#include <linux/tracepoint.h>
48	#include <linux/ftrace.h>
49	#include <linux/livepatch.h>
50	#include <linux/async.h>
51	#include <linux/percpu.h>
52	#include <linux/kmemleak.h>
53	#include <linux/jump_label.h>
54	#include <linux/pfn.h>
55	#include <linux/bsearch.h>
56	#include <linux/dynamic_debug.h>
57	#include <linux/audit.h>
58	#include <linux/cfi.h>
59	#include <linux/codetag.h>
60	#include <linux/debugfs.h>
61	#include <linux/execmem.h>
62	#include <uapi/linux/module.h>
63	#include "internal.h"
64
65	#define CREATE_TRACE_POINTS
66	#include <trace/events/module.h>
67
68	/*
69	* Mutex protects:
70	* 1) List of modules (also safely readable with preempt_disable),
71	* 2) module_use links,
72	* 3) mod_tree.addr_min/mod_tree.addr_max.
73	* (delete and add uses RCU list operations).
74	*/
75	DEFINE_MUTEX(module_mutex);
76	LIST_HEAD(modules);
77
78	/* Work queue for freeing init sections in success case */
79	static void do_free_init(struct work_struct *w);
80	static DECLARE_WORK(init_free_wq, do_free_init);
81	static LLIST_HEAD(init_free_list);
82
83	struct mod_tree_root mod_tree __cacheline_aligned = {
84	.addr_min = -1UL,
85	};
86
87	struct symsearch {
88	const struct kernel_symbol start, stop;
89	const s32 *crcs;
90	enum mod_license license;
91	};
92
93	/*
94	* Bounds of module memory, for speeding up __module_address.
95	* Protected by module_mutex.
96	*/
97	static void __mod_update_bounds(enum mod_mem_type type __maybe_unused, void *base,
98	unsigned int size, struct mod_tree_root *tree)
99	{
100	unsigned long min = (unsigned long)base;
101	unsigned long max = min + size;
102
103	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
104	if (mod_mem_type_is_core_data(type)) {
105	if (min < tree->data_addr_min)
106	tree->data_addr_min = min;
107	if (max > tree->data_addr_max)
108	tree->data_addr_max = max;
109	return;
110	}
111	#endif
112	if (min < tree->addr_min)
113	tree->addr_min = min;
114	if (max > tree->addr_max)
115	tree->addr_max = max;
116	}
117
118	static void mod_update_bounds(struct module *mod)
119	{
120	for_each_mod_mem_type(type) {
121	struct module_memory *mod_mem = &mod->mem[type];
122
123	if (mod_mem->size)
124	__mod_update_bounds(type, mod_mem->base, mod_mem->size, &mod_tree);
125	}
126	}
127
128	/* Block module loading/unloading? */
129	int modules_disabled;
130	core_param(nomodule, modules_disabled, bint, 0);
131
132	/* Waiting for a module to finish initializing? */
133	static DECLARE_WAIT_QUEUE_HEAD(module_wq);
134
135	static BLOCKING_NOTIFIER_HEAD(module_notify_list);
136
137	int register_module_notifier(struct notifier_block *nb)
138	{
139	return blocking_notifier_chain_register(&module_notify_list, nb);
140	}
141	EXPORT_SYMBOL(register_module_notifier);
142
143	int unregister_module_notifier(struct notifier_block *nb)
144	{
145	return blocking_notifier_chain_unregister(&module_notify_list, nb);
146	}
147	EXPORT_SYMBOL(unregister_module_notifier);
148
149	/*
150	* We require a truly strong try_module_get(): 0 means success.
151	* Otherwise an error is returned due to ongoing or failed
152	* initialization etc.
153	*/
154	static inline int strong_try_module_get(struct module *mod)
155	{
156	BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
157	if (mod && mod->state == MODULE_STATE_COMING)
158	return -EBUSY;
159	if (try_module_get(mod))
160	return 0;
161	else
162	return -ENOENT;
163	}
164
165	static inline void add_taint_module(struct module *mod, unsigned flag,
166	enum lockdep_ok lockdep_ok)
167	{
168	add_taint(flag, lockdep_ok);
169	set_bit(flag, &mod->taints);
170	}
171
172	/*
173	* A thread that wants to hold a reference to a module only while it
174	* is running can call this to safely exit.
175	*/
176	void __noreturn __module_put_and_kthread_exit(struct module *mod, long code)
177	{
178	module_put(mod);
179	kthread_exit(code);
180	}
181	EXPORT_SYMBOL(__module_put_and_kthread_exit);
182
183	/* Find a module section: 0 means not found. */
184	static unsigned int find_sec(const struct load_info info, const char name)
185	{
186	unsigned int i;
187
188	for (i = 1; i < info->hdr->e_shnum; i++) {
189	Elf_Shdr *shdr = &info->sechdrs[i];
190	/* Alloc bit cleared means "ignore it." */
191	if ((shdr->sh_flags & SHF_ALLOC)
192	&& strcmp(info->secstrings + shdr->sh_name, name) == 0)
193	return i;
194	}
195	return 0;
196	}
197
198	/* Find a module section, or NULL. */
199	static void section_addr(const struct load_info info, const char *name)
200	{
201	/* Section 0 has sh_addr 0. */
202	return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
203	}
204
205	/* Find a module section, or NULL. Fill in number of "objects" in section. */
206	static void section_objs(const struct load_info info,
207	const char *name,
208	size_t object_size,
209	unsigned int *num)
210	{
211	unsigned int sec = find_sec(info, name);
212
213	/* Section 0 has sh_addr 0 and sh_size 0. */
214	*num = info->sechdrs[sec].sh_size / object_size;
215	return (void *)info->sechdrs[sec].sh_addr;
216	}
217
218	/* Find a module section: 0 means not found. Ignores SHF_ALLOC flag. */
219	static unsigned int find_any_sec(const struct load_info info, const char name)
220	{
221	unsigned int i;
222
223	for (i = 1; i < info->hdr->e_shnum; i++) {
224	Elf_Shdr *shdr = &info->sechdrs[i];
225	if (strcmp(info->secstrings + shdr->sh_name, name) == 0)
226	return i;
227	}
228	return 0;
229	}
230
231	/*
232	* Find a module section, or NULL. Fill in number of "objects" in section.
233	* Ignores SHF_ALLOC flag.
234	*/
235	static __maybe_unused void any_section_objs(const struct load_info info,
236	const char *name,
237	size_t object_size,
238	unsigned int *num)
239	{
240	unsigned int sec = find_any_sec(info, name);
241
242	/* Section 0 has sh_addr 0 and sh_size 0. */
243	*num = info->sechdrs[sec].sh_size / object_size;
244	return (void *)info->sechdrs[sec].sh_addr;
245	}
246
247	#ifndef CONFIG_MODVERSIONS
248	#define symversion(base, idx) NULL
249	#else
250	#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
251	#endif
252
253	static const char kernel_symbol_name(const struct kernel_symbol sym)
254	{
255	#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
256	return offset_to_ptr(&sym->name_offset);
257	#else
258	return sym->name;
259	#endif
260	}
261
262	static const char kernel_symbol_namespace(const struct kernel_symbol sym)
263	{
264	#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
265	if (!sym->namespace_offset)
266	return NULL;
267	return offset_to_ptr(&sym->namespace_offset);
268	#else
269	return sym->namespace;
270	#endif
271	}
272
273	int cmp_name(const void name, const void sym)
274	{
275	return strcmp(name, kernel_symbol_name(sym));
276	}
277
278	static bool find_exported_symbol_in_section(const struct symsearch *syms,
279	struct module *owner,
280	struct find_symbol_arg *fsa)
281	{
282	struct kernel_symbol *sym;
283
284	if (!fsa->gplok && syms->license == GPL_ONLY)
285	return false;
286
287	sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
288	sizeof(struct kernel_symbol), cmp_name);
289	if (!sym)
290	return false;
291
292	fsa->owner = owner;
293	fsa->crc = symversion(syms->crcs, sym - syms->start);
294	fsa->sym = sym;
295	fsa->license = syms->license;
296
297	return true;
298	}
299
300	/*
301	* Find an exported symbol and return it, along with, (optional) crc and
302	* (optional) module which owns it. Needs preempt disabled or module_mutex.
303	*/
304	bool find_symbol(struct find_symbol_arg *fsa)
305	{
306	static const struct symsearch arr[] = {
307	{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,
308	NOT_GPL_ONLY },
309	{ __start___ksymtab_gpl, __stop___ksymtab_gpl,
310	__start___kcrctab_gpl,
311	GPL_ONLY },
312	};
313	struct module *mod;
314	unsigned int i;
315
316	module_assert_mutex_or_preempt();
317
318	for (i = 0; i < ARRAY_SIZE(arr); i++)
319	if (find_exported_symbol_in_section(&arr[i], NULL, fsa))
320	return true;
321
322	list_for_each_entry_rcu(mod, &modules, list,
323	lockdep_is_held(&module_mutex)) {
324	struct symsearch arr[] = {
325	{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
326	NOT_GPL_ONLY },
327	{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
328	mod->gpl_crcs,
329	GPL_ONLY },
330	};
331
332	if (mod->state == MODULE_STATE_UNFORMED)
333	continue;
334
335	for (i = 0; i < ARRAY_SIZE(arr); i++)
336	if (find_exported_symbol_in_section(&arr[i], mod, fsa))
337	return true;
338	}
339
340	pr_debug("Failed to find symbol %s\n", fsa->name);
341	return false;
342	}
343
344	/*
345	* Search for module by name: must hold module_mutex (or preempt disabled
346	* for read-only access).
347	*/
348	struct module find_module_all(const char name, size_t len,
349	bool even_unformed)
350	{
351	struct module *mod;
352
353	module_assert_mutex_or_preempt();
354
355	list_for_each_entry_rcu(mod, &modules, list,
356	lockdep_is_held(&module_mutex)) {
357	if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
358	continue;
359	if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
360	return mod;
361	}
362	return NULL;
363	}
364
365	struct module find_module(const char name)
366	{
367	return find_module_all(name, strlen(name), false);
368	}
369
370	#ifdef CONFIG_SMP
371
372	static inline void __percpu mod_percpu(struct module mod)
373	{
374	return mod->percpu;
375	}
376
377	static int percpu_modalloc(struct module mod, struct load_info info)
378	{
379	Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
380	unsigned long align = pcpusec->sh_addralign;
381
382	if (!pcpusec->sh_size)
383	return 0;
384
385	if (align > PAGE_SIZE) {
386	pr_warn("%s: per-cpu alignment %li > %li\n",
387	mod->name, align, PAGE_SIZE);
388	align = PAGE_SIZE;
389	}
390
391	mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
392	if (!mod->percpu) {
393	pr_warn("%s: Could not allocate %lu bytes percpu data\n",
394	mod->name, (unsigned long)pcpusec->sh_size);
395	return -ENOMEM;
396	}
397	mod->percpu_size = pcpusec->sh_size;
398	return 0;
399	}
400
401	static void percpu_modfree(struct module *mod)
402	{
403	free_percpu(mod->percpu);
404	}
405
406	static unsigned int find_pcpusec(struct load_info *info)
407	{
408	return find_sec(info, ".data..percpu");
409	}
410
411	static void percpu_modcopy(struct module *mod,
412	const void *from, unsigned long size)
413	{
414	int cpu;
415
416	for_each_possible_cpu(cpu)
417	memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
418	}
419
420	bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
421	{
422	struct module *mod;
423	unsigned int cpu;
424
425	preempt_disable();
426
427	list_for_each_entry_rcu(mod, &modules, list) {
428	if (mod->state == MODULE_STATE_UNFORMED)
429	continue;
430	if (!mod->percpu_size)
431	continue;
432	for_each_possible_cpu(cpu) {
433	void *start = per_cpu_ptr(mod->percpu, cpu);
434	void va = (void )addr;
435
436	if (va >= start && va < start + mod->percpu_size) {
437	if (can_addr) {
438	*can_addr = (unsigned long) (va - start);
439	*can_addr += (unsigned long)
440	per_cpu_ptr(mod->percpu,
441	get_boot_cpu_id());
442	}
443	preempt_enable();
444	return true;
445	}
446	}
447	}
448
449	preempt_enable();
450	return false;
451	}
452
453	/**
454	* is_module_percpu_address() - test whether address is from module static percpu
455	* @addr: address to test
456	*
457	* Test whether @addr belongs to module static percpu area.
458	*
459	* Return: %true if @addr is from module static percpu area
460	*/
461	bool is_module_percpu_address(unsigned long addr)
462	{
463	return __is_module_percpu_address(addr, NULL);
464	}
465
466	#else /* ... !CONFIG_SMP */
467
468	static inline void __percpu mod_percpu(struct module mod)
469	{
470	return NULL;
471	}
472	static int percpu_modalloc(struct module mod, struct load_info info)
473	{
474	/* UP modules shouldn't have this section: ENOMEM isn't quite right */
475	if (info->sechdrs[info->index.pcpu].sh_size != 0)
476	return -ENOMEM;
477	return 0;
478	}
479	static inline void percpu_modfree(struct module *mod)
480	{
481	}
482	static unsigned int find_pcpusec(struct load_info *info)
483	{
484	return 0;
485	}
486	static inline void percpu_modcopy(struct module *mod,
487	const void *from, unsigned long size)
488	{
489	/* pcpusec should be 0, and size of that section should be 0. */
490	BUG_ON(size != 0);
491	}
492	bool is_module_percpu_address(unsigned long addr)
493	{
494	return false;
495	}
496
497	bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
498	{
499	return false;
500	}
501
502	#endif /* CONFIG_SMP */
503
504	#define MODINFO_ATTR(field) \
505	static void setup_modinfo_##field(struct module mod, const char s) \
506	{ \
507	mod->field = kstrdup(s, GFP_KERNEL); \
508	} \
509	static ssize_t show_modinfo_##field(struct module_attribute *mattr, \
510	struct module_kobject mk, char buffer) \
511	{ \
512	return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
513	} \
514	static int modinfo_##field##_exists(struct module *mod) \
515	{ \
516	return mod->field != NULL; \
517	} \
518	static void free_modinfo_##field(struct module *mod) \
519	{ \
520	kfree(mod->field); \
521	mod->field = NULL; \
522	} \
523	static struct module_attribute modinfo_##field = { \
524	.attr = { .name = __stringify(field), .mode = 0444 }, \
525	.show = show_modinfo_##field, \
526	.setup = setup_modinfo_##field, \
527	.test = modinfo_##field##_exists, \
528	.free = free_modinfo_##field, \
529	};
530
531	MODINFO_ATTR(version);
532	MODINFO_ATTR(srcversion);
533
534	static struct {
535	char name[MODULE_NAME_LEN + 1];
536	char taints[MODULE_FLAGS_BUF_SIZE];
537	} last_unloaded_module;
538
539	#ifdef CONFIG_MODULE_UNLOAD
540
541	EXPORT_TRACEPOINT_SYMBOL(module_get);
542
543	/* MODULE_REF_BASE is the base reference count by kmodule loader. */
544	#define MODULE_REF_BASE 1
545
546	/* Init the unload section of the module. */
547	static int module_unload_init(struct module *mod)
548	{
549	/*
550	* Initialize reference counter to MODULE_REF_BASE.
551	* refcnt == 0 means module is going.
552	*/
553	atomic_set(&mod->refcnt, MODULE_REF_BASE);
554
555	INIT_LIST_HEAD(&mod->source_list);
556	INIT_LIST_HEAD(&mod->target_list);
557
558	/* Hold reference count during initialization. */
559	atomic_inc(&mod->refcnt);
560
561	return 0;
562	}
563
564	/* Does a already use b? */
565	static int already_uses(struct module a, struct module b)
566	{
567	struct module_use *use;
568
569	list_for_each_entry(use, &b->source_list, source_list) {
570	if (use->source == a)
571	return 1;
572	}
573	pr_debug("%s does not use %s!\n", a->name, b->name);
574	return 0;
575	}
576
577	/*
578	* Module a uses b
579	* - we add 'a' as a "source", 'b' as a "target" of module use
580	* - the module_use is added to the list of 'b' sources (so
581	* 'b' can walk the list to see who sourced them), and of 'a'
582	* targets (so 'a' can see what modules it targets).
583	*/
584	static int add_module_usage(struct module a, struct module b)
585	{
586	struct module_use *use;
587
588	pr_debug("Allocating new usage for %s.\n", a->name);
589	use = kmalloc(sizeof(*use), GFP_ATOMIC);
590	if (!use)
591	return -ENOMEM;
592
593	use->source = a;
594	use->target = b;
595	list_add(&use->source_list, &b->source_list);
596	list_add(&use->target_list, &a->target_list);
597	return 0;
598	}
599
600	/* Module a uses b: caller needs module_mutex() */
601	static int ref_module(struct module a, struct module b)
602	{
603	int err;
604
605	if (b == NULL \|\| already_uses(a, b))
606	return 0;
607
608	/* If module isn't available, we fail. */
609	err = strong_try_module_get(b);
610	if (err)
611	return err;
612
613	err = add_module_usage(a, b);
614	if (err) {
615	module_put(b);
616	return err;
617	}
618	return 0;
619	}
620
621	/* Clear the unload stuff of the module. */
622	static void module_unload_free(struct module *mod)
623	{
624	struct module_use use, tmp;
625
626	mutex_lock(&module_mutex);
627	list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
628	struct module *i = use->target;
629	pr_debug("%s unusing %s\n", mod->name, i->name);
630	module_put(i);
631	list_del(&use->source_list);
632	list_del(&use->target_list);
633	kfree(use);
634	}
635	mutex_unlock(&module_mutex);
636	}
637
638	#ifdef CONFIG_MODULE_FORCE_UNLOAD
639	static inline int try_force_unload(unsigned int flags)
640	{
641	int ret = (flags & O_TRUNC);
642	if (ret)
643	add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
644	return ret;
645	}
646	#else
647	static inline int try_force_unload(unsigned int flags)
648	{
649	return 0;
650	}
651	#endif /* CONFIG_MODULE_FORCE_UNLOAD */
652
653	/* Try to release refcount of module, 0 means success. */
654	static int try_release_module_ref(struct module *mod)
655	{
656	int ret;
657
658	/* Try to decrement refcnt which we set at loading */
659	ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
660	BUG_ON(ret < 0);
661	if (ret)
662	/* Someone can put this right now, recover with checking */
663	ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
664
665	return ret;
666	}
667
668	static int try_stop_module(struct module mod, int flags, int forced)
669	{
670	/* If it's not unused, quit unless we're forcing. */
671	if (try_release_module_ref(mod) != 0) {
672	*forced = try_force_unload(flags);
673	if (!(*forced))
674	return -EWOULDBLOCK;
675	}
676
677	/* Mark it as dying. */
678	mod->state = MODULE_STATE_GOING;
679
680	return 0;
681	}
682
683	/**
684	* module_refcount() - return the refcount or -1 if unloading
685	* @mod: the module we're checking
686	*
687	* Return:
688	* -1 if the module is in the process of unloading
689	* otherwise the number of references in the kernel to the module
690	*/
691	int module_refcount(struct module *mod)
692	{
693	return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
694	}
695	EXPORT_SYMBOL(module_refcount);
696
697	/* This exists whether we can unload or not */
698	static void free_module(struct module *mod);
699
700	SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
701	unsigned int, flags)
702	{
703	struct module *mod;
704	char name[MODULE_NAME_LEN];
705	char buf[MODULE_FLAGS_BUF_SIZE];
706	int ret, forced = 0;
707
708	if (!capable(CAP_SYS_MODULE) \|\| modules_disabled)
709	return -EPERM;
710
711	if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
712	return -EFAULT;
713	name[MODULE_NAME_LEN-1] = '\0';
714
715	audit_log_kern_module(name);
716
717	if (mutex_lock_interruptible(&module_mutex) != 0)
718	return -EINTR;
719
720	mod = find_module(name);
721	if (!mod) {
722	ret = -ENOENT;
723	goto out;
724	}
725
726	if (!list_empty(&mod->source_list)) {
727	/* Other modules depend on us: get rid of them first. */
728	ret = -EWOULDBLOCK;
729	goto out;
730	}
731
732	/* Doing init or already dying? */
733	if (mod->state != MODULE_STATE_LIVE) {
734	/* FIXME: if (force), slam module count damn the torpedoes */
735	pr_debug("%s already dying\n", mod->name);
736	ret = -EBUSY;
737	goto out;
738	}
739
740	/* If it has an init func, it must have an exit func to unload */
741	if (mod->init && !mod->exit) {
742	forced = try_force_unload(flags);
743	if (!forced) {
744	/* This module can't be removed */
745	ret = -EBUSY;
746	goto out;
747	}
748	}
749
750	ret = try_stop_module(mod, flags, &forced);
751	if (ret != 0)
752	goto out;
753
754	mutex_unlock(&module_mutex);
755	/* Final destruction now no one is using it. */
756	if (mod->exit != NULL)
757	mod->exit();
758	blocking_notifier_call_chain(&module_notify_list,
759	MODULE_STATE_GOING, mod);
760	klp_module_going(mod);
761	ftrace_release_mod(mod);
762
763	async_synchronize_full();
764
765	/* Store the name and taints of the last unloaded module for diagnostic purposes */
766	strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name));
767	strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.taints));
768
769	free_module(mod);
770	/* someone could wait for the module in add_unformed_module() */
771	wake_up_all(&module_wq);
772	return 0;
773	out:
774	mutex_unlock(&module_mutex);
775	return ret;
776	}
777
778	void __symbol_put(const char *symbol)
779	{
780	struct find_symbol_arg fsa = {
781	.name = symbol,
782	.gplok = true,
783	};
784
785	preempt_disable();
786	BUG_ON(!find_symbol(&fsa));
787	module_put(fsa.owner);
788	preempt_enable();
789	}
790	EXPORT_SYMBOL(__symbol_put);
791
792	/* Note this assumes addr is a function, which it currently always is. */
793	void symbol_put_addr(void *addr)
794	{
795	struct module *modaddr;
796	unsigned long a = (unsigned long)dereference_function_descriptor(addr);
797
798	if (core_kernel_text(a))
799	return;
800
801	/*
802	* Even though we hold a reference on the module; we still need to
803	* disable preemption in order to safely traverse the data structure.
804	*/
805	preempt_disable();
806	modaddr = __module_text_address(a);
807	BUG_ON(!modaddr);
808	module_put(modaddr);
809	preempt_enable();
810	}
811	EXPORT_SYMBOL_GPL(symbol_put_addr);
812
813	static ssize_t show_refcnt(struct module_attribute *mattr,
814	struct module_kobject mk, char buffer)
815	{
816	return sprintf(buffer, "%i\n", module_refcount(mk->mod));
817	}
818
819	static struct module_attribute modinfo_refcnt =
820	__ATTR(refcnt, 0444, show_refcnt, NULL);
821
822	void __module_get(struct module *module)
823	{
824	if (module) {
825	atomic_inc(&module->refcnt);
826	trace_module_get(module, _RET_IP_);
827	}
828	}
829	EXPORT_SYMBOL(__module_get);
830
831	bool try_module_get(struct module *module)
832	{
833	bool ret = true;
834
835	if (module) {
836	/* Note: here, we can fail to get a reference */
837	if (likely(module_is_live(module) &&
838	atomic_inc_not_zero(&module->refcnt) != 0))
839	trace_module_get(module, _RET_IP_);
840	else
841	ret = false;
842	}
843	return ret;
844	}
845	EXPORT_SYMBOL(try_module_get);
846
847	void module_put(struct module *module)
848	{
849	int ret;
850
851	if (module) {
852	ret = atomic_dec_if_positive(&module->refcnt);
853	WARN_ON(ret < 0); /* Failed to put refcount */
854	trace_module_put(module, _RET_IP_);
855	}
856	}
857	EXPORT_SYMBOL(module_put);
858
859	#else /* !CONFIG_MODULE_UNLOAD */
860	static inline void module_unload_free(struct module *mod)
861	{
862	}
863
864	static int ref_module(struct module a, struct module b)
865	{
866	return strong_try_module_get(b);
867	}
868
869	static inline int module_unload_init(struct module *mod)
870	{
871	return 0;
872	}
873	#endif /* CONFIG_MODULE_UNLOAD */
874
875	size_t module_flags_taint(unsigned long taints, char *buf)
876	{
877	size_t l = 0;
878	int i;
879
880	for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
881	if (taint_flags[i].module && test_bit(i, &taints))
882	buf[l++] = taint_flags[i].c_true;
883	}
884
885	return l;
886	}
887
888	static ssize_t show_initstate(struct module_attribute *mattr,
889	struct module_kobject mk, char buffer)
890	{
891	const char *state = "unknown";
892
893	switch (mk->mod->state) {
894	case MODULE_STATE_LIVE:
895	state = "live";
896	break;
897	case MODULE_STATE_COMING:
898	state = "coming";
899	break;
900	case MODULE_STATE_GOING:
901	state = "going";
902	break;
903	default:
904	BUG();
905	}
906	return sprintf(buffer, "%s\n", state);
907	}
908
909	static struct module_attribute modinfo_initstate =
910	__ATTR(initstate, 0444, show_initstate, NULL);
911
912	static ssize_t store_uevent(struct module_attribute *mattr,
913	struct module_kobject *mk,
914	const char *buffer, size_t count)
915	{
916	int rc;
917
918	rc = kobject_synth_uevent(&mk->kobj, buffer, count);
919	return rc ? rc : count;
920	}
921
922	struct module_attribute module_uevent =
923	__ATTR(uevent, 0200, NULL, store_uevent);
924
925	static ssize_t show_coresize(struct module_attribute *mattr,
926	struct module_kobject mk, char buffer)
927	{
928	unsigned int size = mk->mod->mem[MOD_TEXT].size;
929
930	if (!IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC)) {
931	for_class_mod_mem_type(type, core_data)
932	size += mk->mod->mem[type].size;
933	}
934	return sprintf(buffer, "%u\n", size);
935	}
936
937	static struct module_attribute modinfo_coresize =
938	__ATTR(coresize, 0444, show_coresize, NULL);
939
940	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
941	static ssize_t show_datasize(struct module_attribute *mattr,
942	struct module_kobject mk, char buffer)
943	{
944	unsigned int size = 0;
945
946	for_class_mod_mem_type(type, core_data)
947	size += mk->mod->mem[type].size;
948	return sprintf(buffer, "%u\n", size);
949	}
950
951	static struct module_attribute modinfo_datasize =
952	__ATTR(datasize, 0444, show_datasize, NULL);
953	#endif
954
955	static ssize_t show_initsize(struct module_attribute *mattr,
956	struct module_kobject mk, char buffer)
957	{
958	unsigned int size = 0;
959
960	for_class_mod_mem_type(type, init)
961	size += mk->mod->mem[type].size;
962	return sprintf(buffer, "%u\n", size);
963	}
964
965	static struct module_attribute modinfo_initsize =
966	__ATTR(initsize, 0444, show_initsize, NULL);
967
968	static ssize_t show_taint(struct module_attribute *mattr,
969	struct module_kobject mk, char buffer)
970	{
971	size_t l;
972
973	l = module_flags_taint(mk->mod->taints, buffer);
974	buffer[l++] = '\n';
975	return l;
976	}
977
978	static struct module_attribute modinfo_taint =
979	__ATTR(taint, 0444, show_taint, NULL);
980
981	struct module_attribute *modinfo_attrs[] = {
982	&module_uevent,
983	&modinfo_version,
984	&modinfo_srcversion,
985	&modinfo_initstate,
986	&modinfo_coresize,
987	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
988	&modinfo_datasize,
989	#endif
990	&modinfo_initsize,
991	&modinfo_taint,
992	#ifdef CONFIG_MODULE_UNLOAD
993	&modinfo_refcnt,
994	#endif
995	NULL,
996	};
997
998	size_t modinfo_attrs_count = ARRAY_SIZE(modinfo_attrs);
999
1000	static const char vermagic[] = VERMAGIC_STRING;
1001
1002	int try_to_force_load(struct module mod, const char reason)
1003	{
1004	#ifdef CONFIG_MODULE_FORCE_LOAD
1005	if (!test_taint(TAINT_FORCED_MODULE))
1006	pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
1007	add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
1008	return 0;
1009	#else
1010	return -ENOEXEC;
1011	#endif
1012	}
1013
1014	/* Parse tag=value strings from .modinfo section */
1015	char module_next_tag_pair(char string, unsigned long *secsize)
1016	{
1017	/* Skip non-zero chars */
1018	while (string[0]) {
1019	string++;
1020	if ((*secsize)-- <= 1)
1021	return NULL;
1022	}
1023
1024	/* Skip any zero padding. */
1025	while (!string[0]) {
1026	string++;
1027	if ((*secsize)-- <= 1)
1028	return NULL;
1029	}
1030	return string;
1031	}
1032
1033	static char get_next_modinfo(const struct load_info info, const char *tag,
1034	char *prev)
1035	{
1036	char *p;
1037	unsigned int taglen = strlen(tag);
1038	Elf_Shdr *infosec = &info->sechdrs[info->index.info];
1039	unsigned long size = infosec->sh_size;
1040
1041	/*
1042	* get_modinfo() calls made before rewrite_section_headers()
1043	* must use sh_offset, as sh_addr isn't set!
1044	*/
1045	char modinfo = (char )info->hdr + infosec->sh_offset;
1046
1047	if (prev) {
1048	size -= prev - modinfo;
1049	modinfo = module_next_tag_pair(prev, &size);
1050	}
1051
1052	for (p = modinfo; p; p = module_next_tag_pair(p, &size)) {
1053	if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
1054	return p + taglen + 1;
1055	}
1056	return NULL;
1057	}
1058
1059	static char get_modinfo(const struct load_info info, const char *tag)
1060	{
1061	return get_next_modinfo(info, tag, NULL);
1062	}
1063
1064	static int verify_namespace_is_imported(const struct load_info *info,
1065	const struct kernel_symbol *sym,
1066	struct module *mod)
1067	{
1068	const char *namespace;
1069	char *imported_namespace;
1070
1071	namespace = kernel_symbol_namespace(sym);
1072	if (namespace && namespace[0]) {
1073	for_each_modinfo_entry(imported_namespace, info, "import_ns") {
1074	if (strcmp(namespace, imported_namespace) == 0)
1075	return 0;
1076	}
1077	#ifdef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
1078	pr_warn(
1079	#else
1080	pr_err(
1081	#endif
1082	"%s: module uses symbol (%s) from namespace %s, but does not import it.\n",
1083	mod->name, kernel_symbol_name(sym), namespace);
1084	#ifndef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
1085	return -EINVAL;
1086	#endif
1087	}
1088	return 0;
1089	}
1090
1091	static bool inherit_taint(struct module mod, struct module owner, const char *name)
1092	{
1093	if (!owner \|\| !test_bit(TAINT_PROPRIETARY_MODULE, &owner->taints))
1094	return true;
1095
1096	if (mod->using_gplonly_symbols) {
1097	pr_err("%s: module using GPL-only symbols uses symbols %s from proprietary module %s.\n",
1098	mod->name, name, owner->name);
1099	return false;
1100	}
1101
1102	if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) {
1103	pr_warn("%s: module uses symbols %s from proprietary module %s, inheriting taint.\n",
1104	mod->name, name, owner->name);
1105	set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints);
1106	}
1107	return true;
1108	}
1109
1110	/* Resolve a symbol for this module. I.e. if we find one, record usage. */
1111	static const struct kernel_symbol resolve_symbol(struct module mod,
1112	const struct load_info *info,
1113	const char *name,
1114	char ownername[])
1115	{
1116	struct find_symbol_arg fsa = {
1117	.name = name,
1118	.gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)),
1119	.warn = true,
1120	};
1121	int err;
1122
1123	/*
1124	* The module_mutex should not be a heavily contended lock;
1125	* if we get the occasional sleep here, we'll go an extra iteration
1126	* in the wait_event_interruptible(), which is harmless.
1127	*/
1128	sched_annotate_sleep();
1129	mutex_lock(&module_mutex);
1130	if (!find_symbol(&fsa))
1131	goto unlock;
1132
1133	if (fsa.license == GPL_ONLY)
1134	mod->using_gplonly_symbols = true;
1135
1136	if (!inherit_taint(mod, fsa.owner, name)) {
1137	fsa.sym = NULL;
1138	goto getname;
1139	}
1140
1141	if (!check_version(info, name, mod, fsa.crc)) {
1142	fsa.sym = ERR_PTR(-EINVAL);
1143	goto getname;
1144	}
1145
1146	err = verify_namespace_is_imported(info, fsa.sym, mod);
1147	if (err) {
1148	fsa.sym = ERR_PTR(err);
1149	goto getname;
1150	}
1151
1152	err = ref_module(mod, fsa.owner);
1153	if (err) {
1154	fsa.sym = ERR_PTR(err);
1155	goto getname;
1156	}
1157
1158	getname:
1159	/* We must make copy under the lock if we failed to get ref. */
1160	strncpy(ownername, module_name(fsa.owner), MODULE_NAME_LEN);
1161	unlock:
1162	mutex_unlock(&module_mutex);
1163	return fsa.sym;
1164	}
1165
1166	static const struct kernel_symbol *
1167	resolve_symbol_wait(struct module *mod,
1168	const struct load_info *info,
1169	const char *name)
1170	{
1171	const struct kernel_symbol *ksym;
1172	char owner[MODULE_NAME_LEN];
1173
1174	if (wait_event_interruptible_timeout(module_wq,
1175	!IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
1176	\|\| PTR_ERR(ksym) != -EBUSY,
1177	30 * HZ) <= 0) {
1178	pr_warn("%s: gave up waiting for init of module %s.\n",
1179	mod->name, owner);
1180	}
1181	return ksym;
1182	}
1183
1184	void __weak module_arch_cleanup(struct module *mod)
1185	{
1186	}
1187
1188	void __weak module_arch_freeing_init(struct module *mod)
1189	{
1190	}
1191
1192	static int module_memory_alloc(struct module *mod, enum mod_mem_type type)
1193	{
1194	unsigned int size = PAGE_ALIGN(mod->mem[type].size);
1195	enum execmem_type execmem_type;
1196	void *ptr;
1197
1198	mod->mem[type].size = size;
1199
1200	if (mod_mem_type_is_data(type))
1201	execmem_type = EXECMEM_MODULE_DATA;
1202	else
1203	execmem_type = EXECMEM_MODULE_TEXT;
1204
1205	ptr = execmem_alloc(execmem_type, size);
1206	if (!ptr)
1207	return -ENOMEM;
1208
1209	/*
1210	* The pointer to these blocks of memory are stored on the module
1211	* structure and we keep that around so long as the module is
1212	* around. We only free that memory when we unload the module.
1213	* Just mark them as not being a leak then. The .init* ELF
1214	* sections do get freed after boot so we could treat them
1215	* slightly differently with kmemleak_ignore() and only grey
1216	* them out as they work as typical memory allocations which
1217	* do eventually get freed, but let's just keep things simple
1218	* and avoid any false positives.
1219	*/
1220	kmemleak_not_leak(ptr);
1221
1222	memset(ptr, 0, size);
1223	mod->mem[type].base = ptr;
1224
1225	return 0;
1226	}
1227
1228	static void module_memory_free(struct module *mod, enum mod_mem_type type,
1229	bool unload_codetags)
1230	{
1231	void *ptr = mod->mem[type].base;
1232
1233	if (!unload_codetags && mod_mem_type_is_core_data(type))
1234	return;
1235
1236	execmem_free(ptr);
1237	}
1238
1239	static void free_mod_mem(struct module *mod, bool unload_codetags)
1240	{
1241	for_each_mod_mem_type(type) {
1242	struct module_memory *mod_mem = &mod->mem[type];
1243
1244	if (type == MOD_DATA)
1245	continue;
1246
1247	/* Free lock-classes; relies on the preceding sync_rcu(). */
1248	lockdep_free_key_range(mod_mem->base, mod_mem->size);
1249	if (mod_mem->size)
1250	module_memory_free(mod, type, unload_codetags);
1251	}
1252
1253	/* MOD_DATA hosts mod, so free it at last */
1254	lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size);
1255	module_memory_free(mod, MOD_DATA, unload_codetags);
1256	}
1257
1258	/* Free a module, remove from lists, etc. */
1259	static void free_module(struct module *mod)
1260	{
1261	bool unload_codetags;
1262
1263	trace_module_free(mod);
1264
1265	unload_codetags = codetag_unload_module(mod);
1266	if (!unload_codetags)
1267	pr_warn("%s: memory allocation(s) from the module still alive, cannot unload cleanly\n",
1268	mod->name);
1269
1270	mod_sysfs_teardown(mod);
1271
1272	/*
1273	* We leave it in list to prevent duplicate loads, but make sure
1274	* that noone uses it while it's being deconstructed.
1275	*/
1276	mutex_lock(&module_mutex);
1277	mod->state = MODULE_STATE_UNFORMED;
1278	mutex_unlock(&module_mutex);
1279
1280	/* Arch-specific cleanup. */
1281	module_arch_cleanup(mod);
1282
1283	/* Module unload stuff */
1284	module_unload_free(mod);
1285
1286	/* Free any allocated parameters. */
1287	destroy_params(mod->kp, mod->num_kp);
1288
1289	if (is_livepatch_module(mod))
1290	free_module_elf(mod);
1291
1292	/* Now we can delete it from the lists */
1293	mutex_lock(&module_mutex);
1294	/* Unlink carefully: kallsyms could be walking list. */
1295	list_del_rcu(&mod->list);
1296	mod_tree_remove(mod);
1297	/* Remove this module from bug list, this uses list_del_rcu */
1298	module_bug_cleanup(mod);
1299	/* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
1300	synchronize_rcu();
1301	if (try_add_tainted_module(mod))
1302	pr_err("%s: adding tainted module to the unloaded tainted modules list failed.\n",
1303	mod->name);
1304	mutex_unlock(&module_mutex);
1305
1306	/* This may be empty, but that's OK */
1307	module_arch_freeing_init(mod);
1308	kfree(mod->args);
1309	percpu_modfree(mod);
1310
1311	free_mod_mem(mod, unload_codetags);
1312	}
1313
1314	void __symbol_get(const char symbol)
1315	{
1316	struct find_symbol_arg fsa = {
1317	.name = symbol,
1318	.gplok = true,
1319	.warn = true,
1320	};
1321
1322	preempt_disable();
1323	if (!find_symbol(&fsa))
1324	goto fail;
1325	if (fsa.license != GPL_ONLY) {
1326	pr_warn("failing symbol_get of non-GPLONLY symbol %s.\n",
1327	symbol);
1328	goto fail;
1329	}
1330	if (strong_try_module_get(fsa.owner))
1331	goto fail;
1332	preempt_enable();
1333	return (void *)kernel_symbol_value(fsa.sym);
1334	fail:
1335	preempt_enable();
1336	return NULL;
1337	}
1338	EXPORT_SYMBOL_GPL(__symbol_get);
1339
1340	/*
1341	* Ensure that an exported symbol [global namespace] does not already exist
1342	* in the kernel or in some other module's exported symbol table.
1343	*
1344	* You must hold the module_mutex.
1345	*/
1346	static int verify_exported_symbols(struct module *mod)
1347	{
1348	unsigned int i;
1349	const struct kernel_symbol *s;
1350	struct {
1351	const struct kernel_symbol *sym;
1352	unsigned int num;
1353	} arr[] = {
1354	{ mod->syms, mod->num_syms },
1355	{ mod->gpl_syms, mod->num_gpl_syms },
1356	};
1357
1358	for (i = 0; i < ARRAY_SIZE(arr); i++) {
1359	for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
1360	struct find_symbol_arg fsa = {
1361	.name = kernel_symbol_name(s),
1362	.gplok = true,
1363	};
1364	if (find_symbol(&fsa)) {
1365	pr_err("%s: exports duplicate symbol %s"
1366	" (owned by %s)\n",
1367	mod->name, kernel_symbol_name(s),
1368	module_name(fsa.owner));
1369	return -ENOEXEC;
1370	}
1371	}
1372	}
1373	return 0;
1374	}
1375
1376	static bool ignore_undef_symbol(Elf_Half emachine, const char *name)
1377	{
1378	/*
1379	* On x86, PIC code and Clang non-PIC code may have call foo@PLT. GNU as
1380	* before 2.37 produces an unreferenced _GLOBAL_OFFSET_TABLE_ on x86-64.
1381	* i386 has a similar problem but may not deserve a fix.
1382	*
1383	* If we ever have to ignore many symbols, consider refactoring the code to
1384	* only warn if referenced by a relocation.
1385	*/
1386	if (emachine == EM_386 \|\| emachine == EM_X86_64)
1387	return !strcmp(name, "_GLOBAL_OFFSET_TABLE_");
1388	return false;
1389	}
1390
1391	/* Change all symbols so that st_value encodes the pointer directly. */
1392	static int simplify_symbols(struct module mod, const struct load_info info)
1393	{
1394	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
1395	Elf_Sym sym = (void )symsec->sh_addr;
1396	unsigned long secbase;
1397	unsigned int i;
1398	int ret = 0;
1399	const struct kernel_symbol *ksym;
1400
1401	for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
1402	const char *name = info->strtab + sym[i].st_name;
1403
1404	switch (sym[i].st_shndx) {
1405	case SHN_COMMON:
1406	/* Ignore common symbols */
1407	if (!strncmp(name, "__gnu_lto", 9))
1408	break;
1409
1410	/*
1411	* We compiled with -fno-common. These are not
1412	* supposed to happen.
1413	*/
1414	pr_debug("Common symbol: %s\n", name);
1415	pr_warn("%s: please compile with -fno-common\n",
1416	mod->name);
1417	ret = -ENOEXEC;
1418	break;
1419
1420	case SHN_ABS:
1421	/* Don't need to do anything */
1422	pr_debug("Absolute symbol: 0x%08lx %s\n",
1423	(long)sym[i].st_value, name);
1424	break;
1425
1426	case SHN_LIVEPATCH:
1427	/* Livepatch symbols are resolved by livepatch */
1428	break;
1429
1430	case SHN_UNDEF:
1431	ksym = resolve_symbol_wait(mod, info, name);
1432	/* Ok if resolved. */
1433	if (ksym && !IS_ERR(ksym)) {
1434	sym[i].st_value = kernel_symbol_value(ksym);
1435	break;
1436	}
1437
1438	/* Ok if weak or ignored. */
1439	if (!ksym &&
1440	(ELF_ST_BIND(sym[i].st_info) == STB_WEAK \|\|
1441	ignore_undef_symbol(info->hdr->e_machine, name)))
1442	break;
1443
1444	ret = PTR_ERR(ksym) ?: -ENOENT;
1445	pr_warn("%s: Unknown symbol %s (err %d)\n",
1446	mod->name, name, ret);
1447	break;
1448
1449	default:
1450	/* Divert to percpu allocation if a percpu var. */
1451	if (sym[i].st_shndx == info->index.pcpu)
1452	secbase = (unsigned long)mod_percpu(mod);
1453	else
1454	secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
1455	sym[i].st_value += secbase;
1456	break;
1457	}
1458	}
1459
1460	return ret;
1461	}
1462
1463	static int apply_relocations(struct module mod, const struct load_info info)
1464	{
1465	unsigned int i;
1466	int err = 0;
1467
1468	/* Now do relocations. */
1469	for (i = 1; i < info->hdr->e_shnum; i++) {
1470	unsigned int infosec = info->sechdrs[i].sh_info;
1471
1472	/* Not a valid relocation section? */
1473	if (infosec >= info->hdr->e_shnum)
1474	continue;
1475
1476	/* Don't bother with non-allocated sections */
1477	if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
1478	continue;
1479
1480	if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH)
1481	err = klp_apply_section_relocs(mod, info->sechdrs,
1482	info->secstrings,
1483	info->strtab,
1484	info->index.sym, i,
1485	NULL);
1486	else if (info->sechdrs[i].sh_type == SHT_REL)
1487	err = apply_relocate(info->sechdrs, info->strtab,
1488	info->index.sym, i, mod);
1489	else if (info->sechdrs[i].sh_type == SHT_RELA)
1490	err = apply_relocate_add(info->sechdrs, info->strtab,
1491	info->index.sym, i, mod);
1492	if (err < 0)
1493	break;
1494	}
1495	return err;
1496	}
1497
1498	/* Additional bytes needed by arch in front of individual sections */
1499	unsigned int __weak arch_mod_section_prepend(struct module *mod,
1500	unsigned int section)
1501	{
1502	/* default implementation just returns zero */
1503	return 0;
1504	}
1505
1506	long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
1507	Elf_Shdr *sechdr, unsigned int section)
1508	{
1509	long offset;
1510	long mask = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK) << SH_ENTSIZE_TYPE_SHIFT;
1511
1512	mod->mem[type].size += arch_mod_section_prepend(mod, section);
1513	offset = ALIGN(mod->mem[type].size, sechdr->sh_addralign ?: 1);
1514	mod->mem[type].size = offset + sechdr->sh_size;
1515
1516	WARN_ON_ONCE(offset & mask);
1517	return offset \| mask;
1518	}
1519
1520	bool module_init_layout_section(const char *sname)
1521	{
1522	#ifndef CONFIG_MODULE_UNLOAD
1523	if (module_exit_section(sname))
1524	return true;
1525	#endif
1526	return module_init_section(sname);
1527	}
1528
1529	static void __layout_sections(struct module mod, struct load_info info, bool is_init)
1530	{
1531	unsigned int m, i;
1532
1533	static const unsigned long masks[][2] = {
1534	/*
1535	* NOTE: all executable code must be the first section
1536	* in this array; otherwise modify the text_size
1537	* finder in the two loops below
1538	*/
1539	{ SHF_EXECINSTR \| SHF_ALLOC, ARCH_SHF_SMALL },
1540	{ SHF_ALLOC, SHF_WRITE \| ARCH_SHF_SMALL },
1541	{ SHF_RO_AFTER_INIT \| SHF_ALLOC, ARCH_SHF_SMALL },
1542	{ SHF_WRITE \| SHF_ALLOC, ARCH_SHF_SMALL },
1543	{ ARCH_SHF_SMALL \| SHF_ALLOC, 0 }
1544	};
1545	static const int core_m_to_mem_type[] = {
1546	MOD_TEXT,
1547	MOD_RODATA,
1548	MOD_RO_AFTER_INIT,
1549	MOD_DATA,
1550	MOD_DATA,
1551	};
1552	static const int init_m_to_mem_type[] = {
1553	MOD_INIT_TEXT,
1554	MOD_INIT_RODATA,
1555	MOD_INVALID,
1556	MOD_INIT_DATA,
1557	MOD_INIT_DATA,
1558	};
1559
1560	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1561	enum mod_mem_type type = is_init ? init_m_to_mem_type[m] : core_m_to_mem_type[m];
1562
1563	for (i = 0; i < info->hdr->e_shnum; ++i) {
1564	Elf_Shdr *s = &info->sechdrs[i];
1565	const char *sname = info->secstrings + s->sh_name;
1566
1567	if ((s->sh_flags & masks[m][0]) != masks[m][0]
1568	\|\| (s->sh_flags & masks[m][1])
1569	\|\| s->sh_entsize != ~0UL
1570	\|\| is_init != module_init_layout_section(sname))
1571	continue;
1572
1573	if (WARN_ON_ONCE(type == MOD_INVALID))
1574	continue;
1575
1576	s->sh_entsize = module_get_offset_and_type(mod, type, s, i);
1577	pr_debug("\t%s\n", sname);
1578	}
1579	}
1580	}
1581
1582	/*
1583	* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
1584	* might -- code, read-only data, read-write data, small data. Tally
1585	* sizes, and place the offsets into sh_entsize fields: high bit means it
1586	* belongs in init.
1587	*/
1588	static void layout_sections(struct module mod, struct load_info info)
1589	{
1590	unsigned int i;
1591
1592	for (i = 0; i < info->hdr->e_shnum; i++)
1593	info->sechdrs[i].sh_entsize = ~0UL;
1594
1595	pr_debug("Core section allocation order for %s:\n", mod->name);
1596	__layout_sections(mod, info, false);
1597
1598	pr_debug("Init section allocation order for %s:\n", mod->name);
1599	__layout_sections(mod, info, true);
1600	}
1601
1602	static void module_license_taint_check(struct module mod, const char license)
1603	{
1604	if (!license)
1605	license = "unspecified";
1606
1607	if (!license_is_gpl_compatible(license)) {
1608	if (!test_taint(TAINT_PROPRIETARY_MODULE))
1609	pr_warn("%s: module license '%s' taints kernel.\n",
1610	mod->name, license);
1611	add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
1612	LOCKDEP_NOW_UNRELIABLE);
1613	}
1614	}
1615
1616	static void setup_modinfo(struct module mod, struct load_info info)
1617	{
1618	struct module_attribute *attr;
1619	int i;
1620
1621	for (i = 0; (attr = modinfo_attrs[i]); i++) {
1622	if (attr->setup)
1623	attr->setup(mod, get_modinfo(info, attr->attr.name));
1624	}
1625	}
1626
1627	static void free_modinfo(struct module *mod)
1628	{
1629	struct module_attribute *attr;
1630	int i;
1631
1632	for (i = 0; (attr = modinfo_attrs[i]); i++) {
1633	if (attr->free)
1634	attr->free(mod);
1635	}
1636	}
1637
1638	bool __weak module_init_section(const char *name)
1639	{
1640	return strstarts(name, ".init");
1641	}
1642
1643	bool __weak module_exit_section(const char *name)
1644	{
1645	return strstarts(name, ".exit");
1646	}
1647
1648	static int validate_section_offset(struct load_info info, Elf_Shdr shdr)
1649	{
1650	#if defined(CONFIG_64BIT)
1651	unsigned long long secend;
1652	#else
1653	unsigned long secend;
1654	#endif
1655
1656	/*
1657	* Check for both overflow and offset/size being
1658	* too large.
1659	*/
1660	secend = shdr->sh_offset + shdr->sh_size;
1661	if (secend < shdr->sh_offset \|\| secend > info->len)
1662	return -ENOEXEC;
1663
1664	return 0;
1665	}
1666
1667	/*
1668	* Check userspace passed ELF module against our expectations, and cache
1669	* useful variables for further processing as we go.
1670	*
1671	* This does basic validity checks against section offsets and sizes, the
1672	* section name string table, and the indices used for it (sh_name).
1673	*
1674	* As a last step, since we're already checking the ELF sections we cache
1675	* useful variables which will be used later for our convenience:
1676	*
1677	* o pointers to section headers
1678	* o cache the modinfo symbol section
1679	* o cache the string symbol section
1680	* o cache the module section
1681	*
1682	* As a last step we set info->mod to the temporary copy of the module in
1683	* info->hdr. The final one will be allocated in move_module(). Any
1684	* modifications we make to our copy of the module will be carried over
1685	* to the final minted module.
1686	*/
1687	static int elf_validity_cache_copy(struct load_info *info, int flags)
1688	{
1689	unsigned int i;
1690	Elf_Shdr shdr, strhdr;
1691	int err;
1692	unsigned int num_mod_secs = 0, mod_idx;
1693	unsigned int num_info_secs = 0, info_idx;
1694	unsigned int num_sym_secs = 0, sym_idx;
1695
1696	if (info->len < sizeof(*(info->hdr))) {
1697	pr_err("Invalid ELF header len %lu\n", info->len);
1698	goto no_exec;
1699	}
1700
1701	if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0) {
1702	pr_err("Invalid ELF header magic: != %s\n", ELFMAG);
1703	goto no_exec;
1704	}
1705	if (info->hdr->e_type != ET_REL) {
1706	pr_err("Invalid ELF header type: %u != %u\n",
1707	info->hdr->e_type, ET_REL);
1708	goto no_exec;
1709	}
1710	if (!elf_check_arch(info->hdr)) {
1711	pr_err("Invalid architecture in ELF header: %u\n",
1712	info->hdr->e_machine);
1713	goto no_exec;
1714	}
1715	if (!module_elf_check_arch(info->hdr)) {
1716	pr_err("Invalid module architecture in ELF header: %u\n",
1717	info->hdr->e_machine);
1718	goto no_exec;
1719	}
1720	if (info->hdr->e_shentsize != sizeof(Elf_Shdr)) {
1721	pr_err("Invalid ELF section header size\n");
1722	goto no_exec;
1723	}
1724
1725	/*
1726	* e_shnum is 16 bits, and sizeof(Elf_Shdr) is
1727	* known and small. So e_shnum * sizeof(Elf_Shdr)
1728	* will not overflow unsigned long on any platform.
1729	*/
1730	if (info->hdr->e_shoff >= info->len
1731	\|\| (info->hdr->e_shnum * sizeof(Elf_Shdr) >
1732	info->len - info->hdr->e_shoff)) {
1733	pr_err("Invalid ELF section header overflow\n");
1734	goto no_exec;
1735	}
1736
1737	info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
1738
1739	/*
1740	* Verify if the section name table index is valid.
1741	*/
1742	if (info->hdr->e_shstrndx == SHN_UNDEF
1743	\|\| info->hdr->e_shstrndx >= info->hdr->e_shnum) {
1744	pr_err("Invalid ELF section name index: %d \|\| e_shstrndx (%d) >= e_shnum (%d)\n",
1745	info->hdr->e_shstrndx, info->hdr->e_shstrndx,
1746	info->hdr->e_shnum);
1747	goto no_exec;
1748	}
1749
1750	strhdr = &info->sechdrs[info->hdr->e_shstrndx];
1751	err = validate_section_offset(info, strhdr);
1752	if (err < 0) {
1753	pr_err("Invalid ELF section hdr(type %u)\n", strhdr->sh_type);
1754	return err;
1755	}
1756
1757	/*
1758	* The section name table must be NUL-terminated, as required
1759	* by the spec. This makes strcmp and pr_* calls that access
1760	* strings in the section safe.
1761	*/
1762	info->secstrings = (void *)info->hdr + strhdr->sh_offset;
1763	if (strhdr->sh_size == 0) {
1764	pr_err("empty section name table\n");
1765	goto no_exec;
1766	}
1767	if (info->secstrings[strhdr->sh_size - 1] != '\0') {
1768	pr_err("ELF Spec violation: section name table isn't null terminated\n");
1769	goto no_exec;
1770	}
1771
1772	/*
1773	* The code assumes that section 0 has a length of zero and
1774	* an addr of zero, so check for it.
1775	*/
1776	if (info->sechdrs[0].sh_type != SHT_NULL
1777	\|\| info->sechdrs[0].sh_size != 0
1778	\|\| info->sechdrs[0].sh_addr != 0) {
1779	pr_err("ELF Spec violation: section 0 type(%d)!=SH_NULL or non-zero len or addr\n",
1780	info->sechdrs[0].sh_type);
1781	goto no_exec;
1782	}
1783
1784	for (i = 1; i < info->hdr->e_shnum; i++) {
1785	shdr = &info->sechdrs[i];
1786	switch (shdr->sh_type) {
1787	case SHT_NULL:
1788	case SHT_NOBITS:
1789	continue;
1790	case SHT_SYMTAB:
1791	if (shdr->sh_link == SHN_UNDEF
1792	\|\| shdr->sh_link >= info->hdr->e_shnum) {
1793	pr_err("Invalid ELF sh_link!=SHN_UNDEF(%d) or (sh_link(%d) >= hdr->e_shnum(%d)\n",
1794	shdr->sh_link, shdr->sh_link,
1795	info->hdr->e_shnum);
1796	goto no_exec;
1797	}
1798	num_sym_secs++;
1799	sym_idx = i;
1800	fallthrough;
1801	default:
1802	err = validate_section_offset(info, shdr);
1803	if (err < 0) {
1804	pr_err("Invalid ELF section in module (section %u type %u)\n",
1805	i, shdr->sh_type);
1806	return err;
1807	}
1808	if (strcmp(info->secstrings + shdr->sh_name,
1809	".gnu.linkonce.this_module") == 0) {
1810	num_mod_secs++;
1811	mod_idx = i;
1812	} else if (strcmp(info->secstrings + shdr->sh_name,
1813	".modinfo") == 0) {
1814	num_info_secs++;
1815	info_idx = i;
1816	}
1817
1818	if (shdr->sh_flags & SHF_ALLOC) {
1819	if (shdr->sh_name >= strhdr->sh_size) {
1820	pr_err("Invalid ELF section name in module (section %u type %u)\n",
1821	i, shdr->sh_type);
1822	return -ENOEXEC;
1823	}
1824	}
1825	break;
1826	}
1827	}
1828
1829	if (num_info_secs > 1) {
1830	pr_err("Only one .modinfo section must exist.\n");
1831	goto no_exec;
1832	} else if (num_info_secs == 1) {
1833	/* Try to find a name early so we can log errors with a module name */
1834	info->index.info = info_idx;
1835	info->name = get_modinfo(info, "name");
1836	}
1837
1838	if (num_sym_secs != 1) {
1839	pr_warn("%s: module has no symbols (stripped?)\n",
1840	info->name ?: "(missing .modinfo section or name field)");
1841	goto no_exec;
1842	}
1843
1844	/* Sets internal symbols and strings. */
1845	info->index.sym = sym_idx;
1846	shdr = &info->sechdrs[sym_idx];
1847	info->index.str = shdr->sh_link;
1848	info->strtab = (char *)info->hdr + info->sechdrs[info->index.str].sh_offset;
1849
1850	/*
1851	* The ".gnu.linkonce.this_module" ELF section is special. It is
1852	* what modpost uses to refer to __this_module and let's use rely
1853	* on THIS_MODULE to point to &__this_module properly. The kernel's
1854	* modpost declares it on each modules's *.mod.c file. If the struct
1855	* module of the kernel changes a full kernel rebuild is required.
1856	*
1857	* We have a few expectaions for this special section, the following
1858	* code validates all this for us:
1859	*
1860	* o Only one section must exist
1861	* o We expect the kernel to always have to allocate it: SHF_ALLOC
1862	* o The section size must match the kernel's run time's struct module
1863	* size
1864	*/
1865	if (num_mod_secs != 1) {
1866	pr_err("module %s: Only one .gnu.linkonce.this_module section must exist.\n",
1867	info->name ?: "(missing .modinfo section or name field)");
1868	goto no_exec;
1869	}
1870
1871	shdr = &info->sechdrs[mod_idx];
1872
1873	/*
1874	* This is already implied on the switch above, however let's be
1875	* pedantic about it.
1876	*/
1877	if (shdr->sh_type == SHT_NOBITS) {
1878	pr_err("module %s: .gnu.linkonce.this_module section must have a size set\n",
1879	info->name ?: "(missing .modinfo section or name field)");
1880	goto no_exec;
1881	}
1882
1883	if (!(shdr->sh_flags & SHF_ALLOC)) {
1884	pr_err("module %s: .gnu.linkonce.this_module must occupy memory during process execution\n",
1885	info->name ?: "(missing .modinfo section or name field)");
1886	goto no_exec;
1887	}
1888
1889	if (shdr->sh_size != sizeof(struct module)) {
1890	pr_err("module %s: .gnu.linkonce.this_module section size must match the kernel's built struct module size at run time\n",
1891	info->name ?: "(missing .modinfo section or name field)");
1892	goto no_exec;
1893	}
1894
1895	info->index.mod = mod_idx;
1896
1897	/* This is temporary: point mod into copy of data. */
1898	info->mod = (void *)info->hdr + shdr->sh_offset;
1899
1900	/*
1901	* If we didn't load the .modinfo 'name' field earlier, fall back to
1902	* on-disk struct mod 'name' field.
1903	*/
1904	if (!info->name)
1905	info->name = info->mod->name;
1906
1907	if (flags & MODULE_INIT_IGNORE_MODVERSIONS)
1908	info->index.vers = 0; /* Pretend no __versions section! */
1909	else
1910	info->index.vers = find_sec(info, "__versions");
1911
1912	info->index.pcpu = find_pcpusec(info);
1913
1914	return 0;
1915
1916	no_exec:
1917	return -ENOEXEC;
1918	}
1919
1920	#define COPY_CHUNK_SIZE (16*PAGE_SIZE)
1921
1922	static int copy_chunked_from_user(void dst, const void __user usrc, unsigned long len)
1923	{
1924	do {
1925	unsigned long n = min(len, COPY_CHUNK_SIZE);
1926
1927	if (copy_from_user(dst, usrc, n) != 0)
1928	return -EFAULT;
1929	cond_resched();
1930	dst += n;
1931	usrc += n;
1932	len -= n;
1933	} while (len);
1934	return 0;
1935	}
1936
1937	static int check_modinfo_livepatch(struct module mod, struct load_info info)
1938	{
1939	if (!get_modinfo(info, "livepatch"))
1940	/* Nothing more to do */
1941	return 0;
1942
1943	if (set_livepatch_module(mod))
1944	return 0;
1945
1946	pr_err("%s: module is marked as livepatch module, but livepatch support is disabled",
1947	mod->name);
1948	return -ENOEXEC;
1949	}
1950
1951	static void check_modinfo_retpoline(struct module mod, struct load_info info)
1952	{
1953	if (retpoline_module_ok(get_modinfo(info, "retpoline")))
1954	return;
1955
1956	pr_warn("%s: loading module not compiled with retpoline compiler.\n",
1957	mod->name);
1958	}
1959
1960	/* Sets info->hdr and info->len. */
1961	static int copy_module_from_user(const void __user *umod, unsigned long len,
1962	struct load_info *info)
1963	{
1964	int err;
1965
1966	info->len = len;
1967	if (info->len < sizeof(*(info->hdr)))
1968	return -ENOEXEC;
1969
1970	err = security_kernel_load_data(LOADING_MODULE, true);
1971	if (err)
1972	return err;
1973
1974	/* Suck in entire file: we'll want most of it. */
1975	info->hdr = __vmalloc(info->len, GFP_KERNEL \| __GFP_NOWARN);
1976	if (!info->hdr)
1977	return -ENOMEM;
1978
1979	if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
1980	err = -EFAULT;
1981	goto out;
1982	}
1983
1984	err = security_kernel_post_load_data((char *)info->hdr, info->len,
1985	LOADING_MODULE, "init_module");
1986	out:
1987	if (err)
1988	vfree(info->hdr);
1989
1990	return err;
1991	}
1992
1993	static void free_copy(struct load_info *info, int flags)
1994	{
1995	if (flags & MODULE_INIT_COMPRESSED_FILE)
1996	module_decompress_cleanup(info);
1997	else
1998	vfree(info->hdr);
1999	}
2000
2001	static int rewrite_section_headers(struct load_info *info, int flags)
2002	{
2003	unsigned int i;
2004
2005	/* This should always be true, but let's be sure. */
2006	info->sechdrs[0].sh_addr = 0;
2007
2008	for (i = 1; i < info->hdr->e_shnum; i++) {
2009	Elf_Shdr *shdr = &info->sechdrs[i];
2010
2011	/*
2012	* Mark all sections sh_addr with their address in the
2013	* temporary image.
2014	*/
2015	shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
2016
2017	}
2018
2019	/* Track but don't keep modinfo and version sections. */
2020	info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
2021	info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
2022
2023	return 0;
2024	}
2025
2026	/*
2027	* These calls taint the kernel depending certain module circumstances */
2028	static void module_augment_kernel_taints(struct module mod, struct load_info info)
2029	{
2030	int prev_taint = test_taint(TAINT_PROPRIETARY_MODULE);
2031
2032	if (!get_modinfo(info, "intree")) {
2033	if (!test_taint(TAINT_OOT_MODULE))
2034	pr_warn("%s: loading out-of-tree module taints kernel.\n",
2035	mod->name);
2036	add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
2037	}
2038
2039	check_modinfo_retpoline(mod, info);
2040
2041	if (get_modinfo(info, "staging")) {
2042	add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
2043	pr_warn("%s: module is from the staging directory, the quality "
2044	"is unknown, you have been warned.\n", mod->name);
2045	}
2046
2047	if (is_livepatch_module(mod)) {
2048	add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
2049	pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n",
2050	mod->name);
2051	}
2052
2053	module_license_taint_check(mod, get_modinfo(info, "license"));
2054
2055	if (get_modinfo(info, "test")) {
2056	if (!test_taint(TAINT_TEST))
2057	pr_warn("%s: loading test module taints kernel.\n",
2058	mod->name);
2059	add_taint_module(mod, TAINT_TEST, LOCKDEP_STILL_OK);
2060	}
2061	#ifdef CONFIG_MODULE_SIG
2062	mod->sig_ok = info->sig_ok;
2063	if (!mod->sig_ok) {
2064	pr_notice_once("%s: module verification failed: signature "
2065	"and/or required key missing - tainting "
2066	"kernel\n", mod->name);
2067	add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
2068	}
2069	#endif
2070
2071	/*
2072	* ndiswrapper is under GPL by itself, but loads proprietary modules.
2073	* Don't use add_taint_module(), as it would prevent ndiswrapper from
2074	* using GPL-only symbols it needs.
2075	*/
2076	if (strcmp(mod->name, "ndiswrapper") == 0)
2077	add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
2078
2079	/* driverloader was caught wrongly pretending to be under GPL */
2080	if (strcmp(mod->name, "driverloader") == 0)
2081	add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
2082	LOCKDEP_NOW_UNRELIABLE);
2083
2084	/* lve claims to be GPL but upstream won't provide source */
2085	if (strcmp(mod->name, "lve") == 0)
2086	add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
2087	LOCKDEP_NOW_UNRELIABLE);
2088
2089	if (!prev_taint && test_taint(TAINT_PROPRIETARY_MODULE))
2090	pr_warn("%s: module license taints kernel.\n", mod->name);
2091
2092	}
2093
2094	static int check_modinfo(struct module mod, struct load_info info, int flags)
2095	{
2096	const char *modmagic = get_modinfo(info, "vermagic");
2097	int err;
2098
2099	if (flags & MODULE_INIT_IGNORE_VERMAGIC)
2100	modmagic = NULL;
2101
2102	/* This is allowed: modprobe --force will invalidate it. */
2103	if (!modmagic) {
2104	err = try_to_force_load(mod, "bad vermagic");
2105	if (err)
2106	return err;
2107	} else if (!same_magic(modmagic, vermagic, info->index.vers)) {
2108	pr_err("%s: version magic '%s' should be '%s'\n",
2109	info->name, modmagic, vermagic);
2110	return -ENOEXEC;
2111	}
2112
2113	err = check_modinfo_livepatch(mod, info);
2114	if (err)
2115	return err;
2116
2117	return 0;
2118	}
2119
2120	static int find_module_sections(struct module mod, struct load_info info)
2121	{
2122	mod->kp = section_objs(info, "__param",
2123	sizeof(*mod->kp), &mod->num_kp);
2124	mod->syms = section_objs(info, "__ksymtab",
2125	sizeof(*mod->syms), &mod->num_syms);
2126	mod->crcs = section_addr(info, "__kcrctab");
2127	mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
2128	sizeof(*mod->gpl_syms),
2129	&mod->num_gpl_syms);
2130	mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
2131
2132	#ifdef CONFIG_CONSTRUCTORS
2133	mod->ctors = section_objs(info, ".ctors",
2134	sizeof(*mod->ctors), &mod->num_ctors);
2135	if (!mod->ctors)
2136	mod->ctors = section_objs(info, ".init_array",
2137	sizeof(*mod->ctors), &mod->num_ctors);
2138	else if (find_sec(info, ".init_array")) {
2139	/*
2140	* This shouldn't happen with same compiler and binutils
2141	* building all parts of the module.
2142	*/
2143	pr_warn("%s: has both .ctors and .init_array.\n",
2144	mod->name);
2145	return -EINVAL;
2146	}
2147	#endif
2148
2149	mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1,
2150	&mod->noinstr_text_size);
2151
2152	#ifdef CONFIG_TRACEPOINTS
2153	mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
2154	sizeof(*mod->tracepoints_ptrs),
2155	&mod->num_tracepoints);
2156	#endif
2157	#ifdef CONFIG_TREE_SRCU
2158	mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs",
2159	sizeof(*mod->srcu_struct_ptrs),
2160	&mod->num_srcu_structs);
2161	#endif
2162	#ifdef CONFIG_BPF_EVENTS
2163	mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map",
2164	sizeof(*mod->bpf_raw_events),
2165	&mod->num_bpf_raw_events);
2166	#endif
2167	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
2168	mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size);
2169	mod->btf_base_data = any_section_objs(info, ".BTF.base", 1,
2170	&mod->btf_base_data_size);
2171	#endif
2172	#ifdef CONFIG_JUMP_LABEL
2173	mod->jump_entries = section_objs(info, "__jump_table",
2174	sizeof(*mod->jump_entries),
2175	&mod->num_jump_entries);
2176	#endif
2177	#ifdef CONFIG_EVENT_TRACING
2178	mod->trace_events = section_objs(info, "_ftrace_events",
2179	sizeof(*mod->trace_events),
2180	&mod->num_trace_events);
2181	mod->trace_evals = section_objs(info, "_ftrace_eval_map",
2182	sizeof(*mod->trace_evals),
2183	&mod->num_trace_evals);
2184	#endif
2185	#ifdef CONFIG_TRACING
2186	mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
2187	sizeof(*mod->trace_bprintk_fmt_start),
2188	&mod->num_trace_bprintk_fmt);
2189	#endif
2190	#ifdef CONFIG_FTRACE_MCOUNT_RECORD
2191	/* sechdrs[0].sh_size is always zero */
2192	mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION,
2193	sizeof(*mod->ftrace_callsites),
2194	&mod->num_ftrace_callsites);
2195	#endif
2196	#ifdef CONFIG_FUNCTION_ERROR_INJECTION
2197	mod->ei_funcs = section_objs(info, "_error_injection_whitelist",
2198	sizeof(*mod->ei_funcs),
2199	&mod->num_ei_funcs);
2200	#endif
2201	#ifdef CONFIG_KPROBES
2202	mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1,
2203	&mod->kprobes_text_size);
2204	mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist",
2205	sizeof(unsigned long),
2206	&mod->num_kprobe_blacklist);
2207	#endif
2208	#ifdef CONFIG_PRINTK_INDEX
2209	mod->printk_index_start = section_objs(info, ".printk_index",
2210	sizeof(*mod->printk_index_start),
2211	&mod->printk_index_size);
2212	#endif
2213	#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
2214	mod->static_call_sites = section_objs(info, ".static_call_sites",
2215	sizeof(*mod->static_call_sites),
2216	&mod->num_static_call_sites);
2217	#endif
2218	#if IS_ENABLED(CONFIG_KUNIT)
2219	mod->kunit_suites = section_objs(info, ".kunit_test_suites",
2220	sizeof(*mod->kunit_suites),
2221	&mod->num_kunit_suites);
2222	mod->kunit_init_suites = section_objs(info, ".kunit_init_test_suites",
2223	sizeof(*mod->kunit_init_suites),
2224	&mod->num_kunit_init_suites);
2225	#endif
2226
2227	mod->extable = section_objs(info, "__ex_table",
2228	sizeof(*mod->extable), &mod->num_exentries);
2229
2230	if (section_addr(info, "__obsparm"))
2231	pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
2232
2233	#ifdef CONFIG_DYNAMIC_DEBUG_CORE
2234	mod->dyndbg_info.descs = section_objs(info, "__dyndbg",
2235	sizeof(*mod->dyndbg_info.descs),
2236	&mod->dyndbg_info.num_descs);
2237	mod->dyndbg_info.classes = section_objs(info, "__dyndbg_classes",
2238	sizeof(*mod->dyndbg_info.classes),
2239	&mod->dyndbg_info.num_classes);
2240	#endif
2241
2242	return 0;
2243	}
2244
2245	static int move_module(struct module mod, struct load_info info)
2246	{
2247	int i;
2248	enum mod_mem_type t = 0;
2249	int ret = -ENOMEM;
2250
2251	for_each_mod_mem_type(type) {
2252	if (!mod->mem[type].size) {
2253	mod->mem[type].base = NULL;
2254	continue;
2255	}
2256
2257	ret = module_memory_alloc(mod, type);
2258	if (ret) {
2259	t = type;
2260	goto out_enomem;
2261	}
2262	}
2263
2264	/* Transfer each section which specifies SHF_ALLOC */
2265	pr_debug("Final section addresses for %s:\n", mod->name);
2266	for (i = 0; i < info->hdr->e_shnum; i++) {
2267	void *dest;
2268	Elf_Shdr *shdr = &info->sechdrs[i];
2269	enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
2270
2271	if (!(shdr->sh_flags & SHF_ALLOC))
2272	continue;
2273
2274	dest = mod->mem[type].base + (shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK);
2275
2276	if (shdr->sh_type != SHT_NOBITS) {
2277	/*
2278	* Our ELF checker already validated this, but let's
2279	* be pedantic and make the goal clearer. We actually
2280	* end up copying over all modifications made to the
2281	* userspace copy of the entire struct module.
2282	*/
2283	if (i == info->index.mod &&
2284	(WARN_ON_ONCE(shdr->sh_size != sizeof(struct module)))) {
2285	ret = -ENOEXEC;
2286	goto out_enomem;
2287	}
2288	memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
2289	}
2290	/*
2291	* Update the userspace copy's ELF section address to point to
2292	* our newly allocated memory as a pure convenience so that
2293	* users of info can keep taking advantage and using the newly
2294	* minted official memory area.
2295	*/
2296	shdr->sh_addr = (unsigned long)dest;
2297	pr_debug("\t0x%lx 0x%.8lx %s\n", (long)shdr->sh_addr,
2298	(long)shdr->sh_size, info->secstrings + shdr->sh_name);
2299	}
2300
2301	return 0;
2302	out_enomem:
2303	for (t--; t >= 0; t--)
2304	module_memory_free(mod, t, true);
2305	return ret;
2306	}
2307
2308	static int check_export_symbol_versions(struct module *mod)
2309	{
2310	#ifdef CONFIG_MODVERSIONS
2311	if ((mod->num_syms && !mod->crcs) \|\|
2312	(mod->num_gpl_syms && !mod->gpl_crcs)) {
2313	return try_to_force_load(mod,
2314	"no versions for exported symbols");
2315	}
2316	#endif
2317	return 0;
2318	}
2319
2320	static void flush_module_icache(const struct module *mod)
2321	{
2322	/*
2323	* Flush the instruction cache, since we've played with text.
2324	* Do it before processing of module parameters, so the module
2325	* can provide parameter accessor functions of its own.
2326	*/
2327	for_each_mod_mem_type(type) {
2328	const struct module_memory *mod_mem = &mod->mem[type];
2329
2330	if (mod_mem->size) {
2331	flush_icache_range((unsigned long)mod_mem->base,
2332	(unsigned long)mod_mem->base + mod_mem->size);
2333	}
2334	}
2335	}
2336
2337	bool __weak module_elf_check_arch(Elf_Ehdr *hdr)
2338	{
2339	return true;
2340	}
2341
2342	int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
2343	Elf_Shdr *sechdrs,
2344	char *secstrings,
2345	struct module *mod)
2346	{
2347	return 0;
2348	}
2349
2350	/* module_blacklist is a comma-separated list of module names */
2351	static char *module_blacklist;
2352	static bool blacklisted(const char *module_name)
2353	{
2354	const char *p;
2355	size_t len;
2356
2357	if (!module_blacklist)
2358	return false;
2359
2360	for (p = module_blacklist; *p; p += len) {
2361	len = strcspn(p, ",");
2362	if (strlen(module_name) == len && !memcmp(module_name, p, len))
2363	return true;
2364	if (p[len] == ',')
2365	len++;
2366	}
2367	return false;
2368	}
2369	core_param(module_blacklist, module_blacklist, charp, 0400);
2370
2371	static struct module layout_and_allocate(struct load_info info, int flags)
2372	{
2373	struct module *mod;
2374	unsigned int ndx;
2375	int err;
2376
2377	/* Allow arches to frob section contents and sizes. */
2378	err = module_frob_arch_sections(info->hdr, info->sechdrs,
2379	info->secstrings, info->mod);
2380	if (err < 0)
2381	return ERR_PTR(err);
2382
2383	err = module_enforce_rwx_sections(info->hdr, info->sechdrs,
2384	info->secstrings, info->mod);
2385	if (err < 0)
2386	return ERR_PTR(err);
2387
2388	/* We will do a special allocation for per-cpu sections later. */
2389	info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
2390
2391	/*
2392	* Mark ro_after_init section with SHF_RO_AFTER_INIT so that
2393	* layout_sections() can put it in the right place.
2394	* Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set.
2395	*/
2396	ndx = find_sec(info, ".data..ro_after_init");
2397	if (ndx)
2398	info->sechdrs[ndx].sh_flags \|= SHF_RO_AFTER_INIT;
2399	/*
2400	* Mark the __jump_table section as ro_after_init as well: these data
2401	* structures are never modified, with the exception of entries that
2402	* refer to code in the __init section, which are annotated as such
2403	* at module load time.
2404	*/
2405	ndx = find_sec(info, "__jump_table");
2406	if (ndx)
2407	info->sechdrs[ndx].sh_flags \|= SHF_RO_AFTER_INIT;
2408
2409	/*
2410	* Determine total sizes, and put offsets in sh_entsize. For now
2411	* this is done generically; there doesn't appear to be any
2412	* special cases for the architectures.
2413	*/
2414	layout_sections(info->mod, info);
2415	layout_symtab(info->mod, info);
2416
2417	/* Allocate and move to the final place */
2418	err = move_module(info->mod, info);
2419	if (err)
2420	return ERR_PTR(err);
2421
2422	/* Module has been copied to its final place now: return it. */
2423	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2424	kmemleak_load_module(mod, info);
2425	return mod;
2426	}
2427
2428	/* mod is no longer valid after this! */
2429	static void module_deallocate(struct module mod, struct load_info info)
2430	{
2431	percpu_modfree(mod);
2432	module_arch_freeing_init(mod);
2433
2434	free_mod_mem(mod, true);
2435	}
2436
2437	int __weak module_finalize(const Elf_Ehdr *hdr,
2438	const Elf_Shdr *sechdrs,
2439	struct module *me)
2440	{
2441	return 0;
2442	}
2443
2444	static int post_relocation(struct module mod, const struct load_info info)
2445	{
2446	/* Sort exception table now relocations are done. */
2447	sort_extable(mod->extable, mod->extable + mod->num_exentries);
2448
2449	/* Copy relocated percpu area over. */
2450	percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
2451	info->sechdrs[info->index.pcpu].sh_size);
2452
2453	/* Setup kallsyms-specific fields. */
2454	add_kallsyms(mod, info);
2455
2456	/* Arch-specific module finalizing. */
2457	return module_finalize(info->hdr, info->sechdrs, mod);
2458	}
2459
2460	/* Call module constructors. */
2461	static void do_mod_ctors(struct module *mod)
2462	{
2463	#ifdef CONFIG_CONSTRUCTORS
2464	unsigned long i;
2465
2466	for (i = 0; i < mod->num_ctors; i++)
2467	mod->ctors[i]();
2468	#endif
2469	}
2470
2471	/* For freeing module_init on success, in case kallsyms traversing */
2472	struct mod_initfree {
2473	struct llist_node node;
2474	void *init_text;
2475	void *init_data;
2476	void *init_rodata;
2477	};
2478
2479	static void do_free_init(struct work_struct *w)
2480	{
2481	struct llist_node pos, n, *list;
2482	struct mod_initfree *initfree;
2483
2484	list = llist_del_all(&init_free_list);
2485
2486	synchronize_rcu();
2487
2488	llist_for_each_safe(pos, n, list) {
2489	initfree = container_of(pos, struct mod_initfree, node);
2490	execmem_free(initfree->init_text);
2491	execmem_free(initfree->init_data);
2492	execmem_free(initfree->init_rodata);
2493	kfree(initfree);
2494	}
2495	}
2496
2497	void flush_module_init_free_work(void)
2498	{
2499	flush_work(&init_free_wq);
2500	}
2501
2502	#undef MODULE_PARAM_PREFIX
2503	#define MODULE_PARAM_PREFIX "module."
2504	/* Default value for module->async_probe_requested */
2505	static bool async_probe;
2506	module_param(async_probe, bool, 0644);
2507
2508	/*
2509	* This is where the real work happens.
2510	*
2511	* Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
2512	* helper command 'lx-symbols'.
2513	*/
2514	static noinline int do_init_module(struct module *mod)
2515	{
2516	int ret = 0;
2517	struct mod_initfree *freeinit;
2518	#if defined(CONFIG_MODULE_STATS)
2519	unsigned int text_size = 0, total_size = 0;
2520
2521	for_each_mod_mem_type(type) {
2522	const struct module_memory *mod_mem = &mod->mem[type];
2523	if (mod_mem->size) {
2524	total_size += mod_mem->size;
2525	if (type == MOD_TEXT \|\| type == MOD_INIT_TEXT)
2526	text_size += mod_mem->size;
2527	}
2528	}
2529	#endif
2530
2531	freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
2532	if (!freeinit) {
2533	ret = -ENOMEM;
2534	goto fail;
2535	}
2536	freeinit->init_text = mod->mem[MOD_INIT_TEXT].base;
2537	freeinit->init_data = mod->mem[MOD_INIT_DATA].base;
2538	freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base;
2539
2540	do_mod_ctors(mod);
2541	/* Start the module */
2542	if (mod->init != NULL)
2543	ret = do_one_initcall(mod->init);
2544	if (ret < 0) {
2545	goto fail_free_freeinit;
2546	}
2547	if (ret > 0) {
2548	pr_warn("%s: '%s'->init suspiciously returned %d, it should "
2549	"follow 0/-E convention\n"
2550	"%s: loading module anyway...\n",
2551	__func__, mod->name, ret, __func__);
2552	dump_stack();
2553	}
2554
2555	/* Now it's a first class citizen! */
2556	mod->state = MODULE_STATE_LIVE;
2557	blocking_notifier_call_chain(&module_notify_list,
2558	MODULE_STATE_LIVE, mod);
2559
2560	/* Delay uevent until module has finished its init routine */
2561	kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
2562
2563	/*
2564	* We need to finish all async code before the module init sequence
2565	* is done. This has potential to deadlock if synchronous module
2566	* loading is requested from async (which is not allowed!).
2567	*
2568	* See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous
2569	* request_module() from async workers") for more details.
2570	*/
2571	if (!mod->async_probe_requested)
2572	async_synchronize_full();
2573
2574	ftrace_free_mem(mod, mod->mem[MOD_INIT_TEXT].base,
2575	mod->mem[MOD_INIT_TEXT].base + mod->mem[MOD_INIT_TEXT].size);
2576	mutex_lock(&module_mutex);
2577	/* Drop initial reference. */
2578	module_put(mod);
2579	trim_init_extable(mod);
2580	#ifdef CONFIG_KALLSYMS
2581	/* Switch to core kallsyms now init is done: kallsyms may be walking! */
2582	rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
2583	#endif
2584	ret = module_enable_rodata_ro(mod, true);
2585	if (ret)
2586	goto fail_mutex_unlock;
2587	mod_tree_remove_init(mod);
2588	module_arch_freeing_init(mod);
2589	for_class_mod_mem_type(type, init) {
2590	mod->mem[type].base = NULL;
2591	mod->mem[type].size = 0;
2592	}
2593
2594	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
2595	/* .BTF is not SHF_ALLOC and will get removed, so sanitize pointers */
2596	mod->btf_data = NULL;
2597	mod->btf_base_data = NULL;
2598	#endif
2599	/*
2600	* We want to free module_init, but be aware that kallsyms may be
2601	* walking this with preempt disabled. In all the failure paths, we
2602	* call synchronize_rcu(), but we don't want to slow down the success
2603	* path. execmem_free() cannot be called in an interrupt, so do the
2604	* work and call synchronize_rcu() in a work queue.
2605	*
2606	* Note that execmem_alloc() on most architectures creates W+X page
2607	* mappings which won't be cleaned up until do_free_init() runs. Any
2608	* code such as mark_rodata_ro() which depends on those mappings to
2609	* be cleaned up needs to sync with the queued work by invoking
2610	* flush_module_init_free_work().
2611	*/
2612	if (llist_add(&freeinit->node, &init_free_list))
2613	schedule_work(&init_free_wq);
2614
2615	mutex_unlock(&module_mutex);
2616	wake_up_all(&module_wq);
2617
2618	mod_stat_add_long(text_size, &total_text_size);
2619	mod_stat_add_long(total_size, &total_mod_size);
2620
2621	mod_stat_inc(&modcount);
2622
2623	return 0;
2624
2625	fail_mutex_unlock:
2626	mutex_unlock(&module_mutex);
2627	fail_free_freeinit:
2628	kfree(freeinit);
2629	fail:
2630	/* Try to protect us from buggy refcounters. */
2631	mod->state = MODULE_STATE_GOING;
2632	synchronize_rcu();
2633	module_put(mod);
2634	blocking_notifier_call_chain(&module_notify_list,
2635	MODULE_STATE_GOING, mod);
2636	klp_module_going(mod);
2637	ftrace_release_mod(mod);
2638	free_module(mod);
2639	wake_up_all(&module_wq);
2640
2641	return ret;
2642	}
2643
2644	static int may_init_module(void)
2645	{
2646	if (!capable(CAP_SYS_MODULE) \|\| modules_disabled)
2647	return -EPERM;
2648
2649	return 0;
2650	}
2651
2652	/* Is this module of this name done loading? No locks held. */
2653	static bool finished_loading(const char *name)
2654	{
2655	struct module *mod;
2656	bool ret;
2657
2658	/*
2659	* The module_mutex should not be a heavily contended lock;
2660	* if we get the occasional sleep here, we'll go an extra iteration
2661	* in the wait_event_interruptible(), which is harmless.
2662	*/
2663	sched_annotate_sleep();
2664	mutex_lock(&module_mutex);
2665	mod = find_module_all(name, strlen(name), true);
2666	ret = !mod \|\| mod->state == MODULE_STATE_LIVE
2667	\|\| mod->state == MODULE_STATE_GOING;
2668	mutex_unlock(&module_mutex);
2669
2670	return ret;
2671	}
2672
2673	/* Must be called with module_mutex held */
2674	static int module_patient_check_exists(const char *name,
2675	enum fail_dup_mod_reason reason)
2676	{
2677	struct module *old;
2678	int err = 0;
2679
2680	old = find_module_all(name, strlen(name), true);
2681	if (old == NULL)
2682	return 0;
2683
2684	if (old->state == MODULE_STATE_COMING \|\|
2685	old->state == MODULE_STATE_UNFORMED) {
2686	/* Wait in case it fails to load. */
2687	mutex_unlock(&module_mutex);
2688	err = wait_event_interruptible(module_wq,
2689	finished_loading(name));
2690	mutex_lock(&module_mutex);
2691	if (err)
2692	return err;
2693
2694	/* The module might have gone in the meantime. */
2695	old = find_module_all(name, strlen(name), true);
2696	}
2697
2698	if (try_add_failed_module(name, reason))
2699	pr_warn("Could not add fail-tracking for module: %s\n", name);
2700
2701	/*
2702	* We are here only when the same module was being loaded. Do
2703	* not try to load it again right now. It prevents long delays
2704	* caused by serialized module load failures. It might happen
2705	* when more devices of the same type trigger load of
2706	* a particular module.
2707	*/
2708	if (old && old->state == MODULE_STATE_LIVE)
2709	return -EEXIST;
2710	return -EBUSY;
2711	}
2712
2713	/*
2714	* We try to place it in the list now to make sure it's unique before
2715	* we dedicate too many resources. In particular, temporary percpu
2716	* memory exhaustion.
2717	*/
2718	static int add_unformed_module(struct module *mod)
2719	{
2720	int err;
2721
2722	mod->state = MODULE_STATE_UNFORMED;
2723
2724	mutex_lock(&module_mutex);
2725	err = module_patient_check_exists(mod->name, FAIL_DUP_MOD_LOAD);
2726	if (err)
2727	goto out;
2728
2729	mod_update_bounds(mod);
2730	list_add_rcu(&mod->list, &modules);
2731	mod_tree_insert(mod);
2732	err = 0;
2733
2734	out:
2735	mutex_unlock(&module_mutex);
2736	return err;
2737	}
2738
2739	static int complete_formation(struct module mod, struct load_info info)
2740	{
2741	int err;
2742
2743	mutex_lock(&module_mutex);
2744
2745	/* Find duplicate symbols (must be called under lock). */
2746	err = verify_exported_symbols(mod);
2747	if (err < 0)
2748	goto out;
2749
2750	/* These rely on module_mutex for list integrity. */
2751	module_bug_finalize(info->hdr, info->sechdrs, mod);
2752	module_cfi_finalize(info->hdr, info->sechdrs, mod);
2753
2754	err = module_enable_rodata_ro(mod, false);
2755	if (err)
2756	goto out_strict_rwx;
2757	err = module_enable_data_nx(mod);
2758	if (err)
2759	goto out_strict_rwx;
2760	err = module_enable_text_rox(mod);
2761	if (err)
2762	goto out_strict_rwx;
2763
2764	/*
2765	* Mark state as coming so strong_try_module_get() ignores us,
2766	* but kallsyms etc. can see us.
2767	*/
2768	mod->state = MODULE_STATE_COMING;
2769	mutex_unlock(&module_mutex);
2770
2771	return 0;
2772
2773	out_strict_rwx:
2774	module_bug_cleanup(mod);
2775	out:
2776	mutex_unlock(&module_mutex);
2777	return err;
2778	}
2779
2780	static int prepare_coming_module(struct module *mod)
2781	{
2782	int err;
2783
2784	ftrace_module_enable(mod);
2785	err = klp_module_coming(mod);
2786	if (err)
2787	return err;
2788
2789	err = blocking_notifier_call_chain_robust(&module_notify_list,
2790	MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
2791	err = notifier_to_errno(err);
2792	if (err)
2793	klp_module_going(mod);
2794
2795	return err;
2796	}
2797
2798	static int unknown_module_param_cb(char param, char val, const char *modname,
2799	void *arg)
2800	{
2801	struct module *mod = arg;
2802	int ret;
2803
2804	if (strcmp(param, "async_probe") == 0) {
2805	if (kstrtobool(val, &mod->async_probe_requested))
2806	mod->async_probe_requested = true;
2807	return 0;
2808	}
2809
2810	/* Check for magic 'dyndbg' arg */
2811	ret = ddebug_dyndbg_module_param_cb(param, val, modname);
2812	if (ret != 0)
2813	pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
2814	return 0;
2815	}
2816
2817	/* Module within temporary copy, this doesn't do any allocation */
2818	static int early_mod_check(struct load_info *info, int flags)
2819	{
2820	int err;
2821
2822	/*
2823	* Now that we know we have the correct module name, check
2824	* if it's blacklisted.
2825	*/
2826	if (blacklisted(info->name)) {
2827	pr_err("Module %s is blacklisted\n", info->name);
2828	return -EPERM;
2829	}
2830
2831	err = rewrite_section_headers(info, flags);
2832	if (err)
2833	return err;
2834
2835	/* Check module struct version now, before we try to use module. */
2836	if (!check_modstruct_version(info, info->mod))
2837	return -ENOEXEC;
2838
2839	err = check_modinfo(info->mod, info, flags);
2840	if (err)
2841	return err;
2842
2843	mutex_lock(&module_mutex);
2844	err = module_patient_check_exists(info->mod->name, FAIL_DUP_MOD_BECOMING);
2845	mutex_unlock(&module_mutex);
2846
2847	return err;
2848	}
2849
2850	/*
2851	* Allocate and load the module: note that size of section 0 is always
2852	* zero, and we rely on this for optional sections.
2853	*/
2854	static int load_module(struct load_info info, const char __user uargs,
2855	int flags)
2856	{
2857	struct module *mod;
2858	bool module_allocated = false;
2859	long err = 0;
2860	char *after_dashes;
2861
2862	/*
2863	* Do the signature check (if any) first. All that
2864	* the signature check needs is info->len, it does
2865	* not need any of the section info. That can be
2866	* set up later. This will minimize the chances
2867	* of a corrupt module causing problems before
2868	* we even get to the signature check.
2869	*
2870	* The check will also adjust info->len by stripping
2871	* off the sig length at the end of the module, making
2872	* checks against info->len more correct.
2873	*/
2874	err = module_sig_check(info, flags);
2875	if (err)
2876	goto free_copy;
2877
2878	/*
2879	* Do basic sanity checks against the ELF header and
2880	* sections. Cache useful sections and set the
2881	* info->mod to the userspace passed struct module.
2882	*/
2883	err = elf_validity_cache_copy(info, flags);
2884	if (err)
2885	goto free_copy;
2886
2887	err = early_mod_check(info, flags);
2888	if (err)
2889	goto free_copy;
2890
2891	/* Figure out module layout, and allocate all the memory. */
2892	mod = layout_and_allocate(info, flags);
2893	if (IS_ERR(mod)) {
2894	err = PTR_ERR(mod);
2895	goto free_copy;
2896	}
2897
2898	module_allocated = true;
2899
2900	audit_log_kern_module(mod->name);
2901
2902	/* Reserve our place in the list. */
2903	err = add_unformed_module(mod);
2904	if (err)
2905	goto free_module;
2906
2907	/*
2908	* We are tainting your kernel if your module gets into
2909	* the modules linked list somehow.
2910	*/
2911	module_augment_kernel_taints(mod, info);
2912
2913	/* To avoid stressing percpu allocator, do this once we're unique. */
2914	err = percpu_modalloc(mod, info);
2915	if (err)
2916	goto unlink_mod;
2917
2918	/* Now module is in final location, initialize linked lists, etc. */
2919	err = module_unload_init(mod);
2920	if (err)
2921	goto unlink_mod;
2922
2923	init_param_lock(mod);
2924
2925	/*
2926	* Now we've got everything in the final locations, we can
2927	* find optional sections.
2928	*/
2929	err = find_module_sections(mod, info);
2930	if (err)
2931	goto free_unload;
2932
2933	err = check_export_symbol_versions(mod);
2934	if (err)
2935	goto free_unload;
2936
2937	/* Set up MODINFO_ATTR fields */
2938	setup_modinfo(mod, info);
2939
2940	/* Fix up syms, so that st_value is a pointer to location. */
2941	err = simplify_symbols(mod, info);
2942	if (err < 0)
2943	goto free_modinfo;
2944
2945	err = apply_relocations(mod, info);
2946	if (err < 0)
2947	goto free_modinfo;
2948
2949	err = post_relocation(mod, info);
2950	if (err < 0)
2951	goto free_modinfo;
2952
2953	flush_module_icache(mod);
2954
2955	/* Now copy in args */
2956	mod->args = strndup_user(uargs, ~0UL >> 1);
2957	if (IS_ERR(mod->args)) {
2958	err = PTR_ERR(mod->args);
2959	goto free_arch_cleanup;
2960	}
2961
2962	init_build_id(mod, info);
2963
2964	/* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
2965	ftrace_module_init(mod);
2966
2967	/* Finally it's fully formed, ready to start executing. */
2968	err = complete_formation(mod, info);
2969	if (err)
2970	goto ddebug_cleanup;
2971
2972	err = prepare_coming_module(mod);
2973	if (err)
2974	goto bug_cleanup;
2975
2976	mod->async_probe_requested = async_probe;
2977
2978	/* Module is ready to execute: parsing args may do that. */
2979	after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
2980	-32768, 32767, mod,
2981	unknown_module_param_cb);
2982	if (IS_ERR(after_dashes)) {
2983	err = PTR_ERR(after_dashes);
2984	goto coming_cleanup;
2985	} else if (after_dashes) {
2986	pr_warn("%s: parameters '%s' after `--' ignored\n",
2987	mod->name, after_dashes);
2988	}
2989
2990	/* Link in to sysfs. */
2991	err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
2992	if (err < 0)
2993	goto coming_cleanup;
2994
2995	if (is_livepatch_module(mod)) {
2996	err = copy_module_elf(mod, info);
2997	if (err < 0)
2998	goto sysfs_cleanup;
2999	}
3000
3001	/* Get rid of temporary copy. */
3002	free_copy(info, flags);
3003
3004	codetag_load_module(mod);
3005
3006	/* Done! */
3007	trace_module_load(mod);
3008
3009	return do_init_module(mod);
3010
3011	sysfs_cleanup:
3012	mod_sysfs_teardown(mod);
3013	coming_cleanup:
3014	mod->state = MODULE_STATE_GOING;
3015	destroy_params(mod->kp, mod->num_kp);
3016	blocking_notifier_call_chain(&module_notify_list,
3017	MODULE_STATE_GOING, mod);
3018	klp_module_going(mod);
3019	bug_cleanup:
3020	mod->state = MODULE_STATE_GOING;
3021	/* module_bug_cleanup needs module_mutex protection */
3022	mutex_lock(&module_mutex);
3023	module_bug_cleanup(mod);
3024	mutex_unlock(&module_mutex);
3025
3026	ddebug_cleanup:
3027	ftrace_release_mod(mod);
3028	synchronize_rcu();
3029	kfree(mod->args);
3030	free_arch_cleanup:
3031	module_arch_cleanup(mod);
3032	free_modinfo:
3033	free_modinfo(mod);
3034	free_unload:
3035	module_unload_free(mod);
3036	unlink_mod:
3037	mutex_lock(&module_mutex);
3038	/* Unlink carefully: kallsyms could be walking list. */
3039	list_del_rcu(&mod->list);
3040	mod_tree_remove(mod);
3041	wake_up_all(&module_wq);
3042	/* Wait for RCU-sched synchronizing before releasing mod->list. */
3043	synchronize_rcu();
3044	mutex_unlock(&module_mutex);
3045	free_module:
3046	mod_stat_bump_invalid(info, flags);
3047	/* Free lock-classes; relies on the preceding sync_rcu() */
3048	for_class_mod_mem_type(type, core_data) {
3049	lockdep_free_key_range(mod->mem[type].base,
3050	mod->mem[type].size);
3051	}
3052
3053	module_deallocate(mod, info);
3054	free_copy:
3055	/*
3056	* The info->len is always set. We distinguish between
3057	* failures once the proper module was allocated and
3058	* before that.
3059	*/
3060	if (!module_allocated)
3061	mod_stat_bump_becoming(info, flags);
3062	free_copy(info, flags);
3063	return err;
3064	}
3065
3066	SYSCALL_DEFINE3(init_module, void __user *, umod,
3067	unsigned long, len, const char __user *, uargs)
3068	{
3069	int err;
3070	struct load_info info = { };
3071
3072	err = may_init_module();
3073	if (err)
3074	return err;
3075
3076	pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
3077	umod, len, uargs);
3078
3079	err = copy_module_from_user(umod, len, &info);
3080	if (err) {
3081	mod_stat_inc(&failed_kreads);
3082	mod_stat_add_long(len, &invalid_kread_bytes);
3083	return err;
3084	}
3085
3086	return load_module(&info, uargs, 0);
3087	}
3088
3089	struct idempotent {
3090	const void *cookie;
3091	struct hlist_node entry;
3092	struct completion complete;
3093	int ret;
3094	};
3095
3096	#define IDEM_HASH_BITS 8
3097	static struct hlist_head idem_hash[1 << IDEM_HASH_BITS];
3098	static DEFINE_SPINLOCK(idem_lock);
3099
3100	static bool idempotent(struct idempotent u, const void cookie)
3101	{
3102	int hash = hash_ptr(cookie, IDEM_HASH_BITS);
3103	struct hlist_head *head = idem_hash + hash;
3104	struct idempotent *existing;
3105	bool first;
3106
3107	u->ret = -EINTR;
3108	u->cookie = cookie;
3109	init_completion(&u->complete);
3110
3111	spin_lock(&idem_lock);
3112	first = true;
3113	hlist_for_each_entry(existing, head, entry) {
3114	if (existing->cookie != cookie)
3115	continue;
3116	first = false;
3117	break;
3118	}
3119	hlist_add_head(&u->entry, idem_hash + hash);
3120	spin_unlock(&idem_lock);
3121
3122	return !first;
3123	}
3124
3125	/*
3126	* We were the first one with 'cookie' on the list, and we ended
3127	* up completing the operation. We now need to walk the list,
3128	* remove everybody - which includes ourselves - fill in the return
3129	* value, and then complete the operation.
3130	*/
3131	static int idempotent_complete(struct idempotent *u, int ret)
3132	{
3133	const void *cookie = u->cookie;
3134	int hash = hash_ptr(cookie, IDEM_HASH_BITS);
3135	struct hlist_head *head = idem_hash + hash;
3136	struct hlist_node *next;
3137	struct idempotent *pos;
3138
3139	spin_lock(&idem_lock);
3140	hlist_for_each_entry_safe(pos, next, head, entry) {
3141	if (pos->cookie != cookie)
3142	continue;
3143	hlist_del_init(&pos->entry);
3144	pos->ret = ret;
3145	complete(&pos->complete);
3146	}
3147	spin_unlock(&idem_lock);
3148	return ret;
3149	}
3150
3151	/*
3152	* Wait for the idempotent worker.
3153	*
3154	* If we get interrupted, we need to remove ourselves from the
3155	* the idempotent list, and the completion may still come in.
3156	*
3157	* The 'idem_lock' protects against the race, and 'idem.ret' was
3158	* initialized to -EINTR and is thus always the right return
3159	* value even if the idempotent work then completes between
3160	* the wait_for_completion and the cleanup.
3161	*/
3162	static int idempotent_wait_for_completion(struct idempotent *u)
3163	{
3164	if (wait_for_completion_interruptible(&u->complete)) {
3165	spin_lock(&idem_lock);
3166	if (!hlist_unhashed(&u->entry))
3167	hlist_del(&u->entry);
3168	spin_unlock(&idem_lock);
3169	}
3170	return u->ret;
3171	}
3172
3173	static int init_module_from_file(struct file f, const char __user uargs, int flags)
3174	{
3175	struct load_info info = { };
3176	void *buf = NULL;
3177	int len;
3178
3179	len = kernel_read_file(f, 0, &buf, INT_MAX, NULL, READING_MODULE);
3180	if (len < 0) {
3181	mod_stat_inc(&failed_kreads);
3182	return len;
3183	}
3184
3185	if (flags & MODULE_INIT_COMPRESSED_FILE) {
3186	int err = module_decompress(&info, buf, len);
3187	vfree(buf); /* compressed data is no longer needed */
3188	if (err) {
3189	mod_stat_inc(&failed_decompress);
3190	mod_stat_add_long(len, &invalid_decompress_bytes);
3191	return err;
3192	}
3193	} else {
3194	info.hdr = buf;
3195	info.len = len;
3196	}
3197
3198	return load_module(&info, uargs, flags);
3199	}
3200
3201	static int idempotent_init_module(struct file f, const char __user uargs, int flags)
3202	{
3203	struct idempotent idem;
3204
3205	if (!f \|\| !(f->f_mode & FMODE_READ))
3206	return -EBADF;
3207
3208	/* Are we the winners of the race and get to do this? */
3209	if (!idempotent(&idem, file_inode(f))) {
3210	int ret = init_module_from_file(f, uargs, flags);
3211	return idempotent_complete(&idem, ret);
3212	}
3213
3214	/*
3215	* Somebody else won the race and is loading the module.
3216	*/
3217	return idempotent_wait_for_completion(&idem);
3218	}
3219
3220	SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
3221	{
3222	int err;
3223	struct fd f;
3224
3225	err = may_init_module();
3226	if (err)
3227	return err;
3228
3229	pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
3230
3231	if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
3232	\|MODULE_INIT_IGNORE_VERMAGIC
3233	\|MODULE_INIT_COMPRESSED_FILE))
3234	return -EINVAL;
3235
3236	f = fdget(fd);
3237	err = idempotent_init_module(fd_file(f), uargs, flags);
3238	fdput(f);
3239	return err;
3240	}
3241
3242	/* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
3243	char module_flags(struct module mod, char *buf, bool show_state)
3244	{
3245	int bx = 0;
3246
3247	BUG_ON(mod->state == MODULE_STATE_UNFORMED);
3248	if (!mod->taints && !show_state)
3249	goto out;
3250	if (mod->taints \|\|
3251	mod->state == MODULE_STATE_GOING \|\|
3252	mod->state == MODULE_STATE_COMING) {
3253	buf[bx++] = '(';
3254	bx += module_flags_taint(mod->taints, buf + bx);
3255	/* Show a - for module-is-being-unloaded */
3256	if (mod->state == MODULE_STATE_GOING && show_state)
3257	buf[bx++] = '-';
3258	/* Show a + for module-is-being-loaded */
3259	if (mod->state == MODULE_STATE_COMING && show_state)
3260	buf[bx++] = '+';
3261	buf[bx++] = ')';
3262	}
3263	out:
3264	buf[bx] = '\0';
3265
3266	return buf;
3267	}
3268
3269	/* Given an address, look for it in the module exception tables. */
3270	const struct exception_table_entry *search_module_extables(unsigned long addr)
3271	{
3272	const struct exception_table_entry *e = NULL;
3273	struct module *mod;
3274
3275	preempt_disable();
3276	mod = __module_address(addr);
3277	if (!mod)
3278	goto out;
3279
3280	if (!mod->num_exentries)
3281	goto out;
3282
3283	e = search_extable(mod->extable,
3284	mod->num_exentries,
3285	addr);
3286	out:
3287	preempt_enable();
3288
3289	/*
3290	* Now, if we found one, we are running inside it now, hence
3291	* we cannot unload the module, hence no refcnt needed.
3292	*/
3293	return e;
3294	}
3295
3296	/**
3297	* is_module_address() - is this address inside a module?
3298	* @addr: the address to check.
3299	*
3300	* See is_module_text_address() if you simply want to see if the address
3301	* is code (not data).
3302	*/
3303	bool is_module_address(unsigned long addr)
3304	{
3305	bool ret;
3306
3307	preempt_disable();
3308	ret = __module_address(addr) != NULL;
3309	preempt_enable();
3310
3311	return ret;
3312	}
3313
3314	/**
3315	* __module_address() - get the module which contains an address.
3316	* @addr: the address.
3317	*
3318	* Must be called with preempt disabled or module mutex held so that
3319	* module doesn't get freed during this.
3320	*/
3321	struct module *__module_address(unsigned long addr)
3322	{
3323	struct module *mod;
3324
3325	if (addr >= mod_tree.addr_min && addr <= mod_tree.addr_max)
3326	goto lookup;
3327
3328	#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
3329	if (addr >= mod_tree.data_addr_min && addr <= mod_tree.data_addr_max)
3330	goto lookup;
3331	#endif
3332
3333	return NULL;
3334
3335	lookup:
3336	module_assert_mutex_or_preempt();
3337
3338	mod = mod_find(addr, &mod_tree);
3339	if (mod) {
3340	BUG_ON(!within_module(addr, mod));
3341	if (mod->state == MODULE_STATE_UNFORMED)
3342	mod = NULL;
3343	}
3344	return mod;
3345	}
3346
3347	/**
3348	* is_module_text_address() - is this address inside module code?
3349	* @addr: the address to check.
3350	*
3351	* See is_module_address() if you simply want to see if the address is
3352	* anywhere in a module. See kernel_text_address() for testing if an
3353	* address corresponds to kernel or module code.
3354	*/
3355	bool is_module_text_address(unsigned long addr)
3356	{
3357	bool ret;
3358
3359	preempt_disable();
3360	ret = __module_text_address(addr) != NULL;
3361	preempt_enable();
3362
3363	return ret;
3364	}
3365
3366	/**
3367	* __module_text_address() - get the module whose code contains an address.
3368	* @addr: the address.
3369	*
3370	* Must be called with preempt disabled or module mutex held so that
3371	* module doesn't get freed during this.
3372	*/
3373	struct module *__module_text_address(unsigned long addr)
3374	{
3375	struct module *mod = __module_address(addr);
3376	if (mod) {
3377	/* Make sure it's within the text section. */
3378	if (!within_module_mem_type(addr, mod, MOD_TEXT) &&
3379	!within_module_mem_type(addr, mod, MOD_INIT_TEXT))
3380	mod = NULL;
3381	}
3382	return mod;
3383	}
3384
3385	/* Don't grab lock, we're oopsing. */
3386	void print_modules(void)
3387	{
3388	struct module *mod;
3389	char buf[MODULE_FLAGS_BUF_SIZE];
3390
3391	printk(KERN_DEFAULT "Modules linked in:");
3392	/* Most callers should already have preempt disabled, but make sure */
3393	preempt_disable();
3394	list_for_each_entry_rcu(mod, &modules, list) {
3395	if (mod->state == MODULE_STATE_UNFORMED)
3396	continue;
3397	pr_cont(" %s%s", mod->name, module_flags(mod, buf, true));
3398	}
3399
3400	print_unloaded_tainted_modules();
3401	preempt_enable();
3402	if (last_unloaded_module.name[0])
3403	pr_cont(" [last unloaded: %s%s]", last_unloaded_module.name,
3404	last_unloaded_module.taints);
3405	pr_cont("\n");
3406	}
3407
3408	#ifdef CONFIG_MODULE_DEBUGFS
3409	struct dentry *mod_debugfs_root;
3410
3411	static int module_debugfs_init(void)
3412	{
3413	mod_debugfs_root = debugfs_create_dir("modules", NULL);
3414	return 0;
3415	}
3416	module_init(module_debugfs_init);
3417	#endif