projects/linux·blob v6.13 · refs

411 lines · 12.0 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/* Module internals
3 *
4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells ([email protected])
6 * Copyright (C) 2023 Luis Chamberlain <[email protected]>
7 */
8
9#include <linux/elf.h>
10#include <linux/compiler.h>
11#include <linux/module.h>
12#include <linux/mutex.h>
13#include <linux/rculist.h>
14#include <linux/rcupdate.h>
15#include <linux/mm.h>
16
17#ifndef ARCH_SHF_SMALL
18#define ARCH_SHF_SMALL 0
19#endif
20
21/*
22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this
23 * section. This leaves 28 bits for offset on 32-bit systems, which is
24 * about 256 MiB (WARN_ON_ONCE if we exceed that).
25 */
26
27#define SH_ENTSIZE_TYPE_BITS 4
28#define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)
29#define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1)
30#define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1)
31
32/* Maximum number of characters written by module_flags() */
33#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
34
35struct kernel_symbol {
36#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
37 int value_offset;
38 int name_offset;
39 int namespace_offset;
40#else
41 unsigned long value;
42 const char *name;
43 const char *namespace;
44#endif
45};
46
47extern struct mutex module_mutex;
48extern struct list_head modules;
49
50extern struct module_attribute *modinfo_attrs[];
51extern size_t modinfo_attrs_count;
52
53/* Provided by the linker */
54extern const struct kernel_symbol __start___ksymtab[];
55extern const struct kernel_symbol __stop___ksymtab[];
56extern const struct kernel_symbol __start___ksymtab_gpl[];
57extern const struct kernel_symbol __stop___ksymtab_gpl[];
58extern const s32 __start___kcrctab[];
59extern const s32 __start___kcrctab_gpl[];
60
61struct load_info {
62 const char *name;
63 /* pointer to module in temporary copy, freed at end of load_module() */
64 struct module *mod;
65 Elf_Ehdr *hdr;
66 unsigned long len;
67 Elf_Shdr *sechdrs;
68 char *secstrings, *strtab;
69 unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs;
70 bool sig_ok;
71#ifdef CONFIG_KALLSYMS
72 unsigned long mod_kallsyms_init_off;
73#endif
74#ifdef CONFIG_MODULE_DECOMPRESS
75#ifdef CONFIG_MODULE_STATS
76 unsigned long compressed_len;
77#endif
78 struct page **pages;
79 unsigned int max_pages;
80 unsigned int used_pages;
81#endif
82 struct {
83 unsigned int sym;
84 unsigned int str;
85 unsigned int mod;
86 unsigned int vers;
87 unsigned int info;
88 unsigned int pcpu;
89 } index;
90};
91
92enum mod_license {
93 NOT_GPL_ONLY,
94 GPL_ONLY,
95};
96
97struct find_symbol_arg {
98 /* Input */
99 const char *name;
100 bool gplok;
101 bool warn;
102
103 /* Output */
104 struct module *owner;
105 const s32 *crc;
106 const struct kernel_symbol *sym;
107 enum mod_license license;
108};
109
110int mod_verify_sig(const void *mod, struct load_info *info);
111int try_to_force_load(struct module *mod, const char *reason);
112bool find_symbol(struct find_symbol_arg *fsa);
113struct module *find_module_all(const char *name, size_t len, bool even_unformed);
114int cmp_name(const void *name, const void *sym);
115long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
116 Elf_Shdr *sechdr, unsigned int section);
117char *module_flags(struct module *mod, char *buf, bool show_state);
118size_t module_flags_taint(unsigned long taints, char *buf);
119
120char *module_next_tag_pair(char *string, unsigned long *secsize);
121
122#define for_each_modinfo_entry(entry, info, name) \
123 for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry))
124
125static inline void module_assert_mutex_or_preempt(void)
126{
127#ifdef CONFIG_LOCKDEP
128 if (unlikely(!debug_locks))
129 return;
130
131 WARN_ON_ONCE(!rcu_read_lock_sched_held() &&
132 !lockdep_is_held(&module_mutex));
133#endif
134}
135
136static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym)
137{
138#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
139 return (unsigned long)offset_to_ptr(&sym->value_offset);
140#else
141 return sym->value;
142#endif
143}
144
145#ifdef CONFIG_LIVEPATCH
146int copy_module_elf(struct module *mod, struct load_info *info);
147void free_module_elf(struct module *mod);
148#else /* !CONFIG_LIVEPATCH */
149static inline int copy_module_elf(struct module *mod, struct load_info *info)
150{
151 return 0;
152}
153
154static inline void free_module_elf(struct module *mod) { }
155#endif /* CONFIG_LIVEPATCH */
156
157static inline bool set_livepatch_module(struct module *mod)
158{
159#ifdef CONFIG_LIVEPATCH
160 mod->klp = true;
161 return true;
162#else
163 return false;
164#endif
165}
166
167/**
168 * enum fail_dup_mod_reason - state at which a duplicate module was detected
169 *
170 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but
171 * we've determined that another module with the same name is already loaded
172 * or being processed on our &modules list. This happens on early_mod_check()
173 * right before layout_and_allocate(). The kernel would have already
174 * vmalloc()'d space for the entire module through finit_module(). If
175 * decompression was used two vmap() spaces were used. These failures can
176 * happen when userspace has not seen the module present on the kernel and
177 * tries to load the module multiple times at same time.
178 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation
179 * checks and the kernel determines that the module was unique and because
180 * of this allocated yet another private kernel copy of the module space in
181 * layout_and_allocate() but after this determined in add_unformed_module()
182 * that another module with the same name is already loaded or being processed.
183 * These failures should be mitigated as much as possible and are indicative
184 * of really fast races in loading modules. Without module decompression
185 * they waste twice as much vmap space. With module decompression three
186 * times the module's size vmap space is wasted.
187 */
188enum fail_dup_mod_reason {
189 FAIL_DUP_MOD_BECOMING = 0,
190 FAIL_DUP_MOD_LOAD,
191};
192
193#ifdef CONFIG_MODULE_DEBUGFS
194extern struct dentry *mod_debugfs_root;
195#endif
196
197#ifdef CONFIG_MODULE_STATS
198
199#define mod_stat_add_long(count, var) atomic_long_add(count, var)
200#define mod_stat_inc(name) atomic_inc(name)
201
202extern atomic_long_t total_mod_size;
203extern atomic_long_t total_text_size;
204extern atomic_long_t invalid_kread_bytes;
205extern atomic_long_t invalid_decompress_bytes;
206
207extern atomic_t modcount;
208extern atomic_t failed_kreads;
209extern atomic_t failed_decompress;
210struct mod_fail_load {
211 struct list_head list;
212 char name[MODULE_NAME_LEN];
213 atomic_long_t count;
214 unsigned long dup_fail_mask;
215};
216
217int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason);
218void mod_stat_bump_invalid(struct load_info *info, int flags);
219void mod_stat_bump_becoming(struct load_info *info, int flags);
220
221#else
222
223#define mod_stat_add_long(name, var)
224#define mod_stat_inc(name)
225
226static inline int try_add_failed_module(const char *name,
227 enum fail_dup_mod_reason reason)
228{
229 return 0;
230}
231
232static inline void mod_stat_bump_invalid(struct load_info *info, int flags)
233{
234}
235
236static inline void mod_stat_bump_becoming(struct load_info *info, int flags)
237{
238}
239
240#endif /* CONFIG_MODULE_STATS */
241
242#ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS
243bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret);
244void kmod_dup_request_announce(char *module_name, int ret);
245#else
246static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret)
247{
248 return false;
249}
250
251static inline void kmod_dup_request_announce(char *module_name, int ret)
252{
253}
254#endif
255
256#ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING
257struct mod_unload_taint {
258 struct list_head list;
259 char name[MODULE_NAME_LEN];
260 unsigned long taints;
261 u64 count;
262};
263
264int try_add_tainted_module(struct module *mod);
265void print_unloaded_tainted_modules(void);
266#else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
267static inline int try_add_tainted_module(struct module *mod)
268{
269 return 0;
270}
271
272static inline void print_unloaded_tainted_modules(void)
273{
274}
275#endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
276
277#ifdef CONFIG_MODULE_DECOMPRESS
278int module_decompress(struct load_info *info, const void *buf, size_t size);
279void module_decompress_cleanup(struct load_info *info);
280#else
281static inline int module_decompress(struct load_info *info,
282 const void *buf, size_t size)
283{
284 return -EOPNOTSUPP;
285}
286
287static inline void module_decompress_cleanup(struct load_info *info)
288{
289}
290#endif
291
292struct mod_tree_root {
293#ifdef CONFIG_MODULES_TREE_LOOKUP
294 struct latch_tree_root root;
295#endif
296 unsigned long addr_min;
297 unsigned long addr_max;
298#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
299 unsigned long data_addr_min;
300 unsigned long data_addr_max;
301#endif
302};
303
304extern struct mod_tree_root mod_tree;
305
306#ifdef CONFIG_MODULES_TREE_LOOKUP
307void mod_tree_insert(struct module *mod);
308void mod_tree_remove_init(struct module *mod);
309void mod_tree_remove(struct module *mod);
310struct module *mod_find(unsigned long addr, struct mod_tree_root *tree);
311#else /* !CONFIG_MODULES_TREE_LOOKUP */
312
313static inline void mod_tree_insert(struct module *mod) { }
314static inline void mod_tree_remove_init(struct module *mod) { }
315static inline void mod_tree_remove(struct module *mod) { }
316static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree)
317{
318 struct module *mod;
319
320 list_for_each_entry_rcu(mod, &modules, list,
321 lockdep_is_held(&module_mutex)) {
322 if (within_module(addr, mod))
323 return mod;
324 }
325
326 return NULL;
327}
328#endif /* CONFIG_MODULES_TREE_LOOKUP */
329
330int module_enable_rodata_ro(const struct module *mod, bool after_init);
331int module_enable_data_nx(const struct module *mod);
332int module_enable_text_rox(const struct module *mod);
333int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
334 char *secstrings, struct module *mod);
335
336#ifdef CONFIG_MODULE_SIG
337int module_sig_check(struct load_info *info, int flags);
338#else /* !CONFIG_MODULE_SIG */
339static inline int module_sig_check(struct load_info *info, int flags)
340{
341 return 0;
342}
343#endif /* !CONFIG_MODULE_SIG */
344
345#ifdef CONFIG_DEBUG_KMEMLEAK
346void kmemleak_load_module(const struct module *mod, const struct load_info *info);
347#else /* !CONFIG_DEBUG_KMEMLEAK */
348static inline void kmemleak_load_module(const struct module *mod,
349 const struct load_info *info) { }
350#endif /* CONFIG_DEBUG_KMEMLEAK */
351
352#ifdef CONFIG_KALLSYMS
353void init_build_id(struct module *mod, const struct load_info *info);
354void layout_symtab(struct module *mod, struct load_info *info);
355void add_kallsyms(struct module *mod, const struct load_info *info);
356
357static inline bool sect_empty(const Elf_Shdr *sect)
358{
359 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
360}
361#else /* !CONFIG_KALLSYMS */
362static inline void init_build_id(struct module *mod, const struct load_info *info) { }
363static inline void layout_symtab(struct module *mod, struct load_info *info) { }
364static inline void add_kallsyms(struct module *mod, const struct load_info *info) { }
365#endif /* CONFIG_KALLSYMS */
366
367#ifdef CONFIG_SYSFS
368int mod_sysfs_setup(struct module *mod, const struct load_info *info,
369 struct kernel_param *kparam, unsigned int num_params);
370void mod_sysfs_teardown(struct module *mod);
371void init_param_lock(struct module *mod);
372#else /* !CONFIG_SYSFS */
373static inline int mod_sysfs_setup(struct module *mod,
374 const struct load_info *info,
375 struct kernel_param *kparam,
376 unsigned int num_params)
377{
378 return 0;
379}
380
381static inline void mod_sysfs_teardown(struct module *mod) { }
382static inline void init_param_lock(struct module *mod) { }
383#endif /* CONFIG_SYSFS */
384
385#ifdef CONFIG_MODVERSIONS
386int check_version(const struct load_info *info,
387 const char *symname, struct module *mod, const s32 *crc);
388void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp,
389 struct kernel_symbol *ks, struct tracepoint * const *tp);
390int check_modstruct_version(const struct load_info *info, struct module *mod);
391int same_magic(const char *amagic, const char *bmagic, bool has_crcs);
392#else /* !CONFIG_MODVERSIONS */
393static inline int check_version(const struct load_info *info,
394 const char *symname,
395 struct module *mod,
396 const s32 *crc)
397{
398 return 1;
399}
400
401static inline int check_modstruct_version(const struct load_info *info,
402 struct module *mod)
403{
404 return 1;
405}
406
407static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs)
408{
409 return strcmp(amagic, bmagic) == 0;
410}
411#endif /* CONFIG_MODVERSIONS */