projects/linux·blob uaccess-cleanups · refs

566 lines · 13.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/init.h>
3#include <linux/static_call.h>
4#include <linux/bug.h>
5#include <linux/smp.h>
6#include <linux/sort.h>
7#include <linux/slab.h>
8#include <linux/module.h>
9#include <linux/cpu.h>
10#include <linux/processor.h>
11#include <asm/sections.h>
12
13extern struct static_call_site __start_static_call_sites[],
14 __stop_static_call_sites[];
15extern struct static_call_tramp_key __start_static_call_tramp_key[],
16 __stop_static_call_tramp_key[];
17
18int static_call_initialized;
19
20/*
21 * Must be called before early_initcall() to be effective.
22 */
23void static_call_force_reinit(void)
24{
25 if (WARN_ON_ONCE(!static_call_initialized))
26 return;
27
28 static_call_initialized++;
29}
30
31/* mutex to protect key modules/sites */
32static DEFINE_MUTEX(static_call_mutex);
33
34static void static_call_lock(void)
35{
36 mutex_lock(&static_call_mutex);
37}
38
39static void static_call_unlock(void)
40{
41 mutex_unlock(&static_call_mutex);
42}
43
44static inline void *static_call_addr(struct static_call_site *site)
45{
46 return (void *)((long)site->addr + (long)&site->addr);
47}
48
49static inline unsigned long __static_call_key(const struct static_call_site *site)
50{
51 return (long)site->key + (long)&site->key;
52}
53
54static inline struct static_call_key *static_call_key(const struct static_call_site *site)
55{
56 return (void *)(__static_call_key(site) & ~STATIC_CALL_SITE_FLAGS);
57}
58
59/* These assume the key is word-aligned. */
60static inline bool static_call_is_init(struct static_call_site *site)
61{
62 return __static_call_key(site) & STATIC_CALL_SITE_INIT;
63}
64
65static inline bool static_call_is_tail(struct static_call_site *site)
66{
67 return __static_call_key(site) & STATIC_CALL_SITE_TAIL;
68}
69
70static inline void static_call_set_init(struct static_call_site *site)
71{
72 site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) -
73 (long)&site->key;
74}
75
76static int static_call_site_cmp(const void *_a, const void *_b)
77{
78 const struct static_call_site *a = _a;
79 const struct static_call_site *b = _b;
80 const struct static_call_key *key_a = static_call_key(a);
81 const struct static_call_key *key_b = static_call_key(b);
82
83 if (key_a < key_b)
84 return -1;
85
86 if (key_a > key_b)
87 return 1;
88
89 return 0;
90}
91
92static void static_call_site_swap(void *_a, void *_b, int size)
93{
94 long delta = (unsigned long)_a - (unsigned long)_b;
95 struct static_call_site *a = _a;
96 struct static_call_site *b = _b;
97 struct static_call_site tmp = *a;
98
99 a->addr = b->addr - delta;
100 a->key = b->key - delta;
101
102 b->addr = tmp.addr + delta;
103 b->key = tmp.key + delta;
104}
105
106static inline void static_call_sort_entries(struct static_call_site *start,
107 struct static_call_site *stop)
108{
109 sort(start, stop - start, sizeof(struct static_call_site),
110 static_call_site_cmp, static_call_site_swap);
111}
112
113static inline bool static_call_key_has_mods(struct static_call_key *key)
114{
115 return !(key->type & 1);
116}
117
118static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
119{
120 if (!static_call_key_has_mods(key))
121 return NULL;
122
123 return key->mods;
124}
125
126static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
127{
128 if (static_call_key_has_mods(key))
129 return NULL;
130
131 return (struct static_call_site *)(key->type & ~1);
132}
133
134void __static_call_update(struct static_call_key *key, void *tramp, void *func)
135{
136 struct static_call_site *site, *stop;
137 struct static_call_mod *site_mod, first;
138
139 cpus_read_lock();
140 static_call_lock();
141
142 if (key->func == func)
143 goto done;
144
145 key->func = func;
146
147 arch_static_call_transform(NULL, tramp, func, false);
148
149 /*
150 * If uninitialized, we'll not update the callsites, but they still
151 * point to the trampoline and we just patched that.
152 */
153 if (WARN_ON_ONCE(!static_call_initialized))
154 goto done;
155
156 first = (struct static_call_mod){
157 .next = static_call_key_next(key),
158 .mod = NULL,
159 .sites = static_call_key_sites(key),
160 };
161
162 for (site_mod = &first; site_mod; site_mod = site_mod->next) {
163 bool init = system_state < SYSTEM_RUNNING;
164 struct module *mod = site_mod->mod;
165
166 if (!site_mod->sites) {
167 /*
168 * This can happen if the static call key is defined in
169 * a module which doesn't use it.
170 *
171 * It also happens in the has_mods case, where the
172 * 'first' entry has no sites associated with it.
173 */
174 continue;
175 }
176
177 stop = __stop_static_call_sites;
178
179 if (mod) {
180#ifdef CONFIG_MODULES
181 stop = mod->static_call_sites +
182 mod->num_static_call_sites;
183 init = mod->state == MODULE_STATE_COMING;
184#endif
185 }
186
187 for (site = site_mod->sites;
188 site < stop && static_call_key(site) == key; site++) {
189 void *site_addr = static_call_addr(site);
190
191 if (!init && static_call_is_init(site))
192 continue;
193
194 if (!kernel_text_address((unsigned long)site_addr)) {
195 /*
196 * This skips patching built-in __exit, which
197 * is part of init_section_contains() but is
198 * not part of kernel_text_address().
199 *
200 * Skipping built-in __exit is fine since it
201 * will never be executed.
202 */
203 WARN_ONCE(!static_call_is_init(site),
204 "can't patch static call site at %pS",
205 site_addr);
206 continue;
207 }
208
209 arch_static_call_transform(site_addr, tramp, func,
210 static_call_is_tail(site));
211 }
212 }
213
214done:
215 static_call_unlock();
216 cpus_read_unlock();
217}
218EXPORT_SYMBOL_GPL(__static_call_update);
219
220static int __static_call_init(struct module *mod,
221 struct static_call_site *start,
222 struct static_call_site *stop)
223{
224 struct static_call_site *site;
225 struct static_call_key *key, *prev_key = NULL;
226 struct static_call_mod *site_mod;
227
228 if (start == stop)
229 return 0;
230
231 static_call_sort_entries(start, stop);
232
233 for (site = start; site < stop; site++) {
234 void *site_addr = static_call_addr(site);
235
236 if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
237 (!mod && init_section_contains(site_addr, 1)))
238 static_call_set_init(site);
239
240 key = static_call_key(site);
241 if (key != prev_key) {
242 prev_key = key;
243
244 /*
245 * For vmlinux (!mod) avoid the allocation by storing
246 * the sites pointer in the key itself. Also see
247 * __static_call_update()'s @first.
248 *
249 * This allows architectures (eg. x86) to call
250 * static_call_init() before memory allocation works.
251 */
252 if (!mod) {
253 key->sites = site;
254 key->type |= 1;
255 goto do_transform;
256 }
257
258 site_mod = kzalloc_obj(*site_mod);
259 if (!site_mod)
260 return -ENOMEM;
261
262 /*
263 * When the key has a direct sites pointer, extract
264 * that into an explicit struct static_call_mod, so we
265 * can have a list of modules.
266 */
267 if (static_call_key_sites(key)) {
268 site_mod->mod = NULL;
269 site_mod->next = NULL;
270 site_mod->sites = static_call_key_sites(key);
271
272 key->mods = site_mod;
273
274 site_mod = kzalloc_obj(*site_mod);
275 if (!site_mod)
276 return -ENOMEM;
277 }
278
279 site_mod->mod = mod;
280 site_mod->sites = site;
281 site_mod->next = static_call_key_next(key);
282 key->mods = site_mod;
283 }
284
285do_transform:
286 arch_static_call_transform(site_addr, NULL, key->func,
287 static_call_is_tail(site));
288 }
289
290 return 0;
291}
292
293static int addr_conflict(struct static_call_site *site, void *start, void *end)
294{
295 unsigned long addr = (unsigned long)static_call_addr(site);
296
297 if (addr <= (unsigned long)end &&
298 addr + CALL_INSN_SIZE > (unsigned long)start)
299 return 1;
300
301 return 0;
302}
303
304static int __static_call_text_reserved(struct static_call_site *iter_start,
305 struct static_call_site *iter_stop,
306 void *start, void *end, bool init)
307{
308 struct static_call_site *iter = iter_start;
309
310 while (iter < iter_stop) {
311 if (init || !static_call_is_init(iter)) {
312 if (addr_conflict(iter, start, end))
313 return 1;
314 }
315 iter++;
316 }
317
318 return 0;
319}
320
321#ifdef CONFIG_MODULES
322
323static int __static_call_mod_text_reserved(void *start, void *end)
324{
325 struct module *mod;
326 int ret;
327
328 scoped_guard(rcu) {
329 mod = __module_text_address((unsigned long)start);
330 WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
331 if (!try_module_get(mod))
332 mod = NULL;
333 }
334 if (!mod)
335 return 0;
336
337 ret = __static_call_text_reserved(mod->static_call_sites,
338 mod->static_call_sites + mod->num_static_call_sites,
339 start, end, mod->state == MODULE_STATE_COMING);
340
341 module_put(mod);
342
343 return ret;
344}
345
346static unsigned long tramp_key_lookup(unsigned long addr)
347{
348 struct static_call_tramp_key *start = __start_static_call_tramp_key;
349 struct static_call_tramp_key *stop = __stop_static_call_tramp_key;
350 struct static_call_tramp_key *tramp_key;
351
352 for (tramp_key = start; tramp_key != stop; tramp_key++) {
353 unsigned long tramp;
354
355 tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp;
356 if (tramp == addr)
357 return (long)tramp_key->key + (long)&tramp_key->key;
358 }
359
360 return 0;
361}
362
363static int static_call_add_module(struct module *mod)
364{
365 struct static_call_site *start = mod->static_call_sites;
366 struct static_call_site *stop = start + mod->num_static_call_sites;
367 struct static_call_site *site;
368
369 for (site = start; site != stop; site++) {
370 unsigned long s_key = __static_call_key(site);
371 unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS;
372 unsigned long key;
373
374 /*
375 * Is the key is exported, 'addr' points to the key, which
376 * means modules are allowed to call static_call_update() on
377 * it.
378 *
379 * Otherwise, the key isn't exported, and 'addr' points to the
380 * trampoline so we need to lookup the key.
381 *
382 * We go through this dance to prevent crazy modules from
383 * abusing sensitive static calls.
384 */
385 if (!kernel_text_address(addr))
386 continue;
387
388 key = tramp_key_lookup(addr);
389 if (!key) {
390 pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n",
391 static_call_addr(site));
392 return -EINVAL;
393 }
394
395 key |= s_key & STATIC_CALL_SITE_FLAGS;
396 site->key = key - (long)&site->key;
397 }
398
399 return __static_call_init(mod, start, stop);
400}
401
402static void static_call_del_module(struct module *mod)
403{
404 struct static_call_site *start = mod->static_call_sites;
405 struct static_call_site *stop = mod->static_call_sites +
406 mod->num_static_call_sites;
407 struct static_call_key *key, *prev_key = NULL;
408 struct static_call_mod *site_mod, **prev;
409 struct static_call_site *site;
410
411 for (site = start; site < stop; site++) {
412 key = static_call_key(site);
413
414 /*
415 * If the key was not updated due to a memory allocation
416 * failure in __static_call_init() then treating key::sites
417 * as key::mods in the code below would cause random memory
418 * access and #GP. In that case all subsequent sites have
419 * not been touched either, so stop iterating.
420 */
421 if (!static_call_key_has_mods(key))
422 break;
423
424 if (key == prev_key)
425 continue;
426
427 prev_key = key;
428
429 for (prev = &key->mods, site_mod = key->mods;
430 site_mod && site_mod->mod != mod;
431 prev = &site_mod->next, site_mod = site_mod->next)
432 ;
433
434 if (!site_mod)
435 continue;
436
437 *prev = site_mod->next;
438 kfree(site_mod);
439 }
440}
441
442static int static_call_module_notify(struct notifier_block *nb,
443 unsigned long val, void *data)
444{
445 struct module *mod = data;
446 int ret = 0;
447
448 cpus_read_lock();
449 static_call_lock();
450
451 switch (val) {
452 case MODULE_STATE_COMING:
453 ret = static_call_add_module(mod);
454 if (ret) {
455 pr_warn("Failed to allocate memory for static calls\n");
456 static_call_del_module(mod);
457 }
458 break;
459 case MODULE_STATE_GOING:
460 static_call_del_module(mod);
461 break;
462 }
463
464 static_call_unlock();
465 cpus_read_unlock();
466
467 return notifier_from_errno(ret);
468}
469
470static struct notifier_block static_call_module_nb = {
471 .notifier_call = static_call_module_notify,
472};
473
474#else
475
476static inline int __static_call_mod_text_reserved(void *start, void *end)
477{
478 return 0;
479}
480
481#endif /* CONFIG_MODULES */
482
483int static_call_text_reserved(void *start, void *end)
484{
485 bool init = system_state < SYSTEM_RUNNING;
486 int ret = __static_call_text_reserved(__start_static_call_sites,
487 __stop_static_call_sites, start, end, init);
488
489 if (ret)
490 return ret;
491
492 return __static_call_mod_text_reserved(start, end);
493}
494
495int __init static_call_init(void)
496{
497 int ret;
498
499 /* See static_call_force_reinit(). */
500 if (static_call_initialized == 1)
501 return 0;
502
503 cpus_read_lock();
504 static_call_lock();
505 ret = __static_call_init(NULL, __start_static_call_sites,
506 __stop_static_call_sites);
507 static_call_unlock();
508 cpus_read_unlock();
509
510 if (ret) {
511 pr_err("Failed to allocate memory for static_call!\n");
512 BUG();
513 }
514
515#ifdef CONFIG_MODULES
516 if (!static_call_initialized)
517 register_module_notifier(&static_call_module_nb);
518#endif
519
520 static_call_initialized = 1;
521 return 0;
522}
523early_initcall(static_call_init);
524
525#ifdef CONFIG_STATIC_CALL_SELFTEST
526
527static int func_a(int x)
528{
529 return x+1;
530}
531
532static int func_b(int x)
533{
534 return x+2;
535}
536
537DEFINE_STATIC_CALL(sc_selftest, func_a);
538
539static struct static_call_data {
540 int (*func)(int);
541 int val;
542 int expect;
543} static_call_data [] __initdata = {
544 { NULL, 2, 3 },
545 { func_b, 2, 4 },
546 { func_a, 2, 3 }
547};
548
549static int __init test_static_call_init(void)
550{
551 int i;
552
553 for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) {
554 struct static_call_data *scd = &static_call_data[i];
555
556 if (scd->func)
557 static_call_update(sc_selftest, scd->func);
558
559 WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect);
560 }
561
562 return 0;
563}
564early_initcall(test_static_call_init);
565
566#endif /* CONFIG_STATIC_CALL_SELFTEST */