projects/linux·blob assoc-array-fixes · refs

597 lines · 14.1 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2002 Richard Henderson
4 * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5 * Copyright (C) 2023 Luis Chamberlain <[email protected]>
6 * Copyright (C) 2024 Mike Rapoport IBM.
7 */
8
9#define pr_fmt(fmt) "execmem: " fmt
10
11#include <linux/mm.h>
12#include <linux/mutex.h>
13#include <linux/vmalloc.h>
14#include <linux/execmem.h>
15#include <linux/maple_tree.h>
16#include <linux/set_memory.h>
17#include <linux/moduleloader.h>
18#include <linux/text-patching.h>
19
20#include <asm/tlbflush.h>
21
22#include "internal.h"
23#include "vmalloc.h"
24
25static struct execmem_info *execmem_info __ro_after_init;
26static struct execmem_info default_execmem_info __ro_after_init;
27
28#ifdef CONFIG_MMU
29static void *execmem_vmalloc(struct execmem_range *range, size_t size,
30 pgprot_t pgprot, unsigned long vm_flags)
31{
32 bool kasan = range->flags & EXECMEM_KASAN_SHADOW;
33 gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN;
34 unsigned int align = range->alignment;
35 unsigned long start = range->start;
36 unsigned long end = range->end;
37 void *p;
38
39 if (kasan)
40 vm_flags |= VM_DEFER_KMEMLEAK;
41
42 p = __vmalloc_node_range(size, align, start, end, gfp_flags,
43 pgprot, vm_flags, NUMA_NO_NODE,
44 __builtin_return_address(0));
45 if (!p && range->fallback_start) {
46 start = range->fallback_start;
47 end = range->fallback_end;
48 p = __vmalloc_node_range(size, align, start, end, gfp_flags,
49 pgprot, vm_flags, NUMA_NO_NODE,
50 __builtin_return_address(0));
51 }
52
53 if (!p) {
54 pr_warn_ratelimited("unable to allocate memory\n");
55 return NULL;
56 }
57
58 if (kasan && (kasan_alloc_module_shadow(p, size, GFP_KERNEL) < 0)) {
59 vfree(p);
60 return NULL;
61 }
62
63 return p;
64}
65
66struct vm_struct *execmem_vmap(size_t size)
67{
68 struct execmem_range *range = &execmem_info->ranges[EXECMEM_MODULE_DATA];
69 struct vm_struct *area;
70
71 area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
72 range->start, range->end, NUMA_NO_NODE,
73 GFP_KERNEL, __builtin_return_address(0));
74 if (!area && range->fallback_start)
75 area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
76 range->fallback_start, range->fallback_end,
77 NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0));
78
79 return area;
80}
81#else
82static void *execmem_vmalloc(struct execmem_range *range, size_t size,
83 pgprot_t pgprot, unsigned long vm_flags)
84{
85 return vmalloc(size);
86}
87#endif /* CONFIG_MMU */
88
89#ifdef CONFIG_ARCH_HAS_EXECMEM_ROX
90struct execmem_cache {
91 struct mutex mutex;
92 struct maple_tree busy_areas;
93 struct maple_tree free_areas;
94 unsigned int pending_free_cnt; /* protected by mutex */
95};
96
97/* delay to schedule asynchronous free if fast path free fails */
98#define FREE_DELAY (msecs_to_jiffies(10))
99
100/* mark entries in busy_areas that should be freed asynchronously */
101#define PENDING_FREE_MASK (1 << (PAGE_SHIFT - 1))
102
103static struct execmem_cache execmem_cache = {
104 .mutex = __MUTEX_INITIALIZER(execmem_cache.mutex),
105 .busy_areas = MTREE_INIT_EXT(busy_areas, MT_FLAGS_LOCK_EXTERN,
106 execmem_cache.mutex),
107 .free_areas = MTREE_INIT_EXT(free_areas, MT_FLAGS_LOCK_EXTERN,
108 execmem_cache.mutex),
109};
110
111static inline unsigned long mas_range_len(struct ma_state *mas)
112{
113 return mas->last - mas->index + 1;
114}
115
116static int execmem_set_direct_map_valid(struct vm_struct *vm, bool valid)
117{
118 unsigned int nr = (1 << get_vm_area_page_order(vm));
119 unsigned int updated = 0;
120 int err = 0;
121
122 for (int i = 0; i < vm->nr_pages; i += nr) {
123 err = set_direct_map_valid_noflush(vm->pages[i], nr, valid);
124 if (err)
125 goto err_restore;
126 updated += nr;
127 }
128
129 return 0;
130
131err_restore:
132 for (int i = 0; i < updated; i += nr)
133 set_direct_map_valid_noflush(vm->pages[i], nr, !valid);
134
135 return err;
136}
137
138static int execmem_force_rw(void *ptr, size_t size)
139{
140 unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT;
141 unsigned long addr = (unsigned long)ptr;
142 int ret;
143
144 ret = set_memory_nx(addr, nr);
145 if (ret)
146 return ret;
147
148 return set_memory_rw(addr, nr);
149}
150
151int execmem_restore_rox(void *ptr, size_t size)
152{
153 unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT;
154 unsigned long addr = (unsigned long)ptr;
155
156 return set_memory_rox(addr, nr);
157}
158
159static void execmem_cache_clean(struct work_struct *work)
160{
161 struct maple_tree *free_areas = &execmem_cache.free_areas;
162 struct mutex *mutex = &execmem_cache.mutex;
163 MA_STATE(mas, free_areas, 0, ULONG_MAX);
164 void *area;
165
166 mutex_lock(mutex);
167 mas_for_each(&mas, area, ULONG_MAX) {
168 size_t size = mas_range_len(&mas);
169
170 if (IS_ALIGNED(size, PMD_SIZE) &&
171 IS_ALIGNED(mas.index, PMD_SIZE)) {
172 struct vm_struct *vm = find_vm_area(area);
173
174 execmem_set_direct_map_valid(vm, true);
175 mas_store_gfp(&mas, NULL, GFP_KERNEL);
176 vfree(area);
177 }
178 }
179 mutex_unlock(mutex);
180}
181
182static DECLARE_WORK(execmem_cache_clean_work, execmem_cache_clean);
183
184static int execmem_cache_add_locked(void *ptr, size_t size, gfp_t gfp_mask)
185{
186 struct maple_tree *free_areas = &execmem_cache.free_areas;
187 unsigned long addr = (unsigned long)ptr;
188 MA_STATE(mas, free_areas, addr - 1, addr + 1);
189 unsigned long lower, upper;
190 void *area = NULL;
191
192 lower = addr;
193 upper = addr + size - 1;
194
195 area = mas_walk(&mas);
196 if (area && mas.last == addr - 1)
197 lower = mas.index;
198
199 area = mas_next(&mas, ULONG_MAX);
200 if (area && mas.index == addr + size)
201 upper = mas.last;
202
203 mas_set_range(&mas, lower, upper);
204 return mas_store_gfp(&mas, (void *)lower, gfp_mask);
205}
206
207static bool within_range(struct execmem_range *range, struct ma_state *mas,
208 size_t size)
209{
210 unsigned long addr = mas->index;
211
212 if (addr >= range->start && addr + size < range->end)
213 return true;
214
215 if (range->fallback_start &&
216 addr >= range->fallback_start && addr + size < range->fallback_end)
217 return true;
218
219 return false;
220}
221
222static void *execmem_cache_alloc_locked(struct execmem_range *range, size_t size)
223{
224 struct maple_tree *free_areas = &execmem_cache.free_areas;
225 struct maple_tree *busy_areas = &execmem_cache.busy_areas;
226 MA_STATE(mas_free, free_areas, 0, ULONG_MAX);
227 MA_STATE(mas_busy, busy_areas, 0, ULONG_MAX);
228 unsigned long addr, last, area_size = 0;
229 void *area, *ptr = NULL;
230 int err;
231
232 mas_for_each(&mas_free, area, ULONG_MAX) {
233 area_size = mas_range_len(&mas_free);
234
235 if (area_size >= size && within_range(range, &mas_free, size))
236 break;
237 }
238
239 if (area_size < size)
240 return NULL;
241
242 addr = mas_free.index;
243 last = mas_free.last;
244
245 /* insert allocated size to busy_areas at range [addr, addr + size) */
246 mas_set_range(&mas_busy, addr, addr + size - 1);
247 err = mas_store_gfp(&mas_busy, (void *)addr, GFP_KERNEL);
248 if (err)
249 return NULL;
250
251 mas_store_gfp(&mas_free, NULL, GFP_KERNEL);
252 if (area_size > size) {
253 void *ptr = (void *)(addr + size);
254
255 /*
256 * re-insert remaining free size to free_areas at range
257 * [addr + size, last]
258 */
259 mas_set_range(&mas_free, addr + size, last);
260 err = mas_store_gfp(&mas_free, ptr, GFP_KERNEL);
261 if (err) {
262 mas_store_gfp(&mas_busy, NULL, GFP_KERNEL);
263 return NULL;
264 }
265 }
266 ptr = (void *)addr;
267
268 return ptr;
269}
270
271static void *__execmem_cache_alloc(struct execmem_range *range, size_t size)
272{
273 guard(mutex)(&execmem_cache.mutex);
274
275 return execmem_cache_alloc_locked(range, size);
276}
277
278static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size)
279{
280 unsigned long vm_flags = VM_ALLOW_HUGE_VMAP;
281 struct mutex *mutex = &execmem_cache.mutex;
282 struct vm_struct *vm;
283 size_t alloc_size;
284 int err = -ENOMEM;
285 void *p;
286
287 alloc_size = round_up(size, PMD_SIZE);
288 p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags);
289 if (!p) {
290 alloc_size = size;
291 p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags);
292 }
293
294 if (!p)
295 return NULL;
296
297 vm = find_vm_area(p);
298 if (!vm)
299 goto err_free_mem;
300
301 /* fill memory with instructions that will trap */
302 execmem_fill_trapping_insns(p, alloc_size);
303
304 err = set_memory_rox((unsigned long)p, vm->nr_pages);
305 if (err)
306 goto err_free_mem;
307
308 /*
309 * New memory blocks must be allocated and added to the cache
310 * as an atomic operation, otherwise they may be consumed
311 * by a parallel call to the execmem_cache_alloc function.
312 */
313 mutex_lock(mutex);
314 err = execmem_cache_add_locked(p, alloc_size, GFP_KERNEL);
315 if (err)
316 goto err_reset_direct_map;
317
318 p = execmem_cache_alloc_locked(range, size);
319
320 mutex_unlock(mutex);
321
322 return p;
323
324err_reset_direct_map:
325 mutex_unlock(mutex);
326 execmem_set_direct_map_valid(vm, true);
327err_free_mem:
328 vfree(p);
329 return NULL;
330}
331
332static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
333{
334 void *p;
335
336 p = __execmem_cache_alloc(range, size);
337 if (p)
338 return p;
339
340 return execmem_cache_populate_alloc(range, size);
341}
342
343static inline bool is_pending_free(void *ptr)
344{
345 return ((unsigned long)ptr & PENDING_FREE_MASK);
346}
347
348static inline void *pending_free_set(void *ptr)
349{
350 return (void *)((unsigned long)ptr | PENDING_FREE_MASK);
351}
352
353static inline void *pending_free_clear(void *ptr)
354{
355 return (void *)((unsigned long)ptr & ~PENDING_FREE_MASK);
356}
357
358static int __execmem_cache_free(struct ma_state *mas, void *ptr, gfp_t gfp_mask)
359{
360 size_t size = mas_range_len(mas);
361 int err;
362
363 err = execmem_force_rw(ptr, size);
364 if (err)
365 return err;
366
367 execmem_fill_trapping_insns(ptr, size);
368 execmem_restore_rox(ptr, size);
369
370 err = execmem_cache_add_locked(ptr, size, gfp_mask);
371 if (err)
372 return err;
373
374 mas_store_gfp(mas, NULL, gfp_mask);
375 return 0;
376}
377
378static void execmem_cache_free_slow(struct work_struct *work);
379static DECLARE_DELAYED_WORK(execmem_cache_free_work, execmem_cache_free_slow);
380
381static void execmem_cache_free_slow(struct work_struct *work)
382{
383 struct maple_tree *busy_areas = &execmem_cache.busy_areas;
384 MA_STATE(mas, busy_areas, 0, ULONG_MAX);
385 void *area;
386
387 guard(mutex)(&execmem_cache.mutex);
388
389 if (!execmem_cache.pending_free_cnt)
390 return;
391
392 mas_for_each(&mas, area, ULONG_MAX) {
393 if (!is_pending_free(area))
394 continue;
395
396 area = pending_free_clear(area);
397 if (__execmem_cache_free(&mas, area, GFP_KERNEL))
398 continue;
399
400 execmem_cache.pending_free_cnt--;
401 }
402
403 if (execmem_cache.pending_free_cnt)
404 schedule_delayed_work(&execmem_cache_free_work, FREE_DELAY);
405 else
406 schedule_work(&execmem_cache_clean_work);
407}
408
409static bool execmem_cache_free(void *ptr)
410{
411 struct maple_tree *busy_areas = &execmem_cache.busy_areas;
412 unsigned long addr = (unsigned long)ptr;
413 MA_STATE(mas, busy_areas, addr, addr);
414 void *area;
415 int err;
416
417 guard(mutex)(&execmem_cache.mutex);
418
419 area = mas_walk(&mas);
420 if (!area)
421 return false;
422
423 err = __execmem_cache_free(&mas, area, GFP_KERNEL | __GFP_NORETRY);
424 if (err) {
425 /*
426 * mas points to exact slot we've got the area from, nothing
427 * else can modify the tree because of the mutex, so there
428 * won't be any allocations in mas_store_gfp() and it will just
429 * change the pointer.
430 */
431 area = pending_free_set(area);
432 mas_store_gfp(&mas, area, GFP_KERNEL);
433 execmem_cache.pending_free_cnt++;
434 schedule_delayed_work(&execmem_cache_free_work, FREE_DELAY);
435 return true;
436 }
437
438 schedule_work(&execmem_cache_clean_work);
439
440 return true;
441}
442
443#else /* CONFIG_ARCH_HAS_EXECMEM_ROX */
444/*
445 * when ROX cache is not used the permissions defined by architectures for
446 * execmem ranges that are updated before use (e.g. EXECMEM_MODULE_TEXT) must
447 * be writable anyway
448 */
449static inline int execmem_force_rw(void *ptr, size_t size)
450{
451 return 0;
452}
453
454static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
455{
456 return NULL;
457}
458
459static bool execmem_cache_free(void *ptr)
460{
461 return false;
462}
463#endif /* CONFIG_ARCH_HAS_EXECMEM_ROX */
464
465void *execmem_alloc(enum execmem_type type, size_t size)
466{
467 struct execmem_range *range = &execmem_info->ranges[type];
468 bool use_cache = range->flags & EXECMEM_ROX_CACHE;
469 unsigned long vm_flags = VM_FLUSH_RESET_PERMS;
470 pgprot_t pgprot = range->pgprot;
471 void *p = NULL;
472
473 size = PAGE_ALIGN(size);
474
475 if (use_cache)
476 p = execmem_cache_alloc(range, size);
477 else
478 p = execmem_vmalloc(range, size, pgprot, vm_flags);
479
480 return kasan_reset_tag(p);
481}
482
483void *execmem_alloc_rw(enum execmem_type type, size_t size)
484{
485 void *p __free(execmem) = execmem_alloc(type, size);
486 int err;
487
488 if (!p)
489 return NULL;
490
491 err = execmem_force_rw(p, size);
492 if (err)
493 return NULL;
494
495 return no_free_ptr(p);
496}
497
498void execmem_free(void *ptr)
499{
500 /*
501 * This memory may be RO, and freeing RO memory in an interrupt is not
502 * supported by vmalloc.
503 */
504 WARN_ON(in_interrupt());
505
506 if (!execmem_cache_free(ptr))
507 vfree(ptr);
508}
509
510bool execmem_is_rox(enum execmem_type type)
511{
512 return !!(execmem_info->ranges[type].flags & EXECMEM_ROX_CACHE);
513}
514
515static bool execmem_validate(struct execmem_info *info)
516{
517 struct execmem_range *r = &info->ranges[EXECMEM_DEFAULT];
518
519 if (!r->alignment || !r->start || !r->end || !pgprot_val(r->pgprot)) {
520 pr_crit("Invalid parameters for execmem allocator, module loading will fail");
521 return false;
522 }
523
524 if (!IS_ENABLED(CONFIG_ARCH_HAS_EXECMEM_ROX)) {
525 for (int i = EXECMEM_DEFAULT; i < EXECMEM_TYPE_MAX; i++) {
526 r = &info->ranges[i];
527
528 if (r->flags & EXECMEM_ROX_CACHE) {
529 pr_warn_once("ROX cache is not supported\n");
530 r->flags &= ~EXECMEM_ROX_CACHE;
531 }
532 }
533 }
534
535 return true;
536}
537
538static void execmem_init_missing(struct execmem_info *info)
539{
540 struct execmem_range *default_range = &info->ranges[EXECMEM_DEFAULT];
541
542 for (int i = EXECMEM_DEFAULT + 1; i < EXECMEM_TYPE_MAX; i++) {
543 struct execmem_range *r = &info->ranges[i];
544
545 if (!r->start) {
546 if (i == EXECMEM_MODULE_DATA)
547 r->pgprot = PAGE_KERNEL;
548 else
549 r->pgprot = default_range->pgprot;
550 r->alignment = default_range->alignment;
551 r->start = default_range->start;
552 r->end = default_range->end;
553 r->flags = default_range->flags;
554 r->fallback_start = default_range->fallback_start;
555 r->fallback_end = default_range->fallback_end;
556 }
557 }
558}
559
560struct execmem_info * __weak execmem_arch_setup(void)
561{
562 return NULL;
563}
564
565static void __init __execmem_init(void)
566{
567 struct execmem_info *info = execmem_arch_setup();
568
569 if (!info) {
570 info = execmem_info = &default_execmem_info;
571 info->ranges[EXECMEM_DEFAULT].start = VMALLOC_START;
572 info->ranges[EXECMEM_DEFAULT].end = VMALLOC_END;
573 info->ranges[EXECMEM_DEFAULT].pgprot = PAGE_KERNEL_EXEC;
574 info->ranges[EXECMEM_DEFAULT].alignment = 1;
575 }
576
577 if (!execmem_validate(info))
578 return;
579
580 execmem_init_missing(info);
581
582 execmem_info = info;
583}
584
585#ifdef CONFIG_ARCH_WANTS_EXECMEM_LATE
586static int __init execmem_late_init(void)
587{
588 __execmem_init();
589 return 0;
590}
591core_initcall(execmem_late_init);
592#else
593void __init execmem_init(void)
594{
595 __execmem_init();
596}
597#endif