projects/linux·blob master · refs

125 lines · 3.2 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
4 *
5 * scx_arena_pool: kernel-side sub-allocator over BPF-arena pages.
6 *
7 * Each chunk added to @sch->arena_pool comes from one
8 * bpf_arena_alloc_pages_sleepable() call and is registered at the
9 * kernel-side mapping address.
10 *
11 * Allocations grow the pool on demand. Underlying arena pages are released
12 * when the arena map itself is torn down.
13 *
14 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
15 * Copyright (c) 2026 Tejun Heo <[email protected]>
16 */
17#include <linux/genalloc.h>
18
19#include "internal.h"
20#include "arena.h"
21
22enum scx_arena_consts {
23 SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */
24 SCX_ARENA_GROW_PAGES = 4, /* per growth */
25};
26
27s32 scx_arena_pool_init(struct scx_sched *sch)
28{
29 if (!sch->arena_map)
30 return 0;
31
32 sch->arena_pool = gen_pool_create(SCX_ARENA_MIN_ORDER, NUMA_NO_NODE);
33 if (!sch->arena_pool)
34 return -ENOMEM;
35 return 0;
36}
37
38static void scx_arena_clear_chunk(struct gen_pool *pool, struct gen_pool_chunk *chunk,
39 void *data)
40{
41 int order = pool->min_alloc_order;
42 size_t chunk_sz = chunk->end_addr - chunk->start_addr + 1;
43 unsigned long end_bit = chunk_sz >> order;
44 unsigned long b, e;
45
46 for_each_set_bitrange(b, e, chunk->bits, end_bit)
47 gen_pool_free(pool, chunk->start_addr + (b << order),
48 (e - b) << order);
49}
50
51/*
52 * Tear down the pool. Outstanding gen_pool allocations are freed via
53 * scx_arena_clear_chunk() so gen_pool_destroy() doesn't BUG. The underlying
54 * arena pages are released when the arena map itself is torn down.
55 */
56void scx_arena_pool_destroy(struct scx_sched *sch)
57{
58 if (!sch->arena_pool)
59 return;
60 gen_pool_for_each_chunk(sch->arena_pool, scx_arena_clear_chunk, NULL);
61 gen_pool_destroy(sch->arena_pool);
62 sch->arena_pool = NULL;
63}
64
65/*
66 * Grow the pool by @page_cnt pages. bpf_arena_alloc_pages_sleepable() and
67 * gen_pool_add() (which calls vzalloc(GFP_KERNEL)) require a sleepable
68 * context.
69 */
70static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt)
71{
72 void *p;
73 int ret;
74
75 if (!sch->arena_map || !sch->arena_pool)
76 return -EINVAL;
77
78 p = bpf_arena_alloc_pages_sleepable(sch->arena_map, NULL,
79 page_cnt, NUMA_NO_NODE, 0);
80 if (!p)
81 return -ENOMEM;
82
83 ret = gen_pool_add(sch->arena_pool,
84 (unsigned long)scx_arena_to_kaddr(sch, p),
85 page_cnt * PAGE_SIZE, NUMA_NO_NODE);
86 if (ret) {
87 bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt);
88 return ret;
89 }
90 return 0;
91}
92
93/*
94 * Allocate @size bytes from the arena pool. Returns kernel VA on success, NULL
95 * on failure. May grow the pool via scx_arena_grow() which sleeps. Caller must
96 * be in a GFP_KERNEL context.
97 */
98void *scx_arena_alloc(struct scx_sched *sch, size_t size)
99{
100 unsigned long kern_va;
101 u32 page_cnt;
102
103 might_sleep();
104
105 if (!sch->arena_pool)
106 return NULL;
107
108 while (true) {
109 kern_va = gen_pool_alloc(sch->arena_pool, size);
110 if (kern_va)
111 break;
112 page_cnt = max_t(u32, SCX_ARENA_GROW_PAGES,
113 (size + PAGE_SIZE - 1) >> PAGE_SHIFT);
114 if (scx_arena_grow(sch, page_cnt))
115 return NULL;
116 }
117
118 return (void *)kern_va;
119}
120
121void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size)
122{
123 if (sch->arena_pool && kern_va)
124 gen_pool_free(sch->arena_pool, (unsigned long)kern_va, size);
125}