projects/linux·blob master · refs

1172 lines · 36.3 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
4 *
5 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
6 * Copyright (c) 2026 Tejun Heo <[email protected]>
7 */
8#include <linux/cacheinfo.h>
9
10#include "internal.h"
11#include "cid.h"
12
13/*
14 * cid tables. The cid kfuncs are available whether the root scheduler is
15 * cid-form or cpu-form, the latter to allow gradual migration to cids, so every
16 * root builds a default mapping. Each root enable allocates a fresh set, builds
17 * it privately and publishes the __rcu globals below once the layout is final.
18 * Root disable unpublishes and RCU-frees the set. kfuncs may run before the
19 * tables are published and must check for NULL.
20 */
21u32 scx_nr_cid_shards;
22s16 __rcu *scx_cid_to_cpu_tbl;
23s16 __rcu *scx_cpu_to_cid_tbl;
24s32 __rcu *scx_cid_to_shard;
25s32 __rcu *scx_shard_node;
26struct scx_cid_shard __rcu *scx_cid_shard_ranges;
27struct scx_cid_topo __rcu *scx_cid_topo;
28
29static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */
30
31#define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \
32 .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \
33 .node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \
34}
35
36/*
37 * Return @cpu's LLC shared_cpu_map. If cacheinfo isn't populated (offline or
38 * !present), record @cpu in @fallbacks and return its node mask instead - the
39 * worst that can happen is that the cpu's LLC becomes coarser than reality.
40 */
41static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks)
42{
43 struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu);
44
45 if (!ci || !ci->info_list || !ci->num_leaves) {
46 cpumask_set_cpu(cpu, fallbacks);
47 return cpumask_of_node(cpu_to_node(cpu));
48 }
49 return &ci->info_list[ci->num_leaves - 1].shared_cpu_map;
50}
51
52/*
53 * Compute per-LLC shard layout. Each shard holds at most @shard_size cids, and
54 * in any case no more than SCX_CID_SHARD_MAX_CPUS. Cores are spread as evenly
55 * as possible across shards so cpu count is balanced: the first *@nr_large_p
56 * shards get (*@cores_per_shard_p + 1) cores, the rest get *@cores_per_shard_p.
57 */
58static void calc_shard_layout(const struct cpumask *llc_cpus, u32 shard_size,
59 u32 *cores_per_shard_p, u32 *nr_large_p)
60{
61 u32 nr_cores = 0, nr_cpus = 0, nr_shards;
62 int cpu;
63
64 for_each_cpu(cpu, llc_cpus) {
65 nr_cpus++;
66 if (cpumask_first(topology_sibling_cpumask(cpu)) == cpu)
67 nr_cores++;
68 }
69
70 nr_shards = max_t(u32, 1, DIV_ROUND_UP(nr_cpus, shard_size));
71 nr_shards = max_t(u32, nr_shards,
72 DIV_ROUND_UP(nr_cpus, SCX_CID_SHARD_MAX_CPUS));
73
74 *cores_per_shard_p = nr_cores / nr_shards;
75 *nr_large_p = nr_cores % nr_shards;
76}
77
78static void scx_cid_tables_free(struct scx_cid_tables *tbls)
79{
80 if (!tbls)
81 return;
82 kvfree(tbls->cid_to_cpu);
83 kvfree(tbls->cpu_to_cid);
84 kvfree(tbls->cid_to_shard);
85 kvfree(tbls->shard_node);
86 kvfree(tbls->shard_ranges);
87 kvfree(tbls->topo);
88 kfree(tbls);
89}
90
91static void scx_cid_tables_free_rcufn(struct rcu_head *rcu)
92{
93 scx_cid_tables_free(container_of(rcu, struct scx_cid_tables, rcu));
94}
95
96static struct scx_cid_tables *scx_cid_alloc_tables(void)
97{
98 u32 npossible = num_possible_cpus();
99 struct scx_cid_tables *tbls;
100
101 tbls = kzalloc_obj(*tbls);
102 if (!tbls)
103 return NULL;
104
105 tbls->cid_to_cpu = kvzalloc_objs(*tbls->cid_to_cpu, npossible);
106 tbls->cpu_to_cid = kvzalloc_objs(*tbls->cpu_to_cid, nr_cpu_ids);
107 tbls->cid_to_shard = kvzalloc_objs(*tbls->cid_to_shard, npossible);
108 tbls->shard_node = kvzalloc_objs(*tbls->shard_node, npossible);
109 tbls->shard_ranges = kvzalloc_objs(*tbls->shard_ranges, npossible);
110 tbls->topo = kvzalloc_objs(*tbls->topo, npossible);
111
112 if (!tbls->cid_to_cpu || !tbls->cpu_to_cid || !tbls->cid_to_shard ||
113 !tbls->shard_node || !tbls->shard_ranges || !tbls->topo) {
114 scx_cid_tables_free(tbls);
115 return NULL;
116 }
117
118 return tbls;
119}
120
121/**
122 * scx_cid_publish_tables - Publish the tables scx_cid_init() built
123 *
124 * Called after ops.init_cids() where the layout is final.
125 */
126void scx_cid_publish_tables(void)
127{
128 struct scx_cid_tables *tbls = scx_cid_tables;
129
130 lockdep_assert_held(&scx_enable_mutex);
131
132 scx_nr_cid_shards = tbls->nr_shards;
133 rcu_assign_pointer(scx_cid_to_cpu_tbl, tbls->cid_to_cpu);
134 rcu_assign_pointer(scx_cpu_to_cid_tbl, tbls->cpu_to_cid);
135 rcu_assign_pointer(scx_cid_to_shard, tbls->cid_to_shard);
136 rcu_assign_pointer(scx_shard_node, tbls->shard_node);
137 rcu_assign_pointer(scx_cid_shard_ranges, tbls->shard_ranges);
138 rcu_assign_pointer(scx_cid_topo, tbls->topo);
139}
140
141/**
142 * scx_cid_retire_tables - Unpublish and retire the cid tables
143 *
144 * Called by root disable after the readers which dereference without NULL
145 * checks are drained, inside cpus_read_lock() to exclude the hotplug path.
146 */
147void scx_cid_retire_tables(void)
148{
149 struct scx_cid_tables *tbls = scx_cid_tables;
150
151 lockdep_assert_held(&scx_enable_mutex);
152 lockdep_assert_cpus_held();
153
154 if (!tbls)
155 return;
156
157 scx_cid_tables = NULL;
158 RCU_INIT_POINTER(scx_cid_to_cpu_tbl, NULL);
159 RCU_INIT_POINTER(scx_cpu_to_cid_tbl, NULL);
160 RCU_INIT_POINTER(scx_cid_to_shard, NULL);
161 RCU_INIT_POINTER(scx_shard_node, NULL);
162 RCU_INIT_POINTER(scx_cid_shard_ranges, NULL);
163 RCU_INIT_POINTER(scx_cid_topo, NULL);
164 call_rcu(&tbls->rcu, scx_cid_tables_free_rcufn);
165}
166
167/**
168 * scx_cid_init - build the cid mapping
169 * @sch: the scx_sched being initialized; used as the scx_error() target
170 *
171 * Build a fresh table set. It becomes visible through scx_cid_publish_tables()
172 * and is retired by scx_cid_retire_tables() at disable.
173 *
174 * See "Topological CPU IDs" in cid.h for the model. Walk online cpus by
175 * intersection at each level (parent_scratch & this_level_mask), which keeps
176 * containment correct by construction and naturally splits a physical LLC
177 * straddling two NUMA nodes into two LLC units. The caller must hold
178 * cpus_read_lock.
179 */
180s32 scx_cid_init(struct scx_sched *sch)
181{
182 cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL;
183 cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
184 cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
185 cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
186 cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL;
187 cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL;
188 struct scx_cid_tables *tbls;
189 u32 next_cid = 0;
190 s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0;
191 s32 next_shard_idx = 0;
192 u32 shard_size, max_cids;
193 u32 notopo_in_shard;
194 s32 notopo_shard_cid, notopo_shard_idx;
195 s32 cpu, cid, si;
196
197 /* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */
198 BUILD_BUG_ON(NR_CPUS > 8192);
199
200 lockdep_assert_cpus_held();
201 lockdep_assert_held(&scx_enable_mutex);
202
203 shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL;
204 max_cids = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
205
206 tbls = scx_cid_alloc_tables();
207 if (!tbls)
208 return -ENOMEM;
209
210 scx_cid_tables = tbls;
211
212 for (si = 0; si < num_possible_cpus(); si++)
213 tbls->shard_node[si] = NUMA_NO_NODE;
214
215 if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) ||
216 !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) ||
217 !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) ||
218 !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) ||
219 !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) ||
220 !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL))
221 return -ENOMEM;
222
223 /* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */
224 for (cpu = 0; cpu < nr_cpu_ids; cpu++)
225 tbls->cpu_to_cid[cpu] = -1;
226
227 cpumask_copy(to_walk, cpu_online_mask);
228
229 while (!cpumask_empty(to_walk)) {
230 s32 next_cpu = cpumask_first(to_walk);
231 s32 nid = cpu_to_node(next_cpu);
232 s32 node_cid = next_cid;
233 s32 node_idx;
234
235 /*
236 * No NUMA info: skip and let the tail loop assign a no-topo
237 * cid. cpumask_of_node(-1) is undefined.
238 */
239 if (nid < 0) {
240 cpumask_clear_cpu(next_cpu, to_walk);
241 continue;
242 }
243
244 node_idx = next_node_idx++;
245
246 /* node_scratch = to_walk & this node */
247 cpumask_and(node_scratch, to_walk, cpumask_of_node(nid));
248 if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, node_scratch)))
249 return -EINVAL;
250
251 while (!cpumask_empty(node_scratch)) {
252 s32 ncpu = cpumask_first(node_scratch);
253 const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback);
254 s32 llc_cid = next_cid;
255 s32 llc_idx = next_llc_idx++;
256 u32 cores_per_shard, nr_large;
257 u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
258 s32 shard_cid, shard_idx;
259
260 /* llc_scratch = node_scratch & this llc */
261 cpumask_and(llc_scratch, node_scratch, llc_mask);
262 if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch)))
263 return -EINVAL;
264
265 calc_shard_layout(llc_scratch, shard_size, &cores_per_shard, &nr_large);
266 shard_cid = next_cid;
267 shard_idx = next_shard_idx++;
268 tbls->shard_node[shard_idx] = nid;
269
270 while (!cpumask_empty(llc_scratch)) {
271 s32 lcpu = cpumask_first(llc_scratch);
272 const struct cpumask *sib = topology_sibling_cpumask(lcpu);
273 s32 core_cid = next_cid;
274 s32 core_idx = next_core_idx++;
275 s32 ccpu;
276 u32 max_cores, cids_in_core;
277
278 /* core_scratch = llc_scratch & this core */
279 cpumask_and(core_scratch, llc_scratch, sib);
280 if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch)))
281 return -EINVAL;
282
283 /*
284 * Advance to a new shard when either core or
285 * cid count reaches max. The latter bounds
286 * shard sizes under uneven SMT. Never start an
287 * empty shard.
288 */
289 cids_in_core = cpumask_weight(core_scratch);
290 max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0);
291 if (cores_in_shard &&
292 (cores_in_shard >= max_cores ||
293 cids_in_shard + cids_in_core > max_cids)) {
294 shard_local++;
295 cores_in_shard = 0;
296 cids_in_shard = 0;
297 shard_cid = next_cid;
298 shard_idx = next_shard_idx++;
299 tbls->shard_node[shard_idx] = nid;
300 }
301 cores_in_shard++;
302 cids_in_shard += cids_in_core;
303
304 for_each_cpu(ccpu, core_scratch) {
305 s32 cid = next_cid++;
306
307 tbls->cid_to_cpu[cid] = ccpu;
308 tbls->cpu_to_cid[ccpu] = cid;
309 tbls->cid_to_shard[cid] = shard_idx;
310 tbls->topo[cid] = (struct scx_cid_topo){
311 .core_cid = core_cid,
312 .core_idx = core_idx,
313 .llc_cid = llc_cid,
314 .llc_idx = llc_idx,
315 .node_cid = node_cid,
316 .node_idx = node_idx,
317 .shard_cid = shard_cid,
318 .shard_idx = shard_idx,
319 };
320
321 cpumask_clear_cpu(ccpu, llc_scratch);
322 cpumask_clear_cpu(ccpu, node_scratch);
323 cpumask_clear_cpu(ccpu, to_walk);
324 }
325 }
326 }
327 }
328
329 /*
330 * No-topo section: any possible cpu without a cid - normally just the
331 * not-online ones. Pack into shards of up to min(@shard_size,
332 * SCX_CID_SHARD_MAX_CPUS) cids so that every cid has a valid shard
333 * assignment and the hard cap holds even with a large @shard_size.
334 * Collect any currently-online cpus that land here in @online_no_topo
335 * so we can warn about them at the end.
336 */
337 notopo_in_shard = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
338 notopo_shard_cid = -1;
339 notopo_shard_idx = -1;
340
341 for_each_cpu(cpu, cpu_possible_mask) {
342 if (tbls->cpu_to_cid[cpu] != -1)
343 continue;
344 if (cpu_online(cpu))
345 cpumask_set_cpu(cpu, online_no_topo);
346
347 cid = next_cid++;
348 tbls->cid_to_cpu[cid] = cpu;
349 tbls->cpu_to_cid[cpu] = cid;
350
351 if (notopo_in_shard >= min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS)) {
352 notopo_shard_cid = cid;
353 notopo_shard_idx = next_shard_idx++;
354 notopo_in_shard = 0;
355 }
356 notopo_in_shard++;
357
358 tbls->cid_to_shard[cid] = notopo_shard_idx;
359 tbls->topo[cid] = SCX_CID_TOPO_NEG;
360 tbls->topo[cid].shard_cid = notopo_shard_cid;
361 tbls->topo[cid].shard_idx = notopo_shard_idx;
362 }
363
364 if (!cpumask_empty(llc_fallback))
365 pr_warn("scx_cid: cpus without cacheinfo, using node mask as llc: %*pbl\n",
366 cpumask_pr_args(llc_fallback));
367 if (!cpumask_empty(online_no_topo))
368 pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n",
369 cpumask_pr_args(online_no_topo));
370
371 /*
372 * Fill cid_shard_ranges[] from cid_to_shard[]. Shards are contiguous
373 * cid ranges by construction: base_cid is the first cid landing in a
374 * shard, nr_cids is the count.
375 */
376 for (cid = 0; cid < next_cid; cid++) {
377 s32 sidx = tbls->cid_to_shard[cid];
378
379 if (tbls->shard_ranges[sidx].nr_cids == 0)
380 tbls->shard_ranges[sidx].base_cid = cid;
381 tbls->shard_ranges[sidx].nr_cids++;
382 }
383
384 tbls->nr_shards = next_shard_idx;
385 return 0;
386}
387
388/**
389 * scx_cmask_clear - Zero every bit in @m's active range
390 * @m: cmask to clear
391 *
392 * Storage past the active range is left as is.
393 */
394void scx_cmask_clear(struct scx_cmask *m)
395{
396 u32 nr_words;
397
398 if (!m->nr_cids)
399 return;
400 nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
401 memset(m->bits, 0, nr_words * sizeof(u64));
402}
403
404/**
405 * scx_cmask_fill - Set every bit in @m's active range
406 * @m: cmask to fill
407 *
408 * Counterpart to scx_cmask_clear(). Storage past the active range is left as is.
409 */
410void scx_cmask_fill(struct scx_cmask *m)
411{
412 u32 nr_words, head_bits, tail_bits;
413
414 if (!m->nr_cids)
415 return;
416 nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
417 memset(m->bits, 0xff, nr_words * sizeof(u64));
418
419 /* clear word-0 bits below base */
420 head_bits = m->base & 63;
421 if (head_bits)
422 m->bits[0] &= ~((1ULL << head_bits) - 1);
423
424 /* clear last-word bits at or past base + nr_cids */
425 tail_bits = (m->base + m->nr_cids) & 63;
426 if (tail_bits)
427 m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1;
428}
429
430/*
431 * Return the index of the largest entry in @counts, or NUMA_NO_NODE if all
432 * entries are zero. Ties resolve to the lowest index.
433 */
434static s32 pick_max_node(const u32 *counts, u32 n)
435{
436 s32 best = NUMA_NO_NODE;
437 u32 best_count = 0, i;
438
439 for (i = 0; i < n; i++) {
440 if (counts[i] > best_count) {
441 best_count = counts[i];
442 best = i;
443 }
444 }
445 return best;
446}
447
448__bpf_kfunc_start_defs();
449
450/**
451 * scx_bpf_cid_override - Install an explicit cpu->cid mapping with shard info
452 * @cpu_to_cid__arena: array of nr_cpu_ids s32 entries (cid for each cpu)
453 * @cpu_to_cid_cnt: number of entries, must be nr_cpu_ids
454 * @shard_start__arena: array of first-cid-of-each-shard, one entry per shard
455 * @shard_start_cnt: number of shards
456 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
457 *
458 * May only be called from ops.init_cids() of the root scheduler. Replace the
459 * topology-probed cid mapping and shard layout with caller-provided ones. Each
460 * possible cpu must map to a unique cid in [0, num_possible_cpus()). The shard
461 * starts must be strictly increasing with the first entry 0 and all values <
462 * num_possible_cpus(). The last shard extends to num_possible_cpus() and no
463 * shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info
464 * (core/LLC/node) is cleared and the shard layout is set from the input. On
465 * invalid input, abort the scheduler.
466 */
467__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt,
468 const s32 *shard_start__arena, u32 shard_start_cnt,
469 const struct bpf_prog_aux *aux)
470{
471 cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL;
472 u32 *node_counts __free(kfree) = NULL;
473 s32 *cpu_to_cid __free(kfree) = NULL;
474 s32 *shard_start __free(kfree) = NULL;
475 u32 npossible = num_possible_cpus();
476 struct scx_cid_tables *tbls;
477 struct scx_sched *sch;
478 u32 nr_shards = shard_start_cnt;
479 bool alloced;
480 s32 cpu, cid, si;
481
482 /*
483 * GFP_KERNEL allocs must happen before the rcu read section. Snapshot
484 * the BPF-supplied arrays so a concurrent arena write can't change
485 * them between validation and use.
486 *
487 * The BPF-supplied counts size the snapshots and thus the arena reads.
488 * Gate the copies on the count bounds, reported below once @sch is
489 * available. The bounded reads, at most 32KB, stay within the guard
490 * region that arena fault recovery covers.
491 */
492 alloced = zalloc_cpumask_var(&seen, GFP_KERNEL);
493 node_counts = kzalloc_objs(*node_counts, nr_node_ids);
494 if (cpu_to_cid_cnt == nr_cpu_ids)
495 cpu_to_cid = kmemdup(cpu_to_cid__arena, cpu_to_cid_cnt * sizeof(s32),
496 GFP_KERNEL);
497 if (nr_shards && nr_shards <= npossible)
498 shard_start = kmemdup(shard_start__arena, nr_shards * sizeof(s32),
499 GFP_KERNEL);
500
501 guard(rcu)();
502
503 sch = scx_prog_sched(aux);
504 if (unlikely(!sch))
505 return;
506
507 /* called from ops.init_cids(), so the tables exist and are unpublished */
508 lockdep_assert_held(&scx_enable_mutex);
509 tbls = scx_cid_tables;
510
511 if (cpu_to_cid_cnt != nr_cpu_ids) {
512 scx_error(sch, "scx_bpf_cid_override: cpu_to_cid expected %u entries, got %u",
513 nr_cpu_ids, cpu_to_cid_cnt);
514 return;
515 }
516
517 if (!nr_shards || nr_shards > npossible) {
518 scx_error(sch, "scx_bpf_cid_override: invalid shard_start count %u",
519 nr_shards);
520 return;
521 }
522
523 if (!alloced || !node_counts || !cpu_to_cid || !shard_start) {
524 scx_error(sch, "scx_bpf_cid_override: allocation failed");
525 return;
526 }
527
528 /* validate shard_start[]: starts at 0, strictly increasing, in range */
529 if (shard_start[0] != 0) {
530 scx_error(sch, "scx_bpf_cid_override: shard_start[0] must be 0, got %d",
531 shard_start[0]);
532 return;
533 }
534 for (si = 1; si < nr_shards; si++) {
535 if (shard_start[si] <= shard_start[si - 1]) {
536 scx_error(sch, "scx_bpf_cid_override: shard_start not increasing at [%d]",
537 si);
538 return;
539 }
540 if (shard_start[si] >= npossible) {
541 scx_error(sch, "scx_bpf_cid_override: shard_start[%d]=%d >= %u",
542 si, shard_start[si], npossible);
543 return;
544 }
545 if (shard_start[si] - shard_start[si - 1] > SCX_CID_SHARD_MAX_CPUS) {
546 scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d",
547 si - 1, shard_start[si] - shard_start[si - 1],
548 SCX_CID_SHARD_MAX_CPUS);
549 return;
550 }
551 }
552 if (npossible - shard_start[nr_shards - 1] > SCX_CID_SHARD_MAX_CPUS) {
553 scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d",
554 nr_shards - 1, npossible - shard_start[nr_shards - 1],
555 SCX_CID_SHARD_MAX_CPUS);
556 return;
557 }
558
559 /* validate first so that invalid input leaves the tables untouched */
560 for_each_possible_cpu(cpu) {
561 s32 c = cpu_to_cid[cpu];
562
563 if (!cid_valid(sch, c))
564 return;
565 if (cpumask_test_and_set_cpu(c, seen)) {
566 scx_error(sch, "cid %d assigned to multiple cpus", c);
567 return;
568 }
569 }
570
571 for_each_possible_cpu(cpu) {
572 s32 c = cpu_to_cid[cpu];
573
574 tbls->cpu_to_cid[cpu] = c;
575 tbls->cid_to_cpu[c] = cpu;
576 }
577
578 /*
579 * Derive shard_node[] by majority count: an overridden shard may
580 * span NUMA nodes, so assign each to the node that owns the most cpus.
581 */
582 for (si = 0; si < nr_shards; si++) {
583 u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible;
584
585 memset(node_counts, 0, nr_node_ids * sizeof(*node_counts));
586 for (cid = shard_start[si]; cid < end; cid++) {
587 s32 node = cpu_to_node(tbls->cid_to_cpu[cid]);
588
589 if (numa_valid_node(node))
590 node_counts[node]++;
591 }
592 tbls->shard_node[si] = pick_max_node(node_counts, nr_node_ids);
593 }
594
595 /*
596 * Invalidate stale topo info and install shard layout from
597 * @shard_start. Walk shards to derive shard_cid/shard_idx for each cid.
598 */
599 si = 0;
600 for (cid = 0; cid < npossible; cid++) {
601 if (si + 1 < nr_shards && cid >= shard_start[si + 1])
602 si++;
603 tbls->cid_to_shard[cid] = si;
604 tbls->topo[cid] = SCX_CID_TOPO_NEG;
605 tbls->topo[cid].shard_cid = shard_start[si];
606 tbls->topo[cid].shard_idx = si;
607 }
608
609 /* Rebuild shard_ranges[] for the new layout. */
610 memset(tbls->shard_ranges, 0, npossible * sizeof(*tbls->shard_ranges));
611 for (si = 0; si < nr_shards; si++) {
612 u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible;
613
614 tbls->shard_ranges[si].base_cid = shard_start[si];
615 tbls->shard_ranges[si].nr_cids = end - shard_start[si];
616 }
617
618 tbls->nr_shards = nr_shards;
619}
620
621/**
622 * scx_bpf_cid_to_cpu - Return the raw CPU id for @cid
623 * @cid: cid to look up
624 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
625 *
626 * Return the raw CPU id for @cid. Trigger scx_error() and return -EINVAL if
627 * @cid is invalid. The cid<->cpu mapping is static for the lifetime of the
628 * loaded scheduler, so the BPF side can cache the result to avoid repeated
629 * kfunc invocations.
630 */
631__bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux)
632{
633 struct scx_sched *sch;
634
635 guard(rcu)();
636
637 sch = scx_prog_sched(aux);
638 if (unlikely(!sch))
639 return -EINVAL;
640 return scx_cid_to_cpu(sch, cid);
641}
642
643/**
644 * scx_bpf_cpu_to_cid - Return the cid for @cpu
645 * @cpu: cpu to look up
646 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
647 *
648 * Return the cid for @cpu. Trigger scx_error() and return -EINVAL if @cpu is
649 * invalid. The cid<->cpu mapping is static for the lifetime of the loaded
650 * scheduler, so the BPF side can cache the result to avoid repeated kfunc
651 * invocations.
652 */
653__bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux)
654{
655 struct scx_sched *sch;
656
657 guard(rcu)();
658
659 sch = scx_prog_sched(aux);
660 if (unlikely(!sch))
661 return -EINVAL;
662 return scx_cpu_to_cid(sch, cpu);
663}
664
665/*
666 * Set ops on cmasks. cmask_walk_op2() shares one walk across mutating
667 * (and/or/copy/andnot) and predicate (subset/intersects) two-cmask forms;
668 * cmask_walk_op1() does the same shape over a single cmask range. Every public
669 * entry passes a compile-time-constant @op; cmask_walk_op{1,2}() and
670 * cmask_word_op{1,2}() are __always_inline so the inner switch collapses to the
671 * selected op and cmask_op2_is_pred() folds the predicate early-exit out of
672 * mutating ops.
673 *
674 * Two-cmask ops only touch @dst bits inside the intersection of the two ranges;
675 * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero
676 * @dst bits that lie outside @src's range.
677 *
678 * Word accesses use READ_ONCE/WRITE_ONCE so a caller may read @src
679 * locklessly. Memory ordering against concurrent writers is the caller's
680 * responsibility.
681 */
682enum cmask_op2 {
683 /* mutating */
684 CMASK_OP2_AND,
685 CMASK_OP2_OR,
686 CMASK_OP2_COPY,
687 CMASK_OP2_ANDNOT,
688 /* predicates - short-circuit when the per-word result is true */
689 CMASK_OP2_SUBSET,
690 CMASK_OP2_INTERSECTS,
691 /*
692 * @a is a BPF-arena cmask. Words on @a use READ_ONCE/WRITE_ONCE since
693 * BPF may read/write concurrently. See scx_cmask_ref_or() / _copy().
694 */
695 CMASK_OP2_REF_OR,
696 CMASK_OP2_REF_COPY,
697};
698
699static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op)
700{
701 return op == CMASK_OP2_SUBSET || op == CMASK_OP2_INTERSECTS;
702}
703
704static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask,
705 const enum cmask_op2 op)
706{
707 switch (op) {
708 case CMASK_OP2_AND:
709 WRITE_ONCE(*av, *av & (~mask | READ_ONCE(*bp)));
710 return false;
711 case CMASK_OP2_OR:
712 WRITE_ONCE(*av, *av | (READ_ONCE(*bp) & mask));
713 return false;
714 case CMASK_OP2_COPY:
715 WRITE_ONCE(*av, (*av & ~mask) | (READ_ONCE(*bp) & mask));
716 return false;
717 case CMASK_OP2_ANDNOT:
718 WRITE_ONCE(*av, *av & ~(READ_ONCE(*bp) & mask));
719 return false;
720 case CMASK_OP2_SUBSET:
721 /* stop on the first bit in @sub not set in @super */
722 return (READ_ONCE(*bp) & ~READ_ONCE(*av)) & mask;
723 case CMASK_OP2_INTERSECTS:
724 return (READ_ONCE(*av) & READ_ONCE(*bp)) & mask;
725 case CMASK_OP2_REF_OR:
726 WRITE_ONCE(*av, READ_ONCE(*av) | (READ_ONCE(*bp) & mask));
727 return false;
728 case CMASK_OP2_REF_COPY:
729 WRITE_ONCE(*av, (READ_ONCE(*av) & ~mask) | (READ_ONCE(*bp) & mask));
730 return false;
731 }
732 unreachable();
733}
734
735/*
736 * Walk the intersection of [@a_base, @a_base + @a_nr_cids) with [@b_base,
737 * @b_base + @b_nr_cids) word by word, applying @op. Mutating ops walk all words
738 * and return false; predicates return true on the first word whose per-word
739 * test is true. Empty intersection returns false (matches "no bits to consider"
740 * for both mutate and predicate).
741 *
742 * Base/nr_cids are taken as parameters so callers with snapshotted bounds can
743 * drive the walk with values independent of the cmask's header.
744 */
745static __always_inline bool cmask_walk_op2(u64 *a_bits, u32 a_base, u32 a_nr_cids,
746 const u64 *b_bits, u32 b_base, u32 b_nr_cids,
747 const enum cmask_op2 op)
748{
749 u32 lo = max(a_base, b_base);
750 u32 hi = min(a_base + a_nr_cids, b_base + b_nr_cids);
751 u32 a_word_off = a_base / 64;
752 u32 b_word_off = b_base / 64;
753 u32 lo_word = lo / 64;
754 u32 hi_word = (hi - 1) / 64;
755 u64 head_mask = GENMASK_U64(63, lo & 63);
756 u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0);
757 u32 w;
758
759 if (lo >= hi)
760 return false;
761
762 if (lo_word == hi_word)
763 return cmask_word_op2(&a_bits[lo_word - a_word_off],
764 &b_bits[lo_word - b_word_off],
765 head_mask & tail_mask, op);
766
767 if (cmask_word_op2(&a_bits[lo_word - a_word_off],
768 &b_bits[lo_word - b_word_off], head_mask, op) &&
769 cmask_op2_is_pred(op))
770 return true;
771
772 for (w = lo_word + 1; w < hi_word; w++)
773 if (cmask_word_op2(&a_bits[w - a_word_off],
774 &b_bits[w - b_word_off], ~0ULL, op) &&
775 cmask_op2_is_pred(op))
776 return true;
777
778 return cmask_word_op2(&a_bits[hi_word - a_word_off],
779 &b_bits[hi_word - b_word_off], tail_mask, op);
780}
781
782enum cmask_op1 {
783 CMASK_OP1_ANY_SET,
784};
785
786static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask,
787 const enum cmask_op1 op)
788{
789 switch (op) {
790 case CMASK_OP1_ANY_SET:
791 return READ_ONCE(*ap) & mask;
792 }
793 unreachable();
794}
795
796/*
797 * Walk [@a_base, @a_base + @a_nr_cids) of @a_bits word by word, applying @op.
798 * Returns true on the first word whose per-word test is true; returns false if
799 * no word matches or the range is empty. All current op1s short-circuit on
800 * per-word true; if a non-predicate op1 lands here, add a cmask_op1_is_pred()
801 * guard analogous to cmask_op2_is_pred().
802 */
803static __always_inline bool cmask_walk_op1(const u64 *a_bits, u32 a_base,
804 u32 a_nr_cids,
805 const enum cmask_op1 op)
806{
807 u32 lo = a_base;
808 u32 hi = a_base + a_nr_cids;
809 u32 a_word_off = a_base / 64;
810 u32 lo_word = lo / 64;
811 u32 hi_word = (hi - 1) / 64;
812 u64 head_mask = GENMASK_U64(63, lo & 63);
813 u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0);
814 u32 w;
815
816 if (lo >= hi)
817 return false;
818
819 if (lo_word == hi_word)
820 return cmask_word_op1(&a_bits[lo_word - a_word_off],
821 head_mask & tail_mask, op);
822
823 if (cmask_word_op1(&a_bits[lo_word - a_word_off], head_mask, op))
824 return true;
825 for (w = lo_word + 1; w < hi_word; w++)
826 if (cmask_word_op1(&a_bits[w - a_word_off], ~0ULL, op))
827 return true;
828 return cmask_word_op1(&a_bits[hi_word - a_word_off], tail_mask, op);
829}
830
831void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src)
832{
833 cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
834 src->bits, src->base, src->nr_cids, CMASK_OP2_AND);
835}
836
837void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src)
838{
839 cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
840 src->bits, src->base, src->nr_cids, CMASK_OP2_OR);
841}
842
843void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src)
844{
845 cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
846 src->bits, src->base, src->nr_cids, CMASK_OP2_COPY);
847}
848
849void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src)
850{
851 cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
852 src->bits, src->base, src->nr_cids, CMASK_OP2_ANDNOT);
853}
854
855/*
856 * Return true if @cm has any bit set in [@lo, @hi). Caller must ensure
857 * [@lo, @hi) is contained in @cm's range.
858 */
859static bool cmask_any_set_in_range(const struct scx_cmask *cm, u32 lo, u32 hi)
860{
861 if (lo >= hi)
862 return false;
863 return cmask_walk_op1(&cm->bits[lo / 64 - cm->base / 64], lo, hi - lo,
864 CMASK_OP1_ANY_SET);
865}
866
867/**
868 * scx_cmask_subset - test whether @sub is a subset of @super
869 * @sub: cmask to test
870 * @super: cmask to test against
871 *
872 * Return true iff every set bit of @sub is also set in @super.
873 */
874bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super)
875{
876 u32 super_end = super->base + super->nr_cids;
877 u32 sub_end = sub->base + sub->nr_cids;
878
879 /*
880 * Set bits in @sub outside @super's range can't be in @super, so any
881 * such bit means not a subset. The walk below only visits words
882 * common to both ranges, so these need a separate scan.
883 */
884 if (sub->base < super->base &&
885 cmask_any_set_in_range(sub, sub->base, min(super->base, sub_end)))
886 return false;
887 if (sub_end > super_end &&
888 cmask_any_set_in_range(sub, max(sub->base, super_end), sub_end))
889 return false;
890
891 return !cmask_walk_op2((u64 *)super->bits, super->base, super->nr_cids,
892 sub->bits, sub->base, sub->nr_cids, CMASK_OP2_SUBSET);
893}
894
895bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b)
896{
897 return cmask_walk_op2((u64 *)a->bits, a->base, a->nr_cids,
898 b->bits, b->base, b->nr_cids, CMASK_OP2_INTERSECTS);
899}
900
901/**
902 * scx_cmask_empty - Test whether @m has no bits set
903 * @m: cmask to test
904 *
905 * Return true iff @m's active range has no bits set.
906 */
907bool scx_cmask_empty(const struct scx_cmask *m)
908{
909 return !cmask_any_set_in_range(m, m->base, m->base + m->nr_cids);
910}
911
912/**
913 * scx_bpf_cid_topo - Copy out per-cid topology info
914 * @cid: cid to look up
915 * @out: where to copy the topology info
916 * @out__sz: size of @out, the program's sizeof(struct scx_cid_topo)
917 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
918 *
919 * Fill @out with the topology info for @cid. Trigger scx_error() if @cid is out
920 * of range. If @cid is valid but in the no-topo section, all fields are set to
921 * -1. All fields are also set to -1 when no cid tables have been published yet,
922 * which a program may observe while racing the root enable.
923 *
924 * The program's struct may be older or newer than the kernel's. The smaller of
925 * @out__sz and the kernel's size is copied and the rest of @out is set to -1.
926 */
927__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz,
928 const struct bpf_prog_aux *aux)
929{
930 size_t len = min(out__sz, sizeof(*out));
931 struct scx_cid_topo *topo;
932 struct scx_sched *sch;
933
934 /* the error cases and fields the kernel lacks read as -1 */
935 memset(out, 0xff, out__sz);
936
937 guard(rcu)();
938
939 sch = scx_prog_sched(aux);
940 topo = rcu_dereference(scx_cid_topo);
941 if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo))
942 return;
943
944 memcpy(out, &topo[cid], len);
945}
946
947__bpf_kfunc_end_defs();
948
949BTF_KFUNCS_START(scx_kfunc_ids_init_cids)
950BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE)
951BTF_KFUNCS_END(scx_kfunc_ids_init_cids)
952
953static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = {
954 .owner = THIS_MODULE,
955 .set = &scx_kfunc_ids_init_cids,
956 .filter = scx_kfunc_context_filter,
957};
958
959BTF_KFUNCS_START(scx_kfunc_ids_cid)
960BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS)
961BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS)
962BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS)
963BTF_KFUNCS_END(scx_kfunc_ids_cid)
964
965static const struct btf_kfunc_id_set scx_kfunc_set_cid = {
966 .owner = THIS_MODULE,
967 .set = &scx_kfunc_ids_cid,
968};
969
970/**
971 * scx_cmask_ref_init - Bind a scx_cmask_ref to a BPF-arena cmask
972 * @sch: scheduler whose arena hosts @src
973 * @src: BPF-supplied cmask, rebased to its kernel address
974 * @ref: output ref
975 *
976 * Snapshot @src's @base, @nr_cids and @alloc_words. The snapshot is necessary
977 * because BPF may mutate the live header asynchronously.
978 *
979 * Return 0 on success, -EINVAL if the range is out of bounds or @alloc_words
980 * doesn't cover it.
981 */
982int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src,
983 struct scx_cmask_ref *ref)
984{
985 u32 base, nr_cids, alloc_words, npossible = num_possible_cpus();
986 s32 *cid_to_shard;
987
988 base = READ_ONCE(src->base);
989 nr_cids = READ_ONCE(src->nr_cids);
990 alloc_words = READ_ONCE(src->alloc_words);
991
992 if (unlikely(base >= npossible || nr_cids > npossible - base ||
993 SCX_CMASK_NR_WORDS(nr_cids) > alloc_words))
994 return -EINVAL;
995
996 ref->sch = sch;
997 ref->src = (struct scx_cmask *)src;
998 ref->base = base;
999 ref->nr_cids = nr_cids;
1000
1001 cid_to_shard = rcu_dereference_all(scx_cid_to_shard);
1002 ref->shard_first = cid_to_shard[base];
1003 if (likely(nr_cids))
1004 ref->shard_end = cid_to_shard[base + nr_cids - 1] + 1;
1005 else
1006 ref->shard_end = ref->shard_first;
1007
1008 return 0;
1009}
1010
1011/**
1012 * scx_cmask_ref_init_kern - Bind a scx_cmask_ref to a kernel-owned cmask
1013 * @sch: scheduler the cmask belongs to
1014 * @m: kernel address of the target cmask, storage sized for @nr_cids at @base
1015 * @base: first cid of the active range
1016 * @nr_cids: active range length
1017 * @ref: output ref
1018 *
1019 * Like scx_cmask_ref_init() but the geometry is supplied by the caller, not
1020 * read from @m's header, so a concurrent BPF write to the header can't steer
1021 * later sizing or offsets. Rewrite the header from the trusted geometry and
1022 * bind @ref to it.
1023 */
1024void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m,
1025 u32 base, u32 nr_cids, struct scx_cmask_ref *ref)
1026{
1027 s32 *cid_to_shard;
1028
1029 WRITE_ONCE(m->base, base);
1030 WRITE_ONCE(m->nr_cids, nr_cids);
1031 WRITE_ONCE(m->alloc_words, SCX_CMASK_NR_WORDS(nr_cids));
1032
1033 ref->sch = sch;
1034 ref->src = m;
1035 ref->base = base;
1036 ref->nr_cids = nr_cids;
1037
1038 cid_to_shard = rcu_dereference_all(scx_cid_to_shard);
1039 ref->shard_first = cid_to_shard[base];
1040 if (likely(nr_cids))
1041 ref->shard_end = cid_to_shard[base + nr_cids - 1] + 1;
1042 else
1043 ref->shard_end = ref->shard_first;
1044}
1045
1046/**
1047 * scx_cmask_ref_shard - Read one shard from @ref into @out
1048 * @ref: validated ref
1049 * @shard_idx: target shard, in [@ref->shard_first, @ref->shard_end)
1050 * @out: output cmask whose @out->alloc_words must hold the shard
1051 *
1052 * Set @out to the intersection of @ref's range with @shard_idx's cid range,
1053 * with bits[] read from @ref->src via READ_ONCE. Empty intersection sets
1054 * @out->nr_cids to 0. scx_error()s on @ref's sched if @out can't hold the
1055 * shard.
1056 */
1057void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx,
1058 struct scx_cmask *out)
1059{
1060 const struct scx_cid_shard *shard =
1061 &rcu_dereference_all(scx_cid_shard_ranges)[shard_idx];
1062 u32 shard_base = shard->base_cid;
1063 u32 shard_end = shard_base + shard->nr_cids;
1064 u32 isect_base, isect_end, nr_words, src_off, wi;
1065 u64 head_mask, tail_mask;
1066
1067 isect_base = max(ref->base, shard_base);
1068 isect_end = min(ref->base + ref->nr_cids, shard_end);
1069
1070 if (isect_base >= isect_end) {
1071 out->base = shard_base;
1072 out->nr_cids = 0;
1073 return;
1074 }
1075
1076 nr_words = ((isect_end - 1) / 64) - (isect_base / 64) + 1;
1077 if (nr_words > out->alloc_words) {
1078 scx_error(ref->sch, "scx_cmask_ref_shard: out alloc_words=%u < %u for shard %d",
1079 out->alloc_words, nr_words, shard_idx);
1080 out->base = shard_base;
1081 out->nr_cids = 0;
1082 return;
1083 }
1084
1085 out->base = isect_base;
1086 out->nr_cids = isect_end - isect_base;
1087 src_off = (isect_base / 64) - (ref->base / 64);
1088
1089 for (wi = 0; wi < nr_words; wi++)
1090 out->bits[wi] = READ_ONCE(ref->src->bits[src_off + wi]);
1091
1092 head_mask = GENMASK_U64(63, isect_base & 63);
1093 out->bits[0] &= head_mask;
1094 tail_mask = GENMASK_U64((isect_end - 1) & 63, 0);
1095 out->bits[nr_words - 1] &= tail_mask;
1096}
1097
1098/**
1099 * scx_cmask_ref_or - OR @src into the arena cmask referenced by @ref
1100 * @ref: validated ref
1101 * @src: stable kernel cmask
1102 *
1103 * Bits inside the intersection of @ref's snapshotted range with @src's range
1104 * are OR'd into @ref->src and bits outside are left unchanged. Stores on
1105 * @ref->src use WRITE_ONCE since BPF may read/write concurrently.
1106 */
1107void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src)
1108{
1109 cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids,
1110 src->bits, src->base, src->nr_cids, CMASK_OP2_REF_OR);
1111}
1112
1113/**
1114 * scx_cmask_ref_copy - Copy @src into the arena cmask referenced by @ref
1115 * @ref: validated ref
1116 * @src: stable kernel cmask
1117 *
1118 * Bits inside the intersection of @ref's snapshotted range with @src's range
1119 * take @src's values and bits outside are left unchanged. Stores on @ref->src
1120 * use WRITE_ONCE since BPF may read/write concurrently.
1121 */
1122void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src)
1123{
1124 cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids,
1125 src->bits, src->base, src->nr_cids, CMASK_OP2_REF_COPY);
1126}
1127
1128/**
1129 * scx_cmask_ref_from_cpumask - Populate @ref's arena cmask from a cpumask
1130 * @ref: kern-bound ref, see scx_cmask_ref_init_kern()
1131 * @cpumask: cpus to translate into cids
1132 *
1133 * Write @ref's active range one word at a time, setting each cid's bit when
1134 * its cpu is in @cpumask. Offsets and length come from @ref's trusted geometry
1135 * and stores use WRITE_ONCE since BPF may read concurrently, so the arena
1136 * header is never read.
1137 */
1138void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref,
1139 const struct cpumask *cpumask)
1140{
1141 struct scx_cmask *m = ref->src;
1142 u32 base = ref->base, nr_cids = ref->nr_cids;
1143 u32 wi, nr_words;
1144
1145 if (!nr_cids)
1146 return;
1147
1148 nr_words = (base + nr_cids - 1) / 64 - base / 64 + 1;
1149 for (wi = 0; wi < nr_words; wi++) {
1150 u32 word_first_cid = (base / 64 + wi) * 64;
1151 u64 word = 0;
1152 u32 bit;
1153
1154 for (bit = 0; bit < 64; bit++) {
1155 u32 cid = word_first_cid + bit;
1156
1157 if (cid < base || cid >= base + nr_cids)
1158 continue;
1159 if (cpumask_test_cpu(__scx_cid_to_cpu(cid), cpumask))
1160 word |= BIT_U64(bit);
1161 }
1162 WRITE_ONCE(m->bits[wi], word);
1163 }
1164}
1165
1166int scx_cid_kfunc_init(void)
1167{
1168 return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?:
1169 register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?:
1170 register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?:
1171 register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid);
1172}