projects/linux·blob master · refs

228 lines · 7.4 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Early sched_ext type definitions.
4 *
5 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
6 * Copyright (c) 2026 Tejun Heo <[email protected]>
7 */
8#ifndef _KERNEL_SCHED_EXT_TYPES_H
9#define _KERNEL_SCHED_EXT_TYPES_H
10
11#include <linux/types.h>
12#include <linux/jiffies.h>
13#include <linux/overflow.h>
14#include <linux/time64.h>
15#include <linux/sched/topology.h>
16
17enum scx_consts {
18 SCX_DSP_DFL_MAX_BATCH = 32,
19 SCX_DSP_MAX_LOOPS = 32,
20 SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ,
21
22 /* rescue knob defaults and limits, see scx_rescue_timerfn() */
23 SCX_RESCUE_DFL_BW_PPT = 20, /* parts per thousand, 2% */
24 SCX_RESCUE_MAX_BW_PPT = 250, /* 25% */
25 SCX_RESCUE_DISABLE = U32_MAX, /* disables rescue */
26 SCX_RESCUE_DFL_QUANTUM_US = 5000,
27 SCX_RESCUE_MIN_QUANTUM_US = 1000,
28 SCX_RESCUE_MAX_QUANTUM_US = 100000,
29 SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */
30 SCX_RESCUE_OVERLOAD_MULT = 16, /* overload threshold in funding periods */
31 SCX_RESCUE_MIN_OVERLOAD_MS = 1000,
32 SCX_RESCUE_MAX_OVERLOAD_MS = 15000,
33
34 /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */
35 SCX_TID_CHUNK = 1024,
36
37 SCX_EXIT_BT_LEN = 64,
38 SCX_EXIT_MSG_LEN = 1024,
39 SCX_EXIT_DUMP_DFL_LEN = 32768,
40
41 SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE,
42
43 /*
44 * Iterating all tasks may take a while. Periodically drop
45 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls.
46 */
47 SCX_TASK_ITER_BATCH = 32,
48
49 SCX_BYPASS_HOST_NTH = 2,
50
51 SCX_BYPASS_LB_DFL_INTV_US = 500 * USEC_PER_MSEC,
52 SCX_BYPASS_LB_DONOR_PCT = 125,
53 SCX_BYPASS_LB_MIN_DELTA_DIV = 4,
54 SCX_BYPASS_LB_BATCH = 256,
55
56 SCX_REENQ_MAX_REPEAT = 256,
57
58 SCX_SUB_MAX_DEPTH = 4,
59};
60
61/*
62 * Per-cid topology info. For each topology level (core, LLC, node) and shard,
63 * records the first cid in the unit and its global index. Global indices are
64 * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over
65 * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields
66 * set to -1 but always have valid shard assignments.
67 *
68 * Shards are contiguous CID ranges used as scalable locking/work domains for
69 * sub-scheduler operations. By default each LLC becomes one shard, split into
70 * smaller shards if the LLC exceeds the target size. No-topo cids are packed
71 * into their own max-sized shards.
72 *
73 * New fields are appended, never inserted: scx_bpf_cid_topo() copies this
74 * struct out sized by the program's own layout, and an older program's copy
75 * must stay a prefix of the kernel's.
76 *
77 * @core_cid: first cid of this cid's core (smt-sibling group)
78 * @core_idx: global index of that core, in [0, nr_cores_at_init)
79 * @llc_cid: first cid of this cid's LLC
80 * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init)
81 * @node_cid: first cid of this cid's NUMA node
82 * @node_idx: global index of that node, in [0, nr_nodes_at_init)
83 * @shard_cid: first cid of this cid's shard
84 * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards)
85 */
86struct scx_cid_topo {
87 s32 core_cid;
88 s32 core_idx;
89 s32 llc_cid;
90 s32 llc_idx;
91 s32 node_cid;
92 s32 node_idx;
93 s32 shard_cid;
94 s32 shard_idx;
95};
96
97enum scx_cid_consts {
98 SCX_CID_SHARD_SIZE_DFL = 24,
99 SCX_CID_SHARD_MAX_CPUS = 512,
100};
101
102/*
103 * Per-shard metadata for O(1) shard->cid-range lookup.
104 *
105 * @base_cid: first cid of the shard
106 * @nr_cids: number of cids in the shard
107 */
108struct scx_cid_shard {
109 s32 base_cid;
110 s32 nr_cids;
111};
112
113/*
114 * cmask: variable-length, base-windowed bitmap over cid space
115 * -----------------------------------------------------------
116 *
117 * A cmask covers the cid range [base, base + nr_cids). bits[] is aligned to the
118 * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the
119 * first (base & 63) bits of bits[0] are head padding and the trailing bits of
120 * the last active word past base + nr_cids are tail padding. Both stay zero;
121 * all mutating helpers preserve that. Words past the last active word are not
122 * read by any helper and have no constraint.
123 *
124 * Grid alignment means two cmasks always address bits[] against the same global
125 * 64-cid windows, so cross-cmask word ops (AND, OR, ...) reduce to
126 *
127 * dst->bits[i] OP= src->bits[i - delta]
128 *
129 * with no bit-shifting, regardless of how the two bases relate mod 64.
130 */
131struct scx_cmask {
132 u32 base;
133 u32 nr_cids;
134 u32 alloc_words;
135 u64 bits[];
136};
137
138/*
139 * Number of u64 words of bits[] storage that covers @nr_cids regardless of base
140 * alignment. The +1 absorbs up to 63 bits of head padding when base is not
141 * 64-aligned - always allocating one extra word beats branching on base or
142 * splitting the compute. The u64 cast keeps the +63 from wrapping when @nr_cids
143 * is near U32_MAX, so callers bounds-checking the result against @alloc_words
144 * catch the overflow instead of seeing a small value.
145 */
146#define SCX_CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1))
147
148/**
149 * __SCX_CMASK_DEFINE - Define an on-stack cmask with explicit storage capacity
150 * @NAME: variable name to define
151 * @BASE: first cid of the active range
152 * @NR_CIDS: active range length
153 * @ALLOC_CIDS: storage capacity in cids, at least @NR_CIDS
154 *
155 * @NAME aliases zero-initialized storage with the active range set to
156 * [BASE, BASE + NR_CIDS). Use scx_cmask_reframe() to reshape later, up to
157 * @ALLOC_CIDS.
158 */
159#define __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, ALLOC_CIDS) \
160 _DEFINE_FLEX(struct scx_cmask, NAME, bits, SCX_CMASK_NR_WORDS(ALLOC_CIDS), \
161 = { .base = (BASE), \
162 .nr_cids = (NR_CIDS), \
163 .alloc_words = SCX_CMASK_NR_WORDS(ALLOC_CIDS) })
164
165/**
166 * SCX_CMASK_DEFINE - Define an on-stack cmask on tight storage
167 * @NAME: variable name to define
168 * @BASE: first cid of the active range
169 * @NR_CIDS: active range length, also storage capacity
170 *
171 * @NAME aliases zero-initialized storage with the active range and storage
172 * both [BASE, BASE + NR_CIDS).
173 */
174#define SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS) \
175 __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, NR_CIDS)
176
177/**
178 * SCX_CMASK_DEFINE_SHARD - Define an on-stack cmask sized to one shard
179 * @NAME: variable name to define
180 * @BASE: first cid of the active range
181 * @NR_CIDS: active range length, must be <= SCX_CID_SHARD_MAX_CPUS
182 *
183 * Storage is fixed at SCX_CID_SHARD_MAX_CPUS, active range framed by
184 * (BASE, NR_CIDS). Passing NR_CIDS > SCX_CID_SHARD_MAX_CPUS leaves the
185 * cmask claiming more bits than storage holds and subsequent cmask
186 * operations will overrun.
187 */
188#define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \
189 __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS)
190
191/*
192 * scx_cmask_ref: validated reference to a BPF-arena cmask.
193 *
194 * scx_cmask_ref_init() snapshots @base/@nr_cids. The snapshot is what
195 * downstream code uses for sizing - the live header can be mutated concurrently
196 * by BPF.
197 *
198 * scx_cmask_ref_shard() reads one shard into a cmask. scx_cmask_ref_or() and
199 * scx_cmask_ref_copy() write back into the referenced arena cmask, bounded by
200 * the snapshot.
201 *
202 * Typical input use:
203 *
204 * struct scx_cmask_ref ref;
205 * SCX_CMASK_DEFINE(shard, 0, SCX_CID_SHARD_MAX_CPUS);
206 * s32 idx, ret;
207 *
208 * ret = scx_cmask_ref_init(sch, src, &ref);
209 * if (ret < 0)
210 * return ret;
211 *
212 * for (idx = ref.shard_first; idx < ref.shard_end; idx++) {
213 * scx_cmask_ref_shard(&ref, idx, shard);
214 * if (!shard->nr_cids)
215 * continue;
216 * ... use idx and shard ...
217 * }
218 */
219struct scx_cmask_ref {
220 struct scx_sched *sch;
221 struct scx_cmask *src;
222 u32 base;
223 u32 nr_cids;
224 s32 shard_first;
225 s32 shard_end;
226};
227
228#endif /* _KERNEL_SCHED_EXT_TYPES_H */