projects/linux·blob master · refs

381 lines · 10.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *
9 */
10#include <linux/sched/isolation.h>
11#include <linux/llist.h>
12#include <linux/pci.h>
13#include "sched.h"
14
15enum hk_flags {
16 HK_FLAG_DOMAIN_BOOT = BIT(HK_TYPE_DOMAIN_BOOT),
17 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN),
18 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ),
19 HK_FLAG_KERNEL_NOISE = BIT(HK_TYPE_KERNEL_NOISE),
20};
21
22DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
23EXPORT_SYMBOL_GPL(housekeeping_overridden);
24
25struct housekeeping {
26 struct cpumask __rcu *cpumasks[HK_TYPE_MAX];
27 unsigned long flags;
28};
29
30static struct housekeeping housekeeping;
31static __initdata LLIST_HEAD(memblock_freelist);
32
33bool housekeeping_enabled(enum hk_type type)
34{
35 return !!(READ_ONCE(housekeeping.flags) & BIT(type));
36}
37EXPORT_SYMBOL_GPL(housekeeping_enabled);
38
39static bool housekeeping_dereference_check(enum hk_type type)
40{
41 if (IS_ENABLED(CONFIG_LOCKDEP) && type == HK_TYPE_DOMAIN) {
42 /* Cpuset isn't even writable yet? */
43 if (system_state <= SYSTEM_SCHEDULING)
44 return true;
45
46 /* CPU hotplug write locked, so cpuset partition can't be overwritten */
47 if (IS_ENABLED(CONFIG_HOTPLUG_CPU) && lockdep_is_cpus_write_held())
48 return true;
49
50 /* Cpuset lock held, partitions not writable */
51 if (IS_ENABLED(CONFIG_CPUSETS) && lockdep_is_cpuset_held())
52 return true;
53
54 return false;
55 }
56
57 return true;
58}
59
60static inline struct cpumask *housekeeping_cpumask_dereference(enum hk_type type)
61{
62 return rcu_dereference_all_check(housekeeping.cpumasks[type],
63 housekeeping_dereference_check(type));
64}
65
66const struct cpumask *housekeeping_cpumask(enum hk_type type)
67{
68 const struct cpumask *mask = NULL;
69
70 if (static_branch_unlikely(&housekeeping_overridden)) {
71 if (READ_ONCE(housekeeping.flags) & BIT(type))
72 mask = housekeeping_cpumask_dereference(type);
73 }
74 if (!mask)
75 mask = cpu_possible_mask;
76 return mask;
77}
78EXPORT_SYMBOL_GPL(housekeeping_cpumask);
79
80int housekeeping_any_cpu(enum hk_type type)
81{
82 int cpu;
83
84 if (static_branch_unlikely(&housekeeping_overridden)) {
85 if (housekeeping.flags & BIT(type)) {
86 cpu = sched_numa_find_closest(housekeeping_cpumask(type), smp_processor_id());
87 if (cpu < nr_cpu_ids)
88 return cpu;
89
90 cpu = cpumask_any_and_distribute(housekeeping_cpumask(type), cpu_online_mask);
91 if (likely(cpu < nr_cpu_ids))
92 return cpu;
93 /*
94 * Unless we have another problem this can only happen
95 * at boot time before start_secondary() brings the 1st
96 * housekeeping CPU up.
97 */
98 WARN_ON_ONCE(system_state == SYSTEM_RUNNING ||
99 type != HK_TYPE_TIMER);
100 }
101 }
102 return smp_processor_id();
103}
104EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
105
106void housekeeping_affine(struct task_struct *t, enum hk_type type)
107{
108 if (static_branch_unlikely(&housekeeping_overridden))
109 if (housekeeping.flags & BIT(type))
110 set_cpus_allowed_ptr(t, housekeeping_cpumask(type));
111}
112EXPORT_SYMBOL_GPL(housekeeping_affine);
113
114bool housekeeping_test_cpu(int cpu, enum hk_type type)
115{
116 if (static_branch_unlikely(&housekeeping_overridden) &&
117 READ_ONCE(housekeeping.flags) & BIT(type))
118 return cpumask_test_cpu(cpu, housekeeping_cpumask(type));
119 return true;
120}
121EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
122
123int housekeeping_update(struct cpumask *isol_mask)
124{
125 struct cpumask *trial, *old = NULL;
126 int err;
127
128 trial = kmalloc(cpumask_size(), GFP_KERNEL);
129 if (!trial)
130 return -ENOMEM;
131
132 cpumask_andnot(trial, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT), isol_mask);
133 if (!cpumask_intersects(trial, cpu_online_mask)) {
134 kfree(trial);
135 return -EINVAL;
136 }
137
138 if (!housekeeping.flags)
139 static_branch_enable(&housekeeping_overridden);
140
141 if (housekeeping.flags & HK_FLAG_DOMAIN)
142 old = housekeeping_cpumask_dereference(HK_TYPE_DOMAIN);
143 else
144 WRITE_ONCE(housekeeping.flags, housekeeping.flags | HK_FLAG_DOMAIN);
145 rcu_assign_pointer(housekeeping.cpumasks[HK_TYPE_DOMAIN], trial);
146
147 synchronize_rcu();
148
149 pci_probe_flush_workqueue();
150 mem_cgroup_flush_workqueue();
151 vmstat_flush_workqueue();
152
153 err = workqueue_unbound_housekeeping_update(housekeeping_cpumask(HK_TYPE_DOMAIN));
154 WARN_ON_ONCE(err < 0);
155
156 err = tmigr_isolated_exclude_cpumask(isol_mask);
157 WARN_ON_ONCE(err < 0);
158
159 err = kthreads_update_housekeeping();
160 WARN_ON_ONCE(err < 0);
161
162 kfree(old);
163
164 return 0;
165}
166
167void __init housekeeping_init(void)
168{
169 enum hk_type type;
170
171 if (!housekeeping.flags)
172 return;
173
174 static_branch_enable(&housekeeping_overridden);
175
176 if (housekeeping.flags & HK_FLAG_KERNEL_NOISE)
177 sched_tick_offload_init();
178 /*
179 * Realloc with a proper allocator so that any cpumask update
180 * can indifferently free the old version with kfree().
181 */
182 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
183 struct cpumask *omask, *nmask = kmalloc(cpumask_size(), GFP_KERNEL);
184
185 if (WARN_ON_ONCE(!nmask))
186 return;
187
188 omask = rcu_dereference(housekeeping.cpumasks[type]);
189
190 /* We need at least one CPU to handle housekeeping work */
191 WARN_ON_ONCE(cpumask_empty(omask));
192 cpumask_copy(nmask, omask);
193 RCU_INIT_POINTER(housekeeping.cpumasks[type], nmask);
194 __llist_add((struct llist_node *)omask, &memblock_freelist);
195 }
196}
197
198static int __init housekeeping_late_init(void)
199{
200 struct llist_node *llnode, *pos, *t;
201
202 /* Free allocated memblock memory, if any */
203 llnode = __llist_del_all(&memblock_freelist);
204 llist_for_each_safe(pos, t, llnode)
205 memblock_free(pos, cpumask_size());
206 return 0;
207}
208pure_initcall(housekeeping_late_init);
209
210static void __init housekeeping_setup_type(enum hk_type type,
211 cpumask_var_t housekeeping_staging)
212{
213 struct cpumask *mask = memblock_alloc_or_panic(cpumask_size(), SMP_CACHE_BYTES);
214
215 cpumask_copy(mask, housekeeping_staging);
216 RCU_INIT_POINTER(housekeeping.cpumasks[type], mask);
217}
218
219static int __init housekeeping_setup(char *str, unsigned long flags)
220{
221 cpumask_var_t non_housekeeping_mask, housekeeping_staging;
222 unsigned int first_cpu;
223 int err = 0;
224
225 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) {
226 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
227 pr_warn("Housekeeping: nohz unsupported."
228 " Build with CONFIG_NO_HZ_FULL\n");
229 return 0;
230 }
231 }
232
233 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
234 if (cpulist_parse(str, non_housekeeping_mask) < 0) {
235 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
236 goto free_non_housekeeping_mask;
237 }
238
239 alloc_bootmem_cpumask_var(&housekeeping_staging);
240 cpumask_andnot(housekeeping_staging,
241 cpu_possible_mask, non_housekeeping_mask);
242
243 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging);
244 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) {
245 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
246 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
247 if (!housekeeping.flags) {
248 pr_warn("Housekeeping: must include one present CPU, "
249 "using boot CPU:%d\n", smp_processor_id());
250 }
251 }
252
253 if (cpumask_empty(non_housekeeping_mask))
254 goto free_housekeeping_staging;
255
256 if (!housekeeping.flags) {
257 /* First setup call ("nohz_full=" or "isolcpus=") */
258 enum hk_type type;
259
260 for_each_set_bit(type, &flags, HK_TYPE_MAX)
261 housekeeping_setup_type(type, housekeeping_staging);
262 } else {
263 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
264 enum hk_type type;
265 unsigned long iter_flags = flags & housekeeping.flags;
266
267 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
268 if (!cpumask_equal(housekeeping_staging,
269 housekeeping_cpumask(type))) {
270 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
271 goto free_housekeeping_staging;
272 }
273 }
274
275 /*
276 * Check the combination of nohz_full and isolcpus=domain,
277 * necessary to avoid problems with the timer migration
278 * hierarchy. managed_irq is ignored by this check since it
279 * isn't considered in the timer migration logic.
280 */
281 iter_flags = housekeeping.flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN);
282 type = find_first_bit(&iter_flags, HK_TYPE_MAX);
283 /*
284 * Pass the check if none of these flags were previously set or
285 * are not in the current selection.
286 */
287 iter_flags = flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN);
288 first_cpu = (type == HK_TYPE_MAX || !iter_flags) ? 0 :
289 cpumask_first_and_and(cpu_present_mask,
290 housekeeping_staging, housekeeping_cpumask(type));
291 if (first_cpu >= min(nr_cpu_ids, setup_max_cpus)) {
292 pr_warn("Housekeeping: must include one present CPU "
293 "neither in nohz_full= nor in isolcpus=domain, "
294 "ignoring setting %s\n", str);
295 goto free_housekeeping_staging;
296 }
297
298 iter_flags = flags & ~housekeeping.flags;
299
300 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
301 housekeeping_setup_type(type, housekeeping_staging);
302 }
303
304 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE))
305 tick_nohz_full_setup(non_housekeeping_mask);
306
307 housekeeping.flags |= flags;
308 err = 1;
309
310free_housekeeping_staging:
311 free_bootmem_cpumask_var(housekeeping_staging);
312free_non_housekeeping_mask:
313 free_bootmem_cpumask_var(non_housekeeping_mask);
314
315 return err;
316}
317
318static int __init housekeeping_nohz_full_setup(char *str)
319{
320 unsigned long flags;
321
322 flags = HK_FLAG_KERNEL_NOISE;
323
324 return housekeeping_setup(str, flags);
325}
326__setup("nohz_full=", housekeeping_nohz_full_setup);
327
328static int __init housekeeping_isolcpus_setup(char *str)
329{
330 unsigned long flags = 0;
331 bool illegal = false;
332 char *par;
333 int len;
334
335 while (isalpha(*str)) {
336 /*
337 * isolcpus=nohz is equivalent to nohz_full.
338 */
339 if (!strncmp(str, "nohz,", 5)) {
340 str += 5;
341 flags |= HK_FLAG_KERNEL_NOISE;
342 continue;
343 }
344
345 if (!strncmp(str, "domain,", 7)) {
346 str += 7;
347 flags |= HK_FLAG_DOMAIN | HK_FLAG_DOMAIN_BOOT;
348 continue;
349 }
350
351 if (!strncmp(str, "managed_irq,", 12)) {
352 str += 12;
353 flags |= HK_FLAG_MANAGED_IRQ;
354 continue;
355 }
356
357 /*
358 * Skip unknown sub-parameter and validate that it is not
359 * containing an invalid character.
360 */
361 for (par = str, len = 0; *str && *str != ','; str++, len++) {
362 if (!isalpha(*str) && *str != '_')
363 illegal = true;
364 }
365
366 if (illegal) {
367 pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
368 return 0;
369 }
370
371 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
372 str++;
373 }
374
375 /* Default behaviour for isolcpus without flags */
376 if (!flags)
377 flags |= HK_FLAG_DOMAIN | HK_FLAG_DOMAIN_BOOT;
378
379 return housekeeping_setup(str, flags);
380}
381__setup("isolcpus=", housekeeping_isolcpus_setup);