projects/linux·blob v5.2 · refs

2351 lines · 56.2 KiB

1/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched/signal.h>
11#include <linux/sched/hotplug.h>
12#include <linux/sched/isolation.h>
13#include <linux/sched/task.h>
14#include <linux/sched/smt.h>
15#include <linux/unistd.h>
16#include <linux/cpu.h>
17#include <linux/oom.h>
18#include <linux/rcupdate.h>
19#include <linux/export.h>
20#include <linux/bug.h>
21#include <linux/kthread.h>
22#include <linux/stop_machine.h>
23#include <linux/mutex.h>
24#include <linux/gfp.h>
25#include <linux/suspend.h>
26#include <linux/lockdep.h>
27#include <linux/tick.h>
28#include <linux/irq.h>
29#include <linux/nmi.h>
30#include <linux/smpboot.h>
31#include <linux/relay.h>
32#include <linux/slab.h>
33#include <linux/percpu-rwsem.h>
34
35#include <trace/events/power.h>
36#define CREATE_TRACE_POINTS
37#include <trace/events/cpuhp.h>
38
39#include "smpboot.h"
40
41/**
42 * cpuhp_cpu_state - Per cpu hotplug state storage
43 * @state: The current cpu state
44 * @target: The target state
45 * @thread: Pointer to the hotplug thread
46 * @should_run: Thread should execute
47 * @rollback: Perform a rollback
48 * @single: Single callback invocation
49 * @bringup: Single callback bringup or teardown selector
50 * @cb_state: The state for a single callback (install/uninstall)
51 * @result: Result of the operation
52 * @done_up: Signal completion to the issuer of the task for cpu-up
53 * @done_down: Signal completion to the issuer of the task for cpu-down
54 */
55struct cpuhp_cpu_state {
56 enum cpuhp_state state;
57 enum cpuhp_state target;
58 enum cpuhp_state fail;
59#ifdef CONFIG_SMP
60 struct task_struct *thread;
61 bool should_run;
62 bool rollback;
63 bool single;
64 bool bringup;
65 bool booted_once;
66 struct hlist_node *node;
67 struct hlist_node *last;
68 enum cpuhp_state cb_state;
69 int result;
70 struct completion done_up;
71 struct completion done_down;
72#endif
73};
74
75static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
76 .fail = CPUHP_INVALID,
77};
78
79#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
80static struct lockdep_map cpuhp_state_up_map =
81 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
82static struct lockdep_map cpuhp_state_down_map =
83 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
84
85
86static inline void cpuhp_lock_acquire(bool bringup)
87{
88 lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
89}
90
91static inline void cpuhp_lock_release(bool bringup)
92{
93 lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
94}
95#else
96
97static inline void cpuhp_lock_acquire(bool bringup) { }
98static inline void cpuhp_lock_release(bool bringup) { }
99
100#endif
101
102/**
103 * cpuhp_step - Hotplug state machine step
104 * @name: Name of the step
105 * @startup: Startup function of the step
106 * @teardown: Teardown function of the step
107 * @cant_stop: Bringup/teardown can't be stopped at this step
108 */
109struct cpuhp_step {
110 const char *name;
111 union {
112 int (*single)(unsigned int cpu);
113 int (*multi)(unsigned int cpu,
114 struct hlist_node *node);
115 } startup;
116 union {
117 int (*single)(unsigned int cpu);
118 int (*multi)(unsigned int cpu,
119 struct hlist_node *node);
120 } teardown;
121 struct hlist_head list;
122 bool cant_stop;
123 bool multi_instance;
124};
125
126static DEFINE_MUTEX(cpuhp_state_mutex);
127static struct cpuhp_step cpuhp_hp_states[];
128
129static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
130{
131 return cpuhp_hp_states + state;
132}
133
134/**
135 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
136 * @cpu: The cpu for which the callback should be invoked
137 * @state: The state to do callbacks for
138 * @bringup: True if the bringup callback should be invoked
139 * @node: For multi-instance, do a single entry callback for install/remove
140 * @lastp: For multi-instance rollback, remember how far we got
141 *
142 * Called from cpu hotplug and from the state register machinery.
143 */
144static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
145 bool bringup, struct hlist_node *node,
146 struct hlist_node **lastp)
147{
148 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
149 struct cpuhp_step *step = cpuhp_get_step(state);
150 int (*cbm)(unsigned int cpu, struct hlist_node *node);
151 int (*cb)(unsigned int cpu);
152 int ret, cnt;
153
154 if (st->fail == state) {
155 st->fail = CPUHP_INVALID;
156
157 if (!(bringup ? step->startup.single : step->teardown.single))
158 return 0;
159
160 return -EAGAIN;
161 }
162
163 if (!step->multi_instance) {
164 WARN_ON_ONCE(lastp && *lastp);
165 cb = bringup ? step->startup.single : step->teardown.single;
166 if (!cb)
167 return 0;
168 trace_cpuhp_enter(cpu, st->target, state, cb);
169 ret = cb(cpu);
170 trace_cpuhp_exit(cpu, st->state, state, ret);
171 return ret;
172 }
173 cbm = bringup ? step->startup.multi : step->teardown.multi;
174 if (!cbm)
175 return 0;
176
177 /* Single invocation for instance add/remove */
178 if (node) {
179 WARN_ON_ONCE(lastp && *lastp);
180 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
181 ret = cbm(cpu, node);
182 trace_cpuhp_exit(cpu, st->state, state, ret);
183 return ret;
184 }
185
186 /* State transition. Invoke on all instances */
187 cnt = 0;
188 hlist_for_each(node, &step->list) {
189 if (lastp && node == *lastp)
190 break;
191
192 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
193 ret = cbm(cpu, node);
194 trace_cpuhp_exit(cpu, st->state, state, ret);
195 if (ret) {
196 if (!lastp)
197 goto err;
198
199 *lastp = node;
200 return ret;
201 }
202 cnt++;
203 }
204 if (lastp)
205 *lastp = NULL;
206 return 0;
207err:
208 /* Rollback the instances if one failed */
209 cbm = !bringup ? step->startup.multi : step->teardown.multi;
210 if (!cbm)
211 return ret;
212
213 hlist_for_each(node, &step->list) {
214 if (!cnt--)
215 break;
216
217 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
218 ret = cbm(cpu, node);
219 trace_cpuhp_exit(cpu, st->state, state, ret);
220 /*
221 * Rollback must not fail,
222 */
223 WARN_ON_ONCE(ret);
224 }
225 return ret;
226}
227
228#ifdef CONFIG_SMP
229static bool cpuhp_is_ap_state(enum cpuhp_state state)
230{
231 /*
232 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
233 * purposes as that state is handled explicitly in cpu_down.
234 */
235 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
236}
237
238static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
239{
240 struct completion *done = bringup ? &st->done_up : &st->done_down;
241 wait_for_completion(done);
242}
243
244static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
245{
246 struct completion *done = bringup ? &st->done_up : &st->done_down;
247 complete(done);
248}
249
250/*
251 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
252 */
253static bool cpuhp_is_atomic_state(enum cpuhp_state state)
254{
255 return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
256}
257
258/* Serializes the updates to cpu_online_mask, cpu_present_mask */
259static DEFINE_MUTEX(cpu_add_remove_lock);
260bool cpuhp_tasks_frozen;
261EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
262
263/*
264 * The following two APIs (cpu_maps_update_begin/done) must be used when
265 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
266 */
267void cpu_maps_update_begin(void)
268{
269 mutex_lock(&cpu_add_remove_lock);
270}
271
272void cpu_maps_update_done(void)
273{
274 mutex_unlock(&cpu_add_remove_lock);
275}
276
277/*
278 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
279 * Should always be manipulated under cpu_add_remove_lock
280 */
281static int cpu_hotplug_disabled;
282
283#ifdef CONFIG_HOTPLUG_CPU
284
285DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
286
287void cpus_read_lock(void)
288{
289 percpu_down_read(&cpu_hotplug_lock);
290}
291EXPORT_SYMBOL_GPL(cpus_read_lock);
292
293int cpus_read_trylock(void)
294{
295 return percpu_down_read_trylock(&cpu_hotplug_lock);
296}
297EXPORT_SYMBOL_GPL(cpus_read_trylock);
298
299void cpus_read_unlock(void)
300{
301 percpu_up_read(&cpu_hotplug_lock);
302}
303EXPORT_SYMBOL_GPL(cpus_read_unlock);
304
305void cpus_write_lock(void)
306{
307 percpu_down_write(&cpu_hotplug_lock);
308}
309
310void cpus_write_unlock(void)
311{
312 percpu_up_write(&cpu_hotplug_lock);
313}
314
315void lockdep_assert_cpus_held(void)
316{
317 /*
318 * We can't have hotplug operations before userspace starts running,
319 * and some init codepaths will knowingly not take the hotplug lock.
320 * This is all valid, so mute lockdep until it makes sense to report
321 * unheld locks.
322 */
323 if (system_state < SYSTEM_RUNNING)
324 return;
325
326 percpu_rwsem_assert_held(&cpu_hotplug_lock);
327}
328
329static void lockdep_acquire_cpus_lock(void)
330{
331 rwsem_acquire(&cpu_hotplug_lock.rw_sem.dep_map, 0, 0, _THIS_IP_);
332}
333
334static void lockdep_release_cpus_lock(void)
335{
336 rwsem_release(&cpu_hotplug_lock.rw_sem.dep_map, 1, _THIS_IP_);
337}
338
339/*
340 * Wait for currently running CPU hotplug operations to complete (if any) and
341 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
342 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
343 * hotplug path before performing hotplug operations. So acquiring that lock
344 * guarantees mutual exclusion from any currently running hotplug operations.
345 */
346void cpu_hotplug_disable(void)
347{
348 cpu_maps_update_begin();
349 cpu_hotplug_disabled++;
350 cpu_maps_update_done();
351}
352EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
353
354static void __cpu_hotplug_enable(void)
355{
356 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
357 return;
358 cpu_hotplug_disabled--;
359}
360
361void cpu_hotplug_enable(void)
362{
363 cpu_maps_update_begin();
364 __cpu_hotplug_enable();
365 cpu_maps_update_done();
366}
367EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
368
369#else
370
371static void lockdep_acquire_cpus_lock(void)
372{
373}
374
375static void lockdep_release_cpus_lock(void)
376{
377}
378
379#endif /* CONFIG_HOTPLUG_CPU */
380
381/*
382 * Architectures that need SMT-specific errata handling during SMT hotplug
383 * should override this.
384 */
385void __weak arch_smt_update(void) { }
386
387#ifdef CONFIG_HOTPLUG_SMT
388enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
389
390void __init cpu_smt_disable(bool force)
391{
392 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
393 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
394 return;
395
396 if (force) {
397 pr_info("SMT: Force disabled\n");
398 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
399 } else {
400 pr_info("SMT: disabled\n");
401 cpu_smt_control = CPU_SMT_DISABLED;
402 }
403}
404
405/*
406 * The decision whether SMT is supported can only be done after the full
407 * CPU identification. Called from architecture code.
408 */
409void __init cpu_smt_check_topology(void)
410{
411 if (!topology_smt_supported())
412 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
413}
414
415static int __init smt_cmdline_disable(char *str)
416{
417 cpu_smt_disable(str && !strcmp(str, "force"));
418 return 0;
419}
420early_param("nosmt", smt_cmdline_disable);
421
422static inline bool cpu_smt_allowed(unsigned int cpu)
423{
424 if (cpu_smt_control == CPU_SMT_ENABLED)
425 return true;
426
427 if (topology_is_primary_thread(cpu))
428 return true;
429
430 /*
431 * On x86 it's required to boot all logical CPUs at least once so
432 * that the init code can get a chance to set CR4.MCE on each
433 * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any
434 * core will shutdown the machine.
435 */
436 return !per_cpu(cpuhp_state, cpu).booted_once;
437}
438#else
439static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
440#endif
441
442static inline enum cpuhp_state
443cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
444{
445 enum cpuhp_state prev_state = st->state;
446
447 st->rollback = false;
448 st->last = NULL;
449
450 st->target = target;
451 st->single = false;
452 st->bringup = st->state < target;
453
454 return prev_state;
455}
456
457static inline void
458cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
459{
460 st->rollback = true;
461
462 /*
463 * If we have st->last we need to undo partial multi_instance of this
464 * state first. Otherwise start undo at the previous state.
465 */
466 if (!st->last) {
467 if (st->bringup)
468 st->state--;
469 else
470 st->state++;
471 }
472
473 st->target = prev_state;
474 st->bringup = !st->bringup;
475}
476
477/* Regular hotplug invocation of the AP hotplug thread */
478static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
479{
480 if (!st->single && st->state == st->target)
481 return;
482
483 st->result = 0;
484 /*
485 * Make sure the above stores are visible before should_run becomes
486 * true. Paired with the mb() above in cpuhp_thread_fun()
487 */
488 smp_mb();
489 st->should_run = true;
490 wake_up_process(st->thread);
491 wait_for_ap_thread(st, st->bringup);
492}
493
494static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
495{
496 enum cpuhp_state prev_state;
497 int ret;
498
499 prev_state = cpuhp_set_state(st, target);
500 __cpuhp_kick_ap(st);
501 if ((ret = st->result)) {
502 cpuhp_reset_state(st, prev_state);
503 __cpuhp_kick_ap(st);
504 }
505
506 return ret;
507}
508
509static int bringup_wait_for_ap(unsigned int cpu)
510{
511 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
512
513 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
514 wait_for_ap_thread(st, true);
515 if (WARN_ON_ONCE((!cpu_online(cpu))))
516 return -ECANCELED;
517
518 /* Unpark the stopper thread and the hotplug thread of the target cpu */
519 stop_machine_unpark(cpu);
520 kthread_unpark(st->thread);
521
522 /*
523 * SMT soft disabling on X86 requires to bring the CPU out of the
524 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The
525 * CPU marked itself as booted_once in cpu_notify_starting() so the
526 * cpu_smt_allowed() check will now return false if this is not the
527 * primary sibling.
528 */
529 if (!cpu_smt_allowed(cpu))
530 return -ECANCELED;
531
532 if (st->target <= CPUHP_AP_ONLINE_IDLE)
533 return 0;
534
535 return cpuhp_kick_ap(st, st->target);
536}
537
538static int bringup_cpu(unsigned int cpu)
539{
540 struct task_struct *idle = idle_thread_get(cpu);
541 int ret;
542
543 /*
544 * Some architectures have to walk the irq descriptors to
545 * setup the vector space for the cpu which comes online.
546 * Prevent irq alloc/free across the bringup.
547 */
548 irq_lock_sparse();
549
550 /* Arch-specific enabling code. */
551 ret = __cpu_up(cpu, idle);
552 irq_unlock_sparse();
553 if (ret)
554 return ret;
555 return bringup_wait_for_ap(cpu);
556}
557
558/*
559 * Hotplug state machine related functions
560 */
561
562static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
563{
564 for (st->state--; st->state > st->target; st->state--)
565 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
566}
567
568static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
569{
570 if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
571 return true;
572 /*
573 * When CPU hotplug is disabled, then taking the CPU down is not
574 * possible because takedown_cpu() and the architecture and
575 * subsystem specific mechanisms are not available. So the CPU
576 * which would be completely unplugged again needs to stay around
577 * in the current state.
578 */
579 return st->state <= CPUHP_BRINGUP_CPU;
580}
581
582static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
583 enum cpuhp_state target)
584{
585 enum cpuhp_state prev_state = st->state;
586 int ret = 0;
587
588 while (st->state < target) {
589 st->state++;
590 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
591 if (ret) {
592 if (can_rollback_cpu(st)) {
593 st->target = prev_state;
594 undo_cpu_up(cpu, st);
595 }
596 break;
597 }
598 }
599 return ret;
600}
601
602/*
603 * The cpu hotplug threads manage the bringup and teardown of the cpus
604 */
605static void cpuhp_create(unsigned int cpu)
606{
607 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
608
609 init_completion(&st->done_up);
610 init_completion(&st->done_down);
611}
612
613static int cpuhp_should_run(unsigned int cpu)
614{
615 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
616
617 return st->should_run;
618}
619
620/*
621 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
622 * callbacks when a state gets [un]installed at runtime.
623 *
624 * Each invocation of this function by the smpboot thread does a single AP
625 * state callback.
626 *
627 * It has 3 modes of operation:
628 * - single: runs st->cb_state
629 * - up: runs ++st->state, while st->state < st->target
630 * - down: runs st->state--, while st->state > st->target
631 *
632 * When complete or on error, should_run is cleared and the completion is fired.
633 */
634static void cpuhp_thread_fun(unsigned int cpu)
635{
636 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
637 bool bringup = st->bringup;
638 enum cpuhp_state state;
639
640 if (WARN_ON_ONCE(!st->should_run))
641 return;
642
643 /*
644 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
645 * that if we see ->should_run we also see the rest of the state.
646 */
647 smp_mb();
648
649 /*
650 * The BP holds the hotplug lock, but we're now running on the AP,
651 * ensure that anybody asserting the lock is held, will actually find
652 * it so.
653 */
654 lockdep_acquire_cpus_lock();
655 cpuhp_lock_acquire(bringup);
656
657 if (st->single) {
658 state = st->cb_state;
659 st->should_run = false;
660 } else {
661 if (bringup) {
662 st->state++;
663 state = st->state;
664 st->should_run = (st->state < st->target);
665 WARN_ON_ONCE(st->state > st->target);
666 } else {
667 state = st->state;
668 st->state--;
669 st->should_run = (st->state > st->target);
670 WARN_ON_ONCE(st->state < st->target);
671 }
672 }
673
674 WARN_ON_ONCE(!cpuhp_is_ap_state(state));
675
676 if (cpuhp_is_atomic_state(state)) {
677 local_irq_disable();
678 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
679 local_irq_enable();
680
681 /*
682 * STARTING/DYING must not fail!
683 */
684 WARN_ON_ONCE(st->result);
685 } else {
686 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
687 }
688
689 if (st->result) {
690 /*
691 * If we fail on a rollback, we're up a creek without no
692 * paddle, no way forward, no way back. We loose, thanks for
693 * playing.
694 */
695 WARN_ON_ONCE(st->rollback);
696 st->should_run = false;
697 }
698
699 cpuhp_lock_release(bringup);
700 lockdep_release_cpus_lock();
701
702 if (!st->should_run)
703 complete_ap_thread(st, bringup);
704}
705
706/* Invoke a single callback on a remote cpu */
707static int
708cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
709 struct hlist_node *node)
710{
711 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
712 int ret;
713
714 if (!cpu_online(cpu))
715 return 0;
716
717 cpuhp_lock_acquire(false);
718 cpuhp_lock_release(false);
719
720 cpuhp_lock_acquire(true);
721 cpuhp_lock_release(true);
722
723 /*
724 * If we are up and running, use the hotplug thread. For early calls
725 * we invoke the thread function directly.
726 */
727 if (!st->thread)
728 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
729
730 st->rollback = false;
731 st->last = NULL;
732
733 st->node = node;
734 st->bringup = bringup;
735 st->cb_state = state;
736 st->single = true;
737
738 __cpuhp_kick_ap(st);
739
740 /*
741 * If we failed and did a partial, do a rollback.
742 */
743 if ((ret = st->result) && st->last) {
744 st->rollback = true;
745 st->bringup = !bringup;
746
747 __cpuhp_kick_ap(st);
748 }
749
750 /*
751 * Clean up the leftovers so the next hotplug operation wont use stale
752 * data.
753 */
754 st->node = st->last = NULL;
755 return ret;
756}
757
758static int cpuhp_kick_ap_work(unsigned int cpu)
759{
760 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
761 enum cpuhp_state prev_state = st->state;
762 int ret;
763
764 cpuhp_lock_acquire(false);
765 cpuhp_lock_release(false);
766
767 cpuhp_lock_acquire(true);
768 cpuhp_lock_release(true);
769
770 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
771 ret = cpuhp_kick_ap(st, st->target);
772 trace_cpuhp_exit(cpu, st->state, prev_state, ret);
773
774 return ret;
775}
776
777static struct smp_hotplug_thread cpuhp_threads = {
778 .store = &cpuhp_state.thread,
779 .create = &cpuhp_create,
780 .thread_should_run = cpuhp_should_run,
781 .thread_fn = cpuhp_thread_fun,
782 .thread_comm = "cpuhp/%u",
783 .selfparking = true,
784};
785
786void __init cpuhp_threads_init(void)
787{
788 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
789 kthread_unpark(this_cpu_read(cpuhp_state.thread));
790}
791
792#ifdef CONFIG_HOTPLUG_CPU
793/**
794 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
795 * @cpu: a CPU id
796 *
797 * This function walks all processes, finds a valid mm struct for each one and
798 * then clears a corresponding bit in mm's cpumask. While this all sounds
799 * trivial, there are various non-obvious corner cases, which this function
800 * tries to solve in a safe manner.
801 *
802 * Also note that the function uses a somewhat relaxed locking scheme, so it may
803 * be called only for an already offlined CPU.
804 */
805void clear_tasks_mm_cpumask(int cpu)
806{
807 struct task_struct *p;
808
809 /*
810 * This function is called after the cpu is taken down and marked
811 * offline, so its not like new tasks will ever get this cpu set in
812 * their mm mask. -- Peter Zijlstra
813 * Thus, we may use rcu_read_lock() here, instead of grabbing
814 * full-fledged tasklist_lock.
815 */
816 WARN_ON(cpu_online(cpu));
817 rcu_read_lock();
818 for_each_process(p) {
819 struct task_struct *t;
820
821 /*
822 * Main thread might exit, but other threads may still have
823 * a valid mm. Find one.
824 */
825 t = find_lock_task_mm(p);
826 if (!t)
827 continue;
828 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
829 task_unlock(t);
830 }
831 rcu_read_unlock();
832}
833
834/* Take this CPU down. */
835static int take_cpu_down(void *_param)
836{
837 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
838 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
839 int err, cpu = smp_processor_id();
840 int ret;
841
842 /* Ensure this CPU doesn't handle any more interrupts. */
843 err = __cpu_disable();
844 if (err < 0)
845 return err;
846
847 /*
848 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
849 * do this step again.
850 */
851 WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
852 st->state--;
853 /* Invoke the former CPU_DYING callbacks */
854 for (; st->state > target; st->state--) {
855 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
856 /*
857 * DYING must not fail!
858 */
859 WARN_ON_ONCE(ret);
860 }
861
862 /* Give up timekeeping duties */
863 tick_handover_do_timer();
864 /* Remove CPU from timer broadcasting */
865 tick_offline_cpu(cpu);
866 /* Park the stopper thread */
867 stop_machine_park(cpu);
868 return 0;
869}
870
871static int takedown_cpu(unsigned int cpu)
872{
873 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
874 int err;
875
876 /* Park the smpboot threads */
877 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
878
879 /*
880 * Prevent irq alloc/free while the dying cpu reorganizes the
881 * interrupt affinities.
882 */
883 irq_lock_sparse();
884
885 /*
886 * So now all preempt/rcu users must observe !cpu_active().
887 */
888 err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
889 if (err) {
890 /* CPU refused to die */
891 irq_unlock_sparse();
892 /* Unpark the hotplug thread so we can rollback there */
893 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
894 return err;
895 }
896 BUG_ON(cpu_online(cpu));
897
898 /*
899 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
900 * all runnable tasks from the CPU, there's only the idle task left now
901 * that the migration thread is done doing the stop_machine thing.
902 *
903 * Wait for the stop thread to go away.
904 */
905 wait_for_ap_thread(st, false);
906 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
907
908 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
909 irq_unlock_sparse();
910
911 hotplug_cpu__broadcast_tick_pull(cpu);
912 /* This actually kills the CPU. */
913 __cpu_die(cpu);
914
915 tick_cleanup_dead_cpu(cpu);
916 rcutree_migrate_callbacks(cpu);
917 return 0;
918}
919
920static void cpuhp_complete_idle_dead(void *arg)
921{
922 struct cpuhp_cpu_state *st = arg;
923
924 complete_ap_thread(st, false);
925}
926
927void cpuhp_report_idle_dead(void)
928{
929 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
930
931 BUG_ON(st->state != CPUHP_AP_OFFLINE);
932 rcu_report_dead(smp_processor_id());
933 st->state = CPUHP_AP_IDLE_DEAD;
934 /*
935 * We cannot call complete after rcu_report_dead() so we delegate it
936 * to an online cpu.
937 */
938 smp_call_function_single(cpumask_first(cpu_online_mask),
939 cpuhp_complete_idle_dead, st, 0);
940}
941
942static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
943{
944 for (st->state++; st->state < st->target; st->state++)
945 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
946}
947
948static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
949 enum cpuhp_state target)
950{
951 enum cpuhp_state prev_state = st->state;
952 int ret = 0;
953
954 for (; st->state > target; st->state--) {
955 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
956 if (ret) {
957 st->target = prev_state;
958 if (st->state < prev_state)
959 undo_cpu_down(cpu, st);
960 break;
961 }
962 }
963 return ret;
964}
965
966/* Requires cpu_add_remove_lock to be held */
967static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
968 enum cpuhp_state target)
969{
970 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
971 int prev_state, ret = 0;
972
973 if (num_online_cpus() == 1)
974 return -EBUSY;
975
976 if (!cpu_present(cpu))
977 return -EINVAL;
978
979 cpus_write_lock();
980
981 cpuhp_tasks_frozen = tasks_frozen;
982
983 prev_state = cpuhp_set_state(st, target);
984 /*
985 * If the current CPU state is in the range of the AP hotplug thread,
986 * then we need to kick the thread.
987 */
988 if (st->state > CPUHP_TEARDOWN_CPU) {
989 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
990 ret = cpuhp_kick_ap_work(cpu);
991 /*
992 * The AP side has done the error rollback already. Just
993 * return the error code..
994 */
995 if (ret)
996 goto out;
997
998 /*
999 * We might have stopped still in the range of the AP hotplug
1000 * thread. Nothing to do anymore.
1001 */
1002 if (st->state > CPUHP_TEARDOWN_CPU)
1003 goto out;
1004
1005 st->target = target;
1006 }
1007 /*
1008 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1009 * to do the further cleanups.
1010 */
1011 ret = cpuhp_down_callbacks(cpu, st, target);
1012 if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
1013 cpuhp_reset_state(st, prev_state);
1014 __cpuhp_kick_ap(st);
1015 }
1016
1017out:
1018 cpus_write_unlock();
1019 /*
1020 * Do post unplug cleanup. This is still protected against
1021 * concurrent CPU hotplug via cpu_add_remove_lock.
1022 */
1023 lockup_detector_cleanup();
1024 arch_smt_update();
1025 return ret;
1026}
1027
1028static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1029{
1030 if (cpu_hotplug_disabled)
1031 return -EBUSY;
1032 return _cpu_down(cpu, 0, target);
1033}
1034
1035static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
1036{
1037 int err;
1038
1039 cpu_maps_update_begin();
1040 err = cpu_down_maps_locked(cpu, target);
1041 cpu_maps_update_done();
1042 return err;
1043}
1044
1045int cpu_down(unsigned int cpu)
1046{
1047 return do_cpu_down(cpu, CPUHP_OFFLINE);
1048}
1049EXPORT_SYMBOL(cpu_down);
1050
1051#else
1052#define takedown_cpu NULL
1053#endif /*CONFIG_HOTPLUG_CPU*/
1054
1055/**
1056 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1057 * @cpu: cpu that just started
1058 *
1059 * It must be called by the arch code on the new cpu, before the new cpu
1060 * enables interrupts and before the "boot" cpu returns from __cpu_up().
1061 */
1062void notify_cpu_starting(unsigned int cpu)
1063{
1064 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1065 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1066 int ret;
1067
1068 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
1069 st->booted_once = true;
1070 while (st->state < target) {
1071 st->state++;
1072 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1073 /*
1074 * STARTING must not fail!
1075 */
1076 WARN_ON_ONCE(ret);
1077 }
1078}
1079
1080/*
1081 * Called from the idle task. Wake up the controlling task which brings the
1082 * stopper and the hotplug thread of the upcoming CPU up and then delegates
1083 * the rest of the online bringup to the hotplug thread.
1084 */
1085void cpuhp_online_idle(enum cpuhp_state state)
1086{
1087 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1088
1089 /* Happens for the boot cpu */
1090 if (state != CPUHP_AP_ONLINE_IDLE)
1091 return;
1092
1093 st->state = CPUHP_AP_ONLINE_IDLE;
1094 complete_ap_thread(st, true);
1095}
1096
1097/* Requires cpu_add_remove_lock to be held */
1098static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1099{
1100 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1101 struct task_struct *idle;
1102 int ret = 0;
1103
1104 cpus_write_lock();
1105
1106 if (!cpu_present(cpu)) {
1107 ret = -EINVAL;
1108 goto out;
1109 }
1110
1111 /*
1112 * The caller of do_cpu_up might have raced with another
1113 * caller. Ignore it for now.
1114 */
1115 if (st->state >= target)
1116 goto out;
1117
1118 if (st->state == CPUHP_OFFLINE) {
1119 /* Let it fail before we try to bring the cpu up */
1120 idle = idle_thread_get(cpu);
1121 if (IS_ERR(idle)) {
1122 ret = PTR_ERR(idle);
1123 goto out;
1124 }
1125 }
1126
1127 cpuhp_tasks_frozen = tasks_frozen;
1128
1129 cpuhp_set_state(st, target);
1130 /*
1131 * If the current CPU state is in the range of the AP hotplug thread,
1132 * then we need to kick the thread once more.
1133 */
1134 if (st->state > CPUHP_BRINGUP_CPU) {
1135 ret = cpuhp_kick_ap_work(cpu);
1136 /*
1137 * The AP side has done the error rollback already. Just
1138 * return the error code..
1139 */
1140 if (ret)
1141 goto out;
1142 }
1143
1144 /*
1145 * Try to reach the target state. We max out on the BP at
1146 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1147 * responsible for bringing it up to the target state.
1148 */
1149 target = min((int)target, CPUHP_BRINGUP_CPU);
1150 ret = cpuhp_up_callbacks(cpu, st, target);
1151out:
1152 cpus_write_unlock();
1153 arch_smt_update();
1154 return ret;
1155}
1156
1157static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1158{
1159 int err = 0;
1160
1161 if (!cpu_possible(cpu)) {
1162 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1163 cpu);
1164#if defined(CONFIG_IA64)
1165 pr_err("please check additional_cpus= boot parameter\n");
1166#endif
1167 return -EINVAL;
1168 }
1169
1170 err = try_online_node(cpu_to_node(cpu));
1171 if (err)
1172 return err;
1173
1174 cpu_maps_update_begin();
1175
1176 if (cpu_hotplug_disabled) {
1177 err = -EBUSY;
1178 goto out;
1179 }
1180 if (!cpu_smt_allowed(cpu)) {
1181 err = -EPERM;
1182 goto out;
1183 }
1184
1185 err = _cpu_up(cpu, 0, target);
1186out:
1187 cpu_maps_update_done();
1188 return err;
1189}
1190
1191int cpu_up(unsigned int cpu)
1192{
1193 return do_cpu_up(cpu, CPUHP_ONLINE);
1194}
1195EXPORT_SYMBOL_GPL(cpu_up);
1196
1197#ifdef CONFIG_PM_SLEEP_SMP
1198static cpumask_var_t frozen_cpus;
1199
1200int freeze_secondary_cpus(int primary)
1201{
1202 int cpu, error = 0;
1203
1204 cpu_maps_update_begin();
1205 if (primary == -1) {
1206 primary = cpumask_first(cpu_online_mask);
1207 if (!housekeeping_cpu(primary, HK_FLAG_TIMER))
1208 primary = housekeeping_any_cpu(HK_FLAG_TIMER);
1209 } else {
1210 if (!cpu_online(primary))
1211 primary = cpumask_first(cpu_online_mask);
1212 }
1213
1214 /*
1215 * We take down all of the non-boot CPUs in one shot to avoid races
1216 * with the userspace trying to use the CPU hotplug at the same time
1217 */
1218 cpumask_clear(frozen_cpus);
1219
1220 pr_info("Disabling non-boot CPUs ...\n");
1221 for_each_online_cpu(cpu) {
1222 if (cpu == primary)
1223 continue;
1224 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1225 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1226 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1227 if (!error)
1228 cpumask_set_cpu(cpu, frozen_cpus);
1229 else {
1230 pr_err("Error taking CPU%d down: %d\n", cpu, error);
1231 break;
1232 }
1233 }
1234
1235 if (!error)
1236 BUG_ON(num_online_cpus() > 1);
1237 else
1238 pr_err("Non-boot CPUs are not disabled\n");
1239
1240 /*
1241 * Make sure the CPUs won't be enabled by someone else. We need to do
1242 * this even in case of failure as all disable_nonboot_cpus() users are
1243 * supposed to do enable_nonboot_cpus() on the failure path.
1244 */
1245 cpu_hotplug_disabled++;
1246
1247 cpu_maps_update_done();
1248 return error;
1249}
1250
1251void __weak arch_enable_nonboot_cpus_begin(void)
1252{
1253}
1254
1255void __weak arch_enable_nonboot_cpus_end(void)
1256{
1257}
1258
1259void enable_nonboot_cpus(void)
1260{
1261 int cpu, error;
1262
1263 /* Allow everyone to use the CPU hotplug again */
1264 cpu_maps_update_begin();
1265 __cpu_hotplug_enable();
1266 if (cpumask_empty(frozen_cpus))
1267 goto out;
1268
1269 pr_info("Enabling non-boot CPUs ...\n");
1270
1271 arch_enable_nonboot_cpus_begin();
1272
1273 for_each_cpu(cpu, frozen_cpus) {
1274 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1275 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1276 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1277 if (!error) {
1278 pr_info("CPU%d is up\n", cpu);
1279 continue;
1280 }
1281 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1282 }
1283
1284 arch_enable_nonboot_cpus_end();
1285
1286 cpumask_clear(frozen_cpus);
1287out:
1288 cpu_maps_update_done();
1289}
1290
1291static int __init alloc_frozen_cpus(void)
1292{
1293 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1294 return -ENOMEM;
1295 return 0;
1296}
1297core_initcall(alloc_frozen_cpus);
1298
1299/*
1300 * When callbacks for CPU hotplug notifications are being executed, we must
1301 * ensure that the state of the system with respect to the tasks being frozen
1302 * or not, as reported by the notification, remains unchanged *throughout the
1303 * duration* of the execution of the callbacks.
1304 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1305 *
1306 * This synchronization is implemented by mutually excluding regular CPU
1307 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1308 * Hibernate notifications.
1309 */
1310static int
1311cpu_hotplug_pm_callback(struct notifier_block *nb,
1312 unsigned long action, void *ptr)
1313{
1314 switch (action) {
1315
1316 case PM_SUSPEND_PREPARE:
1317 case PM_HIBERNATION_PREPARE:
1318 cpu_hotplug_disable();
1319 break;
1320
1321 case PM_POST_SUSPEND:
1322 case PM_POST_HIBERNATION:
1323 cpu_hotplug_enable();
1324 break;
1325
1326 default:
1327 return NOTIFY_DONE;
1328 }
1329
1330 return NOTIFY_OK;
1331}
1332
1333
1334static int __init cpu_hotplug_pm_sync_init(void)
1335{
1336 /*
1337 * cpu_hotplug_pm_callback has higher priority than x86
1338 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1339 * to disable cpu hotplug to avoid cpu hotplug race.
1340 */
1341 pm_notifier(cpu_hotplug_pm_callback, 0);
1342 return 0;
1343}
1344core_initcall(cpu_hotplug_pm_sync_init);
1345
1346#endif /* CONFIG_PM_SLEEP_SMP */
1347
1348int __boot_cpu_id;
1349
1350#endif /* CONFIG_SMP */
1351
1352/* Boot processor state steps */
1353static struct cpuhp_step cpuhp_hp_states[] = {
1354 [CPUHP_OFFLINE] = {
1355 .name = "offline",
1356 .startup.single = NULL,
1357 .teardown.single = NULL,
1358 },
1359#ifdef CONFIG_SMP
1360 [CPUHP_CREATE_THREADS]= {
1361 .name = "threads:prepare",
1362 .startup.single = smpboot_create_threads,
1363 .teardown.single = NULL,
1364 .cant_stop = true,
1365 },
1366 [CPUHP_PERF_PREPARE] = {
1367 .name = "perf:prepare",
1368 .startup.single = perf_event_init_cpu,
1369 .teardown.single = perf_event_exit_cpu,
1370 },
1371 [CPUHP_WORKQUEUE_PREP] = {
1372 .name = "workqueue:prepare",
1373 .startup.single = workqueue_prepare_cpu,
1374 .teardown.single = NULL,
1375 },
1376 [CPUHP_HRTIMERS_PREPARE] = {
1377 .name = "hrtimers:prepare",
1378 .startup.single = hrtimers_prepare_cpu,
1379 .teardown.single = hrtimers_dead_cpu,
1380 },
1381 [CPUHP_SMPCFD_PREPARE] = {
1382 .name = "smpcfd:prepare",
1383 .startup.single = smpcfd_prepare_cpu,
1384 .teardown.single = smpcfd_dead_cpu,
1385 },
1386 [CPUHP_RELAY_PREPARE] = {
1387 .name = "relay:prepare",
1388 .startup.single = relay_prepare_cpu,
1389 .teardown.single = NULL,
1390 },
1391 [CPUHP_SLAB_PREPARE] = {
1392 .name = "slab:prepare",
1393 .startup.single = slab_prepare_cpu,
1394 .teardown.single = slab_dead_cpu,
1395 },
1396 [CPUHP_RCUTREE_PREP] = {
1397 .name = "RCU/tree:prepare",
1398 .startup.single = rcutree_prepare_cpu,
1399 .teardown.single = rcutree_dead_cpu,
1400 },
1401 /*
1402 * On the tear-down path, timers_dead_cpu() must be invoked
1403 * before blk_mq_queue_reinit_notify() from notify_dead(),
1404 * otherwise a RCU stall occurs.
1405 */
1406 [CPUHP_TIMERS_PREPARE] = {
1407 .name = "timers:prepare",
1408 .startup.single = timers_prepare_cpu,
1409 .teardown.single = timers_dead_cpu,
1410 },
1411 /* Kicks the plugged cpu into life */
1412 [CPUHP_BRINGUP_CPU] = {
1413 .name = "cpu:bringup",
1414 .startup.single = bringup_cpu,
1415 .teardown.single = NULL,
1416 .cant_stop = true,
1417 },
1418 /* Final state before CPU kills itself */
1419 [CPUHP_AP_IDLE_DEAD] = {
1420 .name = "idle:dead",
1421 },
1422 /*
1423 * Last state before CPU enters the idle loop to die. Transient state
1424 * for synchronization.
1425 */
1426 [CPUHP_AP_OFFLINE] = {
1427 .name = "ap:offline",
1428 .cant_stop = true,
1429 },
1430 /* First state is scheduler control. Interrupts are disabled */
1431 [CPUHP_AP_SCHED_STARTING] = {
1432 .name = "sched:starting",
1433 .startup.single = sched_cpu_starting,
1434 .teardown.single = sched_cpu_dying,
1435 },
1436 [CPUHP_AP_RCUTREE_DYING] = {
1437 .name = "RCU/tree:dying",
1438 .startup.single = NULL,
1439 .teardown.single = rcutree_dying_cpu,
1440 },
1441 [CPUHP_AP_SMPCFD_DYING] = {
1442 .name = "smpcfd:dying",
1443 .startup.single = NULL,
1444 .teardown.single = smpcfd_dying_cpu,
1445 },
1446 /* Entry state on starting. Interrupts enabled from here on. Transient
1447 * state for synchronsization */
1448 [CPUHP_AP_ONLINE] = {
1449 .name = "ap:online",
1450 },
1451 /*
1452 * Handled on controll processor until the plugged processor manages
1453 * this itself.
1454 */
1455 [CPUHP_TEARDOWN_CPU] = {
1456 .name = "cpu:teardown",
1457 .startup.single = NULL,
1458 .teardown.single = takedown_cpu,
1459 .cant_stop = true,
1460 },
1461 /* Handle smpboot threads park/unpark */
1462 [CPUHP_AP_SMPBOOT_THREADS] = {
1463 .name = "smpboot/threads:online",
1464 .startup.single = smpboot_unpark_threads,
1465 .teardown.single = smpboot_park_threads,
1466 },
1467 [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1468 .name = "irq/affinity:online",
1469 .startup.single = irq_affinity_online_cpu,
1470 .teardown.single = NULL,
1471 },
1472 [CPUHP_AP_PERF_ONLINE] = {
1473 .name = "perf:online",
1474 .startup.single = perf_event_init_cpu,
1475 .teardown.single = perf_event_exit_cpu,
1476 },
1477 [CPUHP_AP_WATCHDOG_ONLINE] = {
1478 .name = "lockup_detector:online",
1479 .startup.single = lockup_detector_online_cpu,
1480 .teardown.single = lockup_detector_offline_cpu,
1481 },
1482 [CPUHP_AP_WORKQUEUE_ONLINE] = {
1483 .name = "workqueue:online",
1484 .startup.single = workqueue_online_cpu,
1485 .teardown.single = workqueue_offline_cpu,
1486 },
1487 [CPUHP_AP_RCUTREE_ONLINE] = {
1488 .name = "RCU/tree:online",
1489 .startup.single = rcutree_online_cpu,
1490 .teardown.single = rcutree_offline_cpu,
1491 },
1492#endif
1493 /*
1494 * The dynamically registered state space is here
1495 */
1496
1497#ifdef CONFIG_SMP
1498 /* Last state is scheduler control setting the cpu active */
1499 [CPUHP_AP_ACTIVE] = {
1500 .name = "sched:active",
1501 .startup.single = sched_cpu_activate,
1502 .teardown.single = sched_cpu_deactivate,
1503 },
1504#endif
1505
1506 /* CPU is fully up and running. */
1507 [CPUHP_ONLINE] = {
1508 .name = "online",
1509 .startup.single = NULL,
1510 .teardown.single = NULL,
1511 },
1512};
1513
1514/* Sanity check for callbacks */
1515static int cpuhp_cb_check(enum cpuhp_state state)
1516{
1517 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1518 return -EINVAL;
1519 return 0;
1520}
1521
1522/*
1523 * Returns a free for dynamic slot assignment of the Online state. The states
1524 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1525 * by having no name assigned.
1526 */
1527static int cpuhp_reserve_state(enum cpuhp_state state)
1528{
1529 enum cpuhp_state i, end;
1530 struct cpuhp_step *step;
1531
1532 switch (state) {
1533 case CPUHP_AP_ONLINE_DYN:
1534 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1535 end = CPUHP_AP_ONLINE_DYN_END;
1536 break;
1537 case CPUHP_BP_PREPARE_DYN:
1538 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1539 end = CPUHP_BP_PREPARE_DYN_END;
1540 break;
1541 default:
1542 return -EINVAL;
1543 }
1544
1545 for (i = state; i <= end; i++, step++) {
1546 if (!step->name)
1547 return i;
1548 }
1549 WARN(1, "No more dynamic states available for CPU hotplug\n");
1550 return -ENOSPC;
1551}
1552
1553static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1554 int (*startup)(unsigned int cpu),
1555 int (*teardown)(unsigned int cpu),
1556 bool multi_instance)
1557{
1558 /* (Un)Install the callbacks for further cpu hotplug operations */
1559 struct cpuhp_step *sp;
1560 int ret = 0;
1561
1562 /*
1563 * If name is NULL, then the state gets removed.
1564 *
1565 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1566 * the first allocation from these dynamic ranges, so the removal
1567 * would trigger a new allocation and clear the wrong (already
1568 * empty) state, leaving the callbacks of the to be cleared state
1569 * dangling, which causes wreckage on the next hotplug operation.
1570 */
1571 if (name && (state == CPUHP_AP_ONLINE_DYN ||
1572 state == CPUHP_BP_PREPARE_DYN)) {
1573 ret = cpuhp_reserve_state(state);
1574 if (ret < 0)
1575 return ret;
1576 state = ret;
1577 }
1578 sp = cpuhp_get_step(state);
1579 if (name && sp->name)
1580 return -EBUSY;
1581
1582 sp->startup.single = startup;
1583 sp->teardown.single = teardown;
1584 sp->name = name;
1585 sp->multi_instance = multi_instance;
1586 INIT_HLIST_HEAD(&sp->list);
1587 return ret;
1588}
1589
1590static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1591{
1592 return cpuhp_get_step(state)->teardown.single;
1593}
1594
1595/*
1596 * Call the startup/teardown function for a step either on the AP or
1597 * on the current CPU.
1598 */
1599static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1600 struct hlist_node *node)
1601{
1602 struct cpuhp_step *sp = cpuhp_get_step(state);
1603 int ret;
1604
1605 /*
1606 * If there's nothing to do, we done.
1607 * Relies on the union for multi_instance.
1608 */
1609 if ((bringup && !sp->startup.single) ||
1610 (!bringup && !sp->teardown.single))
1611 return 0;
1612 /*
1613 * The non AP bound callbacks can fail on bringup. On teardown
1614 * e.g. module removal we crash for now.
1615 */
1616#ifdef CONFIG_SMP
1617 if (cpuhp_is_ap_state(state))
1618 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1619 else
1620 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1621#else
1622 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1623#endif
1624 BUG_ON(ret && !bringup);
1625 return ret;
1626}
1627
1628/*
1629 * Called from __cpuhp_setup_state on a recoverable failure.
1630 *
1631 * Note: The teardown callbacks for rollback are not allowed to fail!
1632 */
1633static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1634 struct hlist_node *node)
1635{
1636 int cpu;
1637
1638 /* Roll back the already executed steps on the other cpus */
1639 for_each_present_cpu(cpu) {
1640 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1641 int cpustate = st->state;
1642
1643 if (cpu >= failedcpu)
1644 break;
1645
1646 /* Did we invoke the startup call on that cpu ? */
1647 if (cpustate >= state)
1648 cpuhp_issue_call(cpu, state, false, node);
1649 }
1650}
1651
1652int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1653 struct hlist_node *node,
1654 bool invoke)
1655{
1656 struct cpuhp_step *sp;
1657 int cpu;
1658 int ret;
1659
1660 lockdep_assert_cpus_held();
1661
1662 sp = cpuhp_get_step(state);
1663 if (sp->multi_instance == false)
1664 return -EINVAL;
1665
1666 mutex_lock(&cpuhp_state_mutex);
1667
1668 if (!invoke || !sp->startup.multi)
1669 goto add_node;
1670
1671 /*
1672 * Try to call the startup callback for each present cpu
1673 * depending on the hotplug state of the cpu.
1674 */
1675 for_each_present_cpu(cpu) {
1676 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1677 int cpustate = st->state;
1678
1679 if (cpustate < state)
1680 continue;
1681
1682 ret = cpuhp_issue_call(cpu, state, true, node);
1683 if (ret) {
1684 if (sp->teardown.multi)
1685 cpuhp_rollback_install(cpu, state, node);
1686 goto unlock;
1687 }
1688 }
1689add_node:
1690 ret = 0;
1691 hlist_add_head(node, &sp->list);
1692unlock:
1693 mutex_unlock(&cpuhp_state_mutex);
1694 return ret;
1695}
1696
1697int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1698 bool invoke)
1699{
1700 int ret;
1701
1702 cpus_read_lock();
1703 ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1704 cpus_read_unlock();
1705 return ret;
1706}
1707EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1708
1709/**
1710 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1711 * @state: The state to setup
1712 * @invoke: If true, the startup function is invoked for cpus where
1713 * cpu state >= @state
1714 * @startup: startup callback function
1715 * @teardown: teardown callback function
1716 * @multi_instance: State is set up for multiple instances which get
1717 * added afterwards.
1718 *
1719 * The caller needs to hold cpus read locked while calling this function.
1720 * Returns:
1721 * On success:
1722 * Positive state number if @state is CPUHP_AP_ONLINE_DYN
1723 * 0 for all other states
1724 * On failure: proper (negative) error code
1725 */
1726int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1727 const char *name, bool invoke,
1728 int (*startup)(unsigned int cpu),
1729 int (*teardown)(unsigned int cpu),
1730 bool multi_instance)
1731{
1732 int cpu, ret = 0;
1733 bool dynstate;
1734
1735 lockdep_assert_cpus_held();
1736
1737 if (cpuhp_cb_check(state) || !name)
1738 return -EINVAL;
1739
1740 mutex_lock(&cpuhp_state_mutex);
1741
1742 ret = cpuhp_store_callbacks(state, name, startup, teardown,
1743 multi_instance);
1744
1745 dynstate = state == CPUHP_AP_ONLINE_DYN;
1746 if (ret > 0 && dynstate) {
1747 state = ret;
1748 ret = 0;
1749 }
1750
1751 if (ret || !invoke || !startup)
1752 goto out;
1753
1754 /*
1755 * Try to call the startup callback for each present cpu
1756 * depending on the hotplug state of the cpu.
1757 */
1758 for_each_present_cpu(cpu) {
1759 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1760 int cpustate = st->state;
1761
1762 if (cpustate < state)
1763 continue;
1764
1765 ret = cpuhp_issue_call(cpu, state, true, NULL);
1766 if (ret) {
1767 if (teardown)
1768 cpuhp_rollback_install(cpu, state, NULL);
1769 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1770 goto out;
1771 }
1772 }
1773out:
1774 mutex_unlock(&cpuhp_state_mutex);
1775 /*
1776 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1777 * dynamically allocated state in case of success.
1778 */
1779 if (!ret && dynstate)
1780 return state;
1781 return ret;
1782}
1783EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1784
1785int __cpuhp_setup_state(enum cpuhp_state state,
1786 const char *name, bool invoke,
1787 int (*startup)(unsigned int cpu),
1788 int (*teardown)(unsigned int cpu),
1789 bool multi_instance)
1790{
1791 int ret;
1792
1793 cpus_read_lock();
1794 ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1795 teardown, multi_instance);
1796 cpus_read_unlock();
1797 return ret;
1798}
1799EXPORT_SYMBOL(__cpuhp_setup_state);
1800
1801int __cpuhp_state_remove_instance(enum cpuhp_state state,
1802 struct hlist_node *node, bool invoke)
1803{
1804 struct cpuhp_step *sp = cpuhp_get_step(state);
1805 int cpu;
1806
1807 BUG_ON(cpuhp_cb_check(state));
1808
1809 if (!sp->multi_instance)
1810 return -EINVAL;
1811
1812 cpus_read_lock();
1813 mutex_lock(&cpuhp_state_mutex);
1814
1815 if (!invoke || !cpuhp_get_teardown_cb(state))
1816 goto remove;
1817 /*
1818 * Call the teardown callback for each present cpu depending
1819 * on the hotplug state of the cpu. This function is not
1820 * allowed to fail currently!
1821 */
1822 for_each_present_cpu(cpu) {
1823 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1824 int cpustate = st->state;
1825
1826 if (cpustate >= state)
1827 cpuhp_issue_call(cpu, state, false, node);
1828 }
1829
1830remove:
1831 hlist_del(node);
1832 mutex_unlock(&cpuhp_state_mutex);
1833 cpus_read_unlock();
1834
1835 return 0;
1836}
1837EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1838
1839/**
1840 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1841 * @state: The state to remove
1842 * @invoke: If true, the teardown function is invoked for cpus where
1843 * cpu state >= @state
1844 *
1845 * The caller needs to hold cpus read locked while calling this function.
1846 * The teardown callback is currently not allowed to fail. Think
1847 * about module removal!
1848 */
1849void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1850{
1851 struct cpuhp_step *sp = cpuhp_get_step(state);
1852 int cpu;
1853
1854 BUG_ON(cpuhp_cb_check(state));
1855
1856 lockdep_assert_cpus_held();
1857
1858 mutex_lock(&cpuhp_state_mutex);
1859 if (sp->multi_instance) {
1860 WARN(!hlist_empty(&sp->list),
1861 "Error: Removing state %d which has instances left.\n",
1862 state);
1863 goto remove;
1864 }
1865
1866 if (!invoke || !cpuhp_get_teardown_cb(state))
1867 goto remove;
1868
1869 /*
1870 * Call the teardown callback for each present cpu depending
1871 * on the hotplug state of the cpu. This function is not
1872 * allowed to fail currently!
1873 */
1874 for_each_present_cpu(cpu) {
1875 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1876 int cpustate = st->state;
1877
1878 if (cpustate >= state)
1879 cpuhp_issue_call(cpu, state, false, NULL);
1880 }
1881remove:
1882 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1883 mutex_unlock(&cpuhp_state_mutex);
1884}
1885EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1886
1887void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1888{
1889 cpus_read_lock();
1890 __cpuhp_remove_state_cpuslocked(state, invoke);
1891 cpus_read_unlock();
1892}
1893EXPORT_SYMBOL(__cpuhp_remove_state);
1894
1895#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1896static ssize_t show_cpuhp_state(struct device *dev,
1897 struct device_attribute *attr, char *buf)
1898{
1899 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1900
1901 return sprintf(buf, "%d\n", st->state);
1902}
1903static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1904
1905static ssize_t write_cpuhp_target(struct device *dev,
1906 struct device_attribute *attr,
1907 const char *buf, size_t count)
1908{
1909 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1910 struct cpuhp_step *sp;
1911 int target, ret;
1912
1913 ret = kstrtoint(buf, 10, &target);
1914 if (ret)
1915 return ret;
1916
1917#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1918 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1919 return -EINVAL;
1920#else
1921 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1922 return -EINVAL;
1923#endif
1924
1925 ret = lock_device_hotplug_sysfs();
1926 if (ret)
1927 return ret;
1928
1929 mutex_lock(&cpuhp_state_mutex);
1930 sp = cpuhp_get_step(target);
1931 ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1932 mutex_unlock(&cpuhp_state_mutex);
1933 if (ret)
1934 goto out;
1935
1936 if (st->state < target)
1937 ret = do_cpu_up(dev->id, target);
1938 else
1939 ret = do_cpu_down(dev->id, target);
1940out:
1941 unlock_device_hotplug();
1942 return ret ? ret : count;
1943}
1944
1945static ssize_t show_cpuhp_target(struct device *dev,
1946 struct device_attribute *attr, char *buf)
1947{
1948 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1949
1950 return sprintf(buf, "%d\n", st->target);
1951}
1952static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1953
1954
1955static ssize_t write_cpuhp_fail(struct device *dev,
1956 struct device_attribute *attr,
1957 const char *buf, size_t count)
1958{
1959 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1960 struct cpuhp_step *sp;
1961 int fail, ret;
1962
1963 ret = kstrtoint(buf, 10, &fail);
1964 if (ret)
1965 return ret;
1966
1967 if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
1968 return -EINVAL;
1969
1970 /*
1971 * Cannot fail STARTING/DYING callbacks.
1972 */
1973 if (cpuhp_is_atomic_state(fail))
1974 return -EINVAL;
1975
1976 /*
1977 * Cannot fail anything that doesn't have callbacks.
1978 */
1979 mutex_lock(&cpuhp_state_mutex);
1980 sp = cpuhp_get_step(fail);
1981 if (!sp->startup.single && !sp->teardown.single)
1982 ret = -EINVAL;
1983 mutex_unlock(&cpuhp_state_mutex);
1984 if (ret)
1985 return ret;
1986
1987 st->fail = fail;
1988
1989 return count;
1990}
1991
1992static ssize_t show_cpuhp_fail(struct device *dev,
1993 struct device_attribute *attr, char *buf)
1994{
1995 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1996
1997 return sprintf(buf, "%d\n", st->fail);
1998}
1999
2000static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
2001
2002static struct attribute *cpuhp_cpu_attrs[] = {
2003 &dev_attr_state.attr,
2004 &dev_attr_target.attr,
2005 &dev_attr_fail.attr,
2006 NULL
2007};
2008
2009static const struct attribute_group cpuhp_cpu_attr_group = {
2010 .attrs = cpuhp_cpu_attrs,
2011 .name = "hotplug",
2012 NULL
2013};
2014
2015static ssize_t show_cpuhp_states(struct device *dev,
2016 struct device_attribute *attr, char *buf)
2017{
2018 ssize_t cur, res = 0;
2019 int i;
2020
2021 mutex_lock(&cpuhp_state_mutex);
2022 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2023 struct cpuhp_step *sp = cpuhp_get_step(i);
2024
2025 if (sp->name) {
2026 cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2027 buf += cur;
2028 res += cur;
2029 }
2030 }
2031 mutex_unlock(&cpuhp_state_mutex);
2032 return res;
2033}
2034static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
2035
2036static struct attribute *cpuhp_cpu_root_attrs[] = {
2037 &dev_attr_states.attr,
2038 NULL
2039};
2040
2041static const struct attribute_group cpuhp_cpu_root_attr_group = {
2042 .attrs = cpuhp_cpu_root_attrs,
2043 .name = "hotplug",
2044 NULL
2045};
2046
2047#ifdef CONFIG_HOTPLUG_SMT
2048
2049static void cpuhp_offline_cpu_device(unsigned int cpu)
2050{
2051 struct device *dev = get_cpu_device(cpu);
2052
2053 dev->offline = true;
2054 /* Tell user space about the state change */
2055 kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2056}
2057
2058static void cpuhp_online_cpu_device(unsigned int cpu)
2059{
2060 struct device *dev = get_cpu_device(cpu);
2061
2062 dev->offline = false;
2063 /* Tell user space about the state change */
2064 kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2065}
2066
2067int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2068{
2069 int cpu, ret = 0;
2070
2071 cpu_maps_update_begin();
2072 for_each_online_cpu(cpu) {
2073 if (topology_is_primary_thread(cpu))
2074 continue;
2075 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2076 if (ret)
2077 break;
2078 /*
2079 * As this needs to hold the cpu maps lock it's impossible
2080 * to call device_offline() because that ends up calling
2081 * cpu_down() which takes cpu maps lock. cpu maps lock
2082 * needs to be held as this might race against in kernel
2083 * abusers of the hotplug machinery (thermal management).
2084 *
2085 * So nothing would update device:offline state. That would
2086 * leave the sysfs entry stale and prevent onlining after
2087 * smt control has been changed to 'off' again. This is
2088 * called under the sysfs hotplug lock, so it is properly
2089 * serialized against the regular offline usage.
2090 */
2091 cpuhp_offline_cpu_device(cpu);
2092 }
2093 if (!ret)
2094 cpu_smt_control = ctrlval;
2095 cpu_maps_update_done();
2096 return ret;
2097}
2098
2099int cpuhp_smt_enable(void)
2100{
2101 int cpu, ret = 0;
2102
2103 cpu_maps_update_begin();
2104 cpu_smt_control = CPU_SMT_ENABLED;
2105 for_each_present_cpu(cpu) {
2106 /* Skip online CPUs and CPUs on offline nodes */
2107 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2108 continue;
2109 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2110 if (ret)
2111 break;
2112 /* See comment in cpuhp_smt_disable() */
2113 cpuhp_online_cpu_device(cpu);
2114 }
2115 cpu_maps_update_done();
2116 return ret;
2117}
2118
2119
2120static ssize_t
2121__store_smt_control(struct device *dev, struct device_attribute *attr,
2122 const char *buf, size_t count)
2123{
2124 int ctrlval, ret;
2125
2126 if (sysfs_streq(buf, "on"))
2127 ctrlval = CPU_SMT_ENABLED;
2128 else if (sysfs_streq(buf, "off"))
2129 ctrlval = CPU_SMT_DISABLED;
2130 else if (sysfs_streq(buf, "forceoff"))
2131 ctrlval = CPU_SMT_FORCE_DISABLED;
2132 else
2133 return -EINVAL;
2134
2135 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2136 return -EPERM;
2137
2138 if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2139 return -ENODEV;
2140
2141 ret = lock_device_hotplug_sysfs();
2142 if (ret)
2143 return ret;
2144
2145 if (ctrlval != cpu_smt_control) {
2146 switch (ctrlval) {
2147 case CPU_SMT_ENABLED:
2148 ret = cpuhp_smt_enable();
2149 break;
2150 case CPU_SMT_DISABLED:
2151 case CPU_SMT_FORCE_DISABLED:
2152 ret = cpuhp_smt_disable(ctrlval);
2153 break;
2154 }
2155 }
2156
2157 unlock_device_hotplug();
2158 return ret ? ret : count;
2159}
2160
2161#else /* !CONFIG_HOTPLUG_SMT */
2162static ssize_t
2163__store_smt_control(struct device *dev, struct device_attribute *attr,
2164 const char *buf, size_t count)
2165{
2166 return -ENODEV;
2167}
2168#endif /* CONFIG_HOTPLUG_SMT */
2169
2170static const char *smt_states[] = {
2171 [CPU_SMT_ENABLED] = "on",
2172 [CPU_SMT_DISABLED] = "off",
2173 [CPU_SMT_FORCE_DISABLED] = "forceoff",
2174 [CPU_SMT_NOT_SUPPORTED] = "notsupported",
2175 [CPU_SMT_NOT_IMPLEMENTED] = "notimplemented",
2176};
2177
2178static ssize_t
2179show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2180{
2181 const char *state = smt_states[cpu_smt_control];
2182
2183 return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
2184}
2185
2186static ssize_t
2187store_smt_control(struct device *dev, struct device_attribute *attr,
2188 const char *buf, size_t count)
2189{
2190 return __store_smt_control(dev, attr, buf, count);
2191}
2192static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2193
2194static ssize_t
2195show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2196{
2197 return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
2198}
2199static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2200
2201static struct attribute *cpuhp_smt_attrs[] = {
2202 &dev_attr_control.attr,
2203 &dev_attr_active.attr,
2204 NULL
2205};
2206
2207static const struct attribute_group cpuhp_smt_attr_group = {
2208 .attrs = cpuhp_smt_attrs,
2209 .name = "smt",
2210 NULL
2211};
2212
2213static int __init cpu_smt_sysfs_init(void)
2214{
2215 return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2216 &cpuhp_smt_attr_group);
2217}
2218
2219static int __init cpuhp_sysfs_init(void)
2220{
2221 int cpu, ret;
2222
2223 ret = cpu_smt_sysfs_init();
2224 if (ret)
2225 return ret;
2226
2227 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2228 &cpuhp_cpu_root_attr_group);
2229 if (ret)
2230 return ret;
2231
2232 for_each_possible_cpu(cpu) {
2233 struct device *dev = get_cpu_device(cpu);
2234
2235 if (!dev)
2236 continue;
2237 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2238 if (ret)
2239 return ret;
2240 }
2241 return 0;
2242}
2243device_initcall(cpuhp_sysfs_init);
2244#endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
2245
2246/*
2247 * cpu_bit_bitmap[] is a special, "compressed" data structure that
2248 * represents all NR_CPUS bits binary values of 1<<nr.
2249 *
2250 * It is used by cpumask_of() to get a constant address to a CPU
2251 * mask value that has a single bit set only.
2252 */
2253
2254/* cpu_bit_bitmap[0] is empty - so we can back into it */
2255#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
2256#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2257#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2258#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2259
2260const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2261
2262 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
2263 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
2264#if BITS_PER_LONG > 32
2265 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
2266 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
2267#endif
2268};
2269EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2270
2271const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2272EXPORT_SYMBOL(cpu_all_bits);
2273
2274#ifdef CONFIG_INIT_ALL_POSSIBLE
2275struct cpumask __cpu_possible_mask __read_mostly
2276 = {CPU_BITS_ALL};
2277#else
2278struct cpumask __cpu_possible_mask __read_mostly;
2279#endif
2280EXPORT_SYMBOL(__cpu_possible_mask);
2281
2282struct cpumask __cpu_online_mask __read_mostly;
2283EXPORT_SYMBOL(__cpu_online_mask);
2284
2285struct cpumask __cpu_present_mask __read_mostly;
2286EXPORT_SYMBOL(__cpu_present_mask);
2287
2288struct cpumask __cpu_active_mask __read_mostly;
2289EXPORT_SYMBOL(__cpu_active_mask);
2290
2291void init_cpu_present(const struct cpumask *src)
2292{
2293 cpumask_copy(&__cpu_present_mask, src);
2294}
2295
2296void init_cpu_possible(const struct cpumask *src)
2297{
2298 cpumask_copy(&__cpu_possible_mask, src);
2299}
2300
2301void init_cpu_online(const struct cpumask *src)
2302{
2303 cpumask_copy(&__cpu_online_mask, src);
2304}
2305
2306/*
2307 * Activate the first processor.
2308 */
2309void __init boot_cpu_init(void)
2310{
2311 int cpu = smp_processor_id();
2312
2313 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2314 set_cpu_online(cpu, true);
2315 set_cpu_active(cpu, true);
2316 set_cpu_present(cpu, true);
2317 set_cpu_possible(cpu, true);
2318
2319#ifdef CONFIG_SMP
2320 __boot_cpu_id = cpu;
2321#endif
2322}
2323
2324/*
2325 * Must be called _AFTER_ setting up the per_cpu areas
2326 */
2327void __init boot_cpu_hotplug_init(void)
2328{
2329#ifdef CONFIG_SMP
2330 this_cpu_write(cpuhp_state.booted_once, true);
2331#endif
2332 this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2333}
2334
2335enum cpu_mitigations cpu_mitigations __ro_after_init = CPU_MITIGATIONS_AUTO;
2336
2337static int __init mitigations_parse_cmdline(char *arg)
2338{
2339 if (!strcmp(arg, "off"))
2340 cpu_mitigations = CPU_MITIGATIONS_OFF;
2341 else if (!strcmp(arg, "auto"))
2342 cpu_mitigations = CPU_MITIGATIONS_AUTO;
2343 else if (!strcmp(arg, "auto,nosmt"))
2344 cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2345 else
2346 pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2347 arg);
2348
2349 return 0;
2350}
2351early_param("mitigations", mitigations_parse_cmdline);