projects/linux·blob v5.3-rc6 · refs

2358 lines · 56.4 KiB

1/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched/signal.h>
11#include <linux/sched/hotplug.h>
12#include <linux/sched/isolation.h>
13#include <linux/sched/task.h>
14#include <linux/sched/smt.h>
15#include <linux/unistd.h>
16#include <linux/cpu.h>
17#include <linux/oom.h>
18#include <linux/rcupdate.h>
19#include <linux/export.h>
20#include <linux/bug.h>
21#include <linux/kthread.h>
22#include <linux/stop_machine.h>
23#include <linux/mutex.h>
24#include <linux/gfp.h>
25#include <linux/suspend.h>
26#include <linux/lockdep.h>
27#include <linux/tick.h>
28#include <linux/irq.h>
29#include <linux/nmi.h>
30#include <linux/smpboot.h>
31#include <linux/relay.h>
32#include <linux/slab.h>
33#include <linux/percpu-rwsem.h>
34
35#include <trace/events/power.h>
36#define CREATE_TRACE_POINTS
37#include <trace/events/cpuhp.h>
38
39#include "smpboot.h"
40
41/**
42 * cpuhp_cpu_state - Per cpu hotplug state storage
43 * @state: The current cpu state
44 * @target: The target state
45 * @thread: Pointer to the hotplug thread
46 * @should_run: Thread should execute
47 * @rollback: Perform a rollback
48 * @single: Single callback invocation
49 * @bringup: Single callback bringup or teardown selector
50 * @cb_state: The state for a single callback (install/uninstall)
51 * @result: Result of the operation
52 * @done_up: Signal completion to the issuer of the task for cpu-up
53 * @done_down: Signal completion to the issuer of the task for cpu-down
54 */
55struct cpuhp_cpu_state {
56 enum cpuhp_state state;
57 enum cpuhp_state target;
58 enum cpuhp_state fail;
59#ifdef CONFIG_SMP
60 struct task_struct *thread;
61 bool should_run;
62 bool rollback;
63 bool single;
64 bool bringup;
65 bool booted_once;
66 struct hlist_node *node;
67 struct hlist_node *last;
68 enum cpuhp_state cb_state;
69 int result;
70 struct completion done_up;
71 struct completion done_down;
72#endif
73};
74
75static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
76 .fail = CPUHP_INVALID,
77};
78
79#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
80static struct lockdep_map cpuhp_state_up_map =
81 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
82static struct lockdep_map cpuhp_state_down_map =
83 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
84
85
86static inline void cpuhp_lock_acquire(bool bringup)
87{
88 lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
89}
90
91static inline void cpuhp_lock_release(bool bringup)
92{
93 lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
94}
95#else
96
97static inline void cpuhp_lock_acquire(bool bringup) { }
98static inline void cpuhp_lock_release(bool bringup) { }
99
100#endif
101
102/**
103 * cpuhp_step - Hotplug state machine step
104 * @name: Name of the step
105 * @startup: Startup function of the step
106 * @teardown: Teardown function of the step
107 * @cant_stop: Bringup/teardown can't be stopped at this step
108 */
109struct cpuhp_step {
110 const char *name;
111 union {
112 int (*single)(unsigned int cpu);
113 int (*multi)(unsigned int cpu,
114 struct hlist_node *node);
115 } startup;
116 union {
117 int (*single)(unsigned int cpu);
118 int (*multi)(unsigned int cpu,
119 struct hlist_node *node);
120 } teardown;
121 struct hlist_head list;
122 bool cant_stop;
123 bool multi_instance;
124};
125
126static DEFINE_MUTEX(cpuhp_state_mutex);
127static struct cpuhp_step cpuhp_hp_states[];
128
129static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
130{
131 return cpuhp_hp_states + state;
132}
133
134/**
135 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
136 * @cpu: The cpu for which the callback should be invoked
137 * @state: The state to do callbacks for
138 * @bringup: True if the bringup callback should be invoked
139 * @node: For multi-instance, do a single entry callback for install/remove
140 * @lastp: For multi-instance rollback, remember how far we got
141 *
142 * Called from cpu hotplug and from the state register machinery.
143 */
144static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
145 bool bringup, struct hlist_node *node,
146 struct hlist_node **lastp)
147{
148 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
149 struct cpuhp_step *step = cpuhp_get_step(state);
150 int (*cbm)(unsigned int cpu, struct hlist_node *node);
151 int (*cb)(unsigned int cpu);
152 int ret, cnt;
153
154 if (st->fail == state) {
155 st->fail = CPUHP_INVALID;
156
157 if (!(bringup ? step->startup.single : step->teardown.single))
158 return 0;
159
160 return -EAGAIN;
161 }
162
163 if (!step->multi_instance) {
164 WARN_ON_ONCE(lastp && *lastp);
165 cb = bringup ? step->startup.single : step->teardown.single;
166 if (!cb)
167 return 0;
168 trace_cpuhp_enter(cpu, st->target, state, cb);
169 ret = cb(cpu);
170 trace_cpuhp_exit(cpu, st->state, state, ret);
171 return ret;
172 }
173 cbm = bringup ? step->startup.multi : step->teardown.multi;
174 if (!cbm)
175 return 0;
176
177 /* Single invocation for instance add/remove */
178 if (node) {
179 WARN_ON_ONCE(lastp && *lastp);
180 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
181 ret = cbm(cpu, node);
182 trace_cpuhp_exit(cpu, st->state, state, ret);
183 return ret;
184 }
185
186 /* State transition. Invoke on all instances */
187 cnt = 0;
188 hlist_for_each(node, &step->list) {
189 if (lastp && node == *lastp)
190 break;
191
192 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
193 ret = cbm(cpu, node);
194 trace_cpuhp_exit(cpu, st->state, state, ret);
195 if (ret) {
196 if (!lastp)
197 goto err;
198
199 *lastp = node;
200 return ret;
201 }
202 cnt++;
203 }
204 if (lastp)
205 *lastp = NULL;
206 return 0;
207err:
208 /* Rollback the instances if one failed */
209 cbm = !bringup ? step->startup.multi : step->teardown.multi;
210 if (!cbm)
211 return ret;
212
213 hlist_for_each(node, &step->list) {
214 if (!cnt--)
215 break;
216
217 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
218 ret = cbm(cpu, node);
219 trace_cpuhp_exit(cpu, st->state, state, ret);
220 /*
221 * Rollback must not fail,
222 */
223 WARN_ON_ONCE(ret);
224 }
225 return ret;
226}
227
228#ifdef CONFIG_SMP
229static bool cpuhp_is_ap_state(enum cpuhp_state state)
230{
231 /*
232 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
233 * purposes as that state is handled explicitly in cpu_down.
234 */
235 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
236}
237
238static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
239{
240 struct completion *done = bringup ? &st->done_up : &st->done_down;
241 wait_for_completion(done);
242}
243
244static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
245{
246 struct completion *done = bringup ? &st->done_up : &st->done_down;
247 complete(done);
248}
249
250/*
251 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
252 */
253static bool cpuhp_is_atomic_state(enum cpuhp_state state)
254{
255 return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
256}
257
258/* Serializes the updates to cpu_online_mask, cpu_present_mask */
259static DEFINE_MUTEX(cpu_add_remove_lock);
260bool cpuhp_tasks_frozen;
261EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
262
263/*
264 * The following two APIs (cpu_maps_update_begin/done) must be used when
265 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
266 */
267void cpu_maps_update_begin(void)
268{
269 mutex_lock(&cpu_add_remove_lock);
270}
271
272void cpu_maps_update_done(void)
273{
274 mutex_unlock(&cpu_add_remove_lock);
275}
276
277/*
278 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
279 * Should always be manipulated under cpu_add_remove_lock
280 */
281static int cpu_hotplug_disabled;
282
283#ifdef CONFIG_HOTPLUG_CPU
284
285DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
286
287void cpus_read_lock(void)
288{
289 percpu_down_read(&cpu_hotplug_lock);
290}
291EXPORT_SYMBOL_GPL(cpus_read_lock);
292
293int cpus_read_trylock(void)
294{
295 return percpu_down_read_trylock(&cpu_hotplug_lock);
296}
297EXPORT_SYMBOL_GPL(cpus_read_trylock);
298
299void cpus_read_unlock(void)
300{
301 percpu_up_read(&cpu_hotplug_lock);
302}
303EXPORT_SYMBOL_GPL(cpus_read_unlock);
304
305void cpus_write_lock(void)
306{
307 percpu_down_write(&cpu_hotplug_lock);
308}
309
310void cpus_write_unlock(void)
311{
312 percpu_up_write(&cpu_hotplug_lock);
313}
314
315void lockdep_assert_cpus_held(void)
316{
317 /*
318 * We can't have hotplug operations before userspace starts running,
319 * and some init codepaths will knowingly not take the hotplug lock.
320 * This is all valid, so mute lockdep until it makes sense to report
321 * unheld locks.
322 */
323 if (system_state < SYSTEM_RUNNING)
324 return;
325
326 percpu_rwsem_assert_held(&cpu_hotplug_lock);
327}
328
329static void lockdep_acquire_cpus_lock(void)
330{
331 rwsem_acquire(&cpu_hotplug_lock.rw_sem.dep_map, 0, 0, _THIS_IP_);
332}
333
334static void lockdep_release_cpus_lock(void)
335{
336 rwsem_release(&cpu_hotplug_lock.rw_sem.dep_map, 1, _THIS_IP_);
337}
338
339/*
340 * Wait for currently running CPU hotplug operations to complete (if any) and
341 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
342 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
343 * hotplug path before performing hotplug operations. So acquiring that lock
344 * guarantees mutual exclusion from any currently running hotplug operations.
345 */
346void cpu_hotplug_disable(void)
347{
348 cpu_maps_update_begin();
349 cpu_hotplug_disabled++;
350 cpu_maps_update_done();
351}
352EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
353
354static void __cpu_hotplug_enable(void)
355{
356 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
357 return;
358 cpu_hotplug_disabled--;
359}
360
361void cpu_hotplug_enable(void)
362{
363 cpu_maps_update_begin();
364 __cpu_hotplug_enable();
365 cpu_maps_update_done();
366}
367EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
368
369#else
370
371static void lockdep_acquire_cpus_lock(void)
372{
373}
374
375static void lockdep_release_cpus_lock(void)
376{
377}
378
379#endif /* CONFIG_HOTPLUG_CPU */
380
381/*
382 * Architectures that need SMT-specific errata handling during SMT hotplug
383 * should override this.
384 */
385void __weak arch_smt_update(void) { }
386
387#ifdef CONFIG_HOTPLUG_SMT
388enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
389
390void __init cpu_smt_disable(bool force)
391{
392 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
393 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
394 return;
395
396 if (force) {
397 pr_info("SMT: Force disabled\n");
398 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
399 } else {
400 pr_info("SMT: disabled\n");
401 cpu_smt_control = CPU_SMT_DISABLED;
402 }
403}
404
405/*
406 * The decision whether SMT is supported can only be done after the full
407 * CPU identification. Called from architecture code.
408 */
409void __init cpu_smt_check_topology(void)
410{
411 if (!topology_smt_supported())
412 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
413}
414
415static int __init smt_cmdline_disable(char *str)
416{
417 cpu_smt_disable(str && !strcmp(str, "force"));
418 return 0;
419}
420early_param("nosmt", smt_cmdline_disable);
421
422static inline bool cpu_smt_allowed(unsigned int cpu)
423{
424 if (cpu_smt_control == CPU_SMT_ENABLED)
425 return true;
426
427 if (topology_is_primary_thread(cpu))
428 return true;
429
430 /*
431 * On x86 it's required to boot all logical CPUs at least once so
432 * that the init code can get a chance to set CR4.MCE on each
433 * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any
434 * core will shutdown the machine.
435 */
436 return !per_cpu(cpuhp_state, cpu).booted_once;
437}
438#else
439static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
440#endif
441
442static inline enum cpuhp_state
443cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
444{
445 enum cpuhp_state prev_state = st->state;
446
447 st->rollback = false;
448 st->last = NULL;
449
450 st->target = target;
451 st->single = false;
452 st->bringup = st->state < target;
453
454 return prev_state;
455}
456
457static inline void
458cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
459{
460 st->rollback = true;
461
462 /*
463 * If we have st->last we need to undo partial multi_instance of this
464 * state first. Otherwise start undo at the previous state.
465 */
466 if (!st->last) {
467 if (st->bringup)
468 st->state--;
469 else
470 st->state++;
471 }
472
473 st->target = prev_state;
474 st->bringup = !st->bringup;
475}
476
477/* Regular hotplug invocation of the AP hotplug thread */
478static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
479{
480 if (!st->single && st->state == st->target)
481 return;
482
483 st->result = 0;
484 /*
485 * Make sure the above stores are visible before should_run becomes
486 * true. Paired with the mb() above in cpuhp_thread_fun()
487 */
488 smp_mb();
489 st->should_run = true;
490 wake_up_process(st->thread);
491 wait_for_ap_thread(st, st->bringup);
492}
493
494static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
495{
496 enum cpuhp_state prev_state;
497 int ret;
498
499 prev_state = cpuhp_set_state(st, target);
500 __cpuhp_kick_ap(st);
501 if ((ret = st->result)) {
502 cpuhp_reset_state(st, prev_state);
503 __cpuhp_kick_ap(st);
504 }
505
506 return ret;
507}
508
509static int bringup_wait_for_ap(unsigned int cpu)
510{
511 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
512
513 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
514 wait_for_ap_thread(st, true);
515 if (WARN_ON_ONCE((!cpu_online(cpu))))
516 return -ECANCELED;
517
518 /* Unpark the stopper thread and the hotplug thread of the target cpu */
519 stop_machine_unpark(cpu);
520 kthread_unpark(st->thread);
521
522 /*
523 * SMT soft disabling on X86 requires to bring the CPU out of the
524 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The
525 * CPU marked itself as booted_once in notify_cpu_starting() so the
526 * cpu_smt_allowed() check will now return false if this is not the
527 * primary sibling.
528 */
529 if (!cpu_smt_allowed(cpu))
530 return -ECANCELED;
531
532 if (st->target <= CPUHP_AP_ONLINE_IDLE)
533 return 0;
534
535 return cpuhp_kick_ap(st, st->target);
536}
537
538static int bringup_cpu(unsigned int cpu)
539{
540 struct task_struct *idle = idle_thread_get(cpu);
541 int ret;
542
543 /*
544 * Some architectures have to walk the irq descriptors to
545 * setup the vector space for the cpu which comes online.
546 * Prevent irq alloc/free across the bringup.
547 */
548 irq_lock_sparse();
549
550 /* Arch-specific enabling code. */
551 ret = __cpu_up(cpu, idle);
552 irq_unlock_sparse();
553 if (ret)
554 return ret;
555 return bringup_wait_for_ap(cpu);
556}
557
558/*
559 * Hotplug state machine related functions
560 */
561
562static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
563{
564 for (st->state--; st->state > st->target; st->state--)
565 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
566}
567
568static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
569{
570 if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
571 return true;
572 /*
573 * When CPU hotplug is disabled, then taking the CPU down is not
574 * possible because takedown_cpu() and the architecture and
575 * subsystem specific mechanisms are not available. So the CPU
576 * which would be completely unplugged again needs to stay around
577 * in the current state.
578 */
579 return st->state <= CPUHP_BRINGUP_CPU;
580}
581
582static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
583 enum cpuhp_state target)
584{
585 enum cpuhp_state prev_state = st->state;
586 int ret = 0;
587
588 while (st->state < target) {
589 st->state++;
590 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
591 if (ret) {
592 if (can_rollback_cpu(st)) {
593 st->target = prev_state;
594 undo_cpu_up(cpu, st);
595 }
596 break;
597 }
598 }
599 return ret;
600}
601
602/*
603 * The cpu hotplug threads manage the bringup and teardown of the cpus
604 */
605static void cpuhp_create(unsigned int cpu)
606{
607 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
608
609 init_completion(&st->done_up);
610 init_completion(&st->done_down);
611}
612
613static int cpuhp_should_run(unsigned int cpu)
614{
615 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
616
617 return st->should_run;
618}
619
620/*
621 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
622 * callbacks when a state gets [un]installed at runtime.
623 *
624 * Each invocation of this function by the smpboot thread does a single AP
625 * state callback.
626 *
627 * It has 3 modes of operation:
628 * - single: runs st->cb_state
629 * - up: runs ++st->state, while st->state < st->target
630 * - down: runs st->state--, while st->state > st->target
631 *
632 * When complete or on error, should_run is cleared and the completion is fired.
633 */
634static void cpuhp_thread_fun(unsigned int cpu)
635{
636 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
637 bool bringup = st->bringup;
638 enum cpuhp_state state;
639
640 if (WARN_ON_ONCE(!st->should_run))
641 return;
642
643 /*
644 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
645 * that if we see ->should_run we also see the rest of the state.
646 */
647 smp_mb();
648
649 /*
650 * The BP holds the hotplug lock, but we're now running on the AP,
651 * ensure that anybody asserting the lock is held, will actually find
652 * it so.
653 */
654 lockdep_acquire_cpus_lock();
655 cpuhp_lock_acquire(bringup);
656
657 if (st->single) {
658 state = st->cb_state;
659 st->should_run = false;
660 } else {
661 if (bringup) {
662 st->state++;
663 state = st->state;
664 st->should_run = (st->state < st->target);
665 WARN_ON_ONCE(st->state > st->target);
666 } else {
667 state = st->state;
668 st->state--;
669 st->should_run = (st->state > st->target);
670 WARN_ON_ONCE(st->state < st->target);
671 }
672 }
673
674 WARN_ON_ONCE(!cpuhp_is_ap_state(state));
675
676 if (cpuhp_is_atomic_state(state)) {
677 local_irq_disable();
678 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
679 local_irq_enable();
680
681 /*
682 * STARTING/DYING must not fail!
683 */
684 WARN_ON_ONCE(st->result);
685 } else {
686 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
687 }
688
689 if (st->result) {
690 /*
691 * If we fail on a rollback, we're up a creek without no
692 * paddle, no way forward, no way back. We loose, thanks for
693 * playing.
694 */
695 WARN_ON_ONCE(st->rollback);
696 st->should_run = false;
697 }
698
699 cpuhp_lock_release(bringup);
700 lockdep_release_cpus_lock();
701
702 if (!st->should_run)
703 complete_ap_thread(st, bringup);
704}
705
706/* Invoke a single callback on a remote cpu */
707static int
708cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
709 struct hlist_node *node)
710{
711 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
712 int ret;
713
714 if (!cpu_online(cpu))
715 return 0;
716
717 cpuhp_lock_acquire(false);
718 cpuhp_lock_release(false);
719
720 cpuhp_lock_acquire(true);
721 cpuhp_lock_release(true);
722
723 /*
724 * If we are up and running, use the hotplug thread. For early calls
725 * we invoke the thread function directly.
726 */
727 if (!st->thread)
728 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
729
730 st->rollback = false;
731 st->last = NULL;
732
733 st->node = node;
734 st->bringup = bringup;
735 st->cb_state = state;
736 st->single = true;
737
738 __cpuhp_kick_ap(st);
739
740 /*
741 * If we failed and did a partial, do a rollback.
742 */
743 if ((ret = st->result) && st->last) {
744 st->rollback = true;
745 st->bringup = !bringup;
746
747 __cpuhp_kick_ap(st);
748 }
749
750 /*
751 * Clean up the leftovers so the next hotplug operation wont use stale
752 * data.
753 */
754 st->node = st->last = NULL;
755 return ret;
756}
757
758static int cpuhp_kick_ap_work(unsigned int cpu)
759{
760 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
761 enum cpuhp_state prev_state = st->state;
762 int ret;
763
764 cpuhp_lock_acquire(false);
765 cpuhp_lock_release(false);
766
767 cpuhp_lock_acquire(true);
768 cpuhp_lock_release(true);
769
770 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
771 ret = cpuhp_kick_ap(st, st->target);
772 trace_cpuhp_exit(cpu, st->state, prev_state, ret);
773
774 return ret;
775}
776
777static struct smp_hotplug_thread cpuhp_threads = {
778 .store = &cpuhp_state.thread,
779 .create = &cpuhp_create,
780 .thread_should_run = cpuhp_should_run,
781 .thread_fn = cpuhp_thread_fun,
782 .thread_comm = "cpuhp/%u",
783 .selfparking = true,
784};
785
786void __init cpuhp_threads_init(void)
787{
788 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
789 kthread_unpark(this_cpu_read(cpuhp_state.thread));
790}
791
792#ifdef CONFIG_HOTPLUG_CPU
793/**
794 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
795 * @cpu: a CPU id
796 *
797 * This function walks all processes, finds a valid mm struct for each one and
798 * then clears a corresponding bit in mm's cpumask. While this all sounds
799 * trivial, there are various non-obvious corner cases, which this function
800 * tries to solve in a safe manner.
801 *
802 * Also note that the function uses a somewhat relaxed locking scheme, so it may
803 * be called only for an already offlined CPU.
804 */
805void clear_tasks_mm_cpumask(int cpu)
806{
807 struct task_struct *p;
808
809 /*
810 * This function is called after the cpu is taken down and marked
811 * offline, so its not like new tasks will ever get this cpu set in
812 * their mm mask. -- Peter Zijlstra
813 * Thus, we may use rcu_read_lock() here, instead of grabbing
814 * full-fledged tasklist_lock.
815 */
816 WARN_ON(cpu_online(cpu));
817 rcu_read_lock();
818 for_each_process(p) {
819 struct task_struct *t;
820
821 /*
822 * Main thread might exit, but other threads may still have
823 * a valid mm. Find one.
824 */
825 t = find_lock_task_mm(p);
826 if (!t)
827 continue;
828 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
829 task_unlock(t);
830 }
831 rcu_read_unlock();
832}
833
834/* Take this CPU down. */
835static int take_cpu_down(void *_param)
836{
837 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
838 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
839 int err, cpu = smp_processor_id();
840 int ret;
841
842 /* Ensure this CPU doesn't handle any more interrupts. */
843 err = __cpu_disable();
844 if (err < 0)
845 return err;
846
847 /*
848 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
849 * do this step again.
850 */
851 WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
852 st->state--;
853 /* Invoke the former CPU_DYING callbacks */
854 for (; st->state > target; st->state--) {
855 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
856 /*
857 * DYING must not fail!
858 */
859 WARN_ON_ONCE(ret);
860 }
861
862 /* Give up timekeeping duties */
863 tick_handover_do_timer();
864 /* Remove CPU from timer broadcasting */
865 tick_offline_cpu(cpu);
866 /* Park the stopper thread */
867 stop_machine_park(cpu);
868 return 0;
869}
870
871static int takedown_cpu(unsigned int cpu)
872{
873 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
874 int err;
875
876 /* Park the smpboot threads */
877 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
878
879 /*
880 * Prevent irq alloc/free while the dying cpu reorganizes the
881 * interrupt affinities.
882 */
883 irq_lock_sparse();
884
885 /*
886 * So now all preempt/rcu users must observe !cpu_active().
887 */
888 err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
889 if (err) {
890 /* CPU refused to die */
891 irq_unlock_sparse();
892 /* Unpark the hotplug thread so we can rollback there */
893 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
894 return err;
895 }
896 BUG_ON(cpu_online(cpu));
897
898 /*
899 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
900 * all runnable tasks from the CPU, there's only the idle task left now
901 * that the migration thread is done doing the stop_machine thing.
902 *
903 * Wait for the stop thread to go away.
904 */
905 wait_for_ap_thread(st, false);
906 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
907
908 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
909 irq_unlock_sparse();
910
911 hotplug_cpu__broadcast_tick_pull(cpu);
912 /* This actually kills the CPU. */
913 __cpu_die(cpu);
914
915 tick_cleanup_dead_cpu(cpu);
916 rcutree_migrate_callbacks(cpu);
917 return 0;
918}
919
920static void cpuhp_complete_idle_dead(void *arg)
921{
922 struct cpuhp_cpu_state *st = arg;
923
924 complete_ap_thread(st, false);
925}
926
927void cpuhp_report_idle_dead(void)
928{
929 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
930
931 BUG_ON(st->state != CPUHP_AP_OFFLINE);
932 rcu_report_dead(smp_processor_id());
933 st->state = CPUHP_AP_IDLE_DEAD;
934 /*
935 * We cannot call complete after rcu_report_dead() so we delegate it
936 * to an online cpu.
937 */
938 smp_call_function_single(cpumask_first(cpu_online_mask),
939 cpuhp_complete_idle_dead, st, 0);
940}
941
942static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
943{
944 for (st->state++; st->state < st->target; st->state++)
945 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
946}
947
948static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
949 enum cpuhp_state target)
950{
951 enum cpuhp_state prev_state = st->state;
952 int ret = 0;
953
954 for (; st->state > target; st->state--) {
955 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
956 if (ret) {
957 st->target = prev_state;
958 if (st->state < prev_state)
959 undo_cpu_down(cpu, st);
960 break;
961 }
962 }
963 return ret;
964}
965
966/* Requires cpu_add_remove_lock to be held */
967static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
968 enum cpuhp_state target)
969{
970 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
971 int prev_state, ret = 0;
972
973 if (num_online_cpus() == 1)
974 return -EBUSY;
975
976 if (!cpu_present(cpu))
977 return -EINVAL;
978
979 cpus_write_lock();
980
981 cpuhp_tasks_frozen = tasks_frozen;
982
983 prev_state = cpuhp_set_state(st, target);
984 /*
985 * If the current CPU state is in the range of the AP hotplug thread,
986 * then we need to kick the thread.
987 */
988 if (st->state > CPUHP_TEARDOWN_CPU) {
989 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
990 ret = cpuhp_kick_ap_work(cpu);
991 /*
992 * The AP side has done the error rollback already. Just
993 * return the error code..
994 */
995 if (ret)
996 goto out;
997
998 /*
999 * We might have stopped still in the range of the AP hotplug
1000 * thread. Nothing to do anymore.
1001 */
1002 if (st->state > CPUHP_TEARDOWN_CPU)
1003 goto out;
1004
1005 st->target = target;
1006 }
1007 /*
1008 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1009 * to do the further cleanups.
1010 */
1011 ret = cpuhp_down_callbacks(cpu, st, target);
1012 if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
1013 cpuhp_reset_state(st, prev_state);
1014 __cpuhp_kick_ap(st);
1015 }
1016
1017out:
1018 cpus_write_unlock();
1019 /*
1020 * Do post unplug cleanup. This is still protected against
1021 * concurrent CPU hotplug via cpu_add_remove_lock.
1022 */
1023 lockup_detector_cleanup();
1024 arch_smt_update();
1025 return ret;
1026}
1027
1028static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1029{
1030 if (cpu_hotplug_disabled)
1031 return -EBUSY;
1032 return _cpu_down(cpu, 0, target);
1033}
1034
1035static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
1036{
1037 int err;
1038
1039 cpu_maps_update_begin();
1040 err = cpu_down_maps_locked(cpu, target);
1041 cpu_maps_update_done();
1042 return err;
1043}
1044
1045int cpu_down(unsigned int cpu)
1046{
1047 return do_cpu_down(cpu, CPUHP_OFFLINE);
1048}
1049EXPORT_SYMBOL(cpu_down);
1050
1051#else
1052#define takedown_cpu NULL
1053#endif /*CONFIG_HOTPLUG_CPU*/
1054
1055/**
1056 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1057 * @cpu: cpu that just started
1058 *
1059 * It must be called by the arch code on the new cpu, before the new cpu
1060 * enables interrupts and before the "boot" cpu returns from __cpu_up().
1061 */
1062void notify_cpu_starting(unsigned int cpu)
1063{
1064 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1065 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1066 int ret;
1067
1068 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
1069 st->booted_once = true;
1070 while (st->state < target) {
1071 st->state++;
1072 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1073 /*
1074 * STARTING must not fail!
1075 */
1076 WARN_ON_ONCE(ret);
1077 }
1078}
1079
1080/*
1081 * Called from the idle task. Wake up the controlling task which brings the
1082 * stopper and the hotplug thread of the upcoming CPU up and then delegates
1083 * the rest of the online bringup to the hotplug thread.
1084 */
1085void cpuhp_online_idle(enum cpuhp_state state)
1086{
1087 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1088
1089 /* Happens for the boot cpu */
1090 if (state != CPUHP_AP_ONLINE_IDLE)
1091 return;
1092
1093 st->state = CPUHP_AP_ONLINE_IDLE;
1094 complete_ap_thread(st, true);
1095}
1096
1097/* Requires cpu_add_remove_lock to be held */
1098static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1099{
1100 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1101 struct task_struct *idle;
1102 int ret = 0;
1103
1104 cpus_write_lock();
1105
1106 if (!cpu_present(cpu)) {
1107 ret = -EINVAL;
1108 goto out;
1109 }
1110
1111 /*
1112 * The caller of do_cpu_up might have raced with another
1113 * caller. Ignore it for now.
1114 */
1115 if (st->state >= target)
1116 goto out;
1117
1118 if (st->state == CPUHP_OFFLINE) {
1119 /* Let it fail before we try to bring the cpu up */
1120 idle = idle_thread_get(cpu);
1121 if (IS_ERR(idle)) {
1122 ret = PTR_ERR(idle);
1123 goto out;
1124 }
1125 }
1126
1127 cpuhp_tasks_frozen = tasks_frozen;
1128
1129 cpuhp_set_state(st, target);
1130 /*
1131 * If the current CPU state is in the range of the AP hotplug thread,
1132 * then we need to kick the thread once more.
1133 */
1134 if (st->state > CPUHP_BRINGUP_CPU) {
1135 ret = cpuhp_kick_ap_work(cpu);
1136 /*
1137 * The AP side has done the error rollback already. Just
1138 * return the error code..
1139 */
1140 if (ret)
1141 goto out;
1142 }
1143
1144 /*
1145 * Try to reach the target state. We max out on the BP at
1146 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1147 * responsible for bringing it up to the target state.
1148 */
1149 target = min((int)target, CPUHP_BRINGUP_CPU);
1150 ret = cpuhp_up_callbacks(cpu, st, target);
1151out:
1152 cpus_write_unlock();
1153 arch_smt_update();
1154 return ret;
1155}
1156
1157static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1158{
1159 int err = 0;
1160
1161 if (!cpu_possible(cpu)) {
1162 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1163 cpu);
1164#if defined(CONFIG_IA64)
1165 pr_err("please check additional_cpus= boot parameter\n");
1166#endif
1167 return -EINVAL;
1168 }
1169
1170 err = try_online_node(cpu_to_node(cpu));
1171 if (err)
1172 return err;
1173
1174 cpu_maps_update_begin();
1175
1176 if (cpu_hotplug_disabled) {
1177 err = -EBUSY;
1178 goto out;
1179 }
1180 if (!cpu_smt_allowed(cpu)) {
1181 err = -EPERM;
1182 goto out;
1183 }
1184
1185 err = _cpu_up(cpu, 0, target);
1186out:
1187 cpu_maps_update_done();
1188 return err;
1189}
1190
1191int cpu_up(unsigned int cpu)
1192{
1193 return do_cpu_up(cpu, CPUHP_ONLINE);
1194}
1195EXPORT_SYMBOL_GPL(cpu_up);
1196
1197#ifdef CONFIG_PM_SLEEP_SMP
1198static cpumask_var_t frozen_cpus;
1199
1200int freeze_secondary_cpus(int primary)
1201{
1202 int cpu, error = 0;
1203
1204 cpu_maps_update_begin();
1205 if (primary == -1) {
1206 primary = cpumask_first(cpu_online_mask);
1207 if (!housekeeping_cpu(primary, HK_FLAG_TIMER))
1208 primary = housekeeping_any_cpu(HK_FLAG_TIMER);
1209 } else {
1210 if (!cpu_online(primary))
1211 primary = cpumask_first(cpu_online_mask);
1212 }
1213
1214 /*
1215 * We take down all of the non-boot CPUs in one shot to avoid races
1216 * with the userspace trying to use the CPU hotplug at the same time
1217 */
1218 cpumask_clear(frozen_cpus);
1219
1220 pr_info("Disabling non-boot CPUs ...\n");
1221 for_each_online_cpu(cpu) {
1222 if (cpu == primary)
1223 continue;
1224
1225 if (pm_wakeup_pending()) {
1226 pr_info("Wakeup pending. Abort CPU freeze\n");
1227 error = -EBUSY;
1228 break;
1229 }
1230
1231 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1232 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1233 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1234 if (!error)
1235 cpumask_set_cpu(cpu, frozen_cpus);
1236 else {
1237 pr_err("Error taking CPU%d down: %d\n", cpu, error);
1238 break;
1239 }
1240 }
1241
1242 if (!error)
1243 BUG_ON(num_online_cpus() > 1);
1244 else
1245 pr_err("Non-boot CPUs are not disabled\n");
1246
1247 /*
1248 * Make sure the CPUs won't be enabled by someone else. We need to do
1249 * this even in case of failure as all disable_nonboot_cpus() users are
1250 * supposed to do enable_nonboot_cpus() on the failure path.
1251 */
1252 cpu_hotplug_disabled++;
1253
1254 cpu_maps_update_done();
1255 return error;
1256}
1257
1258void __weak arch_enable_nonboot_cpus_begin(void)
1259{
1260}
1261
1262void __weak arch_enable_nonboot_cpus_end(void)
1263{
1264}
1265
1266void enable_nonboot_cpus(void)
1267{
1268 int cpu, error;
1269
1270 /* Allow everyone to use the CPU hotplug again */
1271 cpu_maps_update_begin();
1272 __cpu_hotplug_enable();
1273 if (cpumask_empty(frozen_cpus))
1274 goto out;
1275
1276 pr_info("Enabling non-boot CPUs ...\n");
1277
1278 arch_enable_nonboot_cpus_begin();
1279
1280 for_each_cpu(cpu, frozen_cpus) {
1281 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1282 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1283 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1284 if (!error) {
1285 pr_info("CPU%d is up\n", cpu);
1286 continue;
1287 }
1288 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1289 }
1290
1291 arch_enable_nonboot_cpus_end();
1292
1293 cpumask_clear(frozen_cpus);
1294out:
1295 cpu_maps_update_done();
1296}
1297
1298static int __init alloc_frozen_cpus(void)
1299{
1300 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1301 return -ENOMEM;
1302 return 0;
1303}
1304core_initcall(alloc_frozen_cpus);
1305
1306/*
1307 * When callbacks for CPU hotplug notifications are being executed, we must
1308 * ensure that the state of the system with respect to the tasks being frozen
1309 * or not, as reported by the notification, remains unchanged *throughout the
1310 * duration* of the execution of the callbacks.
1311 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1312 *
1313 * This synchronization is implemented by mutually excluding regular CPU
1314 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1315 * Hibernate notifications.
1316 */
1317static int
1318cpu_hotplug_pm_callback(struct notifier_block *nb,
1319 unsigned long action, void *ptr)
1320{
1321 switch (action) {
1322
1323 case PM_SUSPEND_PREPARE:
1324 case PM_HIBERNATION_PREPARE:
1325 cpu_hotplug_disable();
1326 break;
1327
1328 case PM_POST_SUSPEND:
1329 case PM_POST_HIBERNATION:
1330 cpu_hotplug_enable();
1331 break;
1332
1333 default:
1334 return NOTIFY_DONE;
1335 }
1336
1337 return NOTIFY_OK;
1338}
1339
1340
1341static int __init cpu_hotplug_pm_sync_init(void)
1342{
1343 /*
1344 * cpu_hotplug_pm_callback has higher priority than x86
1345 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1346 * to disable cpu hotplug to avoid cpu hotplug race.
1347 */
1348 pm_notifier(cpu_hotplug_pm_callback, 0);
1349 return 0;
1350}
1351core_initcall(cpu_hotplug_pm_sync_init);
1352
1353#endif /* CONFIG_PM_SLEEP_SMP */
1354
1355int __boot_cpu_id;
1356
1357#endif /* CONFIG_SMP */
1358
1359/* Boot processor state steps */
1360static struct cpuhp_step cpuhp_hp_states[] = {
1361 [CPUHP_OFFLINE] = {
1362 .name = "offline",
1363 .startup.single = NULL,
1364 .teardown.single = NULL,
1365 },
1366#ifdef CONFIG_SMP
1367 [CPUHP_CREATE_THREADS]= {
1368 .name = "threads:prepare",
1369 .startup.single = smpboot_create_threads,
1370 .teardown.single = NULL,
1371 .cant_stop = true,
1372 },
1373 [CPUHP_PERF_PREPARE] = {
1374 .name = "perf:prepare",
1375 .startup.single = perf_event_init_cpu,
1376 .teardown.single = perf_event_exit_cpu,
1377 },
1378 [CPUHP_WORKQUEUE_PREP] = {
1379 .name = "workqueue:prepare",
1380 .startup.single = workqueue_prepare_cpu,
1381 .teardown.single = NULL,
1382 },
1383 [CPUHP_HRTIMERS_PREPARE] = {
1384 .name = "hrtimers:prepare",
1385 .startup.single = hrtimers_prepare_cpu,
1386 .teardown.single = hrtimers_dead_cpu,
1387 },
1388 [CPUHP_SMPCFD_PREPARE] = {
1389 .name = "smpcfd:prepare",
1390 .startup.single = smpcfd_prepare_cpu,
1391 .teardown.single = smpcfd_dead_cpu,
1392 },
1393 [CPUHP_RELAY_PREPARE] = {
1394 .name = "relay:prepare",
1395 .startup.single = relay_prepare_cpu,
1396 .teardown.single = NULL,
1397 },
1398 [CPUHP_SLAB_PREPARE] = {
1399 .name = "slab:prepare",
1400 .startup.single = slab_prepare_cpu,
1401 .teardown.single = slab_dead_cpu,
1402 },
1403 [CPUHP_RCUTREE_PREP] = {
1404 .name = "RCU/tree:prepare",
1405 .startup.single = rcutree_prepare_cpu,
1406 .teardown.single = rcutree_dead_cpu,
1407 },
1408 /*
1409 * On the tear-down path, timers_dead_cpu() must be invoked
1410 * before blk_mq_queue_reinit_notify() from notify_dead(),
1411 * otherwise a RCU stall occurs.
1412 */
1413 [CPUHP_TIMERS_PREPARE] = {
1414 .name = "timers:prepare",
1415 .startup.single = timers_prepare_cpu,
1416 .teardown.single = timers_dead_cpu,
1417 },
1418 /* Kicks the plugged cpu into life */
1419 [CPUHP_BRINGUP_CPU] = {
1420 .name = "cpu:bringup",
1421 .startup.single = bringup_cpu,
1422 .teardown.single = NULL,
1423 .cant_stop = true,
1424 },
1425 /* Final state before CPU kills itself */
1426 [CPUHP_AP_IDLE_DEAD] = {
1427 .name = "idle:dead",
1428 },
1429 /*
1430 * Last state before CPU enters the idle loop to die. Transient state
1431 * for synchronization.
1432 */
1433 [CPUHP_AP_OFFLINE] = {
1434 .name = "ap:offline",
1435 .cant_stop = true,
1436 },
1437 /* First state is scheduler control. Interrupts are disabled */
1438 [CPUHP_AP_SCHED_STARTING] = {
1439 .name = "sched:starting",
1440 .startup.single = sched_cpu_starting,
1441 .teardown.single = sched_cpu_dying,
1442 },
1443 [CPUHP_AP_RCUTREE_DYING] = {
1444 .name = "RCU/tree:dying",
1445 .startup.single = NULL,
1446 .teardown.single = rcutree_dying_cpu,
1447 },
1448 [CPUHP_AP_SMPCFD_DYING] = {
1449 .name = "smpcfd:dying",
1450 .startup.single = NULL,
1451 .teardown.single = smpcfd_dying_cpu,
1452 },
1453 /* Entry state on starting. Interrupts enabled from here on. Transient
1454 * state for synchronsization */
1455 [CPUHP_AP_ONLINE] = {
1456 .name = "ap:online",
1457 },
1458 /*
1459 * Handled on controll processor until the plugged processor manages
1460 * this itself.
1461 */
1462 [CPUHP_TEARDOWN_CPU] = {
1463 .name = "cpu:teardown",
1464 .startup.single = NULL,
1465 .teardown.single = takedown_cpu,
1466 .cant_stop = true,
1467 },
1468 /* Handle smpboot threads park/unpark */
1469 [CPUHP_AP_SMPBOOT_THREADS] = {
1470 .name = "smpboot/threads:online",
1471 .startup.single = smpboot_unpark_threads,
1472 .teardown.single = smpboot_park_threads,
1473 },
1474 [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1475 .name = "irq/affinity:online",
1476 .startup.single = irq_affinity_online_cpu,
1477 .teardown.single = NULL,
1478 },
1479 [CPUHP_AP_PERF_ONLINE] = {
1480 .name = "perf:online",
1481 .startup.single = perf_event_init_cpu,
1482 .teardown.single = perf_event_exit_cpu,
1483 },
1484 [CPUHP_AP_WATCHDOG_ONLINE] = {
1485 .name = "lockup_detector:online",
1486 .startup.single = lockup_detector_online_cpu,
1487 .teardown.single = lockup_detector_offline_cpu,
1488 },
1489 [CPUHP_AP_WORKQUEUE_ONLINE] = {
1490 .name = "workqueue:online",
1491 .startup.single = workqueue_online_cpu,
1492 .teardown.single = workqueue_offline_cpu,
1493 },
1494 [CPUHP_AP_RCUTREE_ONLINE] = {
1495 .name = "RCU/tree:online",
1496 .startup.single = rcutree_online_cpu,
1497 .teardown.single = rcutree_offline_cpu,
1498 },
1499#endif
1500 /*
1501 * The dynamically registered state space is here
1502 */
1503
1504#ifdef CONFIG_SMP
1505 /* Last state is scheduler control setting the cpu active */
1506 [CPUHP_AP_ACTIVE] = {
1507 .name = "sched:active",
1508 .startup.single = sched_cpu_activate,
1509 .teardown.single = sched_cpu_deactivate,
1510 },
1511#endif
1512
1513 /* CPU is fully up and running. */
1514 [CPUHP_ONLINE] = {
1515 .name = "online",
1516 .startup.single = NULL,
1517 .teardown.single = NULL,
1518 },
1519};
1520
1521/* Sanity check for callbacks */
1522static int cpuhp_cb_check(enum cpuhp_state state)
1523{
1524 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1525 return -EINVAL;
1526 return 0;
1527}
1528
1529/*
1530 * Returns a free for dynamic slot assignment of the Online state. The states
1531 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1532 * by having no name assigned.
1533 */
1534static int cpuhp_reserve_state(enum cpuhp_state state)
1535{
1536 enum cpuhp_state i, end;
1537 struct cpuhp_step *step;
1538
1539 switch (state) {
1540 case CPUHP_AP_ONLINE_DYN:
1541 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1542 end = CPUHP_AP_ONLINE_DYN_END;
1543 break;
1544 case CPUHP_BP_PREPARE_DYN:
1545 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1546 end = CPUHP_BP_PREPARE_DYN_END;
1547 break;
1548 default:
1549 return -EINVAL;
1550 }
1551
1552 for (i = state; i <= end; i++, step++) {
1553 if (!step->name)
1554 return i;
1555 }
1556 WARN(1, "No more dynamic states available for CPU hotplug\n");
1557 return -ENOSPC;
1558}
1559
1560static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1561 int (*startup)(unsigned int cpu),
1562 int (*teardown)(unsigned int cpu),
1563 bool multi_instance)
1564{
1565 /* (Un)Install the callbacks for further cpu hotplug operations */
1566 struct cpuhp_step *sp;
1567 int ret = 0;
1568
1569 /*
1570 * If name is NULL, then the state gets removed.
1571 *
1572 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1573 * the first allocation from these dynamic ranges, so the removal
1574 * would trigger a new allocation and clear the wrong (already
1575 * empty) state, leaving the callbacks of the to be cleared state
1576 * dangling, which causes wreckage on the next hotplug operation.
1577 */
1578 if (name && (state == CPUHP_AP_ONLINE_DYN ||
1579 state == CPUHP_BP_PREPARE_DYN)) {
1580 ret = cpuhp_reserve_state(state);
1581 if (ret < 0)
1582 return ret;
1583 state = ret;
1584 }
1585 sp = cpuhp_get_step(state);
1586 if (name && sp->name)
1587 return -EBUSY;
1588
1589 sp->startup.single = startup;
1590 sp->teardown.single = teardown;
1591 sp->name = name;
1592 sp->multi_instance = multi_instance;
1593 INIT_HLIST_HEAD(&sp->list);
1594 return ret;
1595}
1596
1597static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1598{
1599 return cpuhp_get_step(state)->teardown.single;
1600}
1601
1602/*
1603 * Call the startup/teardown function for a step either on the AP or
1604 * on the current CPU.
1605 */
1606static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1607 struct hlist_node *node)
1608{
1609 struct cpuhp_step *sp = cpuhp_get_step(state);
1610 int ret;
1611
1612 /*
1613 * If there's nothing to do, we done.
1614 * Relies on the union for multi_instance.
1615 */
1616 if ((bringup && !sp->startup.single) ||
1617 (!bringup && !sp->teardown.single))
1618 return 0;
1619 /*
1620 * The non AP bound callbacks can fail on bringup. On teardown
1621 * e.g. module removal we crash for now.
1622 */
1623#ifdef CONFIG_SMP
1624 if (cpuhp_is_ap_state(state))
1625 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1626 else
1627 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1628#else
1629 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1630#endif
1631 BUG_ON(ret && !bringup);
1632 return ret;
1633}
1634
1635/*
1636 * Called from __cpuhp_setup_state on a recoverable failure.
1637 *
1638 * Note: The teardown callbacks for rollback are not allowed to fail!
1639 */
1640static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1641 struct hlist_node *node)
1642{
1643 int cpu;
1644
1645 /* Roll back the already executed steps on the other cpus */
1646 for_each_present_cpu(cpu) {
1647 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1648 int cpustate = st->state;
1649
1650 if (cpu >= failedcpu)
1651 break;
1652
1653 /* Did we invoke the startup call on that cpu ? */
1654 if (cpustate >= state)
1655 cpuhp_issue_call(cpu, state, false, node);
1656 }
1657}
1658
1659int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1660 struct hlist_node *node,
1661 bool invoke)
1662{
1663 struct cpuhp_step *sp;
1664 int cpu;
1665 int ret;
1666
1667 lockdep_assert_cpus_held();
1668
1669 sp = cpuhp_get_step(state);
1670 if (sp->multi_instance == false)
1671 return -EINVAL;
1672
1673 mutex_lock(&cpuhp_state_mutex);
1674
1675 if (!invoke || !sp->startup.multi)
1676 goto add_node;
1677
1678 /*
1679 * Try to call the startup callback for each present cpu
1680 * depending on the hotplug state of the cpu.
1681 */
1682 for_each_present_cpu(cpu) {
1683 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1684 int cpustate = st->state;
1685
1686 if (cpustate < state)
1687 continue;
1688
1689 ret = cpuhp_issue_call(cpu, state, true, node);
1690 if (ret) {
1691 if (sp->teardown.multi)
1692 cpuhp_rollback_install(cpu, state, node);
1693 goto unlock;
1694 }
1695 }
1696add_node:
1697 ret = 0;
1698 hlist_add_head(node, &sp->list);
1699unlock:
1700 mutex_unlock(&cpuhp_state_mutex);
1701 return ret;
1702}
1703
1704int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1705 bool invoke)
1706{
1707 int ret;
1708
1709 cpus_read_lock();
1710 ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1711 cpus_read_unlock();
1712 return ret;
1713}
1714EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1715
1716/**
1717 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1718 * @state: The state to setup
1719 * @invoke: If true, the startup function is invoked for cpus where
1720 * cpu state >= @state
1721 * @startup: startup callback function
1722 * @teardown: teardown callback function
1723 * @multi_instance: State is set up for multiple instances which get
1724 * added afterwards.
1725 *
1726 * The caller needs to hold cpus read locked while calling this function.
1727 * Returns:
1728 * On success:
1729 * Positive state number if @state is CPUHP_AP_ONLINE_DYN
1730 * 0 for all other states
1731 * On failure: proper (negative) error code
1732 */
1733int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1734 const char *name, bool invoke,
1735 int (*startup)(unsigned int cpu),
1736 int (*teardown)(unsigned int cpu),
1737 bool multi_instance)
1738{
1739 int cpu, ret = 0;
1740 bool dynstate;
1741
1742 lockdep_assert_cpus_held();
1743
1744 if (cpuhp_cb_check(state) || !name)
1745 return -EINVAL;
1746
1747 mutex_lock(&cpuhp_state_mutex);
1748
1749 ret = cpuhp_store_callbacks(state, name, startup, teardown,
1750 multi_instance);
1751
1752 dynstate = state == CPUHP_AP_ONLINE_DYN;
1753 if (ret > 0 && dynstate) {
1754 state = ret;
1755 ret = 0;
1756 }
1757
1758 if (ret || !invoke || !startup)
1759 goto out;
1760
1761 /*
1762 * Try to call the startup callback for each present cpu
1763 * depending on the hotplug state of the cpu.
1764 */
1765 for_each_present_cpu(cpu) {
1766 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1767 int cpustate = st->state;
1768
1769 if (cpustate < state)
1770 continue;
1771
1772 ret = cpuhp_issue_call(cpu, state, true, NULL);
1773 if (ret) {
1774 if (teardown)
1775 cpuhp_rollback_install(cpu, state, NULL);
1776 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1777 goto out;
1778 }
1779 }
1780out:
1781 mutex_unlock(&cpuhp_state_mutex);
1782 /*
1783 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1784 * dynamically allocated state in case of success.
1785 */
1786 if (!ret && dynstate)
1787 return state;
1788 return ret;
1789}
1790EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1791
1792int __cpuhp_setup_state(enum cpuhp_state state,
1793 const char *name, bool invoke,
1794 int (*startup)(unsigned int cpu),
1795 int (*teardown)(unsigned int cpu),
1796 bool multi_instance)
1797{
1798 int ret;
1799
1800 cpus_read_lock();
1801 ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1802 teardown, multi_instance);
1803 cpus_read_unlock();
1804 return ret;
1805}
1806EXPORT_SYMBOL(__cpuhp_setup_state);
1807
1808int __cpuhp_state_remove_instance(enum cpuhp_state state,
1809 struct hlist_node *node, bool invoke)
1810{
1811 struct cpuhp_step *sp = cpuhp_get_step(state);
1812 int cpu;
1813
1814 BUG_ON(cpuhp_cb_check(state));
1815
1816 if (!sp->multi_instance)
1817 return -EINVAL;
1818
1819 cpus_read_lock();
1820 mutex_lock(&cpuhp_state_mutex);
1821
1822 if (!invoke || !cpuhp_get_teardown_cb(state))
1823 goto remove;
1824 /*
1825 * Call the teardown callback for each present cpu depending
1826 * on the hotplug state of the cpu. This function is not
1827 * allowed to fail currently!
1828 */
1829 for_each_present_cpu(cpu) {
1830 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1831 int cpustate = st->state;
1832
1833 if (cpustate >= state)
1834 cpuhp_issue_call(cpu, state, false, node);
1835 }
1836
1837remove:
1838 hlist_del(node);
1839 mutex_unlock(&cpuhp_state_mutex);
1840 cpus_read_unlock();
1841
1842 return 0;
1843}
1844EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1845
1846/**
1847 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1848 * @state: The state to remove
1849 * @invoke: If true, the teardown function is invoked for cpus where
1850 * cpu state >= @state
1851 *
1852 * The caller needs to hold cpus read locked while calling this function.
1853 * The teardown callback is currently not allowed to fail. Think
1854 * about module removal!
1855 */
1856void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1857{
1858 struct cpuhp_step *sp = cpuhp_get_step(state);
1859 int cpu;
1860
1861 BUG_ON(cpuhp_cb_check(state));
1862
1863 lockdep_assert_cpus_held();
1864
1865 mutex_lock(&cpuhp_state_mutex);
1866 if (sp->multi_instance) {
1867 WARN(!hlist_empty(&sp->list),
1868 "Error: Removing state %d which has instances left.\n",
1869 state);
1870 goto remove;
1871 }
1872
1873 if (!invoke || !cpuhp_get_teardown_cb(state))
1874 goto remove;
1875
1876 /*
1877 * Call the teardown callback for each present cpu depending
1878 * on the hotplug state of the cpu. This function is not
1879 * allowed to fail currently!
1880 */
1881 for_each_present_cpu(cpu) {
1882 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1883 int cpustate = st->state;
1884
1885 if (cpustate >= state)
1886 cpuhp_issue_call(cpu, state, false, NULL);
1887 }
1888remove:
1889 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1890 mutex_unlock(&cpuhp_state_mutex);
1891}
1892EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1893
1894void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1895{
1896 cpus_read_lock();
1897 __cpuhp_remove_state_cpuslocked(state, invoke);
1898 cpus_read_unlock();
1899}
1900EXPORT_SYMBOL(__cpuhp_remove_state);
1901
1902#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1903static ssize_t show_cpuhp_state(struct device *dev,
1904 struct device_attribute *attr, char *buf)
1905{
1906 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1907
1908 return sprintf(buf, "%d\n", st->state);
1909}
1910static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1911
1912static ssize_t write_cpuhp_target(struct device *dev,
1913 struct device_attribute *attr,
1914 const char *buf, size_t count)
1915{
1916 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1917 struct cpuhp_step *sp;
1918 int target, ret;
1919
1920 ret = kstrtoint(buf, 10, &target);
1921 if (ret)
1922 return ret;
1923
1924#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1925 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1926 return -EINVAL;
1927#else
1928 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1929 return -EINVAL;
1930#endif
1931
1932 ret = lock_device_hotplug_sysfs();
1933 if (ret)
1934 return ret;
1935
1936 mutex_lock(&cpuhp_state_mutex);
1937 sp = cpuhp_get_step(target);
1938 ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1939 mutex_unlock(&cpuhp_state_mutex);
1940 if (ret)
1941 goto out;
1942
1943 if (st->state < target)
1944 ret = do_cpu_up(dev->id, target);
1945 else
1946 ret = do_cpu_down(dev->id, target);
1947out:
1948 unlock_device_hotplug();
1949 return ret ? ret : count;
1950}
1951
1952static ssize_t show_cpuhp_target(struct device *dev,
1953 struct device_attribute *attr, char *buf)
1954{
1955 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1956
1957 return sprintf(buf, "%d\n", st->target);
1958}
1959static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1960
1961
1962static ssize_t write_cpuhp_fail(struct device *dev,
1963 struct device_attribute *attr,
1964 const char *buf, size_t count)
1965{
1966 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1967 struct cpuhp_step *sp;
1968 int fail, ret;
1969
1970 ret = kstrtoint(buf, 10, &fail);
1971 if (ret)
1972 return ret;
1973
1974 if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
1975 return -EINVAL;
1976
1977 /*
1978 * Cannot fail STARTING/DYING callbacks.
1979 */
1980 if (cpuhp_is_atomic_state(fail))
1981 return -EINVAL;
1982
1983 /*
1984 * Cannot fail anything that doesn't have callbacks.
1985 */
1986 mutex_lock(&cpuhp_state_mutex);
1987 sp = cpuhp_get_step(fail);
1988 if (!sp->startup.single && !sp->teardown.single)
1989 ret = -EINVAL;
1990 mutex_unlock(&cpuhp_state_mutex);
1991 if (ret)
1992 return ret;
1993
1994 st->fail = fail;
1995
1996 return count;
1997}
1998
1999static ssize_t show_cpuhp_fail(struct device *dev,
2000 struct device_attribute *attr, char *buf)
2001{
2002 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2003
2004 return sprintf(buf, "%d\n", st->fail);
2005}
2006
2007static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
2008
2009static struct attribute *cpuhp_cpu_attrs[] = {
2010 &dev_attr_state.attr,
2011 &dev_attr_target.attr,
2012 &dev_attr_fail.attr,
2013 NULL
2014};
2015
2016static const struct attribute_group cpuhp_cpu_attr_group = {
2017 .attrs = cpuhp_cpu_attrs,
2018 .name = "hotplug",
2019 NULL
2020};
2021
2022static ssize_t show_cpuhp_states(struct device *dev,
2023 struct device_attribute *attr, char *buf)
2024{
2025 ssize_t cur, res = 0;
2026 int i;
2027
2028 mutex_lock(&cpuhp_state_mutex);
2029 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2030 struct cpuhp_step *sp = cpuhp_get_step(i);
2031
2032 if (sp->name) {
2033 cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2034 buf += cur;
2035 res += cur;
2036 }
2037 }
2038 mutex_unlock(&cpuhp_state_mutex);
2039 return res;
2040}
2041static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
2042
2043static struct attribute *cpuhp_cpu_root_attrs[] = {
2044 &dev_attr_states.attr,
2045 NULL
2046};
2047
2048static const struct attribute_group cpuhp_cpu_root_attr_group = {
2049 .attrs = cpuhp_cpu_root_attrs,
2050 .name = "hotplug",
2051 NULL
2052};
2053
2054#ifdef CONFIG_HOTPLUG_SMT
2055
2056static void cpuhp_offline_cpu_device(unsigned int cpu)
2057{
2058 struct device *dev = get_cpu_device(cpu);
2059
2060 dev->offline = true;
2061 /* Tell user space about the state change */
2062 kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2063}
2064
2065static void cpuhp_online_cpu_device(unsigned int cpu)
2066{
2067 struct device *dev = get_cpu_device(cpu);
2068
2069 dev->offline = false;
2070 /* Tell user space about the state change */
2071 kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2072}
2073
2074int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2075{
2076 int cpu, ret = 0;
2077
2078 cpu_maps_update_begin();
2079 for_each_online_cpu(cpu) {
2080 if (topology_is_primary_thread(cpu))
2081 continue;
2082 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2083 if (ret)
2084 break;
2085 /*
2086 * As this needs to hold the cpu maps lock it's impossible
2087 * to call device_offline() because that ends up calling
2088 * cpu_down() which takes cpu maps lock. cpu maps lock
2089 * needs to be held as this might race against in kernel
2090 * abusers of the hotplug machinery (thermal management).
2091 *
2092 * So nothing would update device:offline state. That would
2093 * leave the sysfs entry stale and prevent onlining after
2094 * smt control has been changed to 'off' again. This is
2095 * called under the sysfs hotplug lock, so it is properly
2096 * serialized against the regular offline usage.
2097 */
2098 cpuhp_offline_cpu_device(cpu);
2099 }
2100 if (!ret)
2101 cpu_smt_control = ctrlval;
2102 cpu_maps_update_done();
2103 return ret;
2104}
2105
2106int cpuhp_smt_enable(void)
2107{
2108 int cpu, ret = 0;
2109
2110 cpu_maps_update_begin();
2111 cpu_smt_control = CPU_SMT_ENABLED;
2112 for_each_present_cpu(cpu) {
2113 /* Skip online CPUs and CPUs on offline nodes */
2114 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2115 continue;
2116 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2117 if (ret)
2118 break;
2119 /* See comment in cpuhp_smt_disable() */
2120 cpuhp_online_cpu_device(cpu);
2121 }
2122 cpu_maps_update_done();
2123 return ret;
2124}
2125
2126
2127static ssize_t
2128__store_smt_control(struct device *dev, struct device_attribute *attr,
2129 const char *buf, size_t count)
2130{
2131 int ctrlval, ret;
2132
2133 if (sysfs_streq(buf, "on"))
2134 ctrlval = CPU_SMT_ENABLED;
2135 else if (sysfs_streq(buf, "off"))
2136 ctrlval = CPU_SMT_DISABLED;
2137 else if (sysfs_streq(buf, "forceoff"))
2138 ctrlval = CPU_SMT_FORCE_DISABLED;
2139 else
2140 return -EINVAL;
2141
2142 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2143 return -EPERM;
2144
2145 if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2146 return -ENODEV;
2147
2148 ret = lock_device_hotplug_sysfs();
2149 if (ret)
2150 return ret;
2151
2152 if (ctrlval != cpu_smt_control) {
2153 switch (ctrlval) {
2154 case CPU_SMT_ENABLED:
2155 ret = cpuhp_smt_enable();
2156 break;
2157 case CPU_SMT_DISABLED:
2158 case CPU_SMT_FORCE_DISABLED:
2159 ret = cpuhp_smt_disable(ctrlval);
2160 break;
2161 }
2162 }
2163
2164 unlock_device_hotplug();
2165 return ret ? ret : count;
2166}
2167
2168#else /* !CONFIG_HOTPLUG_SMT */
2169static ssize_t
2170__store_smt_control(struct device *dev, struct device_attribute *attr,
2171 const char *buf, size_t count)
2172{
2173 return -ENODEV;
2174}
2175#endif /* CONFIG_HOTPLUG_SMT */
2176
2177static const char *smt_states[] = {
2178 [CPU_SMT_ENABLED] = "on",
2179 [CPU_SMT_DISABLED] = "off",
2180 [CPU_SMT_FORCE_DISABLED] = "forceoff",
2181 [CPU_SMT_NOT_SUPPORTED] = "notsupported",
2182 [CPU_SMT_NOT_IMPLEMENTED] = "notimplemented",
2183};
2184
2185static ssize_t
2186show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2187{
2188 const char *state = smt_states[cpu_smt_control];
2189
2190 return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
2191}
2192
2193static ssize_t
2194store_smt_control(struct device *dev, struct device_attribute *attr,
2195 const char *buf, size_t count)
2196{
2197 return __store_smt_control(dev, attr, buf, count);
2198}
2199static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2200
2201static ssize_t
2202show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2203{
2204 return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
2205}
2206static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2207
2208static struct attribute *cpuhp_smt_attrs[] = {
2209 &dev_attr_control.attr,
2210 &dev_attr_active.attr,
2211 NULL
2212};
2213
2214static const struct attribute_group cpuhp_smt_attr_group = {
2215 .attrs = cpuhp_smt_attrs,
2216 .name = "smt",
2217 NULL
2218};
2219
2220static int __init cpu_smt_sysfs_init(void)
2221{
2222 return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2223 &cpuhp_smt_attr_group);
2224}
2225
2226static int __init cpuhp_sysfs_init(void)
2227{
2228 int cpu, ret;
2229
2230 ret = cpu_smt_sysfs_init();
2231 if (ret)
2232 return ret;
2233
2234 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2235 &cpuhp_cpu_root_attr_group);
2236 if (ret)
2237 return ret;
2238
2239 for_each_possible_cpu(cpu) {
2240 struct device *dev = get_cpu_device(cpu);
2241
2242 if (!dev)
2243 continue;
2244 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2245 if (ret)
2246 return ret;
2247 }
2248 return 0;
2249}
2250device_initcall(cpuhp_sysfs_init);
2251#endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
2252
2253/*
2254 * cpu_bit_bitmap[] is a special, "compressed" data structure that
2255 * represents all NR_CPUS bits binary values of 1<<nr.
2256 *
2257 * It is used by cpumask_of() to get a constant address to a CPU
2258 * mask value that has a single bit set only.
2259 */
2260
2261/* cpu_bit_bitmap[0] is empty - so we can back into it */
2262#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
2263#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2264#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2265#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2266
2267const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2268
2269 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
2270 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
2271#if BITS_PER_LONG > 32
2272 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
2273 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
2274#endif
2275};
2276EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2277
2278const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2279EXPORT_SYMBOL(cpu_all_bits);
2280
2281#ifdef CONFIG_INIT_ALL_POSSIBLE
2282struct cpumask __cpu_possible_mask __read_mostly
2283 = {CPU_BITS_ALL};
2284#else
2285struct cpumask __cpu_possible_mask __read_mostly;
2286#endif
2287EXPORT_SYMBOL(__cpu_possible_mask);
2288
2289struct cpumask __cpu_online_mask __read_mostly;
2290EXPORT_SYMBOL(__cpu_online_mask);
2291
2292struct cpumask __cpu_present_mask __read_mostly;
2293EXPORT_SYMBOL(__cpu_present_mask);
2294
2295struct cpumask __cpu_active_mask __read_mostly;
2296EXPORT_SYMBOL(__cpu_active_mask);
2297
2298void init_cpu_present(const struct cpumask *src)
2299{
2300 cpumask_copy(&__cpu_present_mask, src);
2301}
2302
2303void init_cpu_possible(const struct cpumask *src)
2304{
2305 cpumask_copy(&__cpu_possible_mask, src);
2306}
2307
2308void init_cpu_online(const struct cpumask *src)
2309{
2310 cpumask_copy(&__cpu_online_mask, src);
2311}
2312
2313/*
2314 * Activate the first processor.
2315 */
2316void __init boot_cpu_init(void)
2317{
2318 int cpu = smp_processor_id();
2319
2320 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2321 set_cpu_online(cpu, true);
2322 set_cpu_active(cpu, true);
2323 set_cpu_present(cpu, true);
2324 set_cpu_possible(cpu, true);
2325
2326#ifdef CONFIG_SMP
2327 __boot_cpu_id = cpu;
2328#endif
2329}
2330
2331/*
2332 * Must be called _AFTER_ setting up the per_cpu areas
2333 */
2334void __init boot_cpu_hotplug_init(void)
2335{
2336#ifdef CONFIG_SMP
2337 this_cpu_write(cpuhp_state.booted_once, true);
2338#endif
2339 this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2340}
2341
2342enum cpu_mitigations cpu_mitigations __ro_after_init = CPU_MITIGATIONS_AUTO;
2343
2344static int __init mitigations_parse_cmdline(char *arg)
2345{
2346 if (!strcmp(arg, "off"))
2347 cpu_mitigations = CPU_MITIGATIONS_OFF;
2348 else if (!strcmp(arg, "auto"))
2349 cpu_mitigations = CPU_MITIGATIONS_AUTO;
2350 else if (!strcmp(arg, "auto,nosmt"))
2351 cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2352 else
2353 pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2354 arg);
2355
2356 return 0;
2357}
2358early_param("mitigations", mitigations_parse_cmdline);