projects/linux·blob v5.0-rc2 · refs

2309 lines · 55.1 KiB

1/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched/signal.h>
11#include <linux/sched/hotplug.h>
12#include <linux/sched/task.h>
13#include <linux/sched/smt.h>
14#include <linux/unistd.h>
15#include <linux/cpu.h>
16#include <linux/oom.h>
17#include <linux/rcupdate.h>
18#include <linux/export.h>
19#include <linux/bug.h>
20#include <linux/kthread.h>
21#include <linux/stop_machine.h>
22#include <linux/mutex.h>
23#include <linux/gfp.h>
24#include <linux/suspend.h>
25#include <linux/lockdep.h>
26#include <linux/tick.h>
27#include <linux/irq.h>
28#include <linux/nmi.h>
29#include <linux/smpboot.h>
30#include <linux/relay.h>
31#include <linux/slab.h>
32#include <linux/percpu-rwsem.h>
33
34#include <trace/events/power.h>
35#define CREATE_TRACE_POINTS
36#include <trace/events/cpuhp.h>
37
38#include "smpboot.h"
39
40/**
41 * cpuhp_cpu_state - Per cpu hotplug state storage
42 * @state: The current cpu state
43 * @target: The target state
44 * @thread: Pointer to the hotplug thread
45 * @should_run: Thread should execute
46 * @rollback: Perform a rollback
47 * @single: Single callback invocation
48 * @bringup: Single callback bringup or teardown selector
49 * @cb_state: The state for a single callback (install/uninstall)
50 * @result: Result of the operation
51 * @done_up: Signal completion to the issuer of the task for cpu-up
52 * @done_down: Signal completion to the issuer of the task for cpu-down
53 */
54struct cpuhp_cpu_state {
55 enum cpuhp_state state;
56 enum cpuhp_state target;
57 enum cpuhp_state fail;
58#ifdef CONFIG_SMP
59 struct task_struct *thread;
60 bool should_run;
61 bool rollback;
62 bool single;
63 bool bringup;
64 bool booted_once;
65 struct hlist_node *node;
66 struct hlist_node *last;
67 enum cpuhp_state cb_state;
68 int result;
69 struct completion done_up;
70 struct completion done_down;
71#endif
72};
73
74static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
75 .fail = CPUHP_INVALID,
76};
77
78#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
79static struct lockdep_map cpuhp_state_up_map =
80 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
81static struct lockdep_map cpuhp_state_down_map =
82 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
83
84
85static inline void cpuhp_lock_acquire(bool bringup)
86{
87 lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
88}
89
90static inline void cpuhp_lock_release(bool bringup)
91{
92 lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
93}
94#else
95
96static inline void cpuhp_lock_acquire(bool bringup) { }
97static inline void cpuhp_lock_release(bool bringup) { }
98
99#endif
100
101/**
102 * cpuhp_step - Hotplug state machine step
103 * @name: Name of the step
104 * @startup: Startup function of the step
105 * @teardown: Teardown function of the step
106 * @cant_stop: Bringup/teardown can't be stopped at this step
107 */
108struct cpuhp_step {
109 const char *name;
110 union {
111 int (*single)(unsigned int cpu);
112 int (*multi)(unsigned int cpu,
113 struct hlist_node *node);
114 } startup;
115 union {
116 int (*single)(unsigned int cpu);
117 int (*multi)(unsigned int cpu,
118 struct hlist_node *node);
119 } teardown;
120 struct hlist_head list;
121 bool cant_stop;
122 bool multi_instance;
123};
124
125static DEFINE_MUTEX(cpuhp_state_mutex);
126static struct cpuhp_step cpuhp_hp_states[];
127
128static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
129{
130 return cpuhp_hp_states + state;
131}
132
133/**
134 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
135 * @cpu: The cpu for which the callback should be invoked
136 * @state: The state to do callbacks for
137 * @bringup: True if the bringup callback should be invoked
138 * @node: For multi-instance, do a single entry callback for install/remove
139 * @lastp: For multi-instance rollback, remember how far we got
140 *
141 * Called from cpu hotplug and from the state register machinery.
142 */
143static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
144 bool bringup, struct hlist_node *node,
145 struct hlist_node **lastp)
146{
147 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
148 struct cpuhp_step *step = cpuhp_get_step(state);
149 int (*cbm)(unsigned int cpu, struct hlist_node *node);
150 int (*cb)(unsigned int cpu);
151 int ret, cnt;
152
153 if (st->fail == state) {
154 st->fail = CPUHP_INVALID;
155
156 if (!(bringup ? step->startup.single : step->teardown.single))
157 return 0;
158
159 return -EAGAIN;
160 }
161
162 if (!step->multi_instance) {
163 WARN_ON_ONCE(lastp && *lastp);
164 cb = bringup ? step->startup.single : step->teardown.single;
165 if (!cb)
166 return 0;
167 trace_cpuhp_enter(cpu, st->target, state, cb);
168 ret = cb(cpu);
169 trace_cpuhp_exit(cpu, st->state, state, ret);
170 return ret;
171 }
172 cbm = bringup ? step->startup.multi : step->teardown.multi;
173 if (!cbm)
174 return 0;
175
176 /* Single invocation for instance add/remove */
177 if (node) {
178 WARN_ON_ONCE(lastp && *lastp);
179 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
180 ret = cbm(cpu, node);
181 trace_cpuhp_exit(cpu, st->state, state, ret);
182 return ret;
183 }
184
185 /* State transition. Invoke on all instances */
186 cnt = 0;
187 hlist_for_each(node, &step->list) {
188 if (lastp && node == *lastp)
189 break;
190
191 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
192 ret = cbm(cpu, node);
193 trace_cpuhp_exit(cpu, st->state, state, ret);
194 if (ret) {
195 if (!lastp)
196 goto err;
197
198 *lastp = node;
199 return ret;
200 }
201 cnt++;
202 }
203 if (lastp)
204 *lastp = NULL;
205 return 0;
206err:
207 /* Rollback the instances if one failed */
208 cbm = !bringup ? step->startup.multi : step->teardown.multi;
209 if (!cbm)
210 return ret;
211
212 hlist_for_each(node, &step->list) {
213 if (!cnt--)
214 break;
215
216 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
217 ret = cbm(cpu, node);
218 trace_cpuhp_exit(cpu, st->state, state, ret);
219 /*
220 * Rollback must not fail,
221 */
222 WARN_ON_ONCE(ret);
223 }
224 return ret;
225}
226
227#ifdef CONFIG_SMP
228static bool cpuhp_is_ap_state(enum cpuhp_state state)
229{
230 /*
231 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
232 * purposes as that state is handled explicitly in cpu_down.
233 */
234 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
235}
236
237static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
238{
239 struct completion *done = bringup ? &st->done_up : &st->done_down;
240 wait_for_completion(done);
241}
242
243static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
244{
245 struct completion *done = bringup ? &st->done_up : &st->done_down;
246 complete(done);
247}
248
249/*
250 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
251 */
252static bool cpuhp_is_atomic_state(enum cpuhp_state state)
253{
254 return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
255}
256
257/* Serializes the updates to cpu_online_mask, cpu_present_mask */
258static DEFINE_MUTEX(cpu_add_remove_lock);
259bool cpuhp_tasks_frozen;
260EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
261
262/*
263 * The following two APIs (cpu_maps_update_begin/done) must be used when
264 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
265 */
266void cpu_maps_update_begin(void)
267{
268 mutex_lock(&cpu_add_remove_lock);
269}
270
271void cpu_maps_update_done(void)
272{
273 mutex_unlock(&cpu_add_remove_lock);
274}
275
276/*
277 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
278 * Should always be manipulated under cpu_add_remove_lock
279 */
280static int cpu_hotplug_disabled;
281
282#ifdef CONFIG_HOTPLUG_CPU
283
284DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
285
286void cpus_read_lock(void)
287{
288 percpu_down_read(&cpu_hotplug_lock);
289}
290EXPORT_SYMBOL_GPL(cpus_read_lock);
291
292int cpus_read_trylock(void)
293{
294 return percpu_down_read_trylock(&cpu_hotplug_lock);
295}
296EXPORT_SYMBOL_GPL(cpus_read_trylock);
297
298void cpus_read_unlock(void)
299{
300 percpu_up_read(&cpu_hotplug_lock);
301}
302EXPORT_SYMBOL_GPL(cpus_read_unlock);
303
304void cpus_write_lock(void)
305{
306 percpu_down_write(&cpu_hotplug_lock);
307}
308
309void cpus_write_unlock(void)
310{
311 percpu_up_write(&cpu_hotplug_lock);
312}
313
314void lockdep_assert_cpus_held(void)
315{
316 percpu_rwsem_assert_held(&cpu_hotplug_lock);
317}
318
319static void lockdep_acquire_cpus_lock(void)
320{
321 rwsem_acquire(&cpu_hotplug_lock.rw_sem.dep_map, 0, 0, _THIS_IP_);
322}
323
324static void lockdep_release_cpus_lock(void)
325{
326 rwsem_release(&cpu_hotplug_lock.rw_sem.dep_map, 1, _THIS_IP_);
327}
328
329/*
330 * Wait for currently running CPU hotplug operations to complete (if any) and
331 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
332 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
333 * hotplug path before performing hotplug operations. So acquiring that lock
334 * guarantees mutual exclusion from any currently running hotplug operations.
335 */
336void cpu_hotplug_disable(void)
337{
338 cpu_maps_update_begin();
339 cpu_hotplug_disabled++;
340 cpu_maps_update_done();
341}
342EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
343
344static void __cpu_hotplug_enable(void)
345{
346 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
347 return;
348 cpu_hotplug_disabled--;
349}
350
351void cpu_hotplug_enable(void)
352{
353 cpu_maps_update_begin();
354 __cpu_hotplug_enable();
355 cpu_maps_update_done();
356}
357EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
358
359#else
360
361static void lockdep_acquire_cpus_lock(void)
362{
363}
364
365static void lockdep_release_cpus_lock(void)
366{
367}
368
369#endif /* CONFIG_HOTPLUG_CPU */
370
371/*
372 * Architectures that need SMT-specific errata handling during SMT hotplug
373 * should override this.
374 */
375void __weak arch_smt_update(void) { }
376
377#ifdef CONFIG_HOTPLUG_SMT
378enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
379EXPORT_SYMBOL_GPL(cpu_smt_control);
380
381static bool cpu_smt_available __read_mostly;
382
383void __init cpu_smt_disable(bool force)
384{
385 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
386 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
387 return;
388
389 if (force) {
390 pr_info("SMT: Force disabled\n");
391 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
392 } else {
393 pr_info("SMT: disabled\n");
394 cpu_smt_control = CPU_SMT_DISABLED;
395 }
396}
397
398/*
399 * The decision whether SMT is supported can only be done after the full
400 * CPU identification. Called from architecture code before non boot CPUs
401 * are brought up.
402 */
403void __init cpu_smt_check_topology_early(void)
404{
405 if (!topology_smt_supported())
406 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
407}
408
409/*
410 * If SMT was disabled by BIOS, detect it here, after the CPUs have been
411 * brought online. This ensures the smt/l1tf sysfs entries are consistent
412 * with reality. cpu_smt_available is set to true during the bringup of non
413 * boot CPUs when a SMT sibling is detected. Note, this may overwrite
414 * cpu_smt_control's previous setting.
415 */
416void __init cpu_smt_check_topology(void)
417{
418 if (!cpu_smt_available)
419 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
420}
421
422static int __init smt_cmdline_disable(char *str)
423{
424 cpu_smt_disable(str && !strcmp(str, "force"));
425 return 0;
426}
427early_param("nosmt", smt_cmdline_disable);
428
429static inline bool cpu_smt_allowed(unsigned int cpu)
430{
431 if (topology_is_primary_thread(cpu))
432 return true;
433
434 /*
435 * If the CPU is not a 'primary' thread and the booted_once bit is
436 * set then the processor has SMT support. Store this information
437 * for the late check of SMT support in cpu_smt_check_topology().
438 */
439 if (per_cpu(cpuhp_state, cpu).booted_once)
440 cpu_smt_available = true;
441
442 if (cpu_smt_control == CPU_SMT_ENABLED)
443 return true;
444
445 /*
446 * On x86 it's required to boot all logical CPUs at least once so
447 * that the init code can get a chance to set CR4.MCE on each
448 * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any
449 * core will shutdown the machine.
450 */
451 return !per_cpu(cpuhp_state, cpu).booted_once;
452}
453#else
454static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
455#endif
456
457static inline enum cpuhp_state
458cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
459{
460 enum cpuhp_state prev_state = st->state;
461
462 st->rollback = false;
463 st->last = NULL;
464
465 st->target = target;
466 st->single = false;
467 st->bringup = st->state < target;
468
469 return prev_state;
470}
471
472static inline void
473cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
474{
475 st->rollback = true;
476
477 /*
478 * If we have st->last we need to undo partial multi_instance of this
479 * state first. Otherwise start undo at the previous state.
480 */
481 if (!st->last) {
482 if (st->bringup)
483 st->state--;
484 else
485 st->state++;
486 }
487
488 st->target = prev_state;
489 st->bringup = !st->bringup;
490}
491
492/* Regular hotplug invocation of the AP hotplug thread */
493static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
494{
495 if (!st->single && st->state == st->target)
496 return;
497
498 st->result = 0;
499 /*
500 * Make sure the above stores are visible before should_run becomes
501 * true. Paired with the mb() above in cpuhp_thread_fun()
502 */
503 smp_mb();
504 st->should_run = true;
505 wake_up_process(st->thread);
506 wait_for_ap_thread(st, st->bringup);
507}
508
509static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
510{
511 enum cpuhp_state prev_state;
512 int ret;
513
514 prev_state = cpuhp_set_state(st, target);
515 __cpuhp_kick_ap(st);
516 if ((ret = st->result)) {
517 cpuhp_reset_state(st, prev_state);
518 __cpuhp_kick_ap(st);
519 }
520
521 return ret;
522}
523
524static int bringup_wait_for_ap(unsigned int cpu)
525{
526 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
527
528 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
529 wait_for_ap_thread(st, true);
530 if (WARN_ON_ONCE((!cpu_online(cpu))))
531 return -ECANCELED;
532
533 /* Unpark the stopper thread and the hotplug thread of the target cpu */
534 stop_machine_unpark(cpu);
535 kthread_unpark(st->thread);
536
537 /*
538 * SMT soft disabling on X86 requires to bring the CPU out of the
539 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The
540 * CPU marked itself as booted_once in cpu_notify_starting() so the
541 * cpu_smt_allowed() check will now return false if this is not the
542 * primary sibling.
543 */
544 if (!cpu_smt_allowed(cpu))
545 return -ECANCELED;
546
547 if (st->target <= CPUHP_AP_ONLINE_IDLE)
548 return 0;
549
550 return cpuhp_kick_ap(st, st->target);
551}
552
553static int bringup_cpu(unsigned int cpu)
554{
555 struct task_struct *idle = idle_thread_get(cpu);
556 int ret;
557
558 /*
559 * Some architectures have to walk the irq descriptors to
560 * setup the vector space for the cpu which comes online.
561 * Prevent irq alloc/free across the bringup.
562 */
563 irq_lock_sparse();
564
565 /* Arch-specific enabling code. */
566 ret = __cpu_up(cpu, idle);
567 irq_unlock_sparse();
568 if (ret)
569 return ret;
570 return bringup_wait_for_ap(cpu);
571}
572
573/*
574 * Hotplug state machine related functions
575 */
576
577static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
578{
579 for (st->state--; st->state > st->target; st->state--)
580 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
581}
582
583static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
584 enum cpuhp_state target)
585{
586 enum cpuhp_state prev_state = st->state;
587 int ret = 0;
588
589 while (st->state < target) {
590 st->state++;
591 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
592 if (ret) {
593 st->target = prev_state;
594 undo_cpu_up(cpu, st);
595 break;
596 }
597 }
598 return ret;
599}
600
601/*
602 * The cpu hotplug threads manage the bringup and teardown of the cpus
603 */
604static void cpuhp_create(unsigned int cpu)
605{
606 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
607
608 init_completion(&st->done_up);
609 init_completion(&st->done_down);
610}
611
612static int cpuhp_should_run(unsigned int cpu)
613{
614 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
615
616 return st->should_run;
617}
618
619/*
620 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
621 * callbacks when a state gets [un]installed at runtime.
622 *
623 * Each invocation of this function by the smpboot thread does a single AP
624 * state callback.
625 *
626 * It has 3 modes of operation:
627 * - single: runs st->cb_state
628 * - up: runs ++st->state, while st->state < st->target
629 * - down: runs st->state--, while st->state > st->target
630 *
631 * When complete or on error, should_run is cleared and the completion is fired.
632 */
633static void cpuhp_thread_fun(unsigned int cpu)
634{
635 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
636 bool bringup = st->bringup;
637 enum cpuhp_state state;
638
639 if (WARN_ON_ONCE(!st->should_run))
640 return;
641
642 /*
643 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
644 * that if we see ->should_run we also see the rest of the state.
645 */
646 smp_mb();
647
648 /*
649 * The BP holds the hotplug lock, but we're now running on the AP,
650 * ensure that anybody asserting the lock is held, will actually find
651 * it so.
652 */
653 lockdep_acquire_cpus_lock();
654 cpuhp_lock_acquire(bringup);
655
656 if (st->single) {
657 state = st->cb_state;
658 st->should_run = false;
659 } else {
660 if (bringup) {
661 st->state++;
662 state = st->state;
663 st->should_run = (st->state < st->target);
664 WARN_ON_ONCE(st->state > st->target);
665 } else {
666 state = st->state;
667 st->state--;
668 st->should_run = (st->state > st->target);
669 WARN_ON_ONCE(st->state < st->target);
670 }
671 }
672
673 WARN_ON_ONCE(!cpuhp_is_ap_state(state));
674
675 if (cpuhp_is_atomic_state(state)) {
676 local_irq_disable();
677 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
678 local_irq_enable();
679
680 /*
681 * STARTING/DYING must not fail!
682 */
683 WARN_ON_ONCE(st->result);
684 } else {
685 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
686 }
687
688 if (st->result) {
689 /*
690 * If we fail on a rollback, we're up a creek without no
691 * paddle, no way forward, no way back. We loose, thanks for
692 * playing.
693 */
694 WARN_ON_ONCE(st->rollback);
695 st->should_run = false;
696 }
697
698 cpuhp_lock_release(bringup);
699 lockdep_release_cpus_lock();
700
701 if (!st->should_run)
702 complete_ap_thread(st, bringup);
703}
704
705/* Invoke a single callback on a remote cpu */
706static int
707cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
708 struct hlist_node *node)
709{
710 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
711 int ret;
712
713 if (!cpu_online(cpu))
714 return 0;
715
716 cpuhp_lock_acquire(false);
717 cpuhp_lock_release(false);
718
719 cpuhp_lock_acquire(true);
720 cpuhp_lock_release(true);
721
722 /*
723 * If we are up and running, use the hotplug thread. For early calls
724 * we invoke the thread function directly.
725 */
726 if (!st->thread)
727 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
728
729 st->rollback = false;
730 st->last = NULL;
731
732 st->node = node;
733 st->bringup = bringup;
734 st->cb_state = state;
735 st->single = true;
736
737 __cpuhp_kick_ap(st);
738
739 /*
740 * If we failed and did a partial, do a rollback.
741 */
742 if ((ret = st->result) && st->last) {
743 st->rollback = true;
744 st->bringup = !bringup;
745
746 __cpuhp_kick_ap(st);
747 }
748
749 /*
750 * Clean up the leftovers so the next hotplug operation wont use stale
751 * data.
752 */
753 st->node = st->last = NULL;
754 return ret;
755}
756
757static int cpuhp_kick_ap_work(unsigned int cpu)
758{
759 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
760 enum cpuhp_state prev_state = st->state;
761 int ret;
762
763 cpuhp_lock_acquire(false);
764 cpuhp_lock_release(false);
765
766 cpuhp_lock_acquire(true);
767 cpuhp_lock_release(true);
768
769 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
770 ret = cpuhp_kick_ap(st, st->target);
771 trace_cpuhp_exit(cpu, st->state, prev_state, ret);
772
773 return ret;
774}
775
776static struct smp_hotplug_thread cpuhp_threads = {
777 .store = &cpuhp_state.thread,
778 .create = &cpuhp_create,
779 .thread_should_run = cpuhp_should_run,
780 .thread_fn = cpuhp_thread_fun,
781 .thread_comm = "cpuhp/%u",
782 .selfparking = true,
783};
784
785void __init cpuhp_threads_init(void)
786{
787 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
788 kthread_unpark(this_cpu_read(cpuhp_state.thread));
789}
790
791#ifdef CONFIG_HOTPLUG_CPU
792/**
793 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
794 * @cpu: a CPU id
795 *
796 * This function walks all processes, finds a valid mm struct for each one and
797 * then clears a corresponding bit in mm's cpumask. While this all sounds
798 * trivial, there are various non-obvious corner cases, which this function
799 * tries to solve in a safe manner.
800 *
801 * Also note that the function uses a somewhat relaxed locking scheme, so it may
802 * be called only for an already offlined CPU.
803 */
804void clear_tasks_mm_cpumask(int cpu)
805{
806 struct task_struct *p;
807
808 /*
809 * This function is called after the cpu is taken down and marked
810 * offline, so its not like new tasks will ever get this cpu set in
811 * their mm mask. -- Peter Zijlstra
812 * Thus, we may use rcu_read_lock() here, instead of grabbing
813 * full-fledged tasklist_lock.
814 */
815 WARN_ON(cpu_online(cpu));
816 rcu_read_lock();
817 for_each_process(p) {
818 struct task_struct *t;
819
820 /*
821 * Main thread might exit, but other threads may still have
822 * a valid mm. Find one.
823 */
824 t = find_lock_task_mm(p);
825 if (!t)
826 continue;
827 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
828 task_unlock(t);
829 }
830 rcu_read_unlock();
831}
832
833/* Take this CPU down. */
834static int take_cpu_down(void *_param)
835{
836 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
837 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
838 int err, cpu = smp_processor_id();
839 int ret;
840
841 /* Ensure this CPU doesn't handle any more interrupts. */
842 err = __cpu_disable();
843 if (err < 0)
844 return err;
845
846 /*
847 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
848 * do this step again.
849 */
850 WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
851 st->state--;
852 /* Invoke the former CPU_DYING callbacks */
853 for (; st->state > target; st->state--) {
854 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
855 /*
856 * DYING must not fail!
857 */
858 WARN_ON_ONCE(ret);
859 }
860
861 /* Give up timekeeping duties */
862 tick_handover_do_timer();
863 /* Park the stopper thread */
864 stop_machine_park(cpu);
865 return 0;
866}
867
868static int takedown_cpu(unsigned int cpu)
869{
870 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
871 int err;
872
873 /* Park the smpboot threads */
874 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
875
876 /*
877 * Prevent irq alloc/free while the dying cpu reorganizes the
878 * interrupt affinities.
879 */
880 irq_lock_sparse();
881
882 /*
883 * So now all preempt/rcu users must observe !cpu_active().
884 */
885 err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
886 if (err) {
887 /* CPU refused to die */
888 irq_unlock_sparse();
889 /* Unpark the hotplug thread so we can rollback there */
890 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
891 return err;
892 }
893 BUG_ON(cpu_online(cpu));
894
895 /*
896 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
897 * all runnable tasks from the CPU, there's only the idle task left now
898 * that the migration thread is done doing the stop_machine thing.
899 *
900 * Wait for the stop thread to go away.
901 */
902 wait_for_ap_thread(st, false);
903 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
904
905 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
906 irq_unlock_sparse();
907
908 hotplug_cpu__broadcast_tick_pull(cpu);
909 /* This actually kills the CPU. */
910 __cpu_die(cpu);
911
912 tick_cleanup_dead_cpu(cpu);
913 rcutree_migrate_callbacks(cpu);
914 return 0;
915}
916
917static void cpuhp_complete_idle_dead(void *arg)
918{
919 struct cpuhp_cpu_state *st = arg;
920
921 complete_ap_thread(st, false);
922}
923
924void cpuhp_report_idle_dead(void)
925{
926 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
927
928 BUG_ON(st->state != CPUHP_AP_OFFLINE);
929 rcu_report_dead(smp_processor_id());
930 st->state = CPUHP_AP_IDLE_DEAD;
931 /*
932 * We cannot call complete after rcu_report_dead() so we delegate it
933 * to an online cpu.
934 */
935 smp_call_function_single(cpumask_first(cpu_online_mask),
936 cpuhp_complete_idle_dead, st, 0);
937}
938
939static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
940{
941 for (st->state++; st->state < st->target; st->state++)
942 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
943}
944
945static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
946 enum cpuhp_state target)
947{
948 enum cpuhp_state prev_state = st->state;
949 int ret = 0;
950
951 for (; st->state > target; st->state--) {
952 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
953 if (ret) {
954 st->target = prev_state;
955 if (st->state < prev_state)
956 undo_cpu_down(cpu, st);
957 break;
958 }
959 }
960 return ret;
961}
962
963/* Requires cpu_add_remove_lock to be held */
964static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
965 enum cpuhp_state target)
966{
967 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
968 int prev_state, ret = 0;
969
970 if (num_online_cpus() == 1)
971 return -EBUSY;
972
973 if (!cpu_present(cpu))
974 return -EINVAL;
975
976 cpus_write_lock();
977
978 cpuhp_tasks_frozen = tasks_frozen;
979
980 prev_state = cpuhp_set_state(st, target);
981 /*
982 * If the current CPU state is in the range of the AP hotplug thread,
983 * then we need to kick the thread.
984 */
985 if (st->state > CPUHP_TEARDOWN_CPU) {
986 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
987 ret = cpuhp_kick_ap_work(cpu);
988 /*
989 * The AP side has done the error rollback already. Just
990 * return the error code..
991 */
992 if (ret)
993 goto out;
994
995 /*
996 * We might have stopped still in the range of the AP hotplug
997 * thread. Nothing to do anymore.
998 */
999 if (st->state > CPUHP_TEARDOWN_CPU)
1000 goto out;
1001
1002 st->target = target;
1003 }
1004 /*
1005 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1006 * to do the further cleanups.
1007 */
1008 ret = cpuhp_down_callbacks(cpu, st, target);
1009 if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
1010 cpuhp_reset_state(st, prev_state);
1011 __cpuhp_kick_ap(st);
1012 }
1013
1014out:
1015 cpus_write_unlock();
1016 /*
1017 * Do post unplug cleanup. This is still protected against
1018 * concurrent CPU hotplug via cpu_add_remove_lock.
1019 */
1020 lockup_detector_cleanup();
1021 arch_smt_update();
1022 return ret;
1023}
1024
1025static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1026{
1027 if (cpu_hotplug_disabled)
1028 return -EBUSY;
1029 return _cpu_down(cpu, 0, target);
1030}
1031
1032static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
1033{
1034 int err;
1035
1036 cpu_maps_update_begin();
1037 err = cpu_down_maps_locked(cpu, target);
1038 cpu_maps_update_done();
1039 return err;
1040}
1041
1042int cpu_down(unsigned int cpu)
1043{
1044 return do_cpu_down(cpu, CPUHP_OFFLINE);
1045}
1046EXPORT_SYMBOL(cpu_down);
1047
1048#else
1049#define takedown_cpu NULL
1050#endif /*CONFIG_HOTPLUG_CPU*/
1051
1052/**
1053 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1054 * @cpu: cpu that just started
1055 *
1056 * It must be called by the arch code on the new cpu, before the new cpu
1057 * enables interrupts and before the "boot" cpu returns from __cpu_up().
1058 */
1059void notify_cpu_starting(unsigned int cpu)
1060{
1061 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1062 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1063 int ret;
1064
1065 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
1066 st->booted_once = true;
1067 while (st->state < target) {
1068 st->state++;
1069 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1070 /*
1071 * STARTING must not fail!
1072 */
1073 WARN_ON_ONCE(ret);
1074 }
1075}
1076
1077/*
1078 * Called from the idle task. Wake up the controlling task which brings the
1079 * stopper and the hotplug thread of the upcoming CPU up and then delegates
1080 * the rest of the online bringup to the hotplug thread.
1081 */
1082void cpuhp_online_idle(enum cpuhp_state state)
1083{
1084 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1085
1086 /* Happens for the boot cpu */
1087 if (state != CPUHP_AP_ONLINE_IDLE)
1088 return;
1089
1090 st->state = CPUHP_AP_ONLINE_IDLE;
1091 complete_ap_thread(st, true);
1092}
1093
1094/* Requires cpu_add_remove_lock to be held */
1095static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1096{
1097 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1098 struct task_struct *idle;
1099 int ret = 0;
1100
1101 cpus_write_lock();
1102
1103 if (!cpu_present(cpu)) {
1104 ret = -EINVAL;
1105 goto out;
1106 }
1107
1108 /*
1109 * The caller of do_cpu_up might have raced with another
1110 * caller. Ignore it for now.
1111 */
1112 if (st->state >= target)
1113 goto out;
1114
1115 if (st->state == CPUHP_OFFLINE) {
1116 /* Let it fail before we try to bring the cpu up */
1117 idle = idle_thread_get(cpu);
1118 if (IS_ERR(idle)) {
1119 ret = PTR_ERR(idle);
1120 goto out;
1121 }
1122 }
1123
1124 cpuhp_tasks_frozen = tasks_frozen;
1125
1126 cpuhp_set_state(st, target);
1127 /*
1128 * If the current CPU state is in the range of the AP hotplug thread,
1129 * then we need to kick the thread once more.
1130 */
1131 if (st->state > CPUHP_BRINGUP_CPU) {
1132 ret = cpuhp_kick_ap_work(cpu);
1133 /*
1134 * The AP side has done the error rollback already. Just
1135 * return the error code..
1136 */
1137 if (ret)
1138 goto out;
1139 }
1140
1141 /*
1142 * Try to reach the target state. We max out on the BP at
1143 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1144 * responsible for bringing it up to the target state.
1145 */
1146 target = min((int)target, CPUHP_BRINGUP_CPU);
1147 ret = cpuhp_up_callbacks(cpu, st, target);
1148out:
1149 cpus_write_unlock();
1150 arch_smt_update();
1151 return ret;
1152}
1153
1154static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1155{
1156 int err = 0;
1157
1158 if (!cpu_possible(cpu)) {
1159 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1160 cpu);
1161#if defined(CONFIG_IA64)
1162 pr_err("please check additional_cpus= boot parameter\n");
1163#endif
1164 return -EINVAL;
1165 }
1166
1167 err = try_online_node(cpu_to_node(cpu));
1168 if (err)
1169 return err;
1170
1171 cpu_maps_update_begin();
1172
1173 if (cpu_hotplug_disabled) {
1174 err = -EBUSY;
1175 goto out;
1176 }
1177 if (!cpu_smt_allowed(cpu)) {
1178 err = -EPERM;
1179 goto out;
1180 }
1181
1182 err = _cpu_up(cpu, 0, target);
1183out:
1184 cpu_maps_update_done();
1185 return err;
1186}
1187
1188int cpu_up(unsigned int cpu)
1189{
1190 return do_cpu_up(cpu, CPUHP_ONLINE);
1191}
1192EXPORT_SYMBOL_GPL(cpu_up);
1193
1194#ifdef CONFIG_PM_SLEEP_SMP
1195static cpumask_var_t frozen_cpus;
1196
1197int freeze_secondary_cpus(int primary)
1198{
1199 int cpu, error = 0;
1200
1201 cpu_maps_update_begin();
1202 if (!cpu_online(primary))
1203 primary = cpumask_first(cpu_online_mask);
1204 /*
1205 * We take down all of the non-boot CPUs in one shot to avoid races
1206 * with the userspace trying to use the CPU hotplug at the same time
1207 */
1208 cpumask_clear(frozen_cpus);
1209
1210 pr_info("Disabling non-boot CPUs ...\n");
1211 for_each_online_cpu(cpu) {
1212 if (cpu == primary)
1213 continue;
1214 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1215 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1216 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1217 if (!error)
1218 cpumask_set_cpu(cpu, frozen_cpus);
1219 else {
1220 pr_err("Error taking CPU%d down: %d\n", cpu, error);
1221 break;
1222 }
1223 }
1224
1225 if (!error)
1226 BUG_ON(num_online_cpus() > 1);
1227 else
1228 pr_err("Non-boot CPUs are not disabled\n");
1229
1230 /*
1231 * Make sure the CPUs won't be enabled by someone else. We need to do
1232 * this even in case of failure as all disable_nonboot_cpus() users are
1233 * supposed to do enable_nonboot_cpus() on the failure path.
1234 */
1235 cpu_hotplug_disabled++;
1236
1237 cpu_maps_update_done();
1238 return error;
1239}
1240
1241void __weak arch_enable_nonboot_cpus_begin(void)
1242{
1243}
1244
1245void __weak arch_enable_nonboot_cpus_end(void)
1246{
1247}
1248
1249void enable_nonboot_cpus(void)
1250{
1251 int cpu, error;
1252
1253 /* Allow everyone to use the CPU hotplug again */
1254 cpu_maps_update_begin();
1255 __cpu_hotplug_enable();
1256 if (cpumask_empty(frozen_cpus))
1257 goto out;
1258
1259 pr_info("Enabling non-boot CPUs ...\n");
1260
1261 arch_enable_nonboot_cpus_begin();
1262
1263 for_each_cpu(cpu, frozen_cpus) {
1264 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1265 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1266 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1267 if (!error) {
1268 pr_info("CPU%d is up\n", cpu);
1269 continue;
1270 }
1271 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1272 }
1273
1274 arch_enable_nonboot_cpus_end();
1275
1276 cpumask_clear(frozen_cpus);
1277out:
1278 cpu_maps_update_done();
1279}
1280
1281static int __init alloc_frozen_cpus(void)
1282{
1283 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1284 return -ENOMEM;
1285 return 0;
1286}
1287core_initcall(alloc_frozen_cpus);
1288
1289/*
1290 * When callbacks for CPU hotplug notifications are being executed, we must
1291 * ensure that the state of the system with respect to the tasks being frozen
1292 * or not, as reported by the notification, remains unchanged *throughout the
1293 * duration* of the execution of the callbacks.
1294 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1295 *
1296 * This synchronization is implemented by mutually excluding regular CPU
1297 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1298 * Hibernate notifications.
1299 */
1300static int
1301cpu_hotplug_pm_callback(struct notifier_block *nb,
1302 unsigned long action, void *ptr)
1303{
1304 switch (action) {
1305
1306 case PM_SUSPEND_PREPARE:
1307 case PM_HIBERNATION_PREPARE:
1308 cpu_hotplug_disable();
1309 break;
1310
1311 case PM_POST_SUSPEND:
1312 case PM_POST_HIBERNATION:
1313 cpu_hotplug_enable();
1314 break;
1315
1316 default:
1317 return NOTIFY_DONE;
1318 }
1319
1320 return NOTIFY_OK;
1321}
1322
1323
1324static int __init cpu_hotplug_pm_sync_init(void)
1325{
1326 /*
1327 * cpu_hotplug_pm_callback has higher priority than x86
1328 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1329 * to disable cpu hotplug to avoid cpu hotplug race.
1330 */
1331 pm_notifier(cpu_hotplug_pm_callback, 0);
1332 return 0;
1333}
1334core_initcall(cpu_hotplug_pm_sync_init);
1335
1336#endif /* CONFIG_PM_SLEEP_SMP */
1337
1338int __boot_cpu_id;
1339
1340#endif /* CONFIG_SMP */
1341
1342/* Boot processor state steps */
1343static struct cpuhp_step cpuhp_hp_states[] = {
1344 [CPUHP_OFFLINE] = {
1345 .name = "offline",
1346 .startup.single = NULL,
1347 .teardown.single = NULL,
1348 },
1349#ifdef CONFIG_SMP
1350 [CPUHP_CREATE_THREADS]= {
1351 .name = "threads:prepare",
1352 .startup.single = smpboot_create_threads,
1353 .teardown.single = NULL,
1354 .cant_stop = true,
1355 },
1356 [CPUHP_PERF_PREPARE] = {
1357 .name = "perf:prepare",
1358 .startup.single = perf_event_init_cpu,
1359 .teardown.single = perf_event_exit_cpu,
1360 },
1361 [CPUHP_WORKQUEUE_PREP] = {
1362 .name = "workqueue:prepare",
1363 .startup.single = workqueue_prepare_cpu,
1364 .teardown.single = NULL,
1365 },
1366 [CPUHP_HRTIMERS_PREPARE] = {
1367 .name = "hrtimers:prepare",
1368 .startup.single = hrtimers_prepare_cpu,
1369 .teardown.single = hrtimers_dead_cpu,
1370 },
1371 [CPUHP_SMPCFD_PREPARE] = {
1372 .name = "smpcfd:prepare",
1373 .startup.single = smpcfd_prepare_cpu,
1374 .teardown.single = smpcfd_dead_cpu,
1375 },
1376 [CPUHP_RELAY_PREPARE] = {
1377 .name = "relay:prepare",
1378 .startup.single = relay_prepare_cpu,
1379 .teardown.single = NULL,
1380 },
1381 [CPUHP_SLAB_PREPARE] = {
1382 .name = "slab:prepare",
1383 .startup.single = slab_prepare_cpu,
1384 .teardown.single = slab_dead_cpu,
1385 },
1386 [CPUHP_RCUTREE_PREP] = {
1387 .name = "RCU/tree:prepare",
1388 .startup.single = rcutree_prepare_cpu,
1389 .teardown.single = rcutree_dead_cpu,
1390 },
1391 /*
1392 * On the tear-down path, timers_dead_cpu() must be invoked
1393 * before blk_mq_queue_reinit_notify() from notify_dead(),
1394 * otherwise a RCU stall occurs.
1395 */
1396 [CPUHP_TIMERS_PREPARE] = {
1397 .name = "timers:prepare",
1398 .startup.single = timers_prepare_cpu,
1399 .teardown.single = timers_dead_cpu,
1400 },
1401 /* Kicks the plugged cpu into life */
1402 [CPUHP_BRINGUP_CPU] = {
1403 .name = "cpu:bringup",
1404 .startup.single = bringup_cpu,
1405 .teardown.single = NULL,
1406 .cant_stop = true,
1407 },
1408 /* Final state before CPU kills itself */
1409 [CPUHP_AP_IDLE_DEAD] = {
1410 .name = "idle:dead",
1411 },
1412 /*
1413 * Last state before CPU enters the idle loop to die. Transient state
1414 * for synchronization.
1415 */
1416 [CPUHP_AP_OFFLINE] = {
1417 .name = "ap:offline",
1418 .cant_stop = true,
1419 },
1420 /* First state is scheduler control. Interrupts are disabled */
1421 [CPUHP_AP_SCHED_STARTING] = {
1422 .name = "sched:starting",
1423 .startup.single = sched_cpu_starting,
1424 .teardown.single = sched_cpu_dying,
1425 },
1426 [CPUHP_AP_RCUTREE_DYING] = {
1427 .name = "RCU/tree:dying",
1428 .startup.single = NULL,
1429 .teardown.single = rcutree_dying_cpu,
1430 },
1431 [CPUHP_AP_SMPCFD_DYING] = {
1432 .name = "smpcfd:dying",
1433 .startup.single = NULL,
1434 .teardown.single = smpcfd_dying_cpu,
1435 },
1436 /* Entry state on starting. Interrupts enabled from here on. Transient
1437 * state for synchronsization */
1438 [CPUHP_AP_ONLINE] = {
1439 .name = "ap:online",
1440 },
1441 /*
1442 * Handled on controll processor until the plugged processor manages
1443 * this itself.
1444 */
1445 [CPUHP_TEARDOWN_CPU] = {
1446 .name = "cpu:teardown",
1447 .startup.single = NULL,
1448 .teardown.single = takedown_cpu,
1449 .cant_stop = true,
1450 },
1451 /* Handle smpboot threads park/unpark */
1452 [CPUHP_AP_SMPBOOT_THREADS] = {
1453 .name = "smpboot/threads:online",
1454 .startup.single = smpboot_unpark_threads,
1455 .teardown.single = smpboot_park_threads,
1456 },
1457 [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1458 .name = "irq/affinity:online",
1459 .startup.single = irq_affinity_online_cpu,
1460 .teardown.single = NULL,
1461 },
1462 [CPUHP_AP_PERF_ONLINE] = {
1463 .name = "perf:online",
1464 .startup.single = perf_event_init_cpu,
1465 .teardown.single = perf_event_exit_cpu,
1466 },
1467 [CPUHP_AP_WATCHDOG_ONLINE] = {
1468 .name = "lockup_detector:online",
1469 .startup.single = lockup_detector_online_cpu,
1470 .teardown.single = lockup_detector_offline_cpu,
1471 },
1472 [CPUHP_AP_WORKQUEUE_ONLINE] = {
1473 .name = "workqueue:online",
1474 .startup.single = workqueue_online_cpu,
1475 .teardown.single = workqueue_offline_cpu,
1476 },
1477 [CPUHP_AP_RCUTREE_ONLINE] = {
1478 .name = "RCU/tree:online",
1479 .startup.single = rcutree_online_cpu,
1480 .teardown.single = rcutree_offline_cpu,
1481 },
1482#endif
1483 /*
1484 * The dynamically registered state space is here
1485 */
1486
1487#ifdef CONFIG_SMP
1488 /* Last state is scheduler control setting the cpu active */
1489 [CPUHP_AP_ACTIVE] = {
1490 .name = "sched:active",
1491 .startup.single = sched_cpu_activate,
1492 .teardown.single = sched_cpu_deactivate,
1493 },
1494#endif
1495
1496 /* CPU is fully up and running. */
1497 [CPUHP_ONLINE] = {
1498 .name = "online",
1499 .startup.single = NULL,
1500 .teardown.single = NULL,
1501 },
1502};
1503
1504/* Sanity check for callbacks */
1505static int cpuhp_cb_check(enum cpuhp_state state)
1506{
1507 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1508 return -EINVAL;
1509 return 0;
1510}
1511
1512/*
1513 * Returns a free for dynamic slot assignment of the Online state. The states
1514 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1515 * by having no name assigned.
1516 */
1517static int cpuhp_reserve_state(enum cpuhp_state state)
1518{
1519 enum cpuhp_state i, end;
1520 struct cpuhp_step *step;
1521
1522 switch (state) {
1523 case CPUHP_AP_ONLINE_DYN:
1524 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1525 end = CPUHP_AP_ONLINE_DYN_END;
1526 break;
1527 case CPUHP_BP_PREPARE_DYN:
1528 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1529 end = CPUHP_BP_PREPARE_DYN_END;
1530 break;
1531 default:
1532 return -EINVAL;
1533 }
1534
1535 for (i = state; i <= end; i++, step++) {
1536 if (!step->name)
1537 return i;
1538 }
1539 WARN(1, "No more dynamic states available for CPU hotplug\n");
1540 return -ENOSPC;
1541}
1542
1543static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1544 int (*startup)(unsigned int cpu),
1545 int (*teardown)(unsigned int cpu),
1546 bool multi_instance)
1547{
1548 /* (Un)Install the callbacks for further cpu hotplug operations */
1549 struct cpuhp_step *sp;
1550 int ret = 0;
1551
1552 /*
1553 * If name is NULL, then the state gets removed.
1554 *
1555 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1556 * the first allocation from these dynamic ranges, so the removal
1557 * would trigger a new allocation and clear the wrong (already
1558 * empty) state, leaving the callbacks of the to be cleared state
1559 * dangling, which causes wreckage on the next hotplug operation.
1560 */
1561 if (name && (state == CPUHP_AP_ONLINE_DYN ||
1562 state == CPUHP_BP_PREPARE_DYN)) {
1563 ret = cpuhp_reserve_state(state);
1564 if (ret < 0)
1565 return ret;
1566 state = ret;
1567 }
1568 sp = cpuhp_get_step(state);
1569 if (name && sp->name)
1570 return -EBUSY;
1571
1572 sp->startup.single = startup;
1573 sp->teardown.single = teardown;
1574 sp->name = name;
1575 sp->multi_instance = multi_instance;
1576 INIT_HLIST_HEAD(&sp->list);
1577 return ret;
1578}
1579
1580static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1581{
1582 return cpuhp_get_step(state)->teardown.single;
1583}
1584
1585/*
1586 * Call the startup/teardown function for a step either on the AP or
1587 * on the current CPU.
1588 */
1589static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1590 struct hlist_node *node)
1591{
1592 struct cpuhp_step *sp = cpuhp_get_step(state);
1593 int ret;
1594
1595 /*
1596 * If there's nothing to do, we done.
1597 * Relies on the union for multi_instance.
1598 */
1599 if ((bringup && !sp->startup.single) ||
1600 (!bringup && !sp->teardown.single))
1601 return 0;
1602 /*
1603 * The non AP bound callbacks can fail on bringup. On teardown
1604 * e.g. module removal we crash for now.
1605 */
1606#ifdef CONFIG_SMP
1607 if (cpuhp_is_ap_state(state))
1608 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1609 else
1610 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1611#else
1612 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1613#endif
1614 BUG_ON(ret && !bringup);
1615 return ret;
1616}
1617
1618/*
1619 * Called from __cpuhp_setup_state on a recoverable failure.
1620 *
1621 * Note: The teardown callbacks for rollback are not allowed to fail!
1622 */
1623static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1624 struct hlist_node *node)
1625{
1626 int cpu;
1627
1628 /* Roll back the already executed steps on the other cpus */
1629 for_each_present_cpu(cpu) {
1630 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1631 int cpustate = st->state;
1632
1633 if (cpu >= failedcpu)
1634 break;
1635
1636 /* Did we invoke the startup call on that cpu ? */
1637 if (cpustate >= state)
1638 cpuhp_issue_call(cpu, state, false, node);
1639 }
1640}
1641
1642int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1643 struct hlist_node *node,
1644 bool invoke)
1645{
1646 struct cpuhp_step *sp;
1647 int cpu;
1648 int ret;
1649
1650 lockdep_assert_cpus_held();
1651
1652 sp = cpuhp_get_step(state);
1653 if (sp->multi_instance == false)
1654 return -EINVAL;
1655
1656 mutex_lock(&cpuhp_state_mutex);
1657
1658 if (!invoke || !sp->startup.multi)
1659 goto add_node;
1660
1661 /*
1662 * Try to call the startup callback for each present cpu
1663 * depending on the hotplug state of the cpu.
1664 */
1665 for_each_present_cpu(cpu) {
1666 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1667 int cpustate = st->state;
1668
1669 if (cpustate < state)
1670 continue;
1671
1672 ret = cpuhp_issue_call(cpu, state, true, node);
1673 if (ret) {
1674 if (sp->teardown.multi)
1675 cpuhp_rollback_install(cpu, state, node);
1676 goto unlock;
1677 }
1678 }
1679add_node:
1680 ret = 0;
1681 hlist_add_head(node, &sp->list);
1682unlock:
1683 mutex_unlock(&cpuhp_state_mutex);
1684 return ret;
1685}
1686
1687int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1688 bool invoke)
1689{
1690 int ret;
1691
1692 cpus_read_lock();
1693 ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1694 cpus_read_unlock();
1695 return ret;
1696}
1697EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1698
1699/**
1700 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1701 * @state: The state to setup
1702 * @invoke: If true, the startup function is invoked for cpus where
1703 * cpu state >= @state
1704 * @startup: startup callback function
1705 * @teardown: teardown callback function
1706 * @multi_instance: State is set up for multiple instances which get
1707 * added afterwards.
1708 *
1709 * The caller needs to hold cpus read locked while calling this function.
1710 * Returns:
1711 * On success:
1712 * Positive state number if @state is CPUHP_AP_ONLINE_DYN
1713 * 0 for all other states
1714 * On failure: proper (negative) error code
1715 */
1716int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1717 const char *name, bool invoke,
1718 int (*startup)(unsigned int cpu),
1719 int (*teardown)(unsigned int cpu),
1720 bool multi_instance)
1721{
1722 int cpu, ret = 0;
1723 bool dynstate;
1724
1725 lockdep_assert_cpus_held();
1726
1727 if (cpuhp_cb_check(state) || !name)
1728 return -EINVAL;
1729
1730 mutex_lock(&cpuhp_state_mutex);
1731
1732 ret = cpuhp_store_callbacks(state, name, startup, teardown,
1733 multi_instance);
1734
1735 dynstate = state == CPUHP_AP_ONLINE_DYN;
1736 if (ret > 0 && dynstate) {
1737 state = ret;
1738 ret = 0;
1739 }
1740
1741 if (ret || !invoke || !startup)
1742 goto out;
1743
1744 /*
1745 * Try to call the startup callback for each present cpu
1746 * depending on the hotplug state of the cpu.
1747 */
1748 for_each_present_cpu(cpu) {
1749 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1750 int cpustate = st->state;
1751
1752 if (cpustate < state)
1753 continue;
1754
1755 ret = cpuhp_issue_call(cpu, state, true, NULL);
1756 if (ret) {
1757 if (teardown)
1758 cpuhp_rollback_install(cpu, state, NULL);
1759 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1760 goto out;
1761 }
1762 }
1763out:
1764 mutex_unlock(&cpuhp_state_mutex);
1765 /*
1766 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1767 * dynamically allocated state in case of success.
1768 */
1769 if (!ret && dynstate)
1770 return state;
1771 return ret;
1772}
1773EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1774
1775int __cpuhp_setup_state(enum cpuhp_state state,
1776 const char *name, bool invoke,
1777 int (*startup)(unsigned int cpu),
1778 int (*teardown)(unsigned int cpu),
1779 bool multi_instance)
1780{
1781 int ret;
1782
1783 cpus_read_lock();
1784 ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1785 teardown, multi_instance);
1786 cpus_read_unlock();
1787 return ret;
1788}
1789EXPORT_SYMBOL(__cpuhp_setup_state);
1790
1791int __cpuhp_state_remove_instance(enum cpuhp_state state,
1792 struct hlist_node *node, bool invoke)
1793{
1794 struct cpuhp_step *sp = cpuhp_get_step(state);
1795 int cpu;
1796
1797 BUG_ON(cpuhp_cb_check(state));
1798
1799 if (!sp->multi_instance)
1800 return -EINVAL;
1801
1802 cpus_read_lock();
1803 mutex_lock(&cpuhp_state_mutex);
1804
1805 if (!invoke || !cpuhp_get_teardown_cb(state))
1806 goto remove;
1807 /*
1808 * Call the teardown callback for each present cpu depending
1809 * on the hotplug state of the cpu. This function is not
1810 * allowed to fail currently!
1811 */
1812 for_each_present_cpu(cpu) {
1813 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1814 int cpustate = st->state;
1815
1816 if (cpustate >= state)
1817 cpuhp_issue_call(cpu, state, false, node);
1818 }
1819
1820remove:
1821 hlist_del(node);
1822 mutex_unlock(&cpuhp_state_mutex);
1823 cpus_read_unlock();
1824
1825 return 0;
1826}
1827EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1828
1829/**
1830 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1831 * @state: The state to remove
1832 * @invoke: If true, the teardown function is invoked for cpus where
1833 * cpu state >= @state
1834 *
1835 * The caller needs to hold cpus read locked while calling this function.
1836 * The teardown callback is currently not allowed to fail. Think
1837 * about module removal!
1838 */
1839void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1840{
1841 struct cpuhp_step *sp = cpuhp_get_step(state);
1842 int cpu;
1843
1844 BUG_ON(cpuhp_cb_check(state));
1845
1846 lockdep_assert_cpus_held();
1847
1848 mutex_lock(&cpuhp_state_mutex);
1849 if (sp->multi_instance) {
1850 WARN(!hlist_empty(&sp->list),
1851 "Error: Removing state %d which has instances left.\n",
1852 state);
1853 goto remove;
1854 }
1855
1856 if (!invoke || !cpuhp_get_teardown_cb(state))
1857 goto remove;
1858
1859 /*
1860 * Call the teardown callback for each present cpu depending
1861 * on the hotplug state of the cpu. This function is not
1862 * allowed to fail currently!
1863 */
1864 for_each_present_cpu(cpu) {
1865 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1866 int cpustate = st->state;
1867
1868 if (cpustate >= state)
1869 cpuhp_issue_call(cpu, state, false, NULL);
1870 }
1871remove:
1872 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1873 mutex_unlock(&cpuhp_state_mutex);
1874}
1875EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1876
1877void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1878{
1879 cpus_read_lock();
1880 __cpuhp_remove_state_cpuslocked(state, invoke);
1881 cpus_read_unlock();
1882}
1883EXPORT_SYMBOL(__cpuhp_remove_state);
1884
1885#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1886static ssize_t show_cpuhp_state(struct device *dev,
1887 struct device_attribute *attr, char *buf)
1888{
1889 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1890
1891 return sprintf(buf, "%d\n", st->state);
1892}
1893static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1894
1895static ssize_t write_cpuhp_target(struct device *dev,
1896 struct device_attribute *attr,
1897 const char *buf, size_t count)
1898{
1899 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1900 struct cpuhp_step *sp;
1901 int target, ret;
1902
1903 ret = kstrtoint(buf, 10, &target);
1904 if (ret)
1905 return ret;
1906
1907#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1908 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1909 return -EINVAL;
1910#else
1911 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1912 return -EINVAL;
1913#endif
1914
1915 ret = lock_device_hotplug_sysfs();
1916 if (ret)
1917 return ret;
1918
1919 mutex_lock(&cpuhp_state_mutex);
1920 sp = cpuhp_get_step(target);
1921 ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1922 mutex_unlock(&cpuhp_state_mutex);
1923 if (ret)
1924 goto out;
1925
1926 if (st->state < target)
1927 ret = do_cpu_up(dev->id, target);
1928 else
1929 ret = do_cpu_down(dev->id, target);
1930out:
1931 unlock_device_hotplug();
1932 return ret ? ret : count;
1933}
1934
1935static ssize_t show_cpuhp_target(struct device *dev,
1936 struct device_attribute *attr, char *buf)
1937{
1938 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1939
1940 return sprintf(buf, "%d\n", st->target);
1941}
1942static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1943
1944
1945static ssize_t write_cpuhp_fail(struct device *dev,
1946 struct device_attribute *attr,
1947 const char *buf, size_t count)
1948{
1949 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1950 struct cpuhp_step *sp;
1951 int fail, ret;
1952
1953 ret = kstrtoint(buf, 10, &fail);
1954 if (ret)
1955 return ret;
1956
1957 /*
1958 * Cannot fail STARTING/DYING callbacks.
1959 */
1960 if (cpuhp_is_atomic_state(fail))
1961 return -EINVAL;
1962
1963 /*
1964 * Cannot fail anything that doesn't have callbacks.
1965 */
1966 mutex_lock(&cpuhp_state_mutex);
1967 sp = cpuhp_get_step(fail);
1968 if (!sp->startup.single && !sp->teardown.single)
1969 ret = -EINVAL;
1970 mutex_unlock(&cpuhp_state_mutex);
1971 if (ret)
1972 return ret;
1973
1974 st->fail = fail;
1975
1976 return count;
1977}
1978
1979static ssize_t show_cpuhp_fail(struct device *dev,
1980 struct device_attribute *attr, char *buf)
1981{
1982 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1983
1984 return sprintf(buf, "%d\n", st->fail);
1985}
1986
1987static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
1988
1989static struct attribute *cpuhp_cpu_attrs[] = {
1990 &dev_attr_state.attr,
1991 &dev_attr_target.attr,
1992 &dev_attr_fail.attr,
1993 NULL
1994};
1995
1996static const struct attribute_group cpuhp_cpu_attr_group = {
1997 .attrs = cpuhp_cpu_attrs,
1998 .name = "hotplug",
1999 NULL
2000};
2001
2002static ssize_t show_cpuhp_states(struct device *dev,
2003 struct device_attribute *attr, char *buf)
2004{
2005 ssize_t cur, res = 0;
2006 int i;
2007
2008 mutex_lock(&cpuhp_state_mutex);
2009 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2010 struct cpuhp_step *sp = cpuhp_get_step(i);
2011
2012 if (sp->name) {
2013 cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2014 buf += cur;
2015 res += cur;
2016 }
2017 }
2018 mutex_unlock(&cpuhp_state_mutex);
2019 return res;
2020}
2021static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
2022
2023static struct attribute *cpuhp_cpu_root_attrs[] = {
2024 &dev_attr_states.attr,
2025 NULL
2026};
2027
2028static const struct attribute_group cpuhp_cpu_root_attr_group = {
2029 .attrs = cpuhp_cpu_root_attrs,
2030 .name = "hotplug",
2031 NULL
2032};
2033
2034#ifdef CONFIG_HOTPLUG_SMT
2035
2036static const char *smt_states[] = {
2037 [CPU_SMT_ENABLED] = "on",
2038 [CPU_SMT_DISABLED] = "off",
2039 [CPU_SMT_FORCE_DISABLED] = "forceoff",
2040 [CPU_SMT_NOT_SUPPORTED] = "notsupported",
2041};
2042
2043static ssize_t
2044show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2045{
2046 return snprintf(buf, PAGE_SIZE - 2, "%s\n", smt_states[cpu_smt_control]);
2047}
2048
2049static void cpuhp_offline_cpu_device(unsigned int cpu)
2050{
2051 struct device *dev = get_cpu_device(cpu);
2052
2053 dev->offline = true;
2054 /* Tell user space about the state change */
2055 kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2056}
2057
2058static void cpuhp_online_cpu_device(unsigned int cpu)
2059{
2060 struct device *dev = get_cpu_device(cpu);
2061
2062 dev->offline = false;
2063 /* Tell user space about the state change */
2064 kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2065}
2066
2067static int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2068{
2069 int cpu, ret = 0;
2070
2071 cpu_maps_update_begin();
2072 for_each_online_cpu(cpu) {
2073 if (topology_is_primary_thread(cpu))
2074 continue;
2075 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2076 if (ret)
2077 break;
2078 /*
2079 * As this needs to hold the cpu maps lock it's impossible
2080 * to call device_offline() because that ends up calling
2081 * cpu_down() which takes cpu maps lock. cpu maps lock
2082 * needs to be held as this might race against in kernel
2083 * abusers of the hotplug machinery (thermal management).
2084 *
2085 * So nothing would update device:offline state. That would
2086 * leave the sysfs entry stale and prevent onlining after
2087 * smt control has been changed to 'off' again. This is
2088 * called under the sysfs hotplug lock, so it is properly
2089 * serialized against the regular offline usage.
2090 */
2091 cpuhp_offline_cpu_device(cpu);
2092 }
2093 if (!ret) {
2094 cpu_smt_control = ctrlval;
2095 arch_smt_update();
2096 }
2097 cpu_maps_update_done();
2098 return ret;
2099}
2100
2101static int cpuhp_smt_enable(void)
2102{
2103 int cpu, ret = 0;
2104
2105 cpu_maps_update_begin();
2106 cpu_smt_control = CPU_SMT_ENABLED;
2107 arch_smt_update();
2108 for_each_present_cpu(cpu) {
2109 /* Skip online CPUs and CPUs on offline nodes */
2110 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2111 continue;
2112 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2113 if (ret)
2114 break;
2115 /* See comment in cpuhp_smt_disable() */
2116 cpuhp_online_cpu_device(cpu);
2117 }
2118 cpu_maps_update_done();
2119 return ret;
2120}
2121
2122static ssize_t
2123store_smt_control(struct device *dev, struct device_attribute *attr,
2124 const char *buf, size_t count)
2125{
2126 int ctrlval, ret;
2127
2128 if (sysfs_streq(buf, "on"))
2129 ctrlval = CPU_SMT_ENABLED;
2130 else if (sysfs_streq(buf, "off"))
2131 ctrlval = CPU_SMT_DISABLED;
2132 else if (sysfs_streq(buf, "forceoff"))
2133 ctrlval = CPU_SMT_FORCE_DISABLED;
2134 else
2135 return -EINVAL;
2136
2137 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2138 return -EPERM;
2139
2140 if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2141 return -ENODEV;
2142
2143 ret = lock_device_hotplug_sysfs();
2144 if (ret)
2145 return ret;
2146
2147 if (ctrlval != cpu_smt_control) {
2148 switch (ctrlval) {
2149 case CPU_SMT_ENABLED:
2150 ret = cpuhp_smt_enable();
2151 break;
2152 case CPU_SMT_DISABLED:
2153 case CPU_SMT_FORCE_DISABLED:
2154 ret = cpuhp_smt_disable(ctrlval);
2155 break;
2156 }
2157 }
2158
2159 unlock_device_hotplug();
2160 return ret ? ret : count;
2161}
2162static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2163
2164static ssize_t
2165show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2166{
2167 bool active = topology_max_smt_threads() > 1;
2168
2169 return snprintf(buf, PAGE_SIZE - 2, "%d\n", active);
2170}
2171static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2172
2173static struct attribute *cpuhp_smt_attrs[] = {
2174 &dev_attr_control.attr,
2175 &dev_attr_active.attr,
2176 NULL
2177};
2178
2179static const struct attribute_group cpuhp_smt_attr_group = {
2180 .attrs = cpuhp_smt_attrs,
2181 .name = "smt",
2182 NULL
2183};
2184
2185static int __init cpu_smt_state_init(void)
2186{
2187 return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2188 &cpuhp_smt_attr_group);
2189}
2190
2191#else
2192static inline int cpu_smt_state_init(void) { return 0; }
2193#endif
2194
2195static int __init cpuhp_sysfs_init(void)
2196{
2197 int cpu, ret;
2198
2199 ret = cpu_smt_state_init();
2200 if (ret)
2201 return ret;
2202
2203 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2204 &cpuhp_cpu_root_attr_group);
2205 if (ret)
2206 return ret;
2207
2208 for_each_possible_cpu(cpu) {
2209 struct device *dev = get_cpu_device(cpu);
2210
2211 if (!dev)
2212 continue;
2213 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2214 if (ret)
2215 return ret;
2216 }
2217 return 0;
2218}
2219device_initcall(cpuhp_sysfs_init);
2220#endif
2221
2222/*
2223 * cpu_bit_bitmap[] is a special, "compressed" data structure that
2224 * represents all NR_CPUS bits binary values of 1<<nr.
2225 *
2226 * It is used by cpumask_of() to get a constant address to a CPU
2227 * mask value that has a single bit set only.
2228 */
2229
2230/* cpu_bit_bitmap[0] is empty - so we can back into it */
2231#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
2232#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2233#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2234#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2235
2236const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2237
2238 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
2239 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
2240#if BITS_PER_LONG > 32
2241 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
2242 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
2243#endif
2244};
2245EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2246
2247const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2248EXPORT_SYMBOL(cpu_all_bits);
2249
2250#ifdef CONFIG_INIT_ALL_POSSIBLE
2251struct cpumask __cpu_possible_mask __read_mostly
2252 = {CPU_BITS_ALL};
2253#else
2254struct cpumask __cpu_possible_mask __read_mostly;
2255#endif
2256EXPORT_SYMBOL(__cpu_possible_mask);
2257
2258struct cpumask __cpu_online_mask __read_mostly;
2259EXPORT_SYMBOL(__cpu_online_mask);
2260
2261struct cpumask __cpu_present_mask __read_mostly;
2262EXPORT_SYMBOL(__cpu_present_mask);
2263
2264struct cpumask __cpu_active_mask __read_mostly;
2265EXPORT_SYMBOL(__cpu_active_mask);
2266
2267void init_cpu_present(const struct cpumask *src)
2268{
2269 cpumask_copy(&__cpu_present_mask, src);
2270}
2271
2272void init_cpu_possible(const struct cpumask *src)
2273{
2274 cpumask_copy(&__cpu_possible_mask, src);
2275}
2276
2277void init_cpu_online(const struct cpumask *src)
2278{
2279 cpumask_copy(&__cpu_online_mask, src);
2280}
2281
2282/*
2283 * Activate the first processor.
2284 */
2285void __init boot_cpu_init(void)
2286{
2287 int cpu = smp_processor_id();
2288
2289 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2290 set_cpu_online(cpu, true);
2291 set_cpu_active(cpu, true);
2292 set_cpu_present(cpu, true);
2293 set_cpu_possible(cpu, true);
2294
2295#ifdef CONFIG_SMP
2296 __boot_cpu_id = cpu;
2297#endif
2298}
2299
2300/*
2301 * Must be called _AFTER_ setting up the per_cpu areas
2302 */
2303void __init boot_cpu_hotplug_init(void)
2304{
2305#ifdef CONFIG_SMP
2306 this_cpu_write(cpuhp_state.booted_once, true);
2307#endif
2308 this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2309}