projects/linux·blob v4.16-rc3 · refs

2028 lines · 48.1 KiB

1/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched/signal.h>
11#include <linux/sched/hotplug.h>
12#include <linux/sched/task.h>
13#include <linux/unistd.h>
14#include <linux/cpu.h>
15#include <linux/oom.h>
16#include <linux/rcupdate.h>
17#include <linux/export.h>
18#include <linux/bug.h>
19#include <linux/kthread.h>
20#include <linux/stop_machine.h>
21#include <linux/mutex.h>
22#include <linux/gfp.h>
23#include <linux/suspend.h>
24#include <linux/lockdep.h>
25#include <linux/tick.h>
26#include <linux/irq.h>
27#include <linux/nmi.h>
28#include <linux/smpboot.h>
29#include <linux/relay.h>
30#include <linux/slab.h>
31#include <linux/percpu-rwsem.h>
32
33#include <trace/events/power.h>
34#define CREATE_TRACE_POINTS
35#include <trace/events/cpuhp.h>
36
37#include "smpboot.h"
38
39/**
40 * cpuhp_cpu_state - Per cpu hotplug state storage
41 * @state: The current cpu state
42 * @target: The target state
43 * @thread: Pointer to the hotplug thread
44 * @should_run: Thread should execute
45 * @rollback: Perform a rollback
46 * @single: Single callback invocation
47 * @bringup: Single callback bringup or teardown selector
48 * @cb_state: The state for a single callback (install/uninstall)
49 * @result: Result of the operation
50 * @done_up: Signal completion to the issuer of the task for cpu-up
51 * @done_down: Signal completion to the issuer of the task for cpu-down
52 */
53struct cpuhp_cpu_state {
54 enum cpuhp_state state;
55 enum cpuhp_state target;
56 enum cpuhp_state fail;
57#ifdef CONFIG_SMP
58 struct task_struct *thread;
59 bool should_run;
60 bool rollback;
61 bool single;
62 bool bringup;
63 struct hlist_node *node;
64 struct hlist_node *last;
65 enum cpuhp_state cb_state;
66 int result;
67 struct completion done_up;
68 struct completion done_down;
69#endif
70};
71
72static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
73 .fail = CPUHP_INVALID,
74};
75
76#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
77static struct lockdep_map cpuhp_state_up_map =
78 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
79static struct lockdep_map cpuhp_state_down_map =
80 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
81
82
83static inline void cpuhp_lock_acquire(bool bringup)
84{
85 lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
86}
87
88static inline void cpuhp_lock_release(bool bringup)
89{
90 lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
91}
92#else
93
94static inline void cpuhp_lock_acquire(bool bringup) { }
95static inline void cpuhp_lock_release(bool bringup) { }
96
97#endif
98
99/**
100 * cpuhp_step - Hotplug state machine step
101 * @name: Name of the step
102 * @startup: Startup function of the step
103 * @teardown: Teardown function of the step
104 * @skip_onerr: Do not invoke the functions on error rollback
105 * Will go away once the notifiers are gone
106 * @cant_stop: Bringup/teardown can't be stopped at this step
107 */
108struct cpuhp_step {
109 const char *name;
110 union {
111 int (*single)(unsigned int cpu);
112 int (*multi)(unsigned int cpu,
113 struct hlist_node *node);
114 } startup;
115 union {
116 int (*single)(unsigned int cpu);
117 int (*multi)(unsigned int cpu,
118 struct hlist_node *node);
119 } teardown;
120 struct hlist_head list;
121 bool skip_onerr;
122 bool cant_stop;
123 bool multi_instance;
124};
125
126static DEFINE_MUTEX(cpuhp_state_mutex);
127static struct cpuhp_step cpuhp_bp_states[];
128static struct cpuhp_step cpuhp_ap_states[];
129
130static bool cpuhp_is_ap_state(enum cpuhp_state state)
131{
132 /*
133 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
134 * purposes as that state is handled explicitly in cpu_down.
135 */
136 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
137}
138
139static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
140{
141 struct cpuhp_step *sp;
142
143 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
144 return sp + state;
145}
146
147/**
148 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
149 * @cpu: The cpu for which the callback should be invoked
150 * @state: The state to do callbacks for
151 * @bringup: True if the bringup callback should be invoked
152 * @node: For multi-instance, do a single entry callback for install/remove
153 * @lastp: For multi-instance rollback, remember how far we got
154 *
155 * Called from cpu hotplug and from the state register machinery.
156 */
157static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
158 bool bringup, struct hlist_node *node,
159 struct hlist_node **lastp)
160{
161 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
162 struct cpuhp_step *step = cpuhp_get_step(state);
163 int (*cbm)(unsigned int cpu, struct hlist_node *node);
164 int (*cb)(unsigned int cpu);
165 int ret, cnt;
166
167 if (st->fail == state) {
168 st->fail = CPUHP_INVALID;
169
170 if (!(bringup ? step->startup.single : step->teardown.single))
171 return 0;
172
173 return -EAGAIN;
174 }
175
176 if (!step->multi_instance) {
177 WARN_ON_ONCE(lastp && *lastp);
178 cb = bringup ? step->startup.single : step->teardown.single;
179 if (!cb)
180 return 0;
181 trace_cpuhp_enter(cpu, st->target, state, cb);
182 ret = cb(cpu);
183 trace_cpuhp_exit(cpu, st->state, state, ret);
184 return ret;
185 }
186 cbm = bringup ? step->startup.multi : step->teardown.multi;
187 if (!cbm)
188 return 0;
189
190 /* Single invocation for instance add/remove */
191 if (node) {
192 WARN_ON_ONCE(lastp && *lastp);
193 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
194 ret = cbm(cpu, node);
195 trace_cpuhp_exit(cpu, st->state, state, ret);
196 return ret;
197 }
198
199 /* State transition. Invoke on all instances */
200 cnt = 0;
201 hlist_for_each(node, &step->list) {
202 if (lastp && node == *lastp)
203 break;
204
205 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
206 ret = cbm(cpu, node);
207 trace_cpuhp_exit(cpu, st->state, state, ret);
208 if (ret) {
209 if (!lastp)
210 goto err;
211
212 *lastp = node;
213 return ret;
214 }
215 cnt++;
216 }
217 if (lastp)
218 *lastp = NULL;
219 return 0;
220err:
221 /* Rollback the instances if one failed */
222 cbm = !bringup ? step->startup.multi : step->teardown.multi;
223 if (!cbm)
224 return ret;
225
226 hlist_for_each(node, &step->list) {
227 if (!cnt--)
228 break;
229
230 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
231 ret = cbm(cpu, node);
232 trace_cpuhp_exit(cpu, st->state, state, ret);
233 /*
234 * Rollback must not fail,
235 */
236 WARN_ON_ONCE(ret);
237 }
238 return ret;
239}
240
241#ifdef CONFIG_SMP
242static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
243{
244 struct completion *done = bringup ? &st->done_up : &st->done_down;
245 wait_for_completion(done);
246}
247
248static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
249{
250 struct completion *done = bringup ? &st->done_up : &st->done_down;
251 complete(done);
252}
253
254/*
255 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
256 */
257static bool cpuhp_is_atomic_state(enum cpuhp_state state)
258{
259 return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
260}
261
262/* Serializes the updates to cpu_online_mask, cpu_present_mask */
263static DEFINE_MUTEX(cpu_add_remove_lock);
264bool cpuhp_tasks_frozen;
265EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
266
267/*
268 * The following two APIs (cpu_maps_update_begin/done) must be used when
269 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
270 */
271void cpu_maps_update_begin(void)
272{
273 mutex_lock(&cpu_add_remove_lock);
274}
275
276void cpu_maps_update_done(void)
277{
278 mutex_unlock(&cpu_add_remove_lock);
279}
280
281/*
282 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
283 * Should always be manipulated under cpu_add_remove_lock
284 */
285static int cpu_hotplug_disabled;
286
287#ifdef CONFIG_HOTPLUG_CPU
288
289DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
290
291void cpus_read_lock(void)
292{
293 percpu_down_read(&cpu_hotplug_lock);
294}
295EXPORT_SYMBOL_GPL(cpus_read_lock);
296
297void cpus_read_unlock(void)
298{
299 percpu_up_read(&cpu_hotplug_lock);
300}
301EXPORT_SYMBOL_GPL(cpus_read_unlock);
302
303void cpus_write_lock(void)
304{
305 percpu_down_write(&cpu_hotplug_lock);
306}
307
308void cpus_write_unlock(void)
309{
310 percpu_up_write(&cpu_hotplug_lock);
311}
312
313void lockdep_assert_cpus_held(void)
314{
315 percpu_rwsem_assert_held(&cpu_hotplug_lock);
316}
317
318/*
319 * Wait for currently running CPU hotplug operations to complete (if any) and
320 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
321 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
322 * hotplug path before performing hotplug operations. So acquiring that lock
323 * guarantees mutual exclusion from any currently running hotplug operations.
324 */
325void cpu_hotplug_disable(void)
326{
327 cpu_maps_update_begin();
328 cpu_hotplug_disabled++;
329 cpu_maps_update_done();
330}
331EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
332
333static void __cpu_hotplug_enable(void)
334{
335 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
336 return;
337 cpu_hotplug_disabled--;
338}
339
340void cpu_hotplug_enable(void)
341{
342 cpu_maps_update_begin();
343 __cpu_hotplug_enable();
344 cpu_maps_update_done();
345}
346EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
347#endif /* CONFIG_HOTPLUG_CPU */
348
349static inline enum cpuhp_state
350cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
351{
352 enum cpuhp_state prev_state = st->state;
353
354 st->rollback = false;
355 st->last = NULL;
356
357 st->target = target;
358 st->single = false;
359 st->bringup = st->state < target;
360
361 return prev_state;
362}
363
364static inline void
365cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
366{
367 st->rollback = true;
368
369 /*
370 * If we have st->last we need to undo partial multi_instance of this
371 * state first. Otherwise start undo at the previous state.
372 */
373 if (!st->last) {
374 if (st->bringup)
375 st->state--;
376 else
377 st->state++;
378 }
379
380 st->target = prev_state;
381 st->bringup = !st->bringup;
382}
383
384/* Regular hotplug invocation of the AP hotplug thread */
385static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
386{
387 if (!st->single && st->state == st->target)
388 return;
389
390 st->result = 0;
391 /*
392 * Make sure the above stores are visible before should_run becomes
393 * true. Paired with the mb() above in cpuhp_thread_fun()
394 */
395 smp_mb();
396 st->should_run = true;
397 wake_up_process(st->thread);
398 wait_for_ap_thread(st, st->bringup);
399}
400
401static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
402{
403 enum cpuhp_state prev_state;
404 int ret;
405
406 prev_state = cpuhp_set_state(st, target);
407 __cpuhp_kick_ap(st);
408 if ((ret = st->result)) {
409 cpuhp_reset_state(st, prev_state);
410 __cpuhp_kick_ap(st);
411 }
412
413 return ret;
414}
415
416static int bringup_wait_for_ap(unsigned int cpu)
417{
418 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
419
420 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
421 wait_for_ap_thread(st, true);
422 if (WARN_ON_ONCE((!cpu_online(cpu))))
423 return -ECANCELED;
424
425 /* Unpark the stopper thread and the hotplug thread of the target cpu */
426 stop_machine_unpark(cpu);
427 kthread_unpark(st->thread);
428
429 if (st->target <= CPUHP_AP_ONLINE_IDLE)
430 return 0;
431
432 return cpuhp_kick_ap(st, st->target);
433}
434
435static int bringup_cpu(unsigned int cpu)
436{
437 struct task_struct *idle = idle_thread_get(cpu);
438 int ret;
439
440 /*
441 * Some architectures have to walk the irq descriptors to
442 * setup the vector space for the cpu which comes online.
443 * Prevent irq alloc/free across the bringup.
444 */
445 irq_lock_sparse();
446
447 /* Arch-specific enabling code. */
448 ret = __cpu_up(cpu, idle);
449 irq_unlock_sparse();
450 if (ret)
451 return ret;
452 return bringup_wait_for_ap(cpu);
453}
454
455/*
456 * Hotplug state machine related functions
457 */
458
459static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
460{
461 for (st->state--; st->state > st->target; st->state--) {
462 struct cpuhp_step *step = cpuhp_get_step(st->state);
463
464 if (!step->skip_onerr)
465 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
466 }
467}
468
469static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
470 enum cpuhp_state target)
471{
472 enum cpuhp_state prev_state = st->state;
473 int ret = 0;
474
475 while (st->state < target) {
476 st->state++;
477 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
478 if (ret) {
479 st->target = prev_state;
480 undo_cpu_up(cpu, st);
481 break;
482 }
483 }
484 return ret;
485}
486
487/*
488 * The cpu hotplug threads manage the bringup and teardown of the cpus
489 */
490static void cpuhp_create(unsigned int cpu)
491{
492 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
493
494 init_completion(&st->done_up);
495 init_completion(&st->done_down);
496}
497
498static int cpuhp_should_run(unsigned int cpu)
499{
500 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
501
502 return st->should_run;
503}
504
505/*
506 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
507 * callbacks when a state gets [un]installed at runtime.
508 *
509 * Each invocation of this function by the smpboot thread does a single AP
510 * state callback.
511 *
512 * It has 3 modes of operation:
513 * - single: runs st->cb_state
514 * - up: runs ++st->state, while st->state < st->target
515 * - down: runs st->state--, while st->state > st->target
516 *
517 * When complete or on error, should_run is cleared and the completion is fired.
518 */
519static void cpuhp_thread_fun(unsigned int cpu)
520{
521 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
522 bool bringup = st->bringup;
523 enum cpuhp_state state;
524
525 /*
526 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
527 * that if we see ->should_run we also see the rest of the state.
528 */
529 smp_mb();
530
531 if (WARN_ON_ONCE(!st->should_run))
532 return;
533
534 cpuhp_lock_acquire(bringup);
535
536 if (st->single) {
537 state = st->cb_state;
538 st->should_run = false;
539 } else {
540 if (bringup) {
541 st->state++;
542 state = st->state;
543 st->should_run = (st->state < st->target);
544 WARN_ON_ONCE(st->state > st->target);
545 } else {
546 state = st->state;
547 st->state--;
548 st->should_run = (st->state > st->target);
549 WARN_ON_ONCE(st->state < st->target);
550 }
551 }
552
553 WARN_ON_ONCE(!cpuhp_is_ap_state(state));
554
555 if (st->rollback) {
556 struct cpuhp_step *step = cpuhp_get_step(state);
557 if (step->skip_onerr)
558 goto next;
559 }
560
561 if (cpuhp_is_atomic_state(state)) {
562 local_irq_disable();
563 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
564 local_irq_enable();
565
566 /*
567 * STARTING/DYING must not fail!
568 */
569 WARN_ON_ONCE(st->result);
570 } else {
571 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
572 }
573
574 if (st->result) {
575 /*
576 * If we fail on a rollback, we're up a creek without no
577 * paddle, no way forward, no way back. We loose, thanks for
578 * playing.
579 */
580 WARN_ON_ONCE(st->rollback);
581 st->should_run = false;
582 }
583
584next:
585 cpuhp_lock_release(bringup);
586
587 if (!st->should_run)
588 complete_ap_thread(st, bringup);
589}
590
591/* Invoke a single callback on a remote cpu */
592static int
593cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
594 struct hlist_node *node)
595{
596 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
597 int ret;
598
599 if (!cpu_online(cpu))
600 return 0;
601
602 cpuhp_lock_acquire(false);
603 cpuhp_lock_release(false);
604
605 cpuhp_lock_acquire(true);
606 cpuhp_lock_release(true);
607
608 /*
609 * If we are up and running, use the hotplug thread. For early calls
610 * we invoke the thread function directly.
611 */
612 if (!st->thread)
613 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
614
615 st->rollback = false;
616 st->last = NULL;
617
618 st->node = node;
619 st->bringup = bringup;
620 st->cb_state = state;
621 st->single = true;
622
623 __cpuhp_kick_ap(st);
624
625 /*
626 * If we failed and did a partial, do a rollback.
627 */
628 if ((ret = st->result) && st->last) {
629 st->rollback = true;
630 st->bringup = !bringup;
631
632 __cpuhp_kick_ap(st);
633 }
634
635 /*
636 * Clean up the leftovers so the next hotplug operation wont use stale
637 * data.
638 */
639 st->node = st->last = NULL;
640 return ret;
641}
642
643static int cpuhp_kick_ap_work(unsigned int cpu)
644{
645 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
646 enum cpuhp_state prev_state = st->state;
647 int ret;
648
649 cpuhp_lock_acquire(false);
650 cpuhp_lock_release(false);
651
652 cpuhp_lock_acquire(true);
653 cpuhp_lock_release(true);
654
655 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
656 ret = cpuhp_kick_ap(st, st->target);
657 trace_cpuhp_exit(cpu, st->state, prev_state, ret);
658
659 return ret;
660}
661
662static struct smp_hotplug_thread cpuhp_threads = {
663 .store = &cpuhp_state.thread,
664 .create = &cpuhp_create,
665 .thread_should_run = cpuhp_should_run,
666 .thread_fn = cpuhp_thread_fun,
667 .thread_comm = "cpuhp/%u",
668 .selfparking = true,
669};
670
671void __init cpuhp_threads_init(void)
672{
673 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
674 kthread_unpark(this_cpu_read(cpuhp_state.thread));
675}
676
677#ifdef CONFIG_HOTPLUG_CPU
678/**
679 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
680 * @cpu: a CPU id
681 *
682 * This function walks all processes, finds a valid mm struct for each one and
683 * then clears a corresponding bit in mm's cpumask. While this all sounds
684 * trivial, there are various non-obvious corner cases, which this function
685 * tries to solve in a safe manner.
686 *
687 * Also note that the function uses a somewhat relaxed locking scheme, so it may
688 * be called only for an already offlined CPU.
689 */
690void clear_tasks_mm_cpumask(int cpu)
691{
692 struct task_struct *p;
693
694 /*
695 * This function is called after the cpu is taken down and marked
696 * offline, so its not like new tasks will ever get this cpu set in
697 * their mm mask. -- Peter Zijlstra
698 * Thus, we may use rcu_read_lock() here, instead of grabbing
699 * full-fledged tasklist_lock.
700 */
701 WARN_ON(cpu_online(cpu));
702 rcu_read_lock();
703 for_each_process(p) {
704 struct task_struct *t;
705
706 /*
707 * Main thread might exit, but other threads may still have
708 * a valid mm. Find one.
709 */
710 t = find_lock_task_mm(p);
711 if (!t)
712 continue;
713 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
714 task_unlock(t);
715 }
716 rcu_read_unlock();
717}
718
719/* Take this CPU down. */
720static int take_cpu_down(void *_param)
721{
722 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
723 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
724 int err, cpu = smp_processor_id();
725 int ret;
726
727 /* Ensure this CPU doesn't handle any more interrupts. */
728 err = __cpu_disable();
729 if (err < 0)
730 return err;
731
732 /*
733 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
734 * do this step again.
735 */
736 WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
737 st->state--;
738 /* Invoke the former CPU_DYING callbacks */
739 for (; st->state > target; st->state--) {
740 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
741 /*
742 * DYING must not fail!
743 */
744 WARN_ON_ONCE(ret);
745 }
746
747 /* Give up timekeeping duties */
748 tick_handover_do_timer();
749 /* Park the stopper thread */
750 stop_machine_park(cpu);
751 return 0;
752}
753
754static int takedown_cpu(unsigned int cpu)
755{
756 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
757 int err;
758
759 /* Park the smpboot threads */
760 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
761 smpboot_park_threads(cpu);
762
763 /*
764 * Prevent irq alloc/free while the dying cpu reorganizes the
765 * interrupt affinities.
766 */
767 irq_lock_sparse();
768
769 /*
770 * So now all preempt/rcu users must observe !cpu_active().
771 */
772 err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
773 if (err) {
774 /* CPU refused to die */
775 irq_unlock_sparse();
776 /* Unpark the hotplug thread so we can rollback there */
777 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
778 return err;
779 }
780 BUG_ON(cpu_online(cpu));
781
782 /*
783 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
784 * all runnable tasks from the CPU, there's only the idle task left now
785 * that the migration thread is done doing the stop_machine thing.
786 *
787 * Wait for the stop thread to go away.
788 */
789 wait_for_ap_thread(st, false);
790 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
791
792 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
793 irq_unlock_sparse();
794
795 hotplug_cpu__broadcast_tick_pull(cpu);
796 /* This actually kills the CPU. */
797 __cpu_die(cpu);
798
799 tick_cleanup_dead_cpu(cpu);
800 rcutree_migrate_callbacks(cpu);
801 return 0;
802}
803
804static void cpuhp_complete_idle_dead(void *arg)
805{
806 struct cpuhp_cpu_state *st = arg;
807
808 complete_ap_thread(st, false);
809}
810
811void cpuhp_report_idle_dead(void)
812{
813 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
814
815 BUG_ON(st->state != CPUHP_AP_OFFLINE);
816 rcu_report_dead(smp_processor_id());
817 st->state = CPUHP_AP_IDLE_DEAD;
818 /*
819 * We cannot call complete after rcu_report_dead() so we delegate it
820 * to an online cpu.
821 */
822 smp_call_function_single(cpumask_first(cpu_online_mask),
823 cpuhp_complete_idle_dead, st, 0);
824}
825
826static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
827{
828 for (st->state++; st->state < st->target; st->state++) {
829 struct cpuhp_step *step = cpuhp_get_step(st->state);
830
831 if (!step->skip_onerr)
832 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
833 }
834}
835
836static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
837 enum cpuhp_state target)
838{
839 enum cpuhp_state prev_state = st->state;
840 int ret = 0;
841
842 for (; st->state > target; st->state--) {
843 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
844 if (ret) {
845 st->target = prev_state;
846 undo_cpu_down(cpu, st);
847 break;
848 }
849 }
850 return ret;
851}
852
853/* Requires cpu_add_remove_lock to be held */
854static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
855 enum cpuhp_state target)
856{
857 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
858 int prev_state, ret = 0;
859
860 if (num_online_cpus() == 1)
861 return -EBUSY;
862
863 if (!cpu_present(cpu))
864 return -EINVAL;
865
866 cpus_write_lock();
867
868 cpuhp_tasks_frozen = tasks_frozen;
869
870 prev_state = cpuhp_set_state(st, target);
871 /*
872 * If the current CPU state is in the range of the AP hotplug thread,
873 * then we need to kick the thread.
874 */
875 if (st->state > CPUHP_TEARDOWN_CPU) {
876 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
877 ret = cpuhp_kick_ap_work(cpu);
878 /*
879 * The AP side has done the error rollback already. Just
880 * return the error code..
881 */
882 if (ret)
883 goto out;
884
885 /*
886 * We might have stopped still in the range of the AP hotplug
887 * thread. Nothing to do anymore.
888 */
889 if (st->state > CPUHP_TEARDOWN_CPU)
890 goto out;
891
892 st->target = target;
893 }
894 /*
895 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
896 * to do the further cleanups.
897 */
898 ret = cpuhp_down_callbacks(cpu, st, target);
899 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) {
900 cpuhp_reset_state(st, prev_state);
901 __cpuhp_kick_ap(st);
902 }
903
904out:
905 cpus_write_unlock();
906 /*
907 * Do post unplug cleanup. This is still protected against
908 * concurrent CPU hotplug via cpu_add_remove_lock.
909 */
910 lockup_detector_cleanup();
911 return ret;
912}
913
914static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
915{
916 int err;
917
918 cpu_maps_update_begin();
919
920 if (cpu_hotplug_disabled) {
921 err = -EBUSY;
922 goto out;
923 }
924
925 err = _cpu_down(cpu, 0, target);
926
927out:
928 cpu_maps_update_done();
929 return err;
930}
931
932int cpu_down(unsigned int cpu)
933{
934 return do_cpu_down(cpu, CPUHP_OFFLINE);
935}
936EXPORT_SYMBOL(cpu_down);
937
938#else
939#define takedown_cpu NULL
940#endif /*CONFIG_HOTPLUG_CPU*/
941
942/**
943 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
944 * @cpu: cpu that just started
945 *
946 * It must be called by the arch code on the new cpu, before the new cpu
947 * enables interrupts and before the "boot" cpu returns from __cpu_up().
948 */
949void notify_cpu_starting(unsigned int cpu)
950{
951 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
952 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
953 int ret;
954
955 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
956 while (st->state < target) {
957 st->state++;
958 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
959 /*
960 * STARTING must not fail!
961 */
962 WARN_ON_ONCE(ret);
963 }
964}
965
966/*
967 * Called from the idle task. Wake up the controlling task which brings the
968 * stopper and the hotplug thread of the upcoming CPU up and then delegates
969 * the rest of the online bringup to the hotplug thread.
970 */
971void cpuhp_online_idle(enum cpuhp_state state)
972{
973 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
974
975 /* Happens for the boot cpu */
976 if (state != CPUHP_AP_ONLINE_IDLE)
977 return;
978
979 st->state = CPUHP_AP_ONLINE_IDLE;
980 complete_ap_thread(st, true);
981}
982
983/* Requires cpu_add_remove_lock to be held */
984static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
985{
986 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
987 struct task_struct *idle;
988 int ret = 0;
989
990 cpus_write_lock();
991
992 if (!cpu_present(cpu)) {
993 ret = -EINVAL;
994 goto out;
995 }
996
997 /*
998 * The caller of do_cpu_up might have raced with another
999 * caller. Ignore it for now.
1000 */
1001 if (st->state >= target)
1002 goto out;
1003
1004 if (st->state == CPUHP_OFFLINE) {
1005 /* Let it fail before we try to bring the cpu up */
1006 idle = idle_thread_get(cpu);
1007 if (IS_ERR(idle)) {
1008 ret = PTR_ERR(idle);
1009 goto out;
1010 }
1011 }
1012
1013 cpuhp_tasks_frozen = tasks_frozen;
1014
1015 cpuhp_set_state(st, target);
1016 /*
1017 * If the current CPU state is in the range of the AP hotplug thread,
1018 * then we need to kick the thread once more.
1019 */
1020 if (st->state > CPUHP_BRINGUP_CPU) {
1021 ret = cpuhp_kick_ap_work(cpu);
1022 /*
1023 * The AP side has done the error rollback already. Just
1024 * return the error code..
1025 */
1026 if (ret)
1027 goto out;
1028 }
1029
1030 /*
1031 * Try to reach the target state. We max out on the BP at
1032 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1033 * responsible for bringing it up to the target state.
1034 */
1035 target = min((int)target, CPUHP_BRINGUP_CPU);
1036 ret = cpuhp_up_callbacks(cpu, st, target);
1037out:
1038 cpus_write_unlock();
1039 return ret;
1040}
1041
1042static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1043{
1044 int err = 0;
1045
1046 if (!cpu_possible(cpu)) {
1047 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1048 cpu);
1049#if defined(CONFIG_IA64)
1050 pr_err("please check additional_cpus= boot parameter\n");
1051#endif
1052 return -EINVAL;
1053 }
1054
1055 err = try_online_node(cpu_to_node(cpu));
1056 if (err)
1057 return err;
1058
1059 cpu_maps_update_begin();
1060
1061 if (cpu_hotplug_disabled) {
1062 err = -EBUSY;
1063 goto out;
1064 }
1065
1066 err = _cpu_up(cpu, 0, target);
1067out:
1068 cpu_maps_update_done();
1069 return err;
1070}
1071
1072int cpu_up(unsigned int cpu)
1073{
1074 return do_cpu_up(cpu, CPUHP_ONLINE);
1075}
1076EXPORT_SYMBOL_GPL(cpu_up);
1077
1078#ifdef CONFIG_PM_SLEEP_SMP
1079static cpumask_var_t frozen_cpus;
1080
1081int freeze_secondary_cpus(int primary)
1082{
1083 int cpu, error = 0;
1084
1085 cpu_maps_update_begin();
1086 if (!cpu_online(primary))
1087 primary = cpumask_first(cpu_online_mask);
1088 /*
1089 * We take down all of the non-boot CPUs in one shot to avoid races
1090 * with the userspace trying to use the CPU hotplug at the same time
1091 */
1092 cpumask_clear(frozen_cpus);
1093
1094 pr_info("Disabling non-boot CPUs ...\n");
1095 for_each_online_cpu(cpu) {
1096 if (cpu == primary)
1097 continue;
1098 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1099 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1100 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1101 if (!error)
1102 cpumask_set_cpu(cpu, frozen_cpus);
1103 else {
1104 pr_err("Error taking CPU%d down: %d\n", cpu, error);
1105 break;
1106 }
1107 }
1108
1109 if (!error)
1110 BUG_ON(num_online_cpus() > 1);
1111 else
1112 pr_err("Non-boot CPUs are not disabled\n");
1113
1114 /*
1115 * Make sure the CPUs won't be enabled by someone else. We need to do
1116 * this even in case of failure as all disable_nonboot_cpus() users are
1117 * supposed to do enable_nonboot_cpus() on the failure path.
1118 */
1119 cpu_hotplug_disabled++;
1120
1121 cpu_maps_update_done();
1122 return error;
1123}
1124
1125void __weak arch_enable_nonboot_cpus_begin(void)
1126{
1127}
1128
1129void __weak arch_enable_nonboot_cpus_end(void)
1130{
1131}
1132
1133void enable_nonboot_cpus(void)
1134{
1135 int cpu, error;
1136
1137 /* Allow everyone to use the CPU hotplug again */
1138 cpu_maps_update_begin();
1139 __cpu_hotplug_enable();
1140 if (cpumask_empty(frozen_cpus))
1141 goto out;
1142
1143 pr_info("Enabling non-boot CPUs ...\n");
1144
1145 arch_enable_nonboot_cpus_begin();
1146
1147 for_each_cpu(cpu, frozen_cpus) {
1148 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1149 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1150 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1151 if (!error) {
1152 pr_info("CPU%d is up\n", cpu);
1153 continue;
1154 }
1155 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1156 }
1157
1158 arch_enable_nonboot_cpus_end();
1159
1160 cpumask_clear(frozen_cpus);
1161out:
1162 cpu_maps_update_done();
1163}
1164
1165static int __init alloc_frozen_cpus(void)
1166{
1167 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1168 return -ENOMEM;
1169 return 0;
1170}
1171core_initcall(alloc_frozen_cpus);
1172
1173/*
1174 * When callbacks for CPU hotplug notifications are being executed, we must
1175 * ensure that the state of the system with respect to the tasks being frozen
1176 * or not, as reported by the notification, remains unchanged *throughout the
1177 * duration* of the execution of the callbacks.
1178 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1179 *
1180 * This synchronization is implemented by mutually excluding regular CPU
1181 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1182 * Hibernate notifications.
1183 */
1184static int
1185cpu_hotplug_pm_callback(struct notifier_block *nb,
1186 unsigned long action, void *ptr)
1187{
1188 switch (action) {
1189
1190 case PM_SUSPEND_PREPARE:
1191 case PM_HIBERNATION_PREPARE:
1192 cpu_hotplug_disable();
1193 break;
1194
1195 case PM_POST_SUSPEND:
1196 case PM_POST_HIBERNATION:
1197 cpu_hotplug_enable();
1198 break;
1199
1200 default:
1201 return NOTIFY_DONE;
1202 }
1203
1204 return NOTIFY_OK;
1205}
1206
1207
1208static int __init cpu_hotplug_pm_sync_init(void)
1209{
1210 /*
1211 * cpu_hotplug_pm_callback has higher priority than x86
1212 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1213 * to disable cpu hotplug to avoid cpu hotplug race.
1214 */
1215 pm_notifier(cpu_hotplug_pm_callback, 0);
1216 return 0;
1217}
1218core_initcall(cpu_hotplug_pm_sync_init);
1219
1220#endif /* CONFIG_PM_SLEEP_SMP */
1221
1222int __boot_cpu_id;
1223
1224#endif /* CONFIG_SMP */
1225
1226/* Boot processor state steps */
1227static struct cpuhp_step cpuhp_bp_states[] = {
1228 [CPUHP_OFFLINE] = {
1229 .name = "offline",
1230 .startup.single = NULL,
1231 .teardown.single = NULL,
1232 },
1233#ifdef CONFIG_SMP
1234 [CPUHP_CREATE_THREADS]= {
1235 .name = "threads:prepare",
1236 .startup.single = smpboot_create_threads,
1237 .teardown.single = NULL,
1238 .cant_stop = true,
1239 },
1240 [CPUHP_PERF_PREPARE] = {
1241 .name = "perf:prepare",
1242 .startup.single = perf_event_init_cpu,
1243 .teardown.single = perf_event_exit_cpu,
1244 },
1245 [CPUHP_WORKQUEUE_PREP] = {
1246 .name = "workqueue:prepare",
1247 .startup.single = workqueue_prepare_cpu,
1248 .teardown.single = NULL,
1249 },
1250 [CPUHP_HRTIMERS_PREPARE] = {
1251 .name = "hrtimers:prepare",
1252 .startup.single = hrtimers_prepare_cpu,
1253 .teardown.single = hrtimers_dead_cpu,
1254 },
1255 [CPUHP_SMPCFD_PREPARE] = {
1256 .name = "smpcfd:prepare",
1257 .startup.single = smpcfd_prepare_cpu,
1258 .teardown.single = smpcfd_dead_cpu,
1259 },
1260 [CPUHP_RELAY_PREPARE] = {
1261 .name = "relay:prepare",
1262 .startup.single = relay_prepare_cpu,
1263 .teardown.single = NULL,
1264 },
1265 [CPUHP_SLAB_PREPARE] = {
1266 .name = "slab:prepare",
1267 .startup.single = slab_prepare_cpu,
1268 .teardown.single = slab_dead_cpu,
1269 },
1270 [CPUHP_RCUTREE_PREP] = {
1271 .name = "RCU/tree:prepare",
1272 .startup.single = rcutree_prepare_cpu,
1273 .teardown.single = rcutree_dead_cpu,
1274 },
1275 /*
1276 * On the tear-down path, timers_dead_cpu() must be invoked
1277 * before blk_mq_queue_reinit_notify() from notify_dead(),
1278 * otherwise a RCU stall occurs.
1279 */
1280 [CPUHP_TIMERS_PREPARE] = {
1281 .name = "timers:dead",
1282 .startup.single = timers_prepare_cpu,
1283 .teardown.single = timers_dead_cpu,
1284 },
1285 /* Kicks the plugged cpu into life */
1286 [CPUHP_BRINGUP_CPU] = {
1287 .name = "cpu:bringup",
1288 .startup.single = bringup_cpu,
1289 .teardown.single = NULL,
1290 .cant_stop = true,
1291 },
1292 /*
1293 * Handled on controll processor until the plugged processor manages
1294 * this itself.
1295 */
1296 [CPUHP_TEARDOWN_CPU] = {
1297 .name = "cpu:teardown",
1298 .startup.single = NULL,
1299 .teardown.single = takedown_cpu,
1300 .cant_stop = true,
1301 },
1302#else
1303 [CPUHP_BRINGUP_CPU] = { },
1304#endif
1305};
1306
1307/* Application processor state steps */
1308static struct cpuhp_step cpuhp_ap_states[] = {
1309#ifdef CONFIG_SMP
1310 /* Final state before CPU kills itself */
1311 [CPUHP_AP_IDLE_DEAD] = {
1312 .name = "idle:dead",
1313 },
1314 /*
1315 * Last state before CPU enters the idle loop to die. Transient state
1316 * for synchronization.
1317 */
1318 [CPUHP_AP_OFFLINE] = {
1319 .name = "ap:offline",
1320 .cant_stop = true,
1321 },
1322 /* First state is scheduler control. Interrupts are disabled */
1323 [CPUHP_AP_SCHED_STARTING] = {
1324 .name = "sched:starting",
1325 .startup.single = sched_cpu_starting,
1326 .teardown.single = sched_cpu_dying,
1327 },
1328 [CPUHP_AP_RCUTREE_DYING] = {
1329 .name = "RCU/tree:dying",
1330 .startup.single = NULL,
1331 .teardown.single = rcutree_dying_cpu,
1332 },
1333 [CPUHP_AP_SMPCFD_DYING] = {
1334 .name = "smpcfd:dying",
1335 .startup.single = NULL,
1336 .teardown.single = smpcfd_dying_cpu,
1337 },
1338 /* Entry state on starting. Interrupts enabled from here on. Transient
1339 * state for synchronsization */
1340 [CPUHP_AP_ONLINE] = {
1341 .name = "ap:online",
1342 },
1343 /* Handle smpboot threads park/unpark */
1344 [CPUHP_AP_SMPBOOT_THREADS] = {
1345 .name = "smpboot/threads:online",
1346 .startup.single = smpboot_unpark_threads,
1347 .teardown.single = NULL,
1348 },
1349 [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1350 .name = "irq/affinity:online",
1351 .startup.single = irq_affinity_online_cpu,
1352 .teardown.single = NULL,
1353 },
1354 [CPUHP_AP_PERF_ONLINE] = {
1355 .name = "perf:online",
1356 .startup.single = perf_event_init_cpu,
1357 .teardown.single = perf_event_exit_cpu,
1358 },
1359 [CPUHP_AP_WORKQUEUE_ONLINE] = {
1360 .name = "workqueue:online",
1361 .startup.single = workqueue_online_cpu,
1362 .teardown.single = workqueue_offline_cpu,
1363 },
1364 [CPUHP_AP_RCUTREE_ONLINE] = {
1365 .name = "RCU/tree:online",
1366 .startup.single = rcutree_online_cpu,
1367 .teardown.single = rcutree_offline_cpu,
1368 },
1369#endif
1370 /*
1371 * The dynamically registered state space is here
1372 */
1373
1374#ifdef CONFIG_SMP
1375 /* Last state is scheduler control setting the cpu active */
1376 [CPUHP_AP_ACTIVE] = {
1377 .name = "sched:active",
1378 .startup.single = sched_cpu_activate,
1379 .teardown.single = sched_cpu_deactivate,
1380 },
1381#endif
1382
1383 /* CPU is fully up and running. */
1384 [CPUHP_ONLINE] = {
1385 .name = "online",
1386 .startup.single = NULL,
1387 .teardown.single = NULL,
1388 },
1389};
1390
1391/* Sanity check for callbacks */
1392static int cpuhp_cb_check(enum cpuhp_state state)
1393{
1394 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1395 return -EINVAL;
1396 return 0;
1397}
1398
1399/*
1400 * Returns a free for dynamic slot assignment of the Online state. The states
1401 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1402 * by having no name assigned.
1403 */
1404static int cpuhp_reserve_state(enum cpuhp_state state)
1405{
1406 enum cpuhp_state i, end;
1407 struct cpuhp_step *step;
1408
1409 switch (state) {
1410 case CPUHP_AP_ONLINE_DYN:
1411 step = cpuhp_ap_states + CPUHP_AP_ONLINE_DYN;
1412 end = CPUHP_AP_ONLINE_DYN_END;
1413 break;
1414 case CPUHP_BP_PREPARE_DYN:
1415 step = cpuhp_bp_states + CPUHP_BP_PREPARE_DYN;
1416 end = CPUHP_BP_PREPARE_DYN_END;
1417 break;
1418 default:
1419 return -EINVAL;
1420 }
1421
1422 for (i = state; i <= end; i++, step++) {
1423 if (!step->name)
1424 return i;
1425 }
1426 WARN(1, "No more dynamic states available for CPU hotplug\n");
1427 return -ENOSPC;
1428}
1429
1430static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1431 int (*startup)(unsigned int cpu),
1432 int (*teardown)(unsigned int cpu),
1433 bool multi_instance)
1434{
1435 /* (Un)Install the callbacks for further cpu hotplug operations */
1436 struct cpuhp_step *sp;
1437 int ret = 0;
1438
1439 /*
1440 * If name is NULL, then the state gets removed.
1441 *
1442 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1443 * the first allocation from these dynamic ranges, so the removal
1444 * would trigger a new allocation and clear the wrong (already
1445 * empty) state, leaving the callbacks of the to be cleared state
1446 * dangling, which causes wreckage on the next hotplug operation.
1447 */
1448 if (name && (state == CPUHP_AP_ONLINE_DYN ||
1449 state == CPUHP_BP_PREPARE_DYN)) {
1450 ret = cpuhp_reserve_state(state);
1451 if (ret < 0)
1452 return ret;
1453 state = ret;
1454 }
1455 sp = cpuhp_get_step(state);
1456 if (name && sp->name)
1457 return -EBUSY;
1458
1459 sp->startup.single = startup;
1460 sp->teardown.single = teardown;
1461 sp->name = name;
1462 sp->multi_instance = multi_instance;
1463 INIT_HLIST_HEAD(&sp->list);
1464 return ret;
1465}
1466
1467static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1468{
1469 return cpuhp_get_step(state)->teardown.single;
1470}
1471
1472/*
1473 * Call the startup/teardown function for a step either on the AP or
1474 * on the current CPU.
1475 */
1476static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1477 struct hlist_node *node)
1478{
1479 struct cpuhp_step *sp = cpuhp_get_step(state);
1480 int ret;
1481
1482 /*
1483 * If there's nothing to do, we done.
1484 * Relies on the union for multi_instance.
1485 */
1486 if ((bringup && !sp->startup.single) ||
1487 (!bringup && !sp->teardown.single))
1488 return 0;
1489 /*
1490 * The non AP bound callbacks can fail on bringup. On teardown
1491 * e.g. module removal we crash for now.
1492 */
1493#ifdef CONFIG_SMP
1494 if (cpuhp_is_ap_state(state))
1495 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1496 else
1497 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1498#else
1499 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1500#endif
1501 BUG_ON(ret && !bringup);
1502 return ret;
1503}
1504
1505/*
1506 * Called from __cpuhp_setup_state on a recoverable failure.
1507 *
1508 * Note: The teardown callbacks for rollback are not allowed to fail!
1509 */
1510static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1511 struct hlist_node *node)
1512{
1513 int cpu;
1514
1515 /* Roll back the already executed steps on the other cpus */
1516 for_each_present_cpu(cpu) {
1517 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1518 int cpustate = st->state;
1519
1520 if (cpu >= failedcpu)
1521 break;
1522
1523 /* Did we invoke the startup call on that cpu ? */
1524 if (cpustate >= state)
1525 cpuhp_issue_call(cpu, state, false, node);
1526 }
1527}
1528
1529int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1530 struct hlist_node *node,
1531 bool invoke)
1532{
1533 struct cpuhp_step *sp;
1534 int cpu;
1535 int ret;
1536
1537 lockdep_assert_cpus_held();
1538
1539 sp = cpuhp_get_step(state);
1540 if (sp->multi_instance == false)
1541 return -EINVAL;
1542
1543 mutex_lock(&cpuhp_state_mutex);
1544
1545 if (!invoke || !sp->startup.multi)
1546 goto add_node;
1547
1548 /*
1549 * Try to call the startup callback for each present cpu
1550 * depending on the hotplug state of the cpu.
1551 */
1552 for_each_present_cpu(cpu) {
1553 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1554 int cpustate = st->state;
1555
1556 if (cpustate < state)
1557 continue;
1558
1559 ret = cpuhp_issue_call(cpu, state, true, node);
1560 if (ret) {
1561 if (sp->teardown.multi)
1562 cpuhp_rollback_install(cpu, state, node);
1563 goto unlock;
1564 }
1565 }
1566add_node:
1567 ret = 0;
1568 hlist_add_head(node, &sp->list);
1569unlock:
1570 mutex_unlock(&cpuhp_state_mutex);
1571 return ret;
1572}
1573
1574int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1575 bool invoke)
1576{
1577 int ret;
1578
1579 cpus_read_lock();
1580 ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1581 cpus_read_unlock();
1582 return ret;
1583}
1584EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1585
1586/**
1587 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1588 * @state: The state to setup
1589 * @invoke: If true, the startup function is invoked for cpus where
1590 * cpu state >= @state
1591 * @startup: startup callback function
1592 * @teardown: teardown callback function
1593 * @multi_instance: State is set up for multiple instances which get
1594 * added afterwards.
1595 *
1596 * The caller needs to hold cpus read locked while calling this function.
1597 * Returns:
1598 * On success:
1599 * Positive state number if @state is CPUHP_AP_ONLINE_DYN
1600 * 0 for all other states
1601 * On failure: proper (negative) error code
1602 */
1603int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1604 const char *name, bool invoke,
1605 int (*startup)(unsigned int cpu),
1606 int (*teardown)(unsigned int cpu),
1607 bool multi_instance)
1608{
1609 int cpu, ret = 0;
1610 bool dynstate;
1611
1612 lockdep_assert_cpus_held();
1613
1614 if (cpuhp_cb_check(state) || !name)
1615 return -EINVAL;
1616
1617 mutex_lock(&cpuhp_state_mutex);
1618
1619 ret = cpuhp_store_callbacks(state, name, startup, teardown,
1620 multi_instance);
1621
1622 dynstate = state == CPUHP_AP_ONLINE_DYN;
1623 if (ret > 0 && dynstate) {
1624 state = ret;
1625 ret = 0;
1626 }
1627
1628 if (ret || !invoke || !startup)
1629 goto out;
1630
1631 /*
1632 * Try to call the startup callback for each present cpu
1633 * depending on the hotplug state of the cpu.
1634 */
1635 for_each_present_cpu(cpu) {
1636 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1637 int cpustate = st->state;
1638
1639 if (cpustate < state)
1640 continue;
1641
1642 ret = cpuhp_issue_call(cpu, state, true, NULL);
1643 if (ret) {
1644 if (teardown)
1645 cpuhp_rollback_install(cpu, state, NULL);
1646 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1647 goto out;
1648 }
1649 }
1650out:
1651 mutex_unlock(&cpuhp_state_mutex);
1652 /*
1653 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1654 * dynamically allocated state in case of success.
1655 */
1656 if (!ret && dynstate)
1657 return state;
1658 return ret;
1659}
1660EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1661
1662int __cpuhp_setup_state(enum cpuhp_state state,
1663 const char *name, bool invoke,
1664 int (*startup)(unsigned int cpu),
1665 int (*teardown)(unsigned int cpu),
1666 bool multi_instance)
1667{
1668 int ret;
1669
1670 cpus_read_lock();
1671 ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1672 teardown, multi_instance);
1673 cpus_read_unlock();
1674 return ret;
1675}
1676EXPORT_SYMBOL(__cpuhp_setup_state);
1677
1678int __cpuhp_state_remove_instance(enum cpuhp_state state,
1679 struct hlist_node *node, bool invoke)
1680{
1681 struct cpuhp_step *sp = cpuhp_get_step(state);
1682 int cpu;
1683
1684 BUG_ON(cpuhp_cb_check(state));
1685
1686 if (!sp->multi_instance)
1687 return -EINVAL;
1688
1689 cpus_read_lock();
1690 mutex_lock(&cpuhp_state_mutex);
1691
1692 if (!invoke || !cpuhp_get_teardown_cb(state))
1693 goto remove;
1694 /*
1695 * Call the teardown callback for each present cpu depending
1696 * on the hotplug state of the cpu. This function is not
1697 * allowed to fail currently!
1698 */
1699 for_each_present_cpu(cpu) {
1700 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1701 int cpustate = st->state;
1702
1703 if (cpustate >= state)
1704 cpuhp_issue_call(cpu, state, false, node);
1705 }
1706
1707remove:
1708 hlist_del(node);
1709 mutex_unlock(&cpuhp_state_mutex);
1710 cpus_read_unlock();
1711
1712 return 0;
1713}
1714EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1715
1716/**
1717 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1718 * @state: The state to remove
1719 * @invoke: If true, the teardown function is invoked for cpus where
1720 * cpu state >= @state
1721 *
1722 * The caller needs to hold cpus read locked while calling this function.
1723 * The teardown callback is currently not allowed to fail. Think
1724 * about module removal!
1725 */
1726void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1727{
1728 struct cpuhp_step *sp = cpuhp_get_step(state);
1729 int cpu;
1730
1731 BUG_ON(cpuhp_cb_check(state));
1732
1733 lockdep_assert_cpus_held();
1734
1735 mutex_lock(&cpuhp_state_mutex);
1736 if (sp->multi_instance) {
1737 WARN(!hlist_empty(&sp->list),
1738 "Error: Removing state %d which has instances left.\n",
1739 state);
1740 goto remove;
1741 }
1742
1743 if (!invoke || !cpuhp_get_teardown_cb(state))
1744 goto remove;
1745
1746 /*
1747 * Call the teardown callback for each present cpu depending
1748 * on the hotplug state of the cpu. This function is not
1749 * allowed to fail currently!
1750 */
1751 for_each_present_cpu(cpu) {
1752 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1753 int cpustate = st->state;
1754
1755 if (cpustate >= state)
1756 cpuhp_issue_call(cpu, state, false, NULL);
1757 }
1758remove:
1759 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1760 mutex_unlock(&cpuhp_state_mutex);
1761}
1762EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1763
1764void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1765{
1766 cpus_read_lock();
1767 __cpuhp_remove_state_cpuslocked(state, invoke);
1768 cpus_read_unlock();
1769}
1770EXPORT_SYMBOL(__cpuhp_remove_state);
1771
1772#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1773static ssize_t show_cpuhp_state(struct device *dev,
1774 struct device_attribute *attr, char *buf)
1775{
1776 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1777
1778 return sprintf(buf, "%d\n", st->state);
1779}
1780static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1781
1782static ssize_t write_cpuhp_target(struct device *dev,
1783 struct device_attribute *attr,
1784 const char *buf, size_t count)
1785{
1786 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1787 struct cpuhp_step *sp;
1788 int target, ret;
1789
1790 ret = kstrtoint(buf, 10, &target);
1791 if (ret)
1792 return ret;
1793
1794#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1795 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1796 return -EINVAL;
1797#else
1798 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1799 return -EINVAL;
1800#endif
1801
1802 ret = lock_device_hotplug_sysfs();
1803 if (ret)
1804 return ret;
1805
1806 mutex_lock(&cpuhp_state_mutex);
1807 sp = cpuhp_get_step(target);
1808 ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1809 mutex_unlock(&cpuhp_state_mutex);
1810 if (ret)
1811 goto out;
1812
1813 if (st->state < target)
1814 ret = do_cpu_up(dev->id, target);
1815 else
1816 ret = do_cpu_down(dev->id, target);
1817out:
1818 unlock_device_hotplug();
1819 return ret ? ret : count;
1820}
1821
1822static ssize_t show_cpuhp_target(struct device *dev,
1823 struct device_attribute *attr, char *buf)
1824{
1825 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1826
1827 return sprintf(buf, "%d\n", st->target);
1828}
1829static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1830
1831
1832static ssize_t write_cpuhp_fail(struct device *dev,
1833 struct device_attribute *attr,
1834 const char *buf, size_t count)
1835{
1836 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1837 struct cpuhp_step *sp;
1838 int fail, ret;
1839
1840 ret = kstrtoint(buf, 10, &fail);
1841 if (ret)
1842 return ret;
1843
1844 /*
1845 * Cannot fail STARTING/DYING callbacks.
1846 */
1847 if (cpuhp_is_atomic_state(fail))
1848 return -EINVAL;
1849
1850 /*
1851 * Cannot fail anything that doesn't have callbacks.
1852 */
1853 mutex_lock(&cpuhp_state_mutex);
1854 sp = cpuhp_get_step(fail);
1855 if (!sp->startup.single && !sp->teardown.single)
1856 ret = -EINVAL;
1857 mutex_unlock(&cpuhp_state_mutex);
1858 if (ret)
1859 return ret;
1860
1861 st->fail = fail;
1862
1863 return count;
1864}
1865
1866static ssize_t show_cpuhp_fail(struct device *dev,
1867 struct device_attribute *attr, char *buf)
1868{
1869 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1870
1871 return sprintf(buf, "%d\n", st->fail);
1872}
1873
1874static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
1875
1876static struct attribute *cpuhp_cpu_attrs[] = {
1877 &dev_attr_state.attr,
1878 &dev_attr_target.attr,
1879 &dev_attr_fail.attr,
1880 NULL
1881};
1882
1883static const struct attribute_group cpuhp_cpu_attr_group = {
1884 .attrs = cpuhp_cpu_attrs,
1885 .name = "hotplug",
1886 NULL
1887};
1888
1889static ssize_t show_cpuhp_states(struct device *dev,
1890 struct device_attribute *attr, char *buf)
1891{
1892 ssize_t cur, res = 0;
1893 int i;
1894
1895 mutex_lock(&cpuhp_state_mutex);
1896 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
1897 struct cpuhp_step *sp = cpuhp_get_step(i);
1898
1899 if (sp->name) {
1900 cur = sprintf(buf, "%3d: %s\n", i, sp->name);
1901 buf += cur;
1902 res += cur;
1903 }
1904 }
1905 mutex_unlock(&cpuhp_state_mutex);
1906 return res;
1907}
1908static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
1909
1910static struct attribute *cpuhp_cpu_root_attrs[] = {
1911 &dev_attr_states.attr,
1912 NULL
1913};
1914
1915static const struct attribute_group cpuhp_cpu_root_attr_group = {
1916 .attrs = cpuhp_cpu_root_attrs,
1917 .name = "hotplug",
1918 NULL
1919};
1920
1921static int __init cpuhp_sysfs_init(void)
1922{
1923 int cpu, ret;
1924
1925 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
1926 &cpuhp_cpu_root_attr_group);
1927 if (ret)
1928 return ret;
1929
1930 for_each_possible_cpu(cpu) {
1931 struct device *dev = get_cpu_device(cpu);
1932
1933 if (!dev)
1934 continue;
1935 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
1936 if (ret)
1937 return ret;
1938 }
1939 return 0;
1940}
1941device_initcall(cpuhp_sysfs_init);
1942#endif
1943
1944/*
1945 * cpu_bit_bitmap[] is a special, "compressed" data structure that
1946 * represents all NR_CPUS bits binary values of 1<<nr.
1947 *
1948 * It is used by cpumask_of() to get a constant address to a CPU
1949 * mask value that has a single bit set only.
1950 */
1951
1952/* cpu_bit_bitmap[0] is empty - so we can back into it */
1953#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
1954#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
1955#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
1956#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
1957
1958const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
1959
1960 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
1961 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
1962#if BITS_PER_LONG > 32
1963 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
1964 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
1965#endif
1966};
1967EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
1968
1969const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
1970EXPORT_SYMBOL(cpu_all_bits);
1971
1972#ifdef CONFIG_INIT_ALL_POSSIBLE
1973struct cpumask __cpu_possible_mask __read_mostly
1974 = {CPU_BITS_ALL};
1975#else
1976struct cpumask __cpu_possible_mask __read_mostly;
1977#endif
1978EXPORT_SYMBOL(__cpu_possible_mask);
1979
1980struct cpumask __cpu_online_mask __read_mostly;
1981EXPORT_SYMBOL(__cpu_online_mask);
1982
1983struct cpumask __cpu_present_mask __read_mostly;
1984EXPORT_SYMBOL(__cpu_present_mask);
1985
1986struct cpumask __cpu_active_mask __read_mostly;
1987EXPORT_SYMBOL(__cpu_active_mask);
1988
1989void init_cpu_present(const struct cpumask *src)
1990{
1991 cpumask_copy(&__cpu_present_mask, src);
1992}
1993
1994void init_cpu_possible(const struct cpumask *src)
1995{
1996 cpumask_copy(&__cpu_possible_mask, src);
1997}
1998
1999void init_cpu_online(const struct cpumask *src)
2000{
2001 cpumask_copy(&__cpu_online_mask, src);
2002}
2003
2004/*
2005 * Activate the first processor.
2006 */
2007void __init boot_cpu_init(void)
2008{
2009 int cpu = smp_processor_id();
2010
2011 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2012 set_cpu_online(cpu, true);
2013 set_cpu_active(cpu, true);
2014 set_cpu_present(cpu, true);
2015 set_cpu_possible(cpu, true);
2016
2017#ifdef CONFIG_SMP
2018 __boot_cpu_id = cpu;
2019#endif
2020}
2021
2022/*
2023 * Must be called _AFTER_ setting up the per_cpu areas
2024 */
2025void __init boot_cpu_state_init(void)
2026{
2027 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE;
2028}