projects/linux·blob v4.4-rc4 · refs

829 lines · 20.1 KiB

1/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/unistd.h>
12#include <linux/cpu.h>
13#include <linux/oom.h>
14#include <linux/rcupdate.h>
15#include <linux/export.h>
16#include <linux/bug.h>
17#include <linux/kthread.h>
18#include <linux/stop_machine.h>
19#include <linux/mutex.h>
20#include <linux/gfp.h>
21#include <linux/suspend.h>
22#include <linux/lockdep.h>
23#include <linux/tick.h>
24#include <linux/irq.h>
25#include <trace/events/power.h>
26
27#include "smpboot.h"
28
29#ifdef CONFIG_SMP
30/* Serializes the updates to cpu_online_mask, cpu_present_mask */
31static DEFINE_MUTEX(cpu_add_remove_lock);
32
33/*
34 * The following two APIs (cpu_maps_update_begin/done) must be used when
35 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
36 * The APIs cpu_notifier_register_begin/done() must be used to protect CPU
37 * hotplug callback (un)registration performed using __register_cpu_notifier()
38 * or __unregister_cpu_notifier().
39 */
40void cpu_maps_update_begin(void)
41{
42 mutex_lock(&cpu_add_remove_lock);
43}
44EXPORT_SYMBOL(cpu_notifier_register_begin);
45
46void cpu_maps_update_done(void)
47{
48 mutex_unlock(&cpu_add_remove_lock);
49}
50EXPORT_SYMBOL(cpu_notifier_register_done);
51
52static RAW_NOTIFIER_HEAD(cpu_chain);
53
54/* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
55 * Should always be manipulated under cpu_add_remove_lock
56 */
57static int cpu_hotplug_disabled;
58
59#ifdef CONFIG_HOTPLUG_CPU
60
61static struct {
62 struct task_struct *active_writer;
63 /* wait queue to wake up the active_writer */
64 wait_queue_head_t wq;
65 /* verifies that no writer will get active while readers are active */
66 struct mutex lock;
67 /*
68 * Also blocks the new readers during
69 * an ongoing cpu hotplug operation.
70 */
71 atomic_t refcount;
72
73#ifdef CONFIG_DEBUG_LOCK_ALLOC
74 struct lockdep_map dep_map;
75#endif
76} cpu_hotplug = {
77 .active_writer = NULL,
78 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
79 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
80#ifdef CONFIG_DEBUG_LOCK_ALLOC
81 .dep_map = {.name = "cpu_hotplug.lock" },
82#endif
83};
84
85/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */
86#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)
87#define cpuhp_lock_acquire_tryread() \
88 lock_map_acquire_tryread(&cpu_hotplug.dep_map)
89#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
90#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
91
92
93void get_online_cpus(void)
94{
95 might_sleep();
96 if (cpu_hotplug.active_writer == current)
97 return;
98 cpuhp_lock_acquire_read();
99 mutex_lock(&cpu_hotplug.lock);
100 atomic_inc(&cpu_hotplug.refcount);
101 mutex_unlock(&cpu_hotplug.lock);
102}
103EXPORT_SYMBOL_GPL(get_online_cpus);
104
105void put_online_cpus(void)
106{
107 int refcount;
108
109 if (cpu_hotplug.active_writer == current)
110 return;
111
112 refcount = atomic_dec_return(&cpu_hotplug.refcount);
113 if (WARN_ON(refcount < 0)) /* try to fix things up */
114 atomic_inc(&cpu_hotplug.refcount);
115
116 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq))
117 wake_up(&cpu_hotplug.wq);
118
119 cpuhp_lock_release();
120
121}
122EXPORT_SYMBOL_GPL(put_online_cpus);
123
124/*
125 * This ensures that the hotplug operation can begin only when the
126 * refcount goes to zero.
127 *
128 * Note that during a cpu-hotplug operation, the new readers, if any,
129 * will be blocked by the cpu_hotplug.lock
130 *
131 * Since cpu_hotplug_begin() is always called after invoking
132 * cpu_maps_update_begin(), we can be sure that only one writer is active.
133 *
134 * Note that theoretically, there is a possibility of a livelock:
135 * - Refcount goes to zero, last reader wakes up the sleeping
136 * writer.
137 * - Last reader unlocks the cpu_hotplug.lock.
138 * - A new reader arrives at this moment, bumps up the refcount.
139 * - The writer acquires the cpu_hotplug.lock finds the refcount
140 * non zero and goes to sleep again.
141 *
142 * However, this is very difficult to achieve in practice since
143 * get_online_cpus() not an api which is called all that often.
144 *
145 */
146void cpu_hotplug_begin(void)
147{
148 DEFINE_WAIT(wait);
149
150 cpu_hotplug.active_writer = current;
151 cpuhp_lock_acquire();
152
153 for (;;) {
154 mutex_lock(&cpu_hotplug.lock);
155 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE);
156 if (likely(!atomic_read(&cpu_hotplug.refcount)))
157 break;
158 mutex_unlock(&cpu_hotplug.lock);
159 schedule();
160 }
161 finish_wait(&cpu_hotplug.wq, &wait);
162}
163
164void cpu_hotplug_done(void)
165{
166 cpu_hotplug.active_writer = NULL;
167 mutex_unlock(&cpu_hotplug.lock);
168 cpuhp_lock_release();
169}
170
171/*
172 * Wait for currently running CPU hotplug operations to complete (if any) and
173 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
174 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
175 * hotplug path before performing hotplug operations. So acquiring that lock
176 * guarantees mutual exclusion from any currently running hotplug operations.
177 */
178void cpu_hotplug_disable(void)
179{
180 cpu_maps_update_begin();
181 cpu_hotplug_disabled++;
182 cpu_maps_update_done();
183}
184EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
185
186void cpu_hotplug_enable(void)
187{
188 cpu_maps_update_begin();
189 WARN_ON(--cpu_hotplug_disabled < 0);
190 cpu_maps_update_done();
191}
192EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
193#endif /* CONFIG_HOTPLUG_CPU */
194
195/* Need to know about CPUs going up/down? */
196int register_cpu_notifier(struct notifier_block *nb)
197{
198 int ret;
199 cpu_maps_update_begin();
200 ret = raw_notifier_chain_register(&cpu_chain, nb);
201 cpu_maps_update_done();
202 return ret;
203}
204
205int __register_cpu_notifier(struct notifier_block *nb)
206{
207 return raw_notifier_chain_register(&cpu_chain, nb);
208}
209
210static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
211 int *nr_calls)
212{
213 int ret;
214
215 ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
216 nr_calls);
217
218 return notifier_to_errno(ret);
219}
220
221static int cpu_notify(unsigned long val, void *v)
222{
223 return __cpu_notify(val, v, -1, NULL);
224}
225
226#ifdef CONFIG_HOTPLUG_CPU
227
228static void cpu_notify_nofail(unsigned long val, void *v)
229{
230 BUG_ON(cpu_notify(val, v));
231}
232EXPORT_SYMBOL(register_cpu_notifier);
233EXPORT_SYMBOL(__register_cpu_notifier);
234
235void unregister_cpu_notifier(struct notifier_block *nb)
236{
237 cpu_maps_update_begin();
238 raw_notifier_chain_unregister(&cpu_chain, nb);
239 cpu_maps_update_done();
240}
241EXPORT_SYMBOL(unregister_cpu_notifier);
242
243void __unregister_cpu_notifier(struct notifier_block *nb)
244{
245 raw_notifier_chain_unregister(&cpu_chain, nb);
246}
247EXPORT_SYMBOL(__unregister_cpu_notifier);
248
249/**
250 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
251 * @cpu: a CPU id
252 *
253 * This function walks all processes, finds a valid mm struct for each one and
254 * then clears a corresponding bit in mm's cpumask. While this all sounds
255 * trivial, there are various non-obvious corner cases, which this function
256 * tries to solve in a safe manner.
257 *
258 * Also note that the function uses a somewhat relaxed locking scheme, so it may
259 * be called only for an already offlined CPU.
260 */
261void clear_tasks_mm_cpumask(int cpu)
262{
263 struct task_struct *p;
264
265 /*
266 * This function is called after the cpu is taken down and marked
267 * offline, so its not like new tasks will ever get this cpu set in
268 * their mm mask. -- Peter Zijlstra
269 * Thus, we may use rcu_read_lock() here, instead of grabbing
270 * full-fledged tasklist_lock.
271 */
272 WARN_ON(cpu_online(cpu));
273 rcu_read_lock();
274 for_each_process(p) {
275 struct task_struct *t;
276
277 /*
278 * Main thread might exit, but other threads may still have
279 * a valid mm. Find one.
280 */
281 t = find_lock_task_mm(p);
282 if (!t)
283 continue;
284 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
285 task_unlock(t);
286 }
287 rcu_read_unlock();
288}
289
290static inline void check_for_tasks(int dead_cpu)
291{
292 struct task_struct *g, *p;
293
294 read_lock(&tasklist_lock);
295 for_each_process_thread(g, p) {
296 if (!p->on_rq)
297 continue;
298 /*
299 * We do the check with unlocked task_rq(p)->lock.
300 * Order the reading to do not warn about a task,
301 * which was running on this cpu in the past, and
302 * it's just been woken on another cpu.
303 */
304 rmb();
305 if (task_cpu(p) != dead_cpu)
306 continue;
307
308 pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n",
309 p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags);
310 }
311 read_unlock(&tasklist_lock);
312}
313
314struct take_cpu_down_param {
315 unsigned long mod;
316 void *hcpu;
317};
318
319/* Take this CPU down. */
320static int take_cpu_down(void *_param)
321{
322 struct take_cpu_down_param *param = _param;
323 int err;
324
325 /* Ensure this CPU doesn't handle any more interrupts. */
326 err = __cpu_disable();
327 if (err < 0)
328 return err;
329
330 cpu_notify(CPU_DYING | param->mod, param->hcpu);
331 /* Give up timekeeping duties */
332 tick_handover_do_timer();
333 /* Park the stopper thread */
334 stop_machine_park((long)param->hcpu);
335 return 0;
336}
337
338/* Requires cpu_add_remove_lock to be held */
339static int _cpu_down(unsigned int cpu, int tasks_frozen)
340{
341 int err, nr_calls = 0;
342 void *hcpu = (void *)(long)cpu;
343 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
344 struct take_cpu_down_param tcd_param = {
345 .mod = mod,
346 .hcpu = hcpu,
347 };
348
349 if (num_online_cpus() == 1)
350 return -EBUSY;
351
352 if (!cpu_online(cpu))
353 return -EINVAL;
354
355 cpu_hotplug_begin();
356
357 err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
358 if (err) {
359 nr_calls--;
360 __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
361 pr_warn("%s: attempt to take down CPU %u failed\n",
362 __func__, cpu);
363 goto out_release;
364 }
365
366 /*
367 * By now we've cleared cpu_active_mask, wait for all preempt-disabled
368 * and RCU users of this state to go away such that all new such users
369 * will observe it.
370 *
371 * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might
372 * not imply sync_sched(), so wait for both.
373 *
374 * Do sync before park smpboot threads to take care the rcu boost case.
375 */
376 if (IS_ENABLED(CONFIG_PREEMPT))
377 synchronize_rcu_mult(call_rcu, call_rcu_sched);
378 else
379 synchronize_rcu();
380
381 smpboot_park_threads(cpu);
382
383 /*
384 * Prevent irq alloc/free while the dying cpu reorganizes the
385 * interrupt affinities.
386 */
387 irq_lock_sparse();
388
389 /*
390 * So now all preempt/rcu users must observe !cpu_active().
391 */
392 err = stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
393 if (err) {
394 /* CPU didn't die: tell everyone. Can't complain. */
395 cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
396 irq_unlock_sparse();
397 goto out_release;
398 }
399 BUG_ON(cpu_online(cpu));
400
401 /*
402 * The migration_call() CPU_DYING callback will have removed all
403 * runnable tasks from the cpu, there's only the idle task left now
404 * that the migration thread is done doing the stop_machine thing.
405 *
406 * Wait for the stop thread to go away.
407 */
408 while (!per_cpu(cpu_dead_idle, cpu))
409 cpu_relax();
410 smp_mb(); /* Read from cpu_dead_idle before __cpu_die(). */
411 per_cpu(cpu_dead_idle, cpu) = false;
412
413 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
414 irq_unlock_sparse();
415
416 hotplug_cpu__broadcast_tick_pull(cpu);
417 /* This actually kills the CPU. */
418 __cpu_die(cpu);
419
420 /* CPU is completely dead: tell everyone. Too late to complain. */
421 tick_cleanup_dead_cpu(cpu);
422 cpu_notify_nofail(CPU_DEAD | mod, hcpu);
423
424 check_for_tasks(cpu);
425
426out_release:
427 cpu_hotplug_done();
428 if (!err)
429 cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
430 return err;
431}
432
433int cpu_down(unsigned int cpu)
434{
435 int err;
436
437 cpu_maps_update_begin();
438
439 if (cpu_hotplug_disabled) {
440 err = -EBUSY;
441 goto out;
442 }
443
444 err = _cpu_down(cpu, 0);
445
446out:
447 cpu_maps_update_done();
448 return err;
449}
450EXPORT_SYMBOL(cpu_down);
451#endif /*CONFIG_HOTPLUG_CPU*/
452
453/*
454 * Unpark per-CPU smpboot kthreads at CPU-online time.
455 */
456static int smpboot_thread_call(struct notifier_block *nfb,
457 unsigned long action, void *hcpu)
458{
459 int cpu = (long)hcpu;
460
461 switch (action & ~CPU_TASKS_FROZEN) {
462
463 case CPU_DOWN_FAILED:
464 case CPU_ONLINE:
465 smpboot_unpark_threads(cpu);
466 break;
467
468 default:
469 break;
470 }
471
472 return NOTIFY_OK;
473}
474
475static struct notifier_block smpboot_thread_notifier = {
476 .notifier_call = smpboot_thread_call,
477 .priority = CPU_PRI_SMPBOOT,
478};
479
480void smpboot_thread_init(void)
481{
482 register_cpu_notifier(&smpboot_thread_notifier);
483}
484
485/* Requires cpu_add_remove_lock to be held */
486static int _cpu_up(unsigned int cpu, int tasks_frozen)
487{
488 int ret, nr_calls = 0;
489 void *hcpu = (void *)(long)cpu;
490 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
491 struct task_struct *idle;
492
493 cpu_hotplug_begin();
494
495 if (cpu_online(cpu) || !cpu_present(cpu)) {
496 ret = -EINVAL;
497 goto out;
498 }
499
500 idle = idle_thread_get(cpu);
501 if (IS_ERR(idle)) {
502 ret = PTR_ERR(idle);
503 goto out;
504 }
505
506 ret = smpboot_create_threads(cpu);
507 if (ret)
508 goto out;
509
510 ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
511 if (ret) {
512 nr_calls--;
513 pr_warn("%s: attempt to bring up CPU %u failed\n",
514 __func__, cpu);
515 goto out_notify;
516 }
517
518 /* Arch-specific enabling code. */
519 ret = __cpu_up(cpu, idle);
520
521 if (ret != 0)
522 goto out_notify;
523 BUG_ON(!cpu_online(cpu));
524
525 /* Now call notifier in preparation. */
526 cpu_notify(CPU_ONLINE | mod, hcpu);
527
528out_notify:
529 if (ret != 0)
530 __cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
531out:
532 cpu_hotplug_done();
533
534 return ret;
535}
536
537int cpu_up(unsigned int cpu)
538{
539 int err = 0;
540
541 if (!cpu_possible(cpu)) {
542 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
543 cpu);
544#if defined(CONFIG_IA64)
545 pr_err("please check additional_cpus= boot parameter\n");
546#endif
547 return -EINVAL;
548 }
549
550 err = try_online_node(cpu_to_node(cpu));
551 if (err)
552 return err;
553
554 cpu_maps_update_begin();
555
556 if (cpu_hotplug_disabled) {
557 err = -EBUSY;
558 goto out;
559 }
560
561 err = _cpu_up(cpu, 0);
562
563out:
564 cpu_maps_update_done();
565 return err;
566}
567EXPORT_SYMBOL_GPL(cpu_up);
568
569#ifdef CONFIG_PM_SLEEP_SMP
570static cpumask_var_t frozen_cpus;
571
572int disable_nonboot_cpus(void)
573{
574 int cpu, first_cpu, error = 0;
575
576 cpu_maps_update_begin();
577 first_cpu = cpumask_first(cpu_online_mask);
578 /*
579 * We take down all of the non-boot CPUs in one shot to avoid races
580 * with the userspace trying to use the CPU hotplug at the same time
581 */
582 cpumask_clear(frozen_cpus);
583
584 pr_info("Disabling non-boot CPUs ...\n");
585 for_each_online_cpu(cpu) {
586 if (cpu == first_cpu)
587 continue;
588 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
589 error = _cpu_down(cpu, 1);
590 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
591 if (!error)
592 cpumask_set_cpu(cpu, frozen_cpus);
593 else {
594 pr_err("Error taking CPU%d down: %d\n", cpu, error);
595 break;
596 }
597 }
598
599 if (!error)
600 BUG_ON(num_online_cpus() > 1);
601 else
602 pr_err("Non-boot CPUs are not disabled\n");
603
604 /*
605 * Make sure the CPUs won't be enabled by someone else. We need to do
606 * this even in case of failure as all disable_nonboot_cpus() users are
607 * supposed to do enable_nonboot_cpus() on the failure path.
608 */
609 cpu_hotplug_disabled++;
610
611 cpu_maps_update_done();
612 return error;
613}
614
615void __weak arch_enable_nonboot_cpus_begin(void)
616{
617}
618
619void __weak arch_enable_nonboot_cpus_end(void)
620{
621}
622
623void enable_nonboot_cpus(void)
624{
625 int cpu, error;
626
627 /* Allow everyone to use the CPU hotplug again */
628 cpu_maps_update_begin();
629 WARN_ON(--cpu_hotplug_disabled < 0);
630 if (cpumask_empty(frozen_cpus))
631 goto out;
632
633 pr_info("Enabling non-boot CPUs ...\n");
634
635 arch_enable_nonboot_cpus_begin();
636
637 for_each_cpu(cpu, frozen_cpus) {
638 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
639 error = _cpu_up(cpu, 1);
640 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
641 if (!error) {
642 pr_info("CPU%d is up\n", cpu);
643 continue;
644 }
645 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
646 }
647
648 arch_enable_nonboot_cpus_end();
649
650 cpumask_clear(frozen_cpus);
651out:
652 cpu_maps_update_done();
653}
654
655static int __init alloc_frozen_cpus(void)
656{
657 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
658 return -ENOMEM;
659 return 0;
660}
661core_initcall(alloc_frozen_cpus);
662
663/*
664 * When callbacks for CPU hotplug notifications are being executed, we must
665 * ensure that the state of the system with respect to the tasks being frozen
666 * or not, as reported by the notification, remains unchanged *throughout the
667 * duration* of the execution of the callbacks.
668 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
669 *
670 * This synchronization is implemented by mutually excluding regular CPU
671 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
672 * Hibernate notifications.
673 */
674static int
675cpu_hotplug_pm_callback(struct notifier_block *nb,
676 unsigned long action, void *ptr)
677{
678 switch (action) {
679
680 case PM_SUSPEND_PREPARE:
681 case PM_HIBERNATION_PREPARE:
682 cpu_hotplug_disable();
683 break;
684
685 case PM_POST_SUSPEND:
686 case PM_POST_HIBERNATION:
687 cpu_hotplug_enable();
688 break;
689
690 default:
691 return NOTIFY_DONE;
692 }
693
694 return NOTIFY_OK;
695}
696
697
698static int __init cpu_hotplug_pm_sync_init(void)
699{
700 /*
701 * cpu_hotplug_pm_callback has higher priority than x86
702 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
703 * to disable cpu hotplug to avoid cpu hotplug race.
704 */
705 pm_notifier(cpu_hotplug_pm_callback, 0);
706 return 0;
707}
708core_initcall(cpu_hotplug_pm_sync_init);
709
710#endif /* CONFIG_PM_SLEEP_SMP */
711
712/**
713 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
714 * @cpu: cpu that just started
715 *
716 * This function calls the cpu_chain notifiers with CPU_STARTING.
717 * It must be called by the arch code on the new cpu, before the new cpu
718 * enables interrupts and before the "boot" cpu returns from __cpu_up().
719 */
720void notify_cpu_starting(unsigned int cpu)
721{
722 unsigned long val = CPU_STARTING;
723
724#ifdef CONFIG_PM_SLEEP_SMP
725 if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
726 val = CPU_STARTING_FROZEN;
727#endif /* CONFIG_PM_SLEEP_SMP */
728 cpu_notify(val, (void *)(long)cpu);
729}
730
731#endif /* CONFIG_SMP */
732
733/*
734 * cpu_bit_bitmap[] is a special, "compressed" data structure that
735 * represents all NR_CPUS bits binary values of 1<<nr.
736 *
737 * It is used by cpumask_of() to get a constant address to a CPU
738 * mask value that has a single bit set only.
739 */
740
741/* cpu_bit_bitmap[0] is empty - so we can back into it */
742#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
743#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
744#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
745#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
746
747const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
748
749 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
750 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
751#if BITS_PER_LONG > 32
752 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
753 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
754#endif
755};
756EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
757
758const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
759EXPORT_SYMBOL(cpu_all_bits);
760
761#ifdef CONFIG_INIT_ALL_POSSIBLE
762static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
763 = CPU_BITS_ALL;
764#else
765static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
766#endif
767const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
768EXPORT_SYMBOL(cpu_possible_mask);
769
770static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
771const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
772EXPORT_SYMBOL(cpu_online_mask);
773
774static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
775const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
776EXPORT_SYMBOL(cpu_present_mask);
777
778static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
779const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
780EXPORT_SYMBOL(cpu_active_mask);
781
782void set_cpu_possible(unsigned int cpu, bool possible)
783{
784 if (possible)
785 cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
786 else
787 cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
788}
789
790void set_cpu_present(unsigned int cpu, bool present)
791{
792 if (present)
793 cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
794 else
795 cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
796}
797
798void set_cpu_online(unsigned int cpu, bool online)
799{
800 if (online) {
801 cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
802 cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
803 } else {
804 cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
805 }
806}
807
808void set_cpu_active(unsigned int cpu, bool active)
809{
810 if (active)
811 cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
812 else
813 cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits));
814}
815
816void init_cpu_present(const struct cpumask *src)
817{
818 cpumask_copy(to_cpumask(cpu_present_bits), src);
819}
820
821void init_cpu_possible(const struct cpumask *src)
822{
823 cpumask_copy(to_cpumask(cpu_possible_bits), src);
824}
825
826void init_cpu_online(const struct cpumask *src)
827{
828 cpumask_copy(to_cpumask(cpu_online_bits), src);
829}