projects/linux·blob v2.6.27-rc6 · refs

481 lines · 11.4 KiB

1/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/unistd.h>
12#include <linux/cpu.h>
13#include <linux/module.h>
14#include <linux/kthread.h>
15#include <linux/stop_machine.h>
16#include <linux/mutex.h>
17
18/*
19 * Represents all cpu's present in the system
20 * In systems capable of hotplug, this map could dynamically grow
21 * as new cpu's are detected in the system via any platform specific
22 * method, such as ACPI for e.g.
23 */
24cpumask_t cpu_present_map __read_mostly;
25EXPORT_SYMBOL(cpu_present_map);
26
27#ifndef CONFIG_SMP
28
29/*
30 * Represents all cpu's that are currently online.
31 */
32cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
33EXPORT_SYMBOL(cpu_online_map);
34
35cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
36EXPORT_SYMBOL(cpu_possible_map);
37
38#else /* CONFIG_SMP */
39
40/* Serializes the updates to cpu_online_map, cpu_present_map */
41static DEFINE_MUTEX(cpu_add_remove_lock);
42
43static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain);
44
45/* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
46 * Should always be manipulated under cpu_add_remove_lock
47 */
48static int cpu_hotplug_disabled;
49
50static struct {
51 struct task_struct *active_writer;
52 struct mutex lock; /* Synchronizes accesses to refcount, */
53 /*
54 * Also blocks the new readers during
55 * an ongoing cpu hotplug operation.
56 */
57 int refcount;
58} cpu_hotplug;
59
60void __init cpu_hotplug_init(void)
61{
62 cpu_hotplug.active_writer = NULL;
63 mutex_init(&cpu_hotplug.lock);
64 cpu_hotplug.refcount = 0;
65}
66
67cpumask_t cpu_active_map;
68
69#ifdef CONFIG_HOTPLUG_CPU
70
71void get_online_cpus(void)
72{
73 might_sleep();
74 if (cpu_hotplug.active_writer == current)
75 return;
76 mutex_lock(&cpu_hotplug.lock);
77 cpu_hotplug.refcount++;
78 mutex_unlock(&cpu_hotplug.lock);
79
80}
81EXPORT_SYMBOL_GPL(get_online_cpus);
82
83void put_online_cpus(void)
84{
85 if (cpu_hotplug.active_writer == current)
86 return;
87 mutex_lock(&cpu_hotplug.lock);
88 if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
89 wake_up_process(cpu_hotplug.active_writer);
90 mutex_unlock(&cpu_hotplug.lock);
91
92}
93EXPORT_SYMBOL_GPL(put_online_cpus);
94
95#endif /* CONFIG_HOTPLUG_CPU */
96
97/*
98 * The following two API's must be used when attempting
99 * to serialize the updates to cpu_online_map, cpu_present_map.
100 */
101void cpu_maps_update_begin(void)
102{
103 mutex_lock(&cpu_add_remove_lock);
104}
105
106void cpu_maps_update_done(void)
107{
108 mutex_unlock(&cpu_add_remove_lock);
109}
110
111/*
112 * This ensures that the hotplug operation can begin only when the
113 * refcount goes to zero.
114 *
115 * Note that during a cpu-hotplug operation, the new readers, if any,
116 * will be blocked by the cpu_hotplug.lock
117 *
118 * Since cpu_hotplug_begin() is always called after invoking
119 * cpu_maps_update_begin(), we can be sure that only one writer is active.
120 *
121 * Note that theoretically, there is a possibility of a livelock:
122 * - Refcount goes to zero, last reader wakes up the sleeping
123 * writer.
124 * - Last reader unlocks the cpu_hotplug.lock.
125 * - A new reader arrives at this moment, bumps up the refcount.
126 * - The writer acquires the cpu_hotplug.lock finds the refcount
127 * non zero and goes to sleep again.
128 *
129 * However, this is very difficult to achieve in practice since
130 * get_online_cpus() not an api which is called all that often.
131 *
132 */
133static void cpu_hotplug_begin(void)
134{
135 cpu_hotplug.active_writer = current;
136
137 for (;;) {
138 mutex_lock(&cpu_hotplug.lock);
139 if (likely(!cpu_hotplug.refcount))
140 break;
141 __set_current_state(TASK_UNINTERRUPTIBLE);
142 mutex_unlock(&cpu_hotplug.lock);
143 schedule();
144 }
145}
146
147static void cpu_hotplug_done(void)
148{
149 cpu_hotplug.active_writer = NULL;
150 mutex_unlock(&cpu_hotplug.lock);
151}
152/* Need to know about CPUs going up/down? */
153int __ref register_cpu_notifier(struct notifier_block *nb)
154{
155 int ret;
156 cpu_maps_update_begin();
157 ret = raw_notifier_chain_register(&cpu_chain, nb);
158 cpu_maps_update_done();
159 return ret;
160}
161
162#ifdef CONFIG_HOTPLUG_CPU
163
164EXPORT_SYMBOL(register_cpu_notifier);
165
166void __ref unregister_cpu_notifier(struct notifier_block *nb)
167{
168 cpu_maps_update_begin();
169 raw_notifier_chain_unregister(&cpu_chain, nb);
170 cpu_maps_update_done();
171}
172EXPORT_SYMBOL(unregister_cpu_notifier);
173
174static inline void check_for_tasks(int cpu)
175{
176 struct task_struct *p;
177
178 write_lock_irq(&tasklist_lock);
179 for_each_process(p) {
180 if (task_cpu(p) == cpu &&
181 (!cputime_eq(p->utime, cputime_zero) ||
182 !cputime_eq(p->stime, cputime_zero)))
183 printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d\
184 (state = %ld, flags = %x) \n",
185 p->comm, task_pid_nr(p), cpu,
186 p->state, p->flags);
187 }
188 write_unlock_irq(&tasklist_lock);
189}
190
191struct take_cpu_down_param {
192 unsigned long mod;
193 void *hcpu;
194};
195
196/* Take this CPU down. */
197static int __ref take_cpu_down(void *_param)
198{
199 struct take_cpu_down_param *param = _param;
200 int err;
201
202 raw_notifier_call_chain(&cpu_chain, CPU_DYING | param->mod,
203 param->hcpu);
204 /* Ensure this CPU doesn't handle any more interrupts. */
205 err = __cpu_disable();
206 if (err < 0)
207 return err;
208
209 /* Force idle task to run as soon as we yield: it should
210 immediately notice cpu is offline and die quickly. */
211 sched_idle_next();
212 return 0;
213}
214
215/* Requires cpu_add_remove_lock to be held */
216static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
217{
218 int err, nr_calls = 0;
219 cpumask_t old_allowed, tmp;
220 void *hcpu = (void *)(long)cpu;
221 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
222 struct take_cpu_down_param tcd_param = {
223 .mod = mod,
224 .hcpu = hcpu,
225 };
226
227 if (num_online_cpus() == 1)
228 return -EBUSY;
229
230 if (!cpu_online(cpu))
231 return -EINVAL;
232
233 cpu_hotplug_begin();
234 err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod,
235 hcpu, -1, &nr_calls);
236 if (err == NOTIFY_BAD) {
237 nr_calls--;
238 __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod,
239 hcpu, nr_calls, NULL);
240 printk("%s: attempt to take down CPU %u failed\n",
241 __func__, cpu);
242 err = -EINVAL;
243 goto out_release;
244 }
245
246 /* Ensure that we are not runnable on dying cpu */
247 old_allowed = current->cpus_allowed;
248 cpus_setall(tmp);
249 cpu_clear(cpu, tmp);
250 set_cpus_allowed_ptr(current, &tmp);
251 tmp = cpumask_of_cpu(cpu);
252
253 err = __stop_machine(take_cpu_down, &tcd_param, &tmp);
254 if (err) {
255 /* CPU didn't die: tell everyone. Can't complain. */
256 if (raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod,
257 hcpu) == NOTIFY_BAD)
258 BUG();
259
260 goto out_allowed;
261 }
262 BUG_ON(cpu_online(cpu));
263
264 /* Wait for it to sleep (leaving idle task). */
265 while (!idle_cpu(cpu))
266 yield();
267
268 /* This actually kills the CPU. */
269 __cpu_die(cpu);
270
271 /* CPU is completely dead: tell everyone. Too late to complain. */
272 if (raw_notifier_call_chain(&cpu_chain, CPU_DEAD | mod,
273 hcpu) == NOTIFY_BAD)
274 BUG();
275
276 check_for_tasks(cpu);
277
278out_allowed:
279 set_cpus_allowed_ptr(current, &old_allowed);
280out_release:
281 cpu_hotplug_done();
282 if (!err) {
283 if (raw_notifier_call_chain(&cpu_chain, CPU_POST_DEAD | mod,
284 hcpu) == NOTIFY_BAD)
285 BUG();
286 }
287 return err;
288}
289
290int __ref cpu_down(unsigned int cpu)
291{
292 int err = 0;
293
294 cpu_maps_update_begin();
295
296 if (cpu_hotplug_disabled) {
297 err = -EBUSY;
298 goto out;
299 }
300
301 cpu_clear(cpu, cpu_active_map);
302
303 /*
304 * Make sure the all cpus did the reschedule and are not
305 * using stale version of the cpu_active_map.
306 * This is not strictly necessary becuase stop_machine()
307 * that we run down the line already provides the required
308 * synchronization. But it's really a side effect and we do not
309 * want to depend on the innards of the stop_machine here.
310 */
311 synchronize_sched();
312
313 err = _cpu_down(cpu, 0);
314
315 if (cpu_online(cpu))
316 cpu_set(cpu, cpu_active_map);
317
318out:
319 cpu_maps_update_done();
320 return err;
321}
322EXPORT_SYMBOL(cpu_down);
323#endif /*CONFIG_HOTPLUG_CPU*/
324
325/* Requires cpu_add_remove_lock to be held */
326static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
327{
328 int ret, nr_calls = 0;
329 void *hcpu = (void *)(long)cpu;
330 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
331
332 if (cpu_online(cpu) || !cpu_present(cpu))
333 return -EINVAL;
334
335 cpu_hotplug_begin();
336 ret = __raw_notifier_call_chain(&cpu_chain, CPU_UP_PREPARE | mod, hcpu,
337 -1, &nr_calls);
338 if (ret == NOTIFY_BAD) {
339 nr_calls--;
340 printk("%s: attempt to bring up CPU %u failed\n",
341 __func__, cpu);
342 ret = -EINVAL;
343 goto out_notify;
344 }
345
346 /* Arch-specific enabling code. */
347 ret = __cpu_up(cpu);
348 if (ret != 0)
349 goto out_notify;
350 BUG_ON(!cpu_online(cpu));
351
352 cpu_set(cpu, cpu_active_map);
353
354 /* Now call notifier in preparation. */
355 raw_notifier_call_chain(&cpu_chain, CPU_ONLINE | mod, hcpu);
356
357out_notify:
358 if (ret != 0)
359 __raw_notifier_call_chain(&cpu_chain,
360 CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
361 cpu_hotplug_done();
362
363 return ret;
364}
365
366int __cpuinit cpu_up(unsigned int cpu)
367{
368 int err = 0;
369 if (!cpu_isset(cpu, cpu_possible_map)) {
370 printk(KERN_ERR "can't online cpu %d because it is not "
371 "configured as may-hotadd at boot time\n", cpu);
372#if defined(CONFIG_IA64) || defined(CONFIG_X86_64)
373 printk(KERN_ERR "please check additional_cpus= boot "
374 "parameter\n");
375#endif
376 return -EINVAL;
377 }
378
379 cpu_maps_update_begin();
380
381 if (cpu_hotplug_disabled) {
382 err = -EBUSY;
383 goto out;
384 }
385
386 err = _cpu_up(cpu, 0);
387
388out:
389 cpu_maps_update_done();
390 return err;
391}
392
393#ifdef CONFIG_PM_SLEEP_SMP
394static cpumask_t frozen_cpus;
395
396int disable_nonboot_cpus(void)
397{
398 int cpu, first_cpu, error = 0;
399
400 cpu_maps_update_begin();
401 first_cpu = first_cpu(cpu_online_map);
402 /* We take down all of the non-boot CPUs in one shot to avoid races
403 * with the userspace trying to use the CPU hotplug at the same time
404 */
405 cpus_clear(frozen_cpus);
406 printk("Disabling non-boot CPUs ...\n");
407 for_each_online_cpu(cpu) {
408 if (cpu == first_cpu)
409 continue;
410 error = _cpu_down(cpu, 1);
411 if (!error) {
412 cpu_set(cpu, frozen_cpus);
413 printk("CPU%d is down\n", cpu);
414 } else {
415 printk(KERN_ERR "Error taking CPU%d down: %d\n",
416 cpu, error);
417 break;
418 }
419 }
420 if (!error) {
421 BUG_ON(num_online_cpus() > 1);
422 /* Make sure the CPUs won't be enabled by someone else */
423 cpu_hotplug_disabled = 1;
424 } else {
425 printk(KERN_ERR "Non-boot CPUs are not disabled\n");
426 }
427 cpu_maps_update_done();
428 return error;
429}
430
431void __ref enable_nonboot_cpus(void)
432{
433 int cpu, error;
434
435 /* Allow everyone to use the CPU hotplug again */
436 cpu_maps_update_begin();
437 cpu_hotplug_disabled = 0;
438 if (cpus_empty(frozen_cpus))
439 goto out;
440
441 printk("Enabling non-boot CPUs ...\n");
442 for_each_cpu_mask_nr(cpu, frozen_cpus) {
443 error = _cpu_up(cpu, 1);
444 if (!error) {
445 printk("CPU%d is up\n", cpu);
446 continue;
447 }
448 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
449 }
450 cpus_clear(frozen_cpus);
451out:
452 cpu_maps_update_done();
453}
454#endif /* CONFIG_PM_SLEEP_SMP */
455
456#endif /* CONFIG_SMP */
457
458/*
459 * cpu_bit_bitmap[] is a special, "compressed" data structure that
460 * represents all NR_CPUS bits binary values of 1<<nr.
461 *
462 * It is used by cpumask_of_cpu() to get a constant address to a CPU
463 * mask value that has a single bit set only.
464 */
465
466/* cpu_bit_bitmap[0] is empty - so we can back into it */
467#define MASK_DECLARE_1(x) [x+1][0] = 1UL << (x)
468#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
469#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
470#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
471
472const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
473
474 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
475 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
476#if BITS_PER_LONG > 32
477 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
478 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
479#endif
480};
481EXPORT_SYMBOL_GPL(cpu_bit_bitmap);