projects/linux·blob v2.6.28-rc3 · refs

163 lines · 3.7 KiB

1/* Copyright 2008, 2005 Rusty Russell [email protected] IBM Corporation.
2 * GPL v2 and any later version.
3 */
4#include <linux/cpu.h>
5#include <linux/err.h>
6#include <linux/kthread.h>
7#include <linux/module.h>
8#include <linux/sched.h>
9#include <linux/stop_machine.h>
10#include <linux/syscalls.h>
11#include <linux/interrupt.h>
12
13#include <asm/atomic.h>
14#include <asm/uaccess.h>
15
16/* This controls the threads on each CPU. */
17enum stopmachine_state {
18 /* Dummy starting state for thread. */
19 STOPMACHINE_NONE,
20 /* Awaiting everyone to be scheduled. */
21 STOPMACHINE_PREPARE,
22 /* Disable interrupts. */
23 STOPMACHINE_DISABLE_IRQ,
24 /* Run the function */
25 STOPMACHINE_RUN,
26 /* Exit */
27 STOPMACHINE_EXIT,
28};
29static enum stopmachine_state state;
30
31struct stop_machine_data {
32 int (*fn)(void *);
33 void *data;
34 int fnret;
35};
36
37/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
38static unsigned int num_threads;
39static atomic_t thread_ack;
40static DEFINE_MUTEX(lock);
41
42static struct workqueue_struct *stop_machine_wq;
43static struct stop_machine_data active, idle;
44static const cpumask_t *active_cpus;
45static void *stop_machine_work;
46
47static void set_state(enum stopmachine_state newstate)
48{
49 /* Reset ack counter. */
50 atomic_set(&thread_ack, num_threads);
51 smp_wmb();
52 state = newstate;
53}
54
55/* Last one to ack a state moves to the next state. */
56static void ack_state(void)
57{
58 if (atomic_dec_and_test(&thread_ack))
59 set_state(state + 1);
60}
61
62/* This is the actual function which stops the CPU. It runs
63 * in the context of a dedicated stopmachine workqueue. */
64static void stop_cpu(struct work_struct *unused)
65{
66 enum stopmachine_state curstate = STOPMACHINE_NONE;
67 struct stop_machine_data *smdata = &idle;
68 int cpu = smp_processor_id();
69 int err;
70
71 if (!active_cpus) {
72 if (cpu == first_cpu(cpu_online_map))
73 smdata = &active;
74 } else {
75 if (cpu_isset(cpu, *active_cpus))
76 smdata = &active;
77 }
78 /* Simple state machine */
79 do {
80 /* Chill out and ensure we re-read stopmachine_state. */
81 cpu_relax();
82 if (state != curstate) {
83 curstate = state;
84 switch (curstate) {
85 case STOPMACHINE_DISABLE_IRQ:
86 local_irq_disable();
87 hard_irq_disable();
88 break;
89 case STOPMACHINE_RUN:
90 /* On multiple CPUs only a single error code
91 * is needed to tell that something failed. */
92 err = smdata->fn(smdata->data);
93 if (err)
94 smdata->fnret = err;
95 break;
96 default:
97 break;
98 }
99 ack_state();
100 }
101 } while (curstate != STOPMACHINE_EXIT);
102
103 local_irq_enable();
104}
105
106/* Callback for CPUs which aren't supposed to do anything. */
107static int chill(void *unused)
108{
109 return 0;
110}
111
112int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus)
113{
114 struct work_struct *sm_work;
115 int i;
116
117 /* Set up initial state. */
118 mutex_lock(&lock);
119 num_threads = num_online_cpus();
120 active_cpus = cpus;
121 active.fn = fn;
122 active.data = data;
123 active.fnret = 0;
124 idle.fn = chill;
125 idle.data = NULL;
126
127 set_state(STOPMACHINE_PREPARE);
128
129 /* Schedule the stop_cpu work on all cpus: hold this CPU so one
130 * doesn't hit this CPU until we're ready. */
131 get_cpu();
132 for_each_online_cpu(i) {
133 sm_work = percpu_ptr(stop_machine_work, i);
134 INIT_WORK(sm_work, stop_cpu);
135 queue_work_on(i, stop_machine_wq, sm_work);
136 }
137 /* This will release the thread on our CPU. */
138 put_cpu();
139 flush_workqueue(stop_machine_wq);
140 mutex_unlock(&lock);
141 return active.fnret;
142}
143
144int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus)
145{
146 int ret;
147
148 /* No CPUs can come up or down during this. */
149 get_online_cpus();
150 ret = __stop_machine(fn, data, cpus);
151 put_online_cpus();
152
153 return ret;
154}
155EXPORT_SYMBOL_GPL(stop_machine);
156
157static int __init stop_machine_init(void)
158{
159 stop_machine_wq = create_rt_workqueue("kstop");
160 stop_machine_work = alloc_percpu(struct work_struct);
161 return 0;
162}
163core_initcall(stop_machine_init);