projects/linux·blob v2.6.16-rc6 · refs

374 lines · 7.9 KiB

1/*
2 * Copyright (2004) Linus Torvalds
3 *
4 * Author: Zwane Mwaikambo <[email protected]>
5 *
6 * Copyright (2004, 2005) Ingo Molnar
7 *
8 * This file contains the spinlock/rwlock implementations for the
9 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
10 */
11
12#include <linux/config.h>
13#include <linux/linkage.h>
14#include <linux/preempt.h>
15#include <linux/spinlock.h>
16#include <linux/interrupt.h>
17#include <linux/module.h>
18
19/*
20 * Generic declaration of the raw read_trylock() function,
21 * architectures are supposed to optimize this:
22 */
23int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock)
24{
25 __raw_read_lock(lock);
26 return 1;
27}
28EXPORT_SYMBOL(generic__raw_read_trylock);
29
30int __lockfunc _spin_trylock(spinlock_t *lock)
31{
32 preempt_disable();
33 if (_raw_spin_trylock(lock))
34 return 1;
35
36 preempt_enable();
37 return 0;
38}
39EXPORT_SYMBOL(_spin_trylock);
40
41int __lockfunc _read_trylock(rwlock_t *lock)
42{
43 preempt_disable();
44 if (_raw_read_trylock(lock))
45 return 1;
46
47 preempt_enable();
48 return 0;
49}
50EXPORT_SYMBOL(_read_trylock);
51
52int __lockfunc _write_trylock(rwlock_t *lock)
53{
54 preempt_disable();
55 if (_raw_write_trylock(lock))
56 return 1;
57
58 preempt_enable();
59 return 0;
60}
61EXPORT_SYMBOL(_write_trylock);
62
63#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP)
64
65void __lockfunc _read_lock(rwlock_t *lock)
66{
67 preempt_disable();
68 _raw_read_lock(lock);
69}
70EXPORT_SYMBOL(_read_lock);
71
72unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
73{
74 unsigned long flags;
75
76 local_irq_save(flags);
77 preempt_disable();
78 _raw_spin_lock_flags(lock, &flags);
79 return flags;
80}
81EXPORT_SYMBOL(_spin_lock_irqsave);
82
83void __lockfunc _spin_lock_irq(spinlock_t *lock)
84{
85 local_irq_disable();
86 preempt_disable();
87 _raw_spin_lock(lock);
88}
89EXPORT_SYMBOL(_spin_lock_irq);
90
91void __lockfunc _spin_lock_bh(spinlock_t *lock)
92{
93 local_bh_disable();
94 preempt_disable();
95 _raw_spin_lock(lock);
96}
97EXPORT_SYMBOL(_spin_lock_bh);
98
99unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
100{
101 unsigned long flags;
102
103 local_irq_save(flags);
104 preempt_disable();
105 _raw_read_lock(lock);
106 return flags;
107}
108EXPORT_SYMBOL(_read_lock_irqsave);
109
110void __lockfunc _read_lock_irq(rwlock_t *lock)
111{
112 local_irq_disable();
113 preempt_disable();
114 _raw_read_lock(lock);
115}
116EXPORT_SYMBOL(_read_lock_irq);
117
118void __lockfunc _read_lock_bh(rwlock_t *lock)
119{
120 local_bh_disable();
121 preempt_disable();
122 _raw_read_lock(lock);
123}
124EXPORT_SYMBOL(_read_lock_bh);
125
126unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
127{
128 unsigned long flags;
129
130 local_irq_save(flags);
131 preempt_disable();
132 _raw_write_lock(lock);
133 return flags;
134}
135EXPORT_SYMBOL(_write_lock_irqsave);
136
137void __lockfunc _write_lock_irq(rwlock_t *lock)
138{
139 local_irq_disable();
140 preempt_disable();
141 _raw_write_lock(lock);
142}
143EXPORT_SYMBOL(_write_lock_irq);
144
145void __lockfunc _write_lock_bh(rwlock_t *lock)
146{
147 local_bh_disable();
148 preempt_disable();
149 _raw_write_lock(lock);
150}
151EXPORT_SYMBOL(_write_lock_bh);
152
153void __lockfunc _spin_lock(spinlock_t *lock)
154{
155 preempt_disable();
156 _raw_spin_lock(lock);
157}
158
159EXPORT_SYMBOL(_spin_lock);
160
161void __lockfunc _write_lock(rwlock_t *lock)
162{
163 preempt_disable();
164 _raw_write_lock(lock);
165}
166
167EXPORT_SYMBOL(_write_lock);
168
169#else /* CONFIG_PREEMPT: */
170
171/*
172 * This could be a long-held lock. We both prepare to spin for a long
173 * time (making _this_ CPU preemptable if possible), and we also signal
174 * towards that other CPU that it should break the lock ASAP.
175 *
176 * (We do this in a function because inlining it would be excessive.)
177 */
178
179#define BUILD_LOCK_OPS(op, locktype) \
180void __lockfunc _##op##_lock(locktype##_t *lock) \
181{ \
182 preempt_disable(); \
183 for (;;) { \
184 if (likely(_raw_##op##_trylock(lock))) \
185 break; \
186 preempt_enable(); \
187 if (!(lock)->break_lock) \
188 (lock)->break_lock = 1; \
189 while (!op##_can_lock(lock) && (lock)->break_lock) \
190 cpu_relax(); \
191 preempt_disable(); \
192 } \
193 (lock)->break_lock = 0; \
194} \
195 \
196EXPORT_SYMBOL(_##op##_lock); \
197 \
198unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \
199{ \
200 unsigned long flags; \
201 \
202 preempt_disable(); \
203 for (;;) { \
204 local_irq_save(flags); \
205 if (likely(_raw_##op##_trylock(lock))) \
206 break; \
207 local_irq_restore(flags); \
208 \
209 preempt_enable(); \
210 if (!(lock)->break_lock) \
211 (lock)->break_lock = 1; \
212 while (!op##_can_lock(lock) && (lock)->break_lock) \
213 cpu_relax(); \
214 preempt_disable(); \
215 } \
216 (lock)->break_lock = 0; \
217 return flags; \
218} \
219 \
220EXPORT_SYMBOL(_##op##_lock_irqsave); \
221 \
222void __lockfunc _##op##_lock_irq(locktype##_t *lock) \
223{ \
224 _##op##_lock_irqsave(lock); \
225} \
226 \
227EXPORT_SYMBOL(_##op##_lock_irq); \
228 \
229void __lockfunc _##op##_lock_bh(locktype##_t *lock) \
230{ \
231 unsigned long flags; \
232 \
233 /* */ \
234 /* Careful: we must exclude softirqs too, hence the */ \
235 /* irq-disabling. We use the generic preemption-aware */ \
236 /* function: */ \
237 /**/ \
238 flags = _##op##_lock_irqsave(lock); \
239 local_bh_disable(); \
240 local_irq_restore(flags); \
241} \
242 \
243EXPORT_SYMBOL(_##op##_lock_bh)
244
245/*
246 * Build preemption-friendly versions of the following
247 * lock-spinning functions:
248 *
249 * _[spin|read|write]_lock()
250 * _[spin|read|write]_lock_irq()
251 * _[spin|read|write]_lock_irqsave()
252 * _[spin|read|write]_lock_bh()
253 */
254BUILD_LOCK_OPS(spin, spinlock);
255BUILD_LOCK_OPS(read, rwlock);
256BUILD_LOCK_OPS(write, rwlock);
257
258#endif /* CONFIG_PREEMPT */
259
260void __lockfunc _spin_unlock(spinlock_t *lock)
261{
262 _raw_spin_unlock(lock);
263 preempt_enable();
264}
265EXPORT_SYMBOL(_spin_unlock);
266
267void __lockfunc _write_unlock(rwlock_t *lock)
268{
269 _raw_write_unlock(lock);
270 preempt_enable();
271}
272EXPORT_SYMBOL(_write_unlock);
273
274void __lockfunc _read_unlock(rwlock_t *lock)
275{
276 _raw_read_unlock(lock);
277 preempt_enable();
278}
279EXPORT_SYMBOL(_read_unlock);
280
281void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
282{
283 _raw_spin_unlock(lock);
284 local_irq_restore(flags);
285 preempt_enable();
286}
287EXPORT_SYMBOL(_spin_unlock_irqrestore);
288
289void __lockfunc _spin_unlock_irq(spinlock_t *lock)
290{
291 _raw_spin_unlock(lock);
292 local_irq_enable();
293 preempt_enable();
294}
295EXPORT_SYMBOL(_spin_unlock_irq);
296
297void __lockfunc _spin_unlock_bh(spinlock_t *lock)
298{
299 _raw_spin_unlock(lock);
300 preempt_enable_no_resched();
301 local_bh_enable();
302}
303EXPORT_SYMBOL(_spin_unlock_bh);
304
305void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
306{
307 _raw_read_unlock(lock);
308 local_irq_restore(flags);
309 preempt_enable();
310}
311EXPORT_SYMBOL(_read_unlock_irqrestore);
312
313void __lockfunc _read_unlock_irq(rwlock_t *lock)
314{
315 _raw_read_unlock(lock);
316 local_irq_enable();
317 preempt_enable();
318}
319EXPORT_SYMBOL(_read_unlock_irq);
320
321void __lockfunc _read_unlock_bh(rwlock_t *lock)
322{
323 _raw_read_unlock(lock);
324 preempt_enable_no_resched();
325 local_bh_enable();
326}
327EXPORT_SYMBOL(_read_unlock_bh);
328
329void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
330{
331 _raw_write_unlock(lock);
332 local_irq_restore(flags);
333 preempt_enable();
334}
335EXPORT_SYMBOL(_write_unlock_irqrestore);
336
337void __lockfunc _write_unlock_irq(rwlock_t *lock)
338{
339 _raw_write_unlock(lock);
340 local_irq_enable();
341 preempt_enable();
342}
343EXPORT_SYMBOL(_write_unlock_irq);
344
345void __lockfunc _write_unlock_bh(rwlock_t *lock)
346{
347 _raw_write_unlock(lock);
348 preempt_enable_no_resched();
349 local_bh_enable();
350}
351EXPORT_SYMBOL(_write_unlock_bh);
352
353int __lockfunc _spin_trylock_bh(spinlock_t *lock)
354{
355 local_bh_disable();
356 preempt_disable();
357 if (_raw_spin_trylock(lock))
358 return 1;
359
360 preempt_enable_no_resched();
361 local_bh_enable();
362 return 0;
363}
364EXPORT_SYMBOL(_spin_trylock_bh);
365
366int in_lock_functions(unsigned long addr)
367{
368 /* Linker adds these: start and end of __lockfunc functions */
369 extern char __lock_text_start[], __lock_text_end[];
370
371 return addr >= (unsigned long)__lock_text_start
372 && addr < (unsigned long)__lock_text_end;
373}
374EXPORT_SYMBOL(in_lock_functions);