projects/linux·blob v2.6.32 · refs

387 lines · 8.6 KiB

1/*
2 * Copyright (2004) Linus Torvalds
3 *
4 * Author: Zwane Mwaikambo <[email protected]>
5 *
6 * Copyright (2004, 2005) Ingo Molnar
7 *
8 * This file contains the spinlock/rwlock implementations for the
9 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
10 *
11 * Note that some architectures have special knowledge about the
12 * stack frames of these functions in their profile_pc. If you
13 * change anything significant here that could change the stack
14 * frame contact the architecture maintainers.
15 */
16
17#include <linux/linkage.h>
18#include <linux/preempt.h>
19#include <linux/spinlock.h>
20#include <linux/interrupt.h>
21#include <linux/debug_locks.h>
22#include <linux/module.h>
23
24#ifndef _spin_trylock
25int __lockfunc _spin_trylock(spinlock_t *lock)
26{
27 return __spin_trylock(lock);
28}
29EXPORT_SYMBOL(_spin_trylock);
30#endif
31
32#ifndef _read_trylock
33int __lockfunc _read_trylock(rwlock_t *lock)
34{
35 return __read_trylock(lock);
36}
37EXPORT_SYMBOL(_read_trylock);
38#endif
39
40#ifndef _write_trylock
41int __lockfunc _write_trylock(rwlock_t *lock)
42{
43 return __write_trylock(lock);
44}
45EXPORT_SYMBOL(_write_trylock);
46#endif
47
48/*
49 * If lockdep is enabled then we use the non-preemption spin-ops
50 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
51 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
52 */
53#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
54
55#ifndef _read_lock
56void __lockfunc _read_lock(rwlock_t *lock)
57{
58 __read_lock(lock);
59}
60EXPORT_SYMBOL(_read_lock);
61#endif
62
63#ifndef _spin_lock_irqsave
64unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
65{
66 return __spin_lock_irqsave(lock);
67}
68EXPORT_SYMBOL(_spin_lock_irqsave);
69#endif
70
71#ifndef _spin_lock_irq
72void __lockfunc _spin_lock_irq(spinlock_t *lock)
73{
74 __spin_lock_irq(lock);
75}
76EXPORT_SYMBOL(_spin_lock_irq);
77#endif
78
79#ifndef _spin_lock_bh
80void __lockfunc _spin_lock_bh(spinlock_t *lock)
81{
82 __spin_lock_bh(lock);
83}
84EXPORT_SYMBOL(_spin_lock_bh);
85#endif
86
87#ifndef _read_lock_irqsave
88unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
89{
90 return __read_lock_irqsave(lock);
91}
92EXPORT_SYMBOL(_read_lock_irqsave);
93#endif
94
95#ifndef _read_lock_irq
96void __lockfunc _read_lock_irq(rwlock_t *lock)
97{
98 __read_lock_irq(lock);
99}
100EXPORT_SYMBOL(_read_lock_irq);
101#endif
102
103#ifndef _read_lock_bh
104void __lockfunc _read_lock_bh(rwlock_t *lock)
105{
106 __read_lock_bh(lock);
107}
108EXPORT_SYMBOL(_read_lock_bh);
109#endif
110
111#ifndef _write_lock_irqsave
112unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
113{
114 return __write_lock_irqsave(lock);
115}
116EXPORT_SYMBOL(_write_lock_irqsave);
117#endif
118
119#ifndef _write_lock_irq
120void __lockfunc _write_lock_irq(rwlock_t *lock)
121{
122 __write_lock_irq(lock);
123}
124EXPORT_SYMBOL(_write_lock_irq);
125#endif
126
127#ifndef _write_lock_bh
128void __lockfunc _write_lock_bh(rwlock_t *lock)
129{
130 __write_lock_bh(lock);
131}
132EXPORT_SYMBOL(_write_lock_bh);
133#endif
134
135#ifndef _spin_lock
136void __lockfunc _spin_lock(spinlock_t *lock)
137{
138 __spin_lock(lock);
139}
140EXPORT_SYMBOL(_spin_lock);
141#endif
142
143#ifndef _write_lock
144void __lockfunc _write_lock(rwlock_t *lock)
145{
146 __write_lock(lock);
147}
148EXPORT_SYMBOL(_write_lock);
149#endif
150
151#else /* CONFIG_PREEMPT: */
152
153/*
154 * This could be a long-held lock. We both prepare to spin for a long
155 * time (making _this_ CPU preemptable if possible), and we also signal
156 * towards that other CPU that it should break the lock ASAP.
157 *
158 * (We do this in a function because inlining it would be excessive.)
159 */
160
161#define BUILD_LOCK_OPS(op, locktype) \
162void __lockfunc _##op##_lock(locktype##_t *lock) \
163{ \
164 for (;;) { \
165 preempt_disable(); \
166 if (likely(_raw_##op##_trylock(lock))) \
167 break; \
168 preempt_enable(); \
169 \
170 if (!(lock)->break_lock) \
171 (lock)->break_lock = 1; \
172 while (!op##_can_lock(lock) && (lock)->break_lock) \
173 _raw_##op##_relax(&lock->raw_lock); \
174 } \
175 (lock)->break_lock = 0; \
176} \
177 \
178EXPORT_SYMBOL(_##op##_lock); \
179 \
180unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \
181{ \
182 unsigned long flags; \
183 \
184 for (;;) { \
185 preempt_disable(); \
186 local_irq_save(flags); \
187 if (likely(_raw_##op##_trylock(lock))) \
188 break; \
189 local_irq_restore(flags); \
190 preempt_enable(); \
191 \
192 if (!(lock)->break_lock) \
193 (lock)->break_lock = 1; \
194 while (!op##_can_lock(lock) && (lock)->break_lock) \
195 _raw_##op##_relax(&lock->raw_lock); \
196 } \
197 (lock)->break_lock = 0; \
198 return flags; \
199} \
200 \
201EXPORT_SYMBOL(_##op##_lock_irqsave); \
202 \
203void __lockfunc _##op##_lock_irq(locktype##_t *lock) \
204{ \
205 _##op##_lock_irqsave(lock); \
206} \
207 \
208EXPORT_SYMBOL(_##op##_lock_irq); \
209 \
210void __lockfunc _##op##_lock_bh(locktype##_t *lock) \
211{ \
212 unsigned long flags; \
213 \
214 /* */ \
215 /* Careful: we must exclude softirqs too, hence the */ \
216 /* irq-disabling. We use the generic preemption-aware */ \
217 /* function: */ \
218 /**/ \
219 flags = _##op##_lock_irqsave(lock); \
220 local_bh_disable(); \
221 local_irq_restore(flags); \
222} \
223 \
224EXPORT_SYMBOL(_##op##_lock_bh)
225
226/*
227 * Build preemption-friendly versions of the following
228 * lock-spinning functions:
229 *
230 * _[spin|read|write]_lock()
231 * _[spin|read|write]_lock_irq()
232 * _[spin|read|write]_lock_irqsave()
233 * _[spin|read|write]_lock_bh()
234 */
235BUILD_LOCK_OPS(spin, spinlock);
236BUILD_LOCK_OPS(read, rwlock);
237BUILD_LOCK_OPS(write, rwlock);
238
239#endif /* CONFIG_PREEMPT */
240
241#ifdef CONFIG_DEBUG_LOCK_ALLOC
242
243void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
244{
245 preempt_disable();
246 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
247 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
248}
249EXPORT_SYMBOL(_spin_lock_nested);
250
251unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
252{
253 unsigned long flags;
254
255 local_irq_save(flags);
256 preempt_disable();
257 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
258 LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock,
259 _raw_spin_lock_flags, &flags);
260 return flags;
261}
262EXPORT_SYMBOL(_spin_lock_irqsave_nested);
263
264void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
265 struct lockdep_map *nest_lock)
266{
267 preempt_disable();
268 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
269 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
270}
271EXPORT_SYMBOL(_spin_lock_nest_lock);
272
273#endif
274
275#ifndef _spin_unlock
276void __lockfunc _spin_unlock(spinlock_t *lock)
277{
278 __spin_unlock(lock);
279}
280EXPORT_SYMBOL(_spin_unlock);
281#endif
282
283#ifndef _write_unlock
284void __lockfunc _write_unlock(rwlock_t *lock)
285{
286 __write_unlock(lock);
287}
288EXPORT_SYMBOL(_write_unlock);
289#endif
290
291#ifndef _read_unlock
292void __lockfunc _read_unlock(rwlock_t *lock)
293{
294 __read_unlock(lock);
295}
296EXPORT_SYMBOL(_read_unlock);
297#endif
298
299#ifndef _spin_unlock_irqrestore
300void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
301{
302 __spin_unlock_irqrestore(lock, flags);
303}
304EXPORT_SYMBOL(_spin_unlock_irqrestore);
305#endif
306
307#ifndef _spin_unlock_irq
308void __lockfunc _spin_unlock_irq(spinlock_t *lock)
309{
310 __spin_unlock_irq(lock);
311}
312EXPORT_SYMBOL(_spin_unlock_irq);
313#endif
314
315#ifndef _spin_unlock_bh
316void __lockfunc _spin_unlock_bh(spinlock_t *lock)
317{
318 __spin_unlock_bh(lock);
319}
320EXPORT_SYMBOL(_spin_unlock_bh);
321#endif
322
323#ifndef _read_unlock_irqrestore
324void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
325{
326 __read_unlock_irqrestore(lock, flags);
327}
328EXPORT_SYMBOL(_read_unlock_irqrestore);
329#endif
330
331#ifndef _read_unlock_irq
332void __lockfunc _read_unlock_irq(rwlock_t *lock)
333{
334 __read_unlock_irq(lock);
335}
336EXPORT_SYMBOL(_read_unlock_irq);
337#endif
338
339#ifndef _read_unlock_bh
340void __lockfunc _read_unlock_bh(rwlock_t *lock)
341{
342 __read_unlock_bh(lock);
343}
344EXPORT_SYMBOL(_read_unlock_bh);
345#endif
346
347#ifndef _write_unlock_irqrestore
348void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
349{
350 __write_unlock_irqrestore(lock, flags);
351}
352EXPORT_SYMBOL(_write_unlock_irqrestore);
353#endif
354
355#ifndef _write_unlock_irq
356void __lockfunc _write_unlock_irq(rwlock_t *lock)
357{
358 __write_unlock_irq(lock);
359}
360EXPORT_SYMBOL(_write_unlock_irq);
361#endif
362
363#ifndef _write_unlock_bh
364void __lockfunc _write_unlock_bh(rwlock_t *lock)
365{
366 __write_unlock_bh(lock);
367}
368EXPORT_SYMBOL(_write_unlock_bh);
369#endif
370
371#ifndef _spin_trylock_bh
372int __lockfunc _spin_trylock_bh(spinlock_t *lock)
373{
374 return __spin_trylock_bh(lock);
375}
376EXPORT_SYMBOL(_spin_trylock_bh);
377#endif
378
379notrace int in_lock_functions(unsigned long addr)
380{
381 /* Linker adds these: start and end of __lockfunc functions */
382 extern char __lock_text_start[], __lock_text_end[];
383
384 return addr >= (unsigned long)__lock_text_start
385 && addr < (unsigned long)__lock_text_end;
386}
387EXPORT_SYMBOL(in_lock_functions);