projects/linux·blob v3.10-rc6 · refs

1047 lines · 26.3 KiB

1/*
2 * RT-Mutexes: simple blocking mutual exclusion locks with PI support
3 *
4 * started by Ingo Molnar and Thomas Gleixner.
5 *
6 * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <[email protected]>
7 * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <[email protected]>
8 * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
9 * Copyright (C) 2006 Esben Nielsen
10 *
11 * See Documentation/rt-mutex-design.txt for details.
12 */
13#include <linux/spinlock.h>
14#include <linux/export.h>
15#include <linux/sched.h>
16#include <linux/sched/rt.h>
17#include <linux/timer.h>
18
19#include "rtmutex_common.h"
20
21/*
22 * lock->owner state tracking:
23 *
24 * lock->owner holds the task_struct pointer of the owner. Bit 0
25 * is used to keep track of the "lock has waiters" state.
26 *
27 * owner bit0
28 * NULL 0 lock is free (fast acquire possible)
29 * NULL 1 lock is free and has waiters and the top waiter
30 * is going to take the lock*
31 * taskpointer 0 lock is held (fast release possible)
32 * taskpointer 1 lock is held and has waiters**
33 *
34 * The fast atomic compare exchange based acquire and release is only
35 * possible when bit 0 of lock->owner is 0.
36 *
37 * (*) It also can be a transitional state when grabbing the lock
38 * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
39 * we need to set the bit0 before looking at the lock, and the owner may be
40 * NULL in this small time, hence this can be a transitional state.
41 *
42 * (**) There is a small time when bit 0 is set but there are no
43 * waiters. This can happen when grabbing the lock in the slow path.
44 * To prevent a cmpxchg of the owner releasing the lock, we need to
45 * set this bit before looking at the lock.
46 */
47
48static void
49rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
50{
51 unsigned long val = (unsigned long)owner;
52
53 if (rt_mutex_has_waiters(lock))
54 val |= RT_MUTEX_HAS_WAITERS;
55
56 lock->owner = (struct task_struct *)val;
57}
58
59static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
60{
61 lock->owner = (struct task_struct *)
62 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
63}
64
65static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
66{
67 if (!rt_mutex_has_waiters(lock))
68 clear_rt_mutex_waiters(lock);
69}
70
71/*
72 * We can speed up the acquire/release, if the architecture
73 * supports cmpxchg and if there's no debugging state to be set up
74 */
75#if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)
76# define rt_mutex_cmpxchg(l,c,n) (cmpxchg(&l->owner, c, n) == c)
77static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
78{
79 unsigned long owner, *p = (unsigned long *) &lock->owner;
80
81 do {
82 owner = *p;
83 } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
84}
85#else
86# define rt_mutex_cmpxchg(l,c,n) (0)
87static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
88{
89 lock->owner = (struct task_struct *)
90 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
91}
92#endif
93
94/*
95 * Calculate task priority from the waiter list priority
96 *
97 * Return task->normal_prio when the waiter list is empty or when
98 * the waiter is not allowed to do priority boosting
99 */
100int rt_mutex_getprio(struct task_struct *task)
101{
102 if (likely(!task_has_pi_waiters(task)))
103 return task->normal_prio;
104
105 return min(task_top_pi_waiter(task)->pi_list_entry.prio,
106 task->normal_prio);
107}
108
109/*
110 * Adjust the priority of a task, after its pi_waiters got modified.
111 *
112 * This can be both boosting and unboosting. task->pi_lock must be held.
113 */
114static void __rt_mutex_adjust_prio(struct task_struct *task)
115{
116 int prio = rt_mutex_getprio(task);
117
118 if (task->prio != prio)
119 rt_mutex_setprio(task, prio);
120}
121
122/*
123 * Adjust task priority (undo boosting). Called from the exit path of
124 * rt_mutex_slowunlock() and rt_mutex_slowlock().
125 *
126 * (Note: We do this outside of the protection of lock->wait_lock to
127 * allow the lock to be taken while or before we readjust the priority
128 * of task. We do not use the spin_xx_mutex() variants here as we are
129 * outside of the debug path.)
130 */
131static void rt_mutex_adjust_prio(struct task_struct *task)
132{
133 unsigned long flags;
134
135 raw_spin_lock_irqsave(&task->pi_lock, flags);
136 __rt_mutex_adjust_prio(task);
137 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
138}
139
140/*
141 * Max number of times we'll walk the boosting chain:
142 */
143int max_lock_depth = 1024;
144
145/*
146 * Adjust the priority chain. Also used for deadlock detection.
147 * Decreases task's usage by one - may thus free the task.
148 * Returns 0 or -EDEADLK.
149 */
150static int rt_mutex_adjust_prio_chain(struct task_struct *task,
151 int deadlock_detect,
152 struct rt_mutex *orig_lock,
153 struct rt_mutex_waiter *orig_waiter,
154 struct task_struct *top_task)
155{
156 struct rt_mutex *lock;
157 struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
158 int detect_deadlock, ret = 0, depth = 0;
159 unsigned long flags;
160
161 detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
162 deadlock_detect);
163
164 /*
165 * The (de)boosting is a step by step approach with a lot of
166 * pitfalls. We want this to be preemptible and we want hold a
167 * maximum of two locks per step. So we have to check
168 * carefully whether things change under us.
169 */
170 again:
171 if (++depth > max_lock_depth) {
172 static int prev_max;
173
174 /*
175 * Print this only once. If the admin changes the limit,
176 * print a new message when reaching the limit again.
177 */
178 if (prev_max != max_lock_depth) {
179 prev_max = max_lock_depth;
180 printk(KERN_WARNING "Maximum lock depth %d reached "
181 "task: %s (%d)\n", max_lock_depth,
182 top_task->comm, task_pid_nr(top_task));
183 }
184 put_task_struct(task);
185
186 return deadlock_detect ? -EDEADLK : 0;
187 }
188 retry:
189 /*
190 * Task can not go away as we did a get_task() before !
191 */
192 raw_spin_lock_irqsave(&task->pi_lock, flags);
193
194 waiter = task->pi_blocked_on;
195 /*
196 * Check whether the end of the boosting chain has been
197 * reached or the state of the chain has changed while we
198 * dropped the locks.
199 */
200 if (!waiter)
201 goto out_unlock_pi;
202
203 /*
204 * Check the orig_waiter state. After we dropped the locks,
205 * the previous owner of the lock might have released the lock.
206 */
207 if (orig_waiter && !rt_mutex_owner(orig_lock))
208 goto out_unlock_pi;
209
210 /*
211 * Drop out, when the task has no waiters. Note,
212 * top_waiter can be NULL, when we are in the deboosting
213 * mode!
214 */
215 if (top_waiter && (!task_has_pi_waiters(task) ||
216 top_waiter != task_top_pi_waiter(task)))
217 goto out_unlock_pi;
218
219 /*
220 * When deadlock detection is off then we check, if further
221 * priority adjustment is necessary.
222 */
223 if (!detect_deadlock && waiter->list_entry.prio == task->prio)
224 goto out_unlock_pi;
225
226 lock = waiter->lock;
227 if (!raw_spin_trylock(&lock->wait_lock)) {
228 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
229 cpu_relax();
230 goto retry;
231 }
232
233 /* Deadlock detection */
234 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
235 debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
236 raw_spin_unlock(&lock->wait_lock);
237 ret = deadlock_detect ? -EDEADLK : 0;
238 goto out_unlock_pi;
239 }
240
241 top_waiter = rt_mutex_top_waiter(lock);
242
243 /* Requeue the waiter */
244 plist_del(&waiter->list_entry, &lock->wait_list);
245 waiter->list_entry.prio = task->prio;
246 plist_add(&waiter->list_entry, &lock->wait_list);
247
248 /* Release the task */
249 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
250 if (!rt_mutex_owner(lock)) {
251 /*
252 * If the requeue above changed the top waiter, then we need
253 * to wake the new top waiter up to try to get the lock.
254 */
255
256 if (top_waiter != rt_mutex_top_waiter(lock))
257 wake_up_process(rt_mutex_top_waiter(lock)->task);
258 raw_spin_unlock(&lock->wait_lock);
259 goto out_put_task;
260 }
261 put_task_struct(task);
262
263 /* Grab the next task */
264 task = rt_mutex_owner(lock);
265 get_task_struct(task);
266 raw_spin_lock_irqsave(&task->pi_lock, flags);
267
268 if (waiter == rt_mutex_top_waiter(lock)) {
269 /* Boost the owner */
270 plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
271 waiter->pi_list_entry.prio = waiter->list_entry.prio;
272 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
273 __rt_mutex_adjust_prio(task);
274
275 } else if (top_waiter == waiter) {
276 /* Deboost the owner */
277 plist_del(&waiter->pi_list_entry, &task->pi_waiters);
278 waiter = rt_mutex_top_waiter(lock);
279 waiter->pi_list_entry.prio = waiter->list_entry.prio;
280 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
281 __rt_mutex_adjust_prio(task);
282 }
283
284 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
285
286 top_waiter = rt_mutex_top_waiter(lock);
287 raw_spin_unlock(&lock->wait_lock);
288
289 if (!detect_deadlock && waiter != top_waiter)
290 goto out_put_task;
291
292 goto again;
293
294 out_unlock_pi:
295 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
296 out_put_task:
297 put_task_struct(task);
298
299 return ret;
300}
301
302/*
303 * Try to take an rt-mutex
304 *
305 * Must be called with lock->wait_lock held.
306 *
307 * @lock: the lock to be acquired.
308 * @task: the task which wants to acquire the lock
309 * @waiter: the waiter that is queued to the lock's wait list. (could be NULL)
310 */
311static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
312 struct rt_mutex_waiter *waiter)
313{
314 /*
315 * We have to be careful here if the atomic speedups are
316 * enabled, such that, when
317 * - no other waiter is on the lock
318 * - the lock has been released since we did the cmpxchg
319 * the lock can be released or taken while we are doing the
320 * checks and marking the lock with RT_MUTEX_HAS_WAITERS.
321 *
322 * The atomic acquire/release aware variant of
323 * mark_rt_mutex_waiters uses a cmpxchg loop. After setting
324 * the WAITERS bit, the atomic release / acquire can not
325 * happen anymore and lock->wait_lock protects us from the
326 * non-atomic case.
327 *
328 * Note, that this might set lock->owner =
329 * RT_MUTEX_HAS_WAITERS in the case the lock is not contended
330 * any more. This is fixed up when we take the ownership.
331 * This is the transitional state explained at the top of this file.
332 */
333 mark_rt_mutex_waiters(lock);
334
335 if (rt_mutex_owner(lock))
336 return 0;
337
338 /*
339 * It will get the lock because of one of these conditions:
340 * 1) there is no waiter
341 * 2) higher priority than waiters
342 * 3) it is top waiter
343 */
344 if (rt_mutex_has_waiters(lock)) {
345 if (task->prio >= rt_mutex_top_waiter(lock)->list_entry.prio) {
346 if (!waiter || waiter != rt_mutex_top_waiter(lock))
347 return 0;
348 }
349 }
350
351 if (waiter || rt_mutex_has_waiters(lock)) {
352 unsigned long flags;
353 struct rt_mutex_waiter *top;
354
355 raw_spin_lock_irqsave(&task->pi_lock, flags);
356
357 /* remove the queued waiter. */
358 if (waiter) {
359 plist_del(&waiter->list_entry, &lock->wait_list);
360 task->pi_blocked_on = NULL;
361 }
362
363 /*
364 * We have to enqueue the top waiter(if it exists) into
365 * task->pi_waiters list.
366 */
367 if (rt_mutex_has_waiters(lock)) {
368 top = rt_mutex_top_waiter(lock);
369 top->pi_list_entry.prio = top->list_entry.prio;
370 plist_add(&top->pi_list_entry, &task->pi_waiters);
371 }
372 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
373 }
374
375 /* We got the lock. */
376 debug_rt_mutex_lock(lock);
377
378 rt_mutex_set_owner(lock, task);
379
380 rt_mutex_deadlock_account_lock(lock, task);
381
382 return 1;
383}
384
385/*
386 * Task blocks on lock.
387 *
388 * Prepare waiter and propagate pi chain
389 *
390 * This must be called with lock->wait_lock held.
391 */
392static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
393 struct rt_mutex_waiter *waiter,
394 struct task_struct *task,
395 int detect_deadlock)
396{
397 struct task_struct *owner = rt_mutex_owner(lock);
398 struct rt_mutex_waiter *top_waiter = waiter;
399 unsigned long flags;
400 int chain_walk = 0, res;
401
402 raw_spin_lock_irqsave(&task->pi_lock, flags);
403 __rt_mutex_adjust_prio(task);
404 waiter->task = task;
405 waiter->lock = lock;
406 plist_node_init(&waiter->list_entry, task->prio);
407 plist_node_init(&waiter->pi_list_entry, task->prio);
408
409 /* Get the top priority waiter on the lock */
410 if (rt_mutex_has_waiters(lock))
411 top_waiter = rt_mutex_top_waiter(lock);
412 plist_add(&waiter->list_entry, &lock->wait_list);
413
414 task->pi_blocked_on = waiter;
415
416 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
417
418 if (!owner)
419 return 0;
420
421 if (waiter == rt_mutex_top_waiter(lock)) {
422 raw_spin_lock_irqsave(&owner->pi_lock, flags);
423 plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
424 plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
425
426 __rt_mutex_adjust_prio(owner);
427 if (owner->pi_blocked_on)
428 chain_walk = 1;
429 raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
430 }
431 else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
432 chain_walk = 1;
433
434 if (!chain_walk)
435 return 0;
436
437 /*
438 * The owner can't disappear while holding a lock,
439 * so the owner struct is protected by wait_lock.
440 * Gets dropped in rt_mutex_adjust_prio_chain()!
441 */
442 get_task_struct(owner);
443
444 raw_spin_unlock(&lock->wait_lock);
445
446 res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
447 task);
448
449 raw_spin_lock(&lock->wait_lock);
450
451 return res;
452}
453
454/*
455 * Wake up the next waiter on the lock.
456 *
457 * Remove the top waiter from the current tasks waiter list and wake it up.
458 *
459 * Called with lock->wait_lock held.
460 */
461static void wakeup_next_waiter(struct rt_mutex *lock)
462{
463 struct rt_mutex_waiter *waiter;
464 unsigned long flags;
465
466 raw_spin_lock_irqsave(&current->pi_lock, flags);
467
468 waiter = rt_mutex_top_waiter(lock);
469
470 /*
471 * Remove it from current->pi_waiters. We do not adjust a
472 * possible priority boost right now. We execute wakeup in the
473 * boosted mode and go back to normal after releasing
474 * lock->wait_lock.
475 */
476 plist_del(&waiter->pi_list_entry, &current->pi_waiters);
477
478 rt_mutex_set_owner(lock, NULL);
479
480 raw_spin_unlock_irqrestore(&current->pi_lock, flags);
481
482 wake_up_process(waiter->task);
483}
484
485/*
486 * Remove a waiter from a lock and give up
487 *
488 * Must be called with lock->wait_lock held and
489 * have just failed to try_to_take_rt_mutex().
490 */
491static void remove_waiter(struct rt_mutex *lock,
492 struct rt_mutex_waiter *waiter)
493{
494 int first = (waiter == rt_mutex_top_waiter(lock));
495 struct task_struct *owner = rt_mutex_owner(lock);
496 unsigned long flags;
497 int chain_walk = 0;
498
499 raw_spin_lock_irqsave(&current->pi_lock, flags);
500 plist_del(&waiter->list_entry, &lock->wait_list);
501 current->pi_blocked_on = NULL;
502 raw_spin_unlock_irqrestore(&current->pi_lock, flags);
503
504 if (!owner)
505 return;
506
507 if (first) {
508
509 raw_spin_lock_irqsave(&owner->pi_lock, flags);
510
511 plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
512
513 if (rt_mutex_has_waiters(lock)) {
514 struct rt_mutex_waiter *next;
515
516 next = rt_mutex_top_waiter(lock);
517 plist_add(&next->pi_list_entry, &owner->pi_waiters);
518 }
519 __rt_mutex_adjust_prio(owner);
520
521 if (owner->pi_blocked_on)
522 chain_walk = 1;
523
524 raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
525 }
526
527 WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
528
529 if (!chain_walk)
530 return;
531
532 /* gets dropped in rt_mutex_adjust_prio_chain()! */
533 get_task_struct(owner);
534
535 raw_spin_unlock(&lock->wait_lock);
536
537 rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
538
539 raw_spin_lock(&lock->wait_lock);
540}
541
542/*
543 * Recheck the pi chain, in case we got a priority setting
544 *
545 * Called from sched_setscheduler
546 */
547void rt_mutex_adjust_pi(struct task_struct *task)
548{
549 struct rt_mutex_waiter *waiter;
550 unsigned long flags;
551
552 raw_spin_lock_irqsave(&task->pi_lock, flags);
553
554 waiter = task->pi_blocked_on;
555 if (!waiter || waiter->list_entry.prio == task->prio) {
556 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
557 return;
558 }
559
560 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
561
562 /* gets dropped in rt_mutex_adjust_prio_chain()! */
563 get_task_struct(task);
564 rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task);
565}
566
567/**
568 * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
569 * @lock: the rt_mutex to take
570 * @state: the state the task should block in (TASK_INTERRUPTIBLE
571 * or TASK_UNINTERRUPTIBLE)
572 * @timeout: the pre-initialized and started timer, or NULL for none
573 * @waiter: the pre-initialized rt_mutex_waiter
574 *
575 * lock->wait_lock must be held by the caller.
576 */
577static int __sched
578__rt_mutex_slowlock(struct rt_mutex *lock, int state,
579 struct hrtimer_sleeper *timeout,
580 struct rt_mutex_waiter *waiter)
581{
582 int ret = 0;
583
584 for (;;) {
585 /* Try to acquire the lock: */
586 if (try_to_take_rt_mutex(lock, current, waiter))
587 break;
588
589 /*
590 * TASK_INTERRUPTIBLE checks for signals and
591 * timeout. Ignored otherwise.
592 */
593 if (unlikely(state == TASK_INTERRUPTIBLE)) {
594 /* Signal pending? */
595 if (signal_pending(current))
596 ret = -EINTR;
597 if (timeout && !timeout->task)
598 ret = -ETIMEDOUT;
599 if (ret)
600 break;
601 }
602
603 raw_spin_unlock(&lock->wait_lock);
604
605 debug_rt_mutex_print_deadlock(waiter);
606
607 schedule_rt_mutex(lock);
608
609 raw_spin_lock(&lock->wait_lock);
610 set_current_state(state);
611 }
612
613 return ret;
614}
615
616/*
617 * Slow path lock function:
618 */
619static int __sched
620rt_mutex_slowlock(struct rt_mutex *lock, int state,
621 struct hrtimer_sleeper *timeout,
622 int detect_deadlock)
623{
624 struct rt_mutex_waiter waiter;
625 int ret = 0;
626
627 debug_rt_mutex_init_waiter(&waiter);
628
629 raw_spin_lock(&lock->wait_lock);
630
631 /* Try to acquire the lock again: */
632 if (try_to_take_rt_mutex(lock, current, NULL)) {
633 raw_spin_unlock(&lock->wait_lock);
634 return 0;
635 }
636
637 set_current_state(state);
638
639 /* Setup the timer, when timeout != NULL */
640 if (unlikely(timeout)) {
641 hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
642 if (!hrtimer_active(&timeout->timer))
643 timeout->task = NULL;
644 }
645
646 ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock);
647
648 if (likely(!ret))
649 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
650
651 set_current_state(TASK_RUNNING);
652
653 if (unlikely(ret))
654 remove_waiter(lock, &waiter);
655
656 /*
657 * try_to_take_rt_mutex() sets the waiter bit
658 * unconditionally. We might have to fix that up.
659 */
660 fixup_rt_mutex_waiters(lock);
661
662 raw_spin_unlock(&lock->wait_lock);
663
664 /* Remove pending timer: */
665 if (unlikely(timeout))
666 hrtimer_cancel(&timeout->timer);
667
668 debug_rt_mutex_free_waiter(&waiter);
669
670 return ret;
671}
672
673/*
674 * Slow path try-lock function:
675 */
676static inline int
677rt_mutex_slowtrylock(struct rt_mutex *lock)
678{
679 int ret = 0;
680
681 raw_spin_lock(&lock->wait_lock);
682
683 if (likely(rt_mutex_owner(lock) != current)) {
684
685 ret = try_to_take_rt_mutex(lock, current, NULL);
686 /*
687 * try_to_take_rt_mutex() sets the lock waiters
688 * bit unconditionally. Clean this up.
689 */
690 fixup_rt_mutex_waiters(lock);
691 }
692
693 raw_spin_unlock(&lock->wait_lock);
694
695 return ret;
696}
697
698/*
699 * Slow path to release a rt-mutex:
700 */
701static void __sched
702rt_mutex_slowunlock(struct rt_mutex *lock)
703{
704 raw_spin_lock(&lock->wait_lock);
705
706 debug_rt_mutex_unlock(lock);
707
708 rt_mutex_deadlock_account_unlock(current);
709
710 if (!rt_mutex_has_waiters(lock)) {
711 lock->owner = NULL;
712 raw_spin_unlock(&lock->wait_lock);
713 return;
714 }
715
716 wakeup_next_waiter(lock);
717
718 raw_spin_unlock(&lock->wait_lock);
719
720 /* Undo pi boosting if necessary: */
721 rt_mutex_adjust_prio(current);
722}
723
724/*
725 * debug aware fast / slowpath lock,trylock,unlock
726 *
727 * The atomic acquire/release ops are compiled away, when either the
728 * architecture does not support cmpxchg or when debugging is enabled.
729 */
730static inline int
731rt_mutex_fastlock(struct rt_mutex *lock, int state,
732 int detect_deadlock,
733 int (*slowfn)(struct rt_mutex *lock, int state,
734 struct hrtimer_sleeper *timeout,
735 int detect_deadlock))
736{
737 if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
738 rt_mutex_deadlock_account_lock(lock, current);
739 return 0;
740 } else
741 return slowfn(lock, state, NULL, detect_deadlock);
742}
743
744static inline int
745rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
746 struct hrtimer_sleeper *timeout, int detect_deadlock,
747 int (*slowfn)(struct rt_mutex *lock, int state,
748 struct hrtimer_sleeper *timeout,
749 int detect_deadlock))
750{
751 if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
752 rt_mutex_deadlock_account_lock(lock, current);
753 return 0;
754 } else
755 return slowfn(lock, state, timeout, detect_deadlock);
756}
757
758static inline int
759rt_mutex_fasttrylock(struct rt_mutex *lock,
760 int (*slowfn)(struct rt_mutex *lock))
761{
762 if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
763 rt_mutex_deadlock_account_lock(lock, current);
764 return 1;
765 }
766 return slowfn(lock);
767}
768
769static inline void
770rt_mutex_fastunlock(struct rt_mutex *lock,
771 void (*slowfn)(struct rt_mutex *lock))
772{
773 if (likely(rt_mutex_cmpxchg(lock, current, NULL)))
774 rt_mutex_deadlock_account_unlock(current);
775 else
776 slowfn(lock);
777}
778
779/**
780 * rt_mutex_lock - lock a rt_mutex
781 *
782 * @lock: the rt_mutex to be locked
783 */
784void __sched rt_mutex_lock(struct rt_mutex *lock)
785{
786 might_sleep();
787
788 rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
789}
790EXPORT_SYMBOL_GPL(rt_mutex_lock);
791
792/**
793 * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
794 *
795 * @lock: the rt_mutex to be locked
796 * @detect_deadlock: deadlock detection on/off
797 *
798 * Returns:
799 * 0 on success
800 * -EINTR when interrupted by a signal
801 * -EDEADLK when the lock would deadlock (when deadlock detection is on)
802 */
803int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
804 int detect_deadlock)
805{
806 might_sleep();
807
808 return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
809 detect_deadlock, rt_mutex_slowlock);
810}
811EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
812
813/**
814 * rt_mutex_timed_lock - lock a rt_mutex interruptible
815 * the timeout structure is provided
816 * by the caller
817 *
818 * @lock: the rt_mutex to be locked
819 * @timeout: timeout structure or NULL (no timeout)
820 * @detect_deadlock: deadlock detection on/off
821 *
822 * Returns:
823 * 0 on success
824 * -EINTR when interrupted by a signal
825 * -ETIMEDOUT when the timeout expired
826 * -EDEADLK when the lock would deadlock (when deadlock detection is on)
827 */
828int
829rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
830 int detect_deadlock)
831{
832 might_sleep();
833
834 return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
835 detect_deadlock, rt_mutex_slowlock);
836}
837EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
838
839/**
840 * rt_mutex_trylock - try to lock a rt_mutex
841 *
842 * @lock: the rt_mutex to be locked
843 *
844 * Returns 1 on success and 0 on contention
845 */
846int __sched rt_mutex_trylock(struct rt_mutex *lock)
847{
848 return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
849}
850EXPORT_SYMBOL_GPL(rt_mutex_trylock);
851
852/**
853 * rt_mutex_unlock - unlock a rt_mutex
854 *
855 * @lock: the rt_mutex to be unlocked
856 */
857void __sched rt_mutex_unlock(struct rt_mutex *lock)
858{
859 rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
860}
861EXPORT_SYMBOL_GPL(rt_mutex_unlock);
862
863/**
864 * rt_mutex_destroy - mark a mutex unusable
865 * @lock: the mutex to be destroyed
866 *
867 * This function marks the mutex uninitialized, and any subsequent
868 * use of the mutex is forbidden. The mutex must not be locked when
869 * this function is called.
870 */
871void rt_mutex_destroy(struct rt_mutex *lock)
872{
873 WARN_ON(rt_mutex_is_locked(lock));
874#ifdef CONFIG_DEBUG_RT_MUTEXES
875 lock->magic = NULL;
876#endif
877}
878
879EXPORT_SYMBOL_GPL(rt_mutex_destroy);
880
881/**
882 * __rt_mutex_init - initialize the rt lock
883 *
884 * @lock: the rt lock to be initialized
885 *
886 * Initialize the rt lock to unlocked state.
887 *
888 * Initializing of a locked rt lock is not allowed
889 */
890void __rt_mutex_init(struct rt_mutex *lock, const char *name)
891{
892 lock->owner = NULL;
893 raw_spin_lock_init(&lock->wait_lock);
894 plist_head_init(&lock->wait_list);
895
896 debug_rt_mutex_init(lock, name);
897}
898EXPORT_SYMBOL_GPL(__rt_mutex_init);
899
900/**
901 * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
902 * proxy owner
903 *
904 * @lock: the rt_mutex to be locked
905 * @proxy_owner:the task to set as owner
906 *
907 * No locking. Caller has to do serializing itself
908 * Special API call for PI-futex support
909 */
910void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
911 struct task_struct *proxy_owner)
912{
913 __rt_mutex_init(lock, NULL);
914 debug_rt_mutex_proxy_lock(lock, proxy_owner);
915 rt_mutex_set_owner(lock, proxy_owner);
916 rt_mutex_deadlock_account_lock(lock, proxy_owner);
917}
918
919/**
920 * rt_mutex_proxy_unlock - release a lock on behalf of owner
921 *
922 * @lock: the rt_mutex to be locked
923 *
924 * No locking. Caller has to do serializing itself
925 * Special API call for PI-futex support
926 */
927void rt_mutex_proxy_unlock(struct rt_mutex *lock,
928 struct task_struct *proxy_owner)
929{
930 debug_rt_mutex_proxy_unlock(lock);
931 rt_mutex_set_owner(lock, NULL);
932 rt_mutex_deadlock_account_unlock(proxy_owner);
933}
934
935/**
936 * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
937 * @lock: the rt_mutex to take
938 * @waiter: the pre-initialized rt_mutex_waiter
939 * @task: the task to prepare
940 * @detect_deadlock: perform deadlock detection (1) or not (0)
941 *
942 * Returns:
943 * 0 - task blocked on lock
944 * 1 - acquired the lock for task, caller should wake it up
945 * <0 - error
946 *
947 * Special API call for FUTEX_REQUEUE_PI support.
948 */
949int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
950 struct rt_mutex_waiter *waiter,
951 struct task_struct *task, int detect_deadlock)
952{
953 int ret;
954
955 raw_spin_lock(&lock->wait_lock);
956
957 if (try_to_take_rt_mutex(lock, task, NULL)) {
958 raw_spin_unlock(&lock->wait_lock);
959 return 1;
960 }
961
962 ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock);
963
964 if (ret && !rt_mutex_owner(lock)) {
965 /*
966 * Reset the return value. We might have
967 * returned with -EDEADLK and the owner
968 * released the lock while we were walking the
969 * pi chain. Let the waiter sort it out.
970 */
971 ret = 0;
972 }
973
974 if (unlikely(ret))
975 remove_waiter(lock, waiter);
976
977 raw_spin_unlock(&lock->wait_lock);
978
979 debug_rt_mutex_print_deadlock(waiter);
980
981 return ret;
982}
983
984/**
985 * rt_mutex_next_owner - return the next owner of the lock
986 *
987 * @lock: the rt lock query
988 *
989 * Returns the next owner of the lock or NULL
990 *
991 * Caller has to serialize against other accessors to the lock
992 * itself.
993 *
994 * Special API call for PI-futex support
995 */
996struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
997{
998 if (!rt_mutex_has_waiters(lock))
999 return NULL;
1000
1001 return rt_mutex_top_waiter(lock)->task;
1002}
1003
1004/**
1005 * rt_mutex_finish_proxy_lock() - Complete lock acquisition
1006 * @lock: the rt_mutex we were woken on
1007 * @to: the timeout, null if none. hrtimer should already have
1008 * been started.
1009 * @waiter: the pre-initialized rt_mutex_waiter
1010 * @detect_deadlock: perform deadlock detection (1) or not (0)
1011 *
1012 * Complete the lock acquisition started our behalf by another thread.
1013 *
1014 * Returns:
1015 * 0 - success
1016 * <0 - error, one of -EINTR, -ETIMEDOUT, or -EDEADLK
1017 *
1018 * Special API call for PI-futex requeue support
1019 */
1020int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
1021 struct hrtimer_sleeper *to,
1022 struct rt_mutex_waiter *waiter,
1023 int detect_deadlock)
1024{
1025 int ret;
1026
1027 raw_spin_lock(&lock->wait_lock);
1028
1029 set_current_state(TASK_INTERRUPTIBLE);
1030
1031 ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
1032
1033 set_current_state(TASK_RUNNING);
1034
1035 if (unlikely(ret))
1036 remove_waiter(lock, waiter);
1037
1038 /*
1039 * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
1040 * have to fix that up.
1041 */
1042 fixup_rt_mutex_waiters(lock);
1043
1044 raw_spin_unlock(&lock->wait_lock);
1045
1046 return ret;
1047}