projects/linux·blob v7.3 · refs

1301 lines · 33.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * kernel/locking/mutex.c
4 *
5 * Mutexes: blocking mutual exclusion locks
6 *
7 * Started by Ingo Molnar:
8 *
9 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <[email protected]>
10 *
11 * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
12 * David Howells for suggestions and improvements.
13 *
14 * - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
15 * from the -rt tree, where it was originally implemented for rtmutexes
16 * by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
17 * and Sven Dietrich.
18 *
19 * Also see Documentation/locking/mutex-design.rst.
20 */
21#include <linux/mutex.h>
22#include <linux/ww_mutex.h>
23#include <linux/sched/signal.h>
24#include <linux/sched/rt.h>
25#include <linux/sched/wake_q.h>
26#include <linux/sched/debug.h>
27#include <linux/export.h>
28#include <linux/spinlock.h>
29#include <linux/interrupt.h>
30#include <linux/debug_locks.h>
31#include <linux/osq_lock.h>
32#include <linux/hung_task.h>
33
34#define CREATE_TRACE_POINTS
35#include <trace/events/lock.h>
36
37#ifndef CONFIG_PREEMPT_RT
38#include "mutex.h"
39
40#ifdef CONFIG_DEBUG_MUTEXES
41# define MUTEX_WARN_ON(cond) DEBUG_LOCKS_WARN_ON(cond)
42#else
43# define MUTEX_WARN_ON(cond)
44#endif
45
46static void __mutex_init_generic(struct mutex *lock)
47{
48 atomic_long_set(&lock->owner, 0);
49 scoped_guard (raw_spinlock_init, &lock->wait_lock) {
50 lock->first_waiter = NULL;
51 }
52#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
53 osq_lock_init(&lock->osq);
54#endif
55 debug_mutex_init(lock);
56}
57
58static inline struct task_struct *__owner_task(unsigned long owner)
59{
60 return (struct task_struct *)(owner & ~MUTEX_FLAGS);
61}
62
63bool mutex_is_locked(struct mutex *lock)
64{
65 return __mutex_owner(lock) != NULL;
66}
67EXPORT_SYMBOL(mutex_is_locked);
68
69static inline unsigned long __owner_flags(unsigned long owner)
70{
71 return owner & MUTEX_FLAGS;
72}
73
74/* Do not use the return value as a pointer directly. */
75unsigned long mutex_get_owner(struct mutex *lock)
76{
77 unsigned long owner = atomic_long_read(&lock->owner);
78
79 return (unsigned long)__owner_task(owner);
80}
81
82/*
83 * Returns: __mutex_owner(lock) on failure or NULL on success.
84 */
85static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff)
86{
87 unsigned long owner, curr = (unsigned long)current;
88
89 owner = atomic_long_read(&lock->owner);
90 for (;;) { /* must loop, can race against a flag */
91 unsigned long flags = __owner_flags(owner);
92 unsigned long task = owner & ~MUTEX_FLAGS;
93
94 if (task) {
95 if (flags & MUTEX_FLAG_PICKUP) {
96 if (task != curr)
97 break;
98 flags &= ~MUTEX_FLAG_PICKUP;
99 } else if (handoff) {
100 if (flags & MUTEX_FLAG_HANDOFF)
101 break;
102 flags |= MUTEX_FLAG_HANDOFF;
103 } else {
104 break;
105 }
106 } else {
107 MUTEX_WARN_ON(flags & (MUTEX_FLAG_HANDOFF | MUTEX_FLAG_PICKUP));
108 task = curr;
109 }
110
111 if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) {
112 if (task == curr)
113 return NULL;
114 break;
115 }
116 }
117
118 return __owner_task(owner);
119}
120
121/*
122 * Trylock or set HANDOFF
123 */
124static inline bool __mutex_trylock_or_handoff(struct mutex *lock, bool handoff)
125{
126 return !__mutex_trylock_common(lock, handoff);
127}
128
129/*
130 * Actual trylock that will work on any unlocked state.
131 */
132static inline bool __mutex_trylock(struct mutex *lock)
133{
134 return !__mutex_trylock_common(lock, false);
135}
136
137#ifndef CONFIG_DEBUG_LOCK_ALLOC
138/*
139 * Lockdep annotations are contained to the slow paths for simplicity.
140 * There is nothing that would stop spreading the lockdep annotations outwards
141 * except more code.
142 */
143void mutex_init_generic(struct mutex *lock)
144{
145 __mutex_init_generic(lock);
146}
147EXPORT_SYMBOL(mutex_init_generic);
148
149/*
150 * Optimistic trylock that only works in the uncontended case. Make sure to
151 * follow with a __mutex_trylock() before failing.
152 */
153static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
154 __cond_acquires(true, lock)
155{
156 unsigned long curr = (unsigned long)current;
157 unsigned long zero = 0UL;
158
159 MUTEX_WARN_ON(lock->magic != lock);
160
161 if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))
162 return true;
163
164 return false;
165}
166
167static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
168 __cond_releases(true, lock)
169{
170 unsigned long curr = (unsigned long)current;
171
172 return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL);
173}
174
175#else /* !CONFIG_DEBUG_LOCK_ALLOC */
176
177void mutex_init_lockdep(struct mutex *lock, const char *name, struct lock_class_key *key)
178{
179 __mutex_init_generic(lock);
180
181 /*
182 * Make sure we are not reinitializing a held lock:
183 */
184 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
185 lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP);
186}
187EXPORT_SYMBOL(mutex_init_lockdep);
188#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
189
190static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)
191{
192 atomic_long_or(flag, &lock->owner);
193}
194
195static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)
196{
197 atomic_long_andnot(flag, &lock->owner);
198}
199
200/*
201 * Add @waiter to the @lock wait_list and set the FLAG_WAITERS flag if it's
202 * the first waiter.
203 *
204 * When @pos, @waiter is added before the waiter indicated by @pos. Otherwise
205 * @waiter will be added to the tail of the list.
206 */
207static void
208__mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
209 struct mutex_waiter *pos)
210 __must_hold(&lock->wait_lock)
211{
212 struct mutex_waiter *first = lock->first_waiter;
213
214 hung_task_set_blocker(lock, BLOCKER_TYPE_MUTEX);
215 debug_mutex_add_waiter(lock, waiter, current);
216
217 if (pos) {
218 /*
219 * Insert @waiter before @pos.
220 */
221 list_add_tail(&waiter->list, &pos->list);
222 /*
223 * If @pos == @first, then @waiter will be the new first.
224 */
225 if (pos == first)
226 lock->first_waiter = waiter;
227 return;
228 }
229
230 if (first) {
231 list_add_tail(&waiter->list, &first->list);
232 return;
233 }
234
235 INIT_LIST_HEAD(&waiter->list);
236 lock->first_waiter = waiter;
237 __mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
238}
239
240static void
241__mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
242 __must_hold(&lock->wait_lock)
243{
244 if (list_empty(&waiter->list)) {
245 __mutex_clear_flag(lock, MUTEX_FLAGS);
246 lock->first_waiter = NULL;
247 } else {
248 if (lock->first_waiter == waiter)
249 lock->first_waiter = list_next_entry(waiter, list);
250 list_del(&waiter->list);
251 }
252
253 debug_mutex_remove_waiter(lock, waiter, current);
254 hung_task_clear_blocker();
255}
256
257/*
258 * Give up ownership to a specific task, when @task = NULL, this is equivalent
259 * to a regular unlock. Sets PICKUP on a handoff, clears HANDOFF, preserves
260 * WAITERS. Provides RELEASE semantics like a regular unlock, the
261 * __mutex_trylock() provides a matching ACQUIRE semantics for the handoff.
262 */
263static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
264{
265 unsigned long owner = atomic_long_read(&lock->owner);
266
267 for (;;) {
268 unsigned long new;
269
270 MUTEX_WARN_ON(__owner_task(owner) != current);
271 MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
272
273 new = (owner & MUTEX_FLAG_WAITERS);
274 new |= (unsigned long)task;
275 if (task)
276 new |= MUTEX_FLAG_PICKUP;
277
278 if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new))
279 break;
280 }
281}
282
283#ifndef CONFIG_DEBUG_LOCK_ALLOC
284/*
285 * We split the mutex lock/unlock logic into separate fastpath and
286 * slowpath functions, to reduce the register pressure on the fastpath.
287 * We also put the fastpath first in the kernel image, to make sure the
288 * branch is predicted by the CPU as default-untaken.
289 */
290static void __sched __mutex_lock_slowpath(struct mutex *lock)
291 __acquires(lock);
292
293/**
294 * mutex_lock - acquire the mutex
295 * @lock: the mutex to be acquired
296 *
297 * Lock the mutex exclusively for this task. If the mutex is not
298 * available right now, it will sleep until it can get it.
299 *
300 * The mutex must later on be released by the same task that
301 * acquired it. Recursive locking is not allowed. The task
302 * may not exit without first unlocking the mutex. Also, kernel
303 * memory where the mutex resides must not be freed with
304 * the mutex still locked. The mutex must first be initialized
305 * (or statically defined) before it can be locked. memset()-ing
306 * the mutex to 0 is not allowed.
307 *
308 * (The CONFIG_DEBUG_MUTEXES .config option turns on debugging
309 * checks that will enforce the restrictions and will also do
310 * deadlock debugging)
311 *
312 * This function is similar to (but not equivalent to) down().
313 */
314void __sched mutex_lock(struct mutex *lock)
315{
316 might_sleep();
317
318 if (!__mutex_trylock_fast(lock))
319 __mutex_lock_slowpath(lock);
320}
321EXPORT_SYMBOL(mutex_lock);
322#endif
323
324#include "ww_mutex.h"
325
326#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
327
328/*
329 * Trylock variant that returns the owning task on failure.
330 */
331static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock)
332{
333 return __mutex_trylock_common(lock, false);
334}
335
336static inline
337bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
338 struct mutex_waiter *waiter)
339{
340 struct ww_mutex *ww;
341
342 ww = container_of(lock, struct ww_mutex, base);
343
344 /*
345 * If ww->ctx is set the contents are undefined, only
346 * by acquiring wait_lock there is a guarantee that
347 * they are not invalid when reading.
348 *
349 * As such, when deadlock detection needs to be
350 * performed the optimistic spinning cannot be done.
351 *
352 * Check this in every inner iteration because we may
353 * be racing against another thread's ww_mutex_lock.
354 */
355 if (ww_ctx->acquired > 0 && READ_ONCE(ww->ctx))
356 return false;
357
358 /*
359 * If we aren't on the wait list yet, cancel the spin
360 * if there are waiters. We want to avoid stealing the
361 * lock from a waiter with an earlier stamp, since the
362 * other thread may already own a lock that we also
363 * need.
364 */
365 if (!waiter && (atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS))
366 return false;
367
368 /*
369 * Similarly, stop spinning if we are no longer the
370 * first waiter.
371 */
372 if (waiter && data_race(lock->first_waiter != waiter))
373 return false;
374
375 return true;
376}
377
378/*
379 * Look out! "owner" is an entirely speculative pointer access and not
380 * reliable.
381 *
382 * "noinline" so that this function shows up on perf profiles.
383 */
384static noinline
385bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,
386 struct ww_acquire_ctx *ww_ctx, struct mutex_waiter *waiter)
387{
388 bool ret = true;
389
390 lockdep_assert_preemption_disabled();
391
392 while (__mutex_owner(lock) == owner) {
393 /*
394 * Ensure we emit the owner->on_cpu, dereference _after_
395 * checking lock->owner still matches owner. And we already
396 * disabled preemption which is equal to the RCU read-side
397 * crital section in optimistic spinning code. Thus the
398 * task_strcut structure won't go away during the spinning
399 * period
400 */
401 barrier();
402
403 /*
404 * Use vcpu_is_preempted to detect lock holder preemption issue.
405 */
406 if (!owner_on_cpu(owner) || need_resched()) {
407 ret = false;
408 break;
409 }
410
411 if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) {
412 ret = false;
413 break;
414 }
415
416 cpu_relax();
417 }
418
419 return ret;
420}
421
422/*
423 * Initial check for entering the mutex spinning loop
424 */
425static inline int mutex_can_spin_on_owner(struct mutex *lock)
426{
427 struct task_struct *owner;
428 int retval = 1;
429
430 lockdep_assert_preemption_disabled();
431
432 if (need_resched())
433 return 0;
434
435 /*
436 * We already disabled preemption which is equal to the RCU read-side
437 * crital section in optimistic spinning code. Thus the task_strcut
438 * structure won't go away during the spinning period.
439 */
440 owner = __mutex_owner(lock);
441 if (owner)
442 retval = owner_on_cpu(owner);
443
444 /*
445 * If lock->owner is not set, the mutex has been released. Return true
446 * such that we'll trylock in the spin path, which is a faster option
447 * than the blocking slow path.
448 */
449 return retval;
450}
451
452/*
453 * Optimistic spinning.
454 *
455 * We try to spin for acquisition when we find that the lock owner
456 * is currently running on a (different) CPU and while we don't
457 * need to reschedule. The rationale is that if the lock owner is
458 * running, it is likely to release the lock soon.
459 *
460 * The mutex spinners are queued up using MCS lock so that only one
461 * spinner can compete for the mutex. However, if mutex spinning isn't
462 * going to happen, there is no point in going through the lock/unlock
463 * overhead.
464 *
465 * Returns true when the lock was taken, otherwise false, indicating
466 * that we need to jump to the slowpath and sleep.
467 *
468 * The waiter flag is set to true if the spinner is a waiter in the wait
469 * queue. The waiter-spinner will spin on the lock directly and concurrently
470 * with the spinner at the head of the OSQ, if present, until the owner is
471 * changed to itself.
472 */
473static __always_inline bool
474mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
475 struct mutex_waiter *waiter)
476{
477 if (!waiter) {
478 /*
479 * The purpose of the mutex_can_spin_on_owner() function is
480 * to eliminate the overhead of osq_lock() and osq_unlock()
481 * in case spinning isn't possible. As a waiter-spinner
482 * is not going to take OSQ lock anyway, there is no need
483 * to call mutex_can_spin_on_owner().
484 */
485 if (!mutex_can_spin_on_owner(lock))
486 goto fail;
487
488 /*
489 * In order to avoid a stampede of mutex spinners trying to
490 * acquire the mutex all at once, the spinners need to take a
491 * MCS (queued) lock first before spinning on the owner field.
492 */
493 if (!osq_lock(&lock->osq))
494 goto fail;
495 }
496
497 for (;;) {
498 struct task_struct *owner;
499
500 /* Try to acquire the mutex... */
501 owner = __mutex_trylock_or_owner(lock);
502 if (!owner)
503 break;
504
505 /*
506 * There's an owner, wait for it to either
507 * release the lock or go to sleep.
508 */
509 if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))
510 goto fail_unlock;
511
512 /*
513 * The cpu_relax() call is a compiler barrier which forces
514 * everything in this loop to be re-loaded. We don't need
515 * memory barriers as we'll eventually observe the right
516 * values at the cost of a few extra spins.
517 */
518 cpu_relax();
519 }
520
521 if (!waiter)
522 osq_unlock(&lock->osq);
523
524 return true;
525
526
527fail_unlock:
528 if (!waiter)
529 osq_unlock(&lock->osq);
530
531fail:
532 /*
533 * If we fell out of the spin path because of need_resched(),
534 * reschedule now, before we try-lock the mutex. This avoids getting
535 * scheduled out right after we obtained the mutex.
536 */
537 if (need_resched()) {
538 /*
539 * We _should_ have TASK_RUNNING here, but just in case
540 * we do not, make it so, otherwise we might get stuck.
541 */
542 __set_current_state(TASK_RUNNING);
543 schedule_preempt_disabled();
544 }
545
546 return false;
547}
548#else
549static __always_inline bool
550mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
551 struct mutex_waiter *waiter)
552{
553 return false;
554}
555#endif
556
557static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
558 __releases(lock);
559
560/**
561 * mutex_unlock - release the mutex
562 * @lock: the mutex to be released
563 *
564 * Unlock a mutex that has been locked by this task previously.
565 *
566 * This function must not be used in interrupt context. Unlocking
567 * of a not locked mutex is not allowed.
568 *
569 * The caller must ensure that the mutex stays alive until this function has
570 * returned - mutex_unlock() can NOT directly be used to release an object such
571 * that another concurrent task can free it.
572 * Mutexes are different from spinlocks & refcounts in this aspect.
573 *
574 * This function is similar to (but not equivalent to) up().
575 */
576void __sched mutex_unlock(struct mutex *lock)
577{
578#ifndef CONFIG_DEBUG_LOCK_ALLOC
579 if (__mutex_unlock_fast(lock))
580 return;
581#endif
582 __mutex_unlock_slowpath(lock, _RET_IP_);
583}
584EXPORT_SYMBOL(mutex_unlock);
585
586/**
587 * ww_mutex_unlock - release the w/w mutex
588 * @lock: the mutex to be released
589 *
590 * Unlock a mutex that has been locked by this task previously with any of the
591 * ww_mutex_lock* functions (with or without an acquire context). It is
592 * forbidden to release the locks after releasing the acquire context.
593 *
594 * This function must not be used in interrupt context. Unlocking
595 * of a unlocked mutex is not allowed.
596 */
597void __sched ww_mutex_unlock(struct ww_mutex *lock)
598 __no_context_analysis
599{
600 __ww_mutex_unlock(lock);
601 mutex_unlock(&lock->base);
602}
603EXPORT_SYMBOL(ww_mutex_unlock);
604
605/*
606 * Lock a mutex (possibly interruptible), slowpath:
607 */
608static __always_inline int __sched
609__mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass,
610 struct lockdep_map *nest_lock, unsigned long ip,
611 struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
612 __cond_acquires(0, lock)
613{
614 DEFINE_WAKE_Q(wake_q);
615 struct mutex_waiter waiter;
616 struct ww_mutex *ww;
617 unsigned long flags;
618 int ret;
619
620 if (!use_ww_ctx)
621 ww_ctx = NULL;
622
623 might_sleep();
624
625 MUTEX_WARN_ON(lock->magic != lock);
626
627 ww = container_of(lock, struct ww_mutex, base);
628 if (ww_ctx) {
629 if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))
630 return -EALREADY;
631
632 /*
633 * Reset the wounded flag after a kill. No other process can
634 * race and wound us here since they can't have a valid owner
635 * pointer if we don't have any locks held.
636 */
637 if (ww_ctx->acquired == 0)
638 ww_ctx->wounded = 0;
639
640#ifdef CONFIG_DEBUG_LOCK_ALLOC
641 nest_lock = &ww_ctx->dep_map;
642#endif
643 }
644
645 preempt_disable();
646 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
647
648 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
649 if (__mutex_trylock(lock) ||
650 mutex_optimistic_spin(lock, ww_ctx, NULL)) {
651 /* got the lock, yay! */
652 lock_acquired(&lock->dep_map, ip);
653 if (ww_ctx)
654 ww_mutex_set_context_fastpath(ww, ww_ctx);
655 trace_contention_end(lock, 0);
656 preempt_enable();
657 return 0;
658 }
659
660 raw_spin_lock_irqsave(&lock->wait_lock, flags);
661 /*
662 * After waiting to acquire the wait_lock, try again.
663 */
664 if (__mutex_trylock(lock)) {
665 if (ww_ctx)
666 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q);
667
668 goto skip_wait;
669 }
670
671 debug_mutex_lock_common(lock, &waiter);
672 waiter.task = current;
673 if (use_ww_ctx)
674 waiter.ww_ctx = ww_ctx;
675
676 lock_contended(&lock->dep_map, ip);
677
678 if (!use_ww_ctx) {
679 /* add waiting tasks to the end of the waitqueue (FIFO): */
680 __mutex_add_waiter(lock, &waiter, NULL);
681 } else {
682 /*
683 * Add in stamp order, waking up waiters that must kill
684 * themselves.
685 */
686 ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx, &wake_q);
687 if (ret)
688 goto err_early_kill;
689 }
690
691 raw_spin_lock(&current->blocked_lock);
692 __set_task_blocked_on(current, lock);
693 set_current_state(state);
694 trace_contention_begin(lock, LCB_F_MUTEX);
695 for (;;) {
696 bool first;
697
698 /*
699 * Once we hold wait_lock, we're serialized against
700 * mutex_unlock() handing the lock off to us, do a trylock
701 * before testing the error conditions to make sure we pick up
702 * the handoff.
703 */
704 if (__mutex_trylock(lock))
705 break;
706
707 raw_spin_unlock(&current->blocked_lock);
708 /*
709 * Check for signals and kill conditions while holding
710 * wait_lock. This ensures the lock cancellation is ordered
711 * against mutex_unlock() and wake-ups do not go missing.
712 */
713 if (signal_pending_state(state, current)) {
714 ret = -EINTR;
715 goto err;
716 }
717
718 if (ww_ctx) {
719 ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);
720 if (ret)
721 goto err;
722 }
723
724 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
725
726 schedule_preempt_disabled();
727
728 first = lock->first_waiter == &waiter;
729
730 raw_spin_lock_irqsave(&lock->wait_lock, flags);
731 raw_spin_lock(&current->blocked_lock);
732 /*
733 * As we likely have been woken up by task
734 * that has cleared our blocked_on state, re-set
735 * it to the lock we are trying to acquire.
736 */
737 __set_task_blocked_on(current, lock);
738 set_current_state(state);
739 /*
740 * Here we order against unlock; we must either see it change
741 * state back to RUNNING and fall through the next schedule(),
742 * or we must see its unlock and acquire.
743 */
744 if (__mutex_trylock_or_handoff(lock, first))
745 break;
746
747 if (first) {
748 bool opt_acquired;
749
750 /*
751 * mutex_optimistic_spin() can call schedule(), so
752 * we need to release these locks before calling it,
753 * and clear blocked on so we don't become unselectable
754 * to run.
755 */
756 __clear_task_blocked_on(current, lock);
757 raw_spin_unlock(&current->blocked_lock);
758 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
759
760 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
761 opt_acquired = mutex_optimistic_spin(lock, ww_ctx, &waiter);
762
763 raw_spin_lock_irqsave(&lock->wait_lock, flags);
764 raw_spin_lock(&current->blocked_lock);
765 __set_task_blocked_on(current, lock);
766 set_current_state(state);
767
768 if (opt_acquired)
769 break;
770 trace_contention_begin(lock, LCB_F_MUTEX);
771 }
772 }
773 __clear_task_blocked_on(current, lock);
774 __set_current_state(TASK_RUNNING);
775 raw_spin_unlock(&current->blocked_lock);
776
777 if (ww_ctx) {
778 /*
779 * Wound-Wait; we stole the lock (!first_waiter), check the
780 * waiters as anyone might want to wound us.
781 */
782 if (!ww_ctx->is_wait_die && lock->first_waiter != &waiter)
783 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q);
784 }
785
786 __mutex_remove_waiter(lock, &waiter);
787
788 debug_mutex_free_waiter(&waiter);
789
790skip_wait:
791 /* got the lock - cleanup and rejoice! */
792 lock_acquired(&lock->dep_map, ip);
793 trace_contention_end(lock, 0);
794
795 if (ww_ctx)
796 ww_mutex_lock_acquired(ww, ww_ctx);
797
798 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
799 preempt_enable();
800 return 0;
801
802err:
803 clear_task_blocked_on(current, lock);
804 __set_current_state(TASK_RUNNING);
805 __mutex_remove_waiter(lock, &waiter);
806err_early_kill:
807 WARN_ON(get_task_blocked_on(current));
808 trace_contention_end(lock, ret);
809 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
810 debug_mutex_free_waiter(&waiter);
811 mutex_release(&lock->dep_map, ip);
812 preempt_enable();
813 return ret;
814}
815
816static int __sched
817__mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
818 struct lockdep_map *nest_lock, unsigned long ip)
819 __cond_acquires(0, lock)
820{
821 return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false);
822}
823
824static int __sched
825__ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
826 unsigned long ip, struct ww_acquire_ctx *ww_ctx)
827 __cond_acquires(0, lock)
828{
829 return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true);
830}
831
832/**
833 * ww_mutex_trylock - tries to acquire the w/w mutex with optional acquire context
834 * @ww: mutex to lock
835 * @ww_ctx: optional w/w acquire context
836 *
837 * Trylocks a mutex with the optional acquire context; no deadlock detection is
838 * possible. Returns 1 if the mutex has been acquired successfully, 0 otherwise.
839 *
840 * Unlike ww_mutex_lock, no deadlock handling is performed. However, if a @ctx is
841 * specified, -EALREADY handling may happen in calls to ww_mutex_trylock.
842 *
843 * A mutex acquired with this function must be released with ww_mutex_unlock.
844 */
845int ww_mutex_trylock(struct ww_mutex *ww, struct ww_acquire_ctx *ww_ctx)
846{
847 if (!ww_ctx)
848 return mutex_trylock(&ww->base);
849
850 MUTEX_WARN_ON(ww->base.magic != &ww->base);
851
852 /*
853 * Reset the wounded flag after a kill. No other process can
854 * race and wound us here, since they can't have a valid owner
855 * pointer if we don't have any locks held.
856 */
857 if (ww_ctx->acquired == 0)
858 ww_ctx->wounded = 0;
859
860 if (__mutex_trylock(&ww->base)) {
861 ww_mutex_set_context_fastpath(ww, ww_ctx);
862 mutex_acquire_nest(&ww->base.dep_map, 0, 1, &ww_ctx->dep_map, _RET_IP_);
863 return 1;
864 }
865
866 return 0;
867}
868EXPORT_SYMBOL(ww_mutex_trylock);
869
870#ifdef CONFIG_DEBUG_LOCK_ALLOC
871void __sched
872mutex_lock_nested(struct mutex *lock, unsigned int subclass)
873{
874 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
875 __acquire(lock);
876}
877
878EXPORT_SYMBOL_GPL(mutex_lock_nested);
879
880void __sched
881_mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
882{
883 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);
884 __acquire(lock);
885}
886EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
887
888int __sched
889_mutex_lock_killable(struct mutex *lock, unsigned int subclass,
890 struct lockdep_map *nest)
891{
892 return __mutex_lock(lock, TASK_KILLABLE, subclass, nest, _RET_IP_);
893}
894EXPORT_SYMBOL_GPL(_mutex_lock_killable);
895
896int __sched
897mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
898{
899 return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);
900}
901EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
902
903void __sched
904mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)
905{
906 int token;
907
908 might_sleep();
909
910 token = io_schedule_prepare();
911 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
912 subclass, NULL, _RET_IP_, NULL, 0);
913 __acquire(lock);
914 io_schedule_finish(token);
915}
916EXPORT_SYMBOL_GPL(mutex_lock_io_nested);
917
918static inline int
919ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
920 __cond_releases(nonzero, lock)
921{
922#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
923 unsigned tmp;
924
925 if (ctx->deadlock_inject_countdown-- == 0) {
926 tmp = ctx->deadlock_inject_interval;
927 if (tmp > UINT_MAX/4)
928 tmp = UINT_MAX;
929 else
930 tmp = tmp*2 + tmp + tmp/2;
931
932 ctx->deadlock_inject_interval = tmp;
933 ctx->deadlock_inject_countdown = tmp;
934 ctx->contending_lock = lock;
935
936 ww_mutex_unlock(lock);
937
938 return -EDEADLK;
939 }
940#endif
941
942 return 0;
943}
944
945int __sched
946ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
947{
948 int ret;
949
950 might_sleep();
951 ret = __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE,
952 0, _RET_IP_, ctx);
953 if (!ret && ctx && ctx->acquired > 1)
954 return ww_mutex_deadlock_injection(lock, ctx);
955
956 return ret;
957}
958EXPORT_SYMBOL_GPL(ww_mutex_lock);
959
960int __sched
961ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
962{
963 int ret;
964
965 might_sleep();
966 ret = __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE,
967 0, _RET_IP_, ctx);
968
969 if (!ret && ctx && ctx->acquired > 1)
970 return ww_mutex_deadlock_injection(lock, ctx);
971
972 return ret;
973}
974EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible);
975
976#endif
977
978/*
979 * Release the lock, slowpath:
980 */
981static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
982 __releases(lock)
983{
984 struct task_struct *donor, *next = NULL;
985 struct mutex_waiter *waiter;
986 unsigned long owner;
987 unsigned long flags;
988
989 mutex_release(&lock->dep_map, ip);
990 __release(lock);
991
992 /*
993 * Ensures the proxy donor stack is stable across unlock and handoff.
994 * Specifically, it avoids the case where current->blocked_donor is
995 * NULL when it is inspected while doing the unlock, but a preemption
996 * before taking the wake_lock would make it set and a hand-off is
997 * missed.
998 */
999 guard(preempt)();
1000 /*
1001 * Release the lock before (potentially) taking the spinlock such that
1002 * other contenders can get on with things ASAP.
1003 *
1004 * Except when HANDOFF, in that case we must not clear the owner field,
1005 * but instead set it to the top waiter.
1006 */
1007 owner = atomic_long_read(&lock->owner);
1008 for (;;) {
1009 MUTEX_WARN_ON(__owner_task(owner) != current);
1010 MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
1011
1012 if (sched_proxy_exec() && current->blocked_donor) {
1013 /* force handoff if we have a blocked_donor */
1014 owner = MUTEX_FLAG_HANDOFF;
1015 break;
1016 }
1017
1018 if (owner & MUTEX_FLAG_HANDOFF)
1019 break;
1020
1021 if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {
1022 if (owner & MUTEX_FLAG_WAITERS)
1023 break;
1024
1025 return;
1026 }
1027 }
1028
1029 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1030 raw_spin_lock(&current->blocked_lock);
1031 debug_mutex_unlock(lock);
1032
1033 if (sched_proxy_exec()) {
1034 /*
1035 * If we have a task boosting current, and that task was boosting
1036 * current through this lock, hand the lock to that task, as that
1037 * is the highest waiter, as selected by the scheduling function.
1038 */
1039 donor = current->blocked_donor;
1040 if (donor) {
1041 struct mutex *next_lock;
1042
1043 raw_spin_lock_nested(&donor->blocked_lock, SINGLE_DEPTH_NESTING);
1044 next_lock = __get_task_blocked_on(donor);
1045 if (next_lock == lock) {
1046 next = get_task_struct(donor);
1047 __clear_task_blocked_on(next, lock);
1048 current->blocked_donor = NULL;
1049 }
1050 raw_spin_unlock(&donor->blocked_lock);
1051 }
1052 }
1053
1054 /*
1055 * Failing that, pick first on the wait list.
1056 */
1057 waiter = lock->first_waiter;
1058 if (!next && waiter) {
1059 next = get_task_struct(waiter->task);
1060
1061 raw_spin_lock_nested(&next->blocked_lock, SINGLE_DEPTH_NESTING);
1062 debug_mutex_wake_waiter(lock, waiter);
1063 __clear_task_blocked_on(next, lock);
1064 raw_spin_unlock(&next->blocked_lock);
1065
1066 }
1067
1068 if (trace_contended_release_enabled() && waiter)
1069 trace_call__contended_release(lock);
1070
1071 if (owner & MUTEX_FLAG_HANDOFF)
1072 __mutex_handoff(lock, next);
1073
1074 raw_spin_unlock(&current->blocked_lock);
1075 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1076 if (next) {
1077 wake_up_process(next);
1078 put_task_struct(next);
1079 }
1080}
1081
1082#ifndef CONFIG_DEBUG_LOCK_ALLOC
1083/*
1084 * Here come the less common (and hence less performance-critical) APIs:
1085 * mutex_lock_interruptible() and mutex_trylock().
1086 */
1087static noinline int __sched
1088__mutex_lock_killable_slowpath(struct mutex *lock);
1089
1090static noinline int __sched
1091__mutex_lock_interruptible_slowpath(struct mutex *lock);
1092
1093/**
1094 * mutex_lock_interruptible() - Acquire the mutex, interruptible by signals.
1095 * @lock: The mutex to be acquired.
1096 *
1097 * Lock the mutex like mutex_lock(). If a signal is delivered while the
1098 * process is sleeping, this function will return without acquiring the
1099 * mutex.
1100 *
1101 * Context: Process context.
1102 * Return: 0 if the lock was successfully acquired or %-EINTR if a
1103 * signal arrived.
1104 */
1105int __sched mutex_lock_interruptible(struct mutex *lock)
1106{
1107 might_sleep();
1108
1109 if (__mutex_trylock_fast(lock))
1110 return 0;
1111
1112 return __mutex_lock_interruptible_slowpath(lock);
1113}
1114
1115EXPORT_SYMBOL(mutex_lock_interruptible);
1116
1117/**
1118 * mutex_lock_killable() - Acquire the mutex, interruptible by fatal signals.
1119 * @lock: The mutex to be acquired.
1120 *
1121 * Lock the mutex like mutex_lock(). If a signal which will be fatal to
1122 * the current process is delivered while the process is sleeping, this
1123 * function will return without acquiring the mutex.
1124 *
1125 * Context: Process context.
1126 * Return: 0 if the lock was successfully acquired or %-EINTR if a
1127 * fatal signal arrived.
1128 */
1129int __sched mutex_lock_killable(struct mutex *lock)
1130{
1131 might_sleep();
1132
1133 if (__mutex_trylock_fast(lock))
1134 return 0;
1135
1136 return __mutex_lock_killable_slowpath(lock);
1137}
1138EXPORT_SYMBOL(mutex_lock_killable);
1139
1140/**
1141 * mutex_lock_io() - Acquire the mutex and mark the process as waiting for I/O
1142 * @lock: The mutex to be acquired.
1143 *
1144 * Lock the mutex like mutex_lock(). While the task is waiting for this
1145 * mutex, it will be accounted as being in the IO wait state by the
1146 * scheduler.
1147 *
1148 * Context: Process context.
1149 */
1150void __sched mutex_lock_io(struct mutex *lock)
1151{
1152 int token;
1153
1154 token = io_schedule_prepare();
1155 mutex_lock(lock);
1156 io_schedule_finish(token);
1157}
1158EXPORT_SYMBOL_GPL(mutex_lock_io);
1159
1160static noinline void __sched
1161__mutex_lock_slowpath(struct mutex *lock)
1162 __acquires(lock)
1163{
1164 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
1165 __acquire(lock);
1166}
1167
1168static noinline int __sched
1169__mutex_lock_killable_slowpath(struct mutex *lock)
1170 __cond_acquires(0, lock)
1171{
1172 return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
1173}
1174
1175static noinline int __sched
1176__mutex_lock_interruptible_slowpath(struct mutex *lock)
1177 __cond_acquires(0, lock)
1178{
1179 return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
1180}
1181
1182static noinline int __sched
1183__ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1184 __cond_acquires(0, lock)
1185{
1186 return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0,
1187 _RET_IP_, ctx);
1188}
1189
1190static noinline int __sched
1191__ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
1192 struct ww_acquire_ctx *ctx)
1193 __cond_acquires(0, lock)
1194{
1195 return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0,
1196 _RET_IP_, ctx);
1197}
1198
1199#endif
1200
1201#ifndef CONFIG_DEBUG_LOCK_ALLOC
1202/**
1203 * mutex_trylock - try to acquire the mutex, without waiting
1204 * @lock: the mutex to be acquired
1205 *
1206 * Try to acquire the mutex atomically. Returns 1 if the mutex
1207 * has been acquired successfully, and 0 on contention.
1208 *
1209 * NOTE: this function follows the spin_trylock() convention, so
1210 * it is negated from the down_trylock() return values! Be careful
1211 * about this when converting semaphore users to mutexes.
1212 *
1213 * This function must not be used in interrupt context. The
1214 * mutex must be released by the same task that acquired it.
1215 */
1216int __sched mutex_trylock(struct mutex *lock)
1217{
1218 MUTEX_WARN_ON(lock->magic != lock);
1219 return __mutex_trylock(lock);
1220}
1221EXPORT_SYMBOL(mutex_trylock);
1222#else
1223int __sched _mutex_trylock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock)
1224{
1225 bool locked;
1226
1227 MUTEX_WARN_ON(lock->magic != lock);
1228 locked = __mutex_trylock(lock);
1229 if (locked)
1230 mutex_acquire_nest(&lock->dep_map, 0, 1, nest_lock, _RET_IP_);
1231
1232 return locked;
1233}
1234EXPORT_SYMBOL(_mutex_trylock_nest_lock);
1235#endif
1236
1237#ifndef CONFIG_DEBUG_LOCK_ALLOC
1238int __sched
1239ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1240{
1241 might_sleep();
1242
1243 if (__mutex_trylock_fast(&lock->base)) {
1244 if (ctx)
1245 ww_mutex_set_context_fastpath(lock, ctx);
1246 return 0;
1247 }
1248
1249 return __ww_mutex_lock_slowpath(lock, ctx);
1250}
1251EXPORT_SYMBOL(ww_mutex_lock);
1252
1253int __sched
1254ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1255{
1256 might_sleep();
1257
1258 if (__mutex_trylock_fast(&lock->base)) {
1259 if (ctx)
1260 ww_mutex_set_context_fastpath(lock, ctx);
1261 return 0;
1262 }
1263
1264 return __ww_mutex_lock_interruptible_slowpath(lock, ctx);
1265}
1266EXPORT_SYMBOL(ww_mutex_lock_interruptible);
1267
1268#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
1269#endif /* !CONFIG_PREEMPT_RT */
1270
1271EXPORT_TRACEPOINT_SYMBOL_GPL(contention_begin);
1272EXPORT_TRACEPOINT_SYMBOL_GPL(contention_end);
1273EXPORT_TRACEPOINT_SYMBOL_GPL(contended_release);
1274
1275__weak int arch_contended_release_trace_reg(void) { return 0; }
1276
1277__weak void arch_contended_release_trace_unreg(void) { }
1278
1279/**
1280 * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
1281 * @cnt: the atomic which we are to dec
1282 * @lock: the mutex to return holding if we dec to 0
1283 *
1284 * return true and hold lock if we dec to 0, return false otherwise
1285 */
1286int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
1287{
1288 /* dec if we can't possibly hit 0 */
1289 if (atomic_add_unless(cnt, -1, 1))
1290 return 0;
1291 /* we might hit 0, so take the lock */
1292 mutex_lock(lock);
1293 if (!atomic_dec_and_test(cnt)) {
1294 /* when we actually did the dec, we didn't hit 0 */
1295 mutex_unlock(lock);
1296 return 0;
1297 }
1298 /* we hit 0, and we hold the lock */
1299 return 1;
1300}
1301EXPORT_SYMBOL(atomic_dec_and_mutex_lock);