projects/linux·blob master · refs

184 lines · 5.7 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Queued spinlock
4 *
5 * A 'generic' spinlock implementation that is based on MCS locks. For an
6 * architecture that's looking for a 'generic' spinlock, please first consider
7 * ticket-lock.h and only come looking here when you've considered all the
8 * constraints below and can show your hardware does actually perform better
9 * with qspinlock.
10 *
11 * qspinlock relies on atomic_*_release()/atomic_*_acquire() to be RCsc (or no
12 * weaker than RCtso if you're power), where regular code only expects atomic_t
13 * to be RCpc.
14 *
15 * qspinlock relies on a far greater (compared to asm-generic/spinlock.h) set
16 * of atomic operations to behave well together, please audit them carefully to
17 * ensure they all have forward progress. Many atomic operations may default to
18 * cmpxchg() loops which will not have good forward progress properties on
19 * LL/SC architectures.
20 *
21 * One notable example is atomic_fetch_or_acquire(), which x86 cannot (cheaply)
22 * do. Carefully read the patches that introduced
23 * queued_fetch_set_pending_acquire().
24 *
25 * qspinlock also heavily relies on mixed size atomic operations, in specific
26 * it requires architectures to have xchg16; something which many LL/SC
27 * architectures need to implement as a 32bit and+or in order to satisfy the
28 * forward progress guarantees mentioned above.
29 *
30 * Further reading on mixed size atomics that might be relevant:
31 *
32 * http://www.cl.cam.ac.uk/~pes20/popl17/mixed-size.pdf
33 *
34 * (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
35 * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
36 *
37 * Authors: Waiman Long <[email protected]>
38 */
39#ifndef __ASM_GENERIC_QSPINLOCK_H
40#define __ASM_GENERIC_QSPINLOCK_H
41
42#include <asm-generic/qspinlock_types.h>
43#include <linux/atomic.h>
44#include <linux/tracepoint-defs.h>
45
46#ifndef queued_spin_is_locked
47/**
48 * queued_spin_is_locked - is the spinlock locked?
49 * @lock: Pointer to queued spinlock structure
50 * Return: 1 if it is locked, 0 otherwise
51 */
52static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
53{
54 /*
55 * Any !0 state indicates it is locked, even if _Q_LOCKED_VAL
56 * isn't immediately observable.
57 */
58 return atomic_read(&lock->val);
59}
60#endif
61
62/**
63 * queued_spin_value_unlocked - is the spinlock structure unlocked?
64 * @lock: queued spinlock structure
65 * Return: 1 if it is unlocked, 0 otherwise
66 *
67 * N.B. Whenever there are tasks waiting for the lock, it is considered
68 * locked wrt the lockref code to avoid lock stealing by the lockref
69 * code and change things underneath the lock. This also allows some
70 * optimizations to be applied without conflict with lockref.
71 */
72static __always_inline int queued_spin_value_unlocked(struct qspinlock lock)
73{
74 return !lock.val.counter;
75}
76
77/**
78 * queued_spin_is_contended - check if the lock is contended
79 * @lock : Pointer to queued spinlock structure
80 * Return: 1 if lock contended, 0 otherwise
81 */
82static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
83{
84 return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
85}
86/**
87 * queued_spin_trylock - try to acquire the queued spinlock
88 * @lock : Pointer to queued spinlock structure
89 * Return: 1 if lock acquired, 0 if failed
90 */
91static __always_inline int queued_spin_trylock(struct qspinlock *lock)
92{
93 int val = atomic_read(&lock->val);
94
95 if (unlikely(val))
96 return 0;
97
98 return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
99}
100
101extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
102
103#ifndef queued_spin_lock
104/**
105 * queued_spin_lock - acquire a queued spinlock
106 * @lock: Pointer to queued spinlock structure
107 */
108static __always_inline void queued_spin_lock(struct qspinlock *lock)
109{
110 int val = 0;
111
112 if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)))
113 return;
114
115 queued_spin_lock_slowpath(lock, val);
116}
117#endif
118
119#ifndef queued_spin_release
120/**
121 * queued_spin_release - release a queued spinlock
122 * @lock : Pointer to queued spinlock structure
123 */
124static __always_inline void queued_spin_release(struct qspinlock *lock)
125{
126 /*
127 * unlock() needs release semantics:
128 */
129 smp_store_release(&lock->locked, 0);
130}
131#endif
132
133#ifndef queued_spin_unlock
134
135DECLARE_TRACEPOINT(contended_release);
136
137extern void queued_spin_release_traced(struct qspinlock *lock);
138
139/**
140 * queued_spin_unlock - unlock a queued spinlock
141 * @lock : Pointer to queued spinlock structure
142 *
143 * Generic tracing wrapper around the arch-overridable
144 * queued_spin_release().
145 */
146static __always_inline void queued_spin_unlock(struct qspinlock *lock)
147{
148 /*
149 * Trace and release are combined in queued_spin_release_traced() so
150 * the compiler does not need to preserve the lock pointer across the
151 * function call, avoiding callee-saved register save/restore on the
152 * hot path. queued_spin_release() is therefore called both here and in
153 * queued_spin_release_traced(). Keep the two in sync.
154 */
155 if (IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE) &&
156 tracepoint_enabled(contended_release)) {
157 queued_spin_release_traced(lock);
158 return;
159 }
160 queued_spin_release(lock);
161}
162#endif
163
164#ifndef virt_spin_lock
165static __always_inline bool virt_spin_lock(struct qspinlock *lock)
166{
167 return false;
168}
169#endif
170
171#ifndef __no_arch_spinlock_redefine
172/*
173 * Remapping spinlock architecture specific functions to the corresponding
174 * queued spinlock functions.
175 */
176#define arch_spin_is_locked(l) queued_spin_is_locked(l)
177#define arch_spin_is_contended(l) queued_spin_is_contended(l)
178#define arch_spin_value_unlocked(l) queued_spin_value_unlocked(l)
179#define arch_spin_lock(l) queued_spin_lock(l)
180#define arch_spin_trylock(l) queued_spin_trylock(l)
181#define arch_spin_unlock(l) queued_spin_unlock(l)
182#endif
183
184#endif /* __ASM_GENERIC_QSPINLOCK_H */