projects/linux·blob v5.2 · refs

304 lines · 8.7 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * The least significant 2 bits of the owner value has the following
4 * meanings when set.
5 * - RWSEM_READER_OWNED (bit 0): The rwsem is owned by readers
6 * - RWSEM_ANONYMOUSLY_OWNED (bit 1): The rwsem is anonymously owned,
7 * i.e. the owner(s) cannot be readily determined. It can be reader
8 * owned or the owning writer is indeterminate.
9 *
10 * When a writer acquires a rwsem, it puts its task_struct pointer
11 * into the owner field. It is cleared after an unlock.
12 *
13 * When a reader acquires a rwsem, it will also puts its task_struct
14 * pointer into the owner field with both the RWSEM_READER_OWNED and
15 * RWSEM_ANONYMOUSLY_OWNED bits set. On unlock, the owner field will
16 * largely be left untouched. So for a free or reader-owned rwsem,
17 * the owner value may contain information about the last reader that
18 * acquires the rwsem. The anonymous bit is set because that particular
19 * reader may or may not still own the lock.
20 *
21 * That information may be helpful in debugging cases where the system
22 * seems to hang on a reader owned rwsem especially if only one reader
23 * is involved. Ideally we would like to track all the readers that own
24 * a rwsem, but the overhead is simply too big.
25 */
26#include "lock_events.h"
27
28#define RWSEM_READER_OWNED (1UL << 0)
29#define RWSEM_ANONYMOUSLY_OWNED (1UL << 1)
30
31#ifdef CONFIG_DEBUG_RWSEMS
32# define DEBUG_RWSEMS_WARN_ON(c, sem) do { \
33 if (!debug_locks_silent && \
34 WARN_ONCE(c, "DEBUG_RWSEMS_WARN_ON(%s): count = 0x%lx, owner = 0x%lx, curr 0x%lx, list %sempty\n",\
35 #c, atomic_long_read(&(sem)->count), \
36 (long)((sem)->owner), (long)current, \
37 list_empty(&(sem)->wait_list) ? "" : "not ")) \
38 debug_locks_off(); \
39 } while (0)
40#else
41# define DEBUG_RWSEMS_WARN_ON(c, sem)
42#endif
43
44/*
45 * R/W semaphores originally for PPC using the stuff in lib/rwsem.c.
46 * Adapted largely from include/asm-i386/rwsem.h
47 * by Paul Mackerras <[email protected]>.
48 */
49
50/*
51 * the semaphore definition
52 */
53#ifdef CONFIG_64BIT
54# define RWSEM_ACTIVE_MASK 0xffffffffL
55#else
56# define RWSEM_ACTIVE_MASK 0x0000ffffL
57#endif
58
59#define RWSEM_ACTIVE_BIAS 0x00000001L
60#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1)
61#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
62#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
63
64#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
65/*
66 * All writes to owner are protected by WRITE_ONCE() to make sure that
67 * store tearing can't happen as optimistic spinners may read and use
68 * the owner value concurrently without lock. Read from owner, however,
69 * may not need READ_ONCE() as long as the pointer value is only used
70 * for comparison and isn't being dereferenced.
71 */
72static inline void rwsem_set_owner(struct rw_semaphore *sem)
73{
74 WRITE_ONCE(sem->owner, current);
75}
76
77static inline void rwsem_clear_owner(struct rw_semaphore *sem)
78{
79 WRITE_ONCE(sem->owner, NULL);
80}
81
82/*
83 * The task_struct pointer of the last owning reader will be left in
84 * the owner field.
85 *
86 * Note that the owner value just indicates the task has owned the rwsem
87 * previously, it may not be the real owner or one of the real owners
88 * anymore when that field is examined, so take it with a grain of salt.
89 */
90static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
91 struct task_struct *owner)
92{
93 unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED
94 | RWSEM_ANONYMOUSLY_OWNED;
95
96 WRITE_ONCE(sem->owner, (struct task_struct *)val);
97}
98
99static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
100{
101 __rwsem_set_reader_owned(sem, current);
102}
103
104/*
105 * Return true if the a rwsem waiter can spin on the rwsem's owner
106 * and steal the lock, i.e. the lock is not anonymously owned.
107 * N.B. !owner is considered spinnable.
108 */
109static inline bool is_rwsem_owner_spinnable(struct task_struct *owner)
110{
111 return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED);
112}
113
114/*
115 * Return true if rwsem is owned by an anonymous writer or readers.
116 */
117static inline bool rwsem_has_anonymous_owner(struct task_struct *owner)
118{
119 return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED;
120}
121
122#ifdef CONFIG_DEBUG_RWSEMS
123/*
124 * With CONFIG_DEBUG_RWSEMS configured, it will make sure that if there
125 * is a task pointer in owner of a reader-owned rwsem, it will be the
126 * real owner or one of the real owners. The only exception is when the
127 * unlock is done by up_read_non_owner().
128 */
129#define rwsem_clear_reader_owned rwsem_clear_reader_owned
130static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
131{
132 unsigned long val = (unsigned long)current | RWSEM_READER_OWNED
133 | RWSEM_ANONYMOUSLY_OWNED;
134 if (READ_ONCE(sem->owner) == (struct task_struct *)val)
135 cmpxchg_relaxed((unsigned long *)&sem->owner, val,
136 RWSEM_READER_OWNED | RWSEM_ANONYMOUSLY_OWNED);
137}
138#endif
139
140#else
141static inline void rwsem_set_owner(struct rw_semaphore *sem)
142{
143}
144
145static inline void rwsem_clear_owner(struct rw_semaphore *sem)
146{
147}
148
149static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
150 struct task_struct *owner)
151{
152}
153
154static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
155{
156}
157#endif
158
159#ifndef rwsem_clear_reader_owned
160static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
161{
162}
163#endif
164
165extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
166extern struct rw_semaphore *rwsem_down_read_failed_killable(struct rw_semaphore *sem);
167extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
168extern struct rw_semaphore *rwsem_down_write_failed_killable(struct rw_semaphore *sem);
169extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem);
170extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);
171
172/*
173 * lock for reading
174 */
175static inline void __down_read(struct rw_semaphore *sem)
176{
177 if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0)) {
178 rwsem_down_read_failed(sem);
179 DEBUG_RWSEMS_WARN_ON(!((unsigned long)sem->owner &
180 RWSEM_READER_OWNED), sem);
181 } else {
182 rwsem_set_reader_owned(sem);
183 }
184}
185
186static inline int __down_read_killable(struct rw_semaphore *sem)
187{
188 if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0)) {
189 if (IS_ERR(rwsem_down_read_failed_killable(sem)))
190 return -EINTR;
191 DEBUG_RWSEMS_WARN_ON(!((unsigned long)sem->owner &
192 RWSEM_READER_OWNED), sem);
193 } else {
194 rwsem_set_reader_owned(sem);
195 }
196 return 0;
197}
198
199static inline int __down_read_trylock(struct rw_semaphore *sem)
200{
201 /*
202 * Optimize for the case when the rwsem is not locked at all.
203 */
204 long tmp = RWSEM_UNLOCKED_VALUE;
205
206 lockevent_inc(rwsem_rtrylock);
207 do {
208 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,
209 tmp + RWSEM_ACTIVE_READ_BIAS)) {
210 rwsem_set_reader_owned(sem);
211 return 1;
212 }
213 } while (tmp >= 0);
214 return 0;
215}
216
217/*
218 * lock for writing
219 */
220static inline void __down_write(struct rw_semaphore *sem)
221{
222 long tmp;
223
224 tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
225 &sem->count);
226 if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
227 rwsem_down_write_failed(sem);
228 rwsem_set_owner(sem);
229}
230
231static inline int __down_write_killable(struct rw_semaphore *sem)
232{
233 long tmp;
234
235 tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
236 &sem->count);
237 if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
238 if (IS_ERR(rwsem_down_write_failed_killable(sem)))
239 return -EINTR;
240 rwsem_set_owner(sem);
241 return 0;
242}
243
244static inline int __down_write_trylock(struct rw_semaphore *sem)
245{
246 long tmp;
247
248 lockevent_inc(rwsem_wtrylock);
249 tmp = atomic_long_cmpxchg_acquire(&sem->count, RWSEM_UNLOCKED_VALUE,
250 RWSEM_ACTIVE_WRITE_BIAS);
251 if (tmp == RWSEM_UNLOCKED_VALUE) {
252 rwsem_set_owner(sem);
253 return true;
254 }
255 return false;
256}
257
258/*
259 * unlock after reading
260 */
261static inline void __up_read(struct rw_semaphore *sem)
262{
263 long tmp;
264
265 DEBUG_RWSEMS_WARN_ON(!((unsigned long)sem->owner & RWSEM_READER_OWNED),
266 sem);
267 rwsem_clear_reader_owned(sem);
268 tmp = atomic_long_dec_return_release(&sem->count);
269 if (unlikely(tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0))
270 rwsem_wake(sem);
271}
272
273/*
274 * unlock after writing
275 */
276static inline void __up_write(struct rw_semaphore *sem)
277{
278 DEBUG_RWSEMS_WARN_ON(sem->owner != current, sem);
279 rwsem_clear_owner(sem);
280 if (unlikely(atomic_long_sub_return_release(RWSEM_ACTIVE_WRITE_BIAS,
281 &sem->count) < 0))
282 rwsem_wake(sem);
283}
284
285/*
286 * downgrade write lock to read lock
287 */
288static inline void __downgrade_write(struct rw_semaphore *sem)
289{
290 long tmp;
291
292 /*
293 * When downgrading from exclusive to shared ownership,
294 * anything inside the write-locked region cannot leak
295 * into the read side. In contrast, anything in the
296 * read-locked region is ok to be re-ordered into the
297 * write side. As such, rely on RELEASE semantics.
298 */
299 DEBUG_RWSEMS_WARN_ON(sem->owner != current, sem);
300 tmp = atomic_long_add_return_release(-RWSEM_WAITING_BIAS, &sem->count);
301 rwsem_set_reader_owned(sem);
302 if (tmp < 0)
303 rwsem_downgrade_wake(sem);
304}