projects/linux·blob master · refs

308 lines · 7.7 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * PREEMPT_RT substitution for spin/rw_locks
4 *
5 * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to
6 * resemble the non RT semantics:
7 *
8 * - Contrary to plain rtmutexes, spinlocks and rwlocks are state
9 * preserving. The task state is saved before blocking on the underlying
10 * rtmutex, and restored when the lock has been acquired. Regular wakeups
11 * during that time are redirected to the saved state so no wake up is
12 * missed.
13 *
14 * - Non RT spin/rwlocks disable preemption and eventually interrupts.
15 * Disabling preemption has the side effect of disabling migration and
16 * preventing RCU grace periods.
17 *
18 * The RT substitutions explicitly disable migration and take
19 * rcu_read_lock() across the lock held section.
20 */
21#include <linux/spinlock.h>
22#include <linux/export.h>
23
24#define RT_MUTEX_BUILD_SPINLOCKS
25#include "rtmutex.c"
26
27/*
28 * __might_resched() skips the state check as rtlocks are state
29 * preserving. Take RCU nesting into account as spin/read/write_lock() can
30 * legitimately nest into an RCU read side critical section.
31 */
32#define RTLOCK_RESCHED_OFFSETS \
33 (rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT)
34
35#define rtlock_might_resched() \
36 __might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS)
37
38static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
39{
40 lockdep_assert(!current->pi_blocked_on);
41
42 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
43 rtlock_slowlock(rtm);
44}
45
46static __always_inline void __rt_spin_lock(spinlock_t *lock)
47{
48 rtlock_might_resched();
49 rtlock_lock(&lock->lock);
50 rcu_read_lock();
51 migrate_disable();
52}
53
54void __sched rt_spin_lock(spinlock_t *lock) __acquires(RCU)
55{
56 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
57 __rt_spin_lock(lock);
58}
59EXPORT_SYMBOL(rt_spin_lock);
60
61#ifdef CONFIG_DEBUG_LOCK_ALLOC
62void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass)
63{
64 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
65 __rt_spin_lock(lock);
66}
67EXPORT_SYMBOL(rt_spin_lock_nested);
68
69void __sched rt_spin_lock_nest_lock(spinlock_t *lock,
70 struct lockdep_map *nest_lock)
71{
72 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
73 __rt_spin_lock(lock);
74}
75EXPORT_SYMBOL(rt_spin_lock_nest_lock);
76#endif
77
78void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU)
79{
80 spin_release(&lock->dep_map, _RET_IP_);
81 migrate_enable();
82
83 if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL)))
84 rt_mutex_slowunlock(&lock->lock);
85
86 /*
87 * This must be last to prevent the following UAF:
88 *
89 * T1 T2
90 * spin_lock(&p->lock); rcu_read_lock();
91 * invalidate(p); p = rcu_dereference(ptr);
92 * rcu_assign_pointer(ptr, NULL); if (!p) return;
93 * spin_unlock(&p->lock); spin_lock(&p->lock);
94 * kfree_rcu(p); rcu_read_unlock();
95 * ....
96 * spin_unlock(&p->lock)
97 * rcu_read_unlock(); // Ends grace period
98 * rcu_do_batch()
99 * kfree(p);
100 * UAF -> rt_mutex_cmpxchg_release(&p->lock.lock...)
101 */
102 rcu_read_unlock();
103}
104EXPORT_SYMBOL(rt_spin_unlock);
105
106/*
107 * Wait for the lock to get unlocked: instead of polling for an unlock
108 * (like raw spinlocks do), lock and unlock, to force the kernel to
109 * schedule if there's contention:
110 */
111void __sched rt_spin_lock_unlock(spinlock_t *lock)
112{
113 spin_lock(lock);
114 spin_unlock(lock);
115}
116EXPORT_SYMBOL(rt_spin_lock_unlock);
117
118static __always_inline int __rt_spin_trylock(spinlock_t *lock)
119{
120 int ret = 1;
121
122 if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current)))
123 ret = rt_mutex_slowtrylock(&lock->lock);
124
125 if (ret) {
126 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
127 rcu_read_lock();
128 migrate_disable();
129 }
130 return ret;
131}
132
133int __sched rt_spin_trylock(spinlock_t *lock)
134{
135 return __rt_spin_trylock(lock);
136}
137EXPORT_SYMBOL(rt_spin_trylock);
138
139int __sched rt_spin_trylock_bh(spinlock_t *lock)
140{
141 int ret;
142
143 local_bh_disable();
144 ret = __rt_spin_trylock(lock);
145 if (!ret)
146 local_bh_enable();
147 return ret;
148}
149EXPORT_SYMBOL(rt_spin_trylock_bh);
150
151#ifdef CONFIG_DEBUG_LOCK_ALLOC
152void __rt_spin_lock_init(spinlock_t *lock, const char *name,
153 struct lock_class_key *key, bool percpu)
154{
155 u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL;
156
157 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
158 lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG,
159 LD_WAIT_INV, type);
160}
161EXPORT_SYMBOL(__rt_spin_lock_init);
162#endif
163
164/*
165 * RT-specific reader/writer locks
166 */
167#define rwbase_set_and_save_current_state(state) \
168 current_save_and_set_rtlock_wait_state()
169
170#define rwbase_restore_current_state() \
171 current_restore_rtlock_saved_state()
172
173static __always_inline int
174rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state)
175{
176 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
177 rtlock_slowlock(rtm);
178 return 0;
179}
180
181static __always_inline int
182rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state,
183 struct wake_q_head *wake_q)
184{
185 rtlock_slowlock_locked(rtm, wake_q);
186 return 0;
187}
188
189static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm)
190{
191 if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL)))
192 return;
193
194 rt_mutex_slowunlock(rtm);
195}
196
197static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm)
198{
199 if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
200 return 1;
201
202 return rt_mutex_slowtrylock(rtm);
203}
204
205#define rwbase_signal_pending_state(state, current) (0)
206
207#define rwbase_pre_schedule()
208
209#define rwbase_schedule() \
210 schedule_rtlock()
211
212#define rwbase_post_schedule()
213
214#include "rwbase_rt.c"
215/*
216 * The common functions which get wrapped into the rwlock API.
217 */
218int __sched rt_read_trylock(rwlock_t *rwlock)
219{
220 int ret;
221
222 ret = rwbase_read_trylock(&rwlock->rwbase);
223 if (ret) {
224 rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_);
225 rcu_read_lock();
226 migrate_disable();
227 }
228 return ret;
229}
230EXPORT_SYMBOL(rt_read_trylock);
231
232int __sched rt_write_trylock(rwlock_t *rwlock)
233{
234 int ret;
235
236 ret = rwbase_write_trylock(&rwlock->rwbase);
237 if (ret) {
238 rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
239 rcu_read_lock();
240 migrate_disable();
241 }
242 return ret;
243}
244EXPORT_SYMBOL(rt_write_trylock);
245
246void __sched rt_read_lock(rwlock_t *rwlock) __acquires(RCU)
247{
248 rtlock_might_resched();
249 rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_);
250 rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
251 rcu_read_lock();
252 migrate_disable();
253}
254EXPORT_SYMBOL(rt_read_lock);
255
256void __sched rt_write_lock(rwlock_t *rwlock) __acquires(RCU)
257{
258 rtlock_might_resched();
259 rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
260 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
261 rcu_read_lock();
262 migrate_disable();
263}
264EXPORT_SYMBOL(rt_write_lock);
265
266#ifdef CONFIG_DEBUG_LOCK_ALLOC
267void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass) __acquires(RCU)
268{
269 rtlock_might_resched();
270 rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_);
271 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
272 rcu_read_lock();
273 migrate_disable();
274}
275EXPORT_SYMBOL(rt_write_lock_nested);
276#endif
277
278void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU)
279{
280 rwlock_release(&rwlock->dep_map, _RET_IP_);
281 migrate_enable();
282 rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
283
284 /* This must be last. See comment in rt_spin_unlock() */
285 rcu_read_unlock();
286}
287EXPORT_SYMBOL(rt_read_unlock);
288
289void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU)
290{
291 rwlock_release(&rwlock->dep_map, _RET_IP_);
292 migrate_enable();
293 rwbase_write_unlock(&rwlock->rwbase);
294
295 /* This must be last. See comment in rt_spin_unlock() */
296 rcu_read_unlock();
297}
298EXPORT_SYMBOL(rt_write_unlock);
299
300#ifdef CONFIG_DEBUG_LOCK_ALLOC
301void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
302 struct lock_class_key *key)
303{
304 debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock));
305 lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG);
306}
307EXPORT_SYMBOL(__rt_rwlock_init);
308#endif