| 1 | |
| 2 | |
| 3 | * PREEMPT_RT substitution for spin/rw_locks
|
| 4 | *
|
| 5 | * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to
|
| 6 | * resemble the non RT semantics:
|
| 7 | *
|
| 8 | * - Contrary to plain rtmutexes, spinlocks and rwlocks are state
|
| 9 | * preserving. The task state is saved before blocking on the underlying
|
| 10 | * rtmutex, and restored when the lock has been acquired. Regular wakeups
|
| 11 | * during that time are redirected to the saved state so no wake up is
|
| 12 | * missed.
|
| 13 | *
|
| 14 | * - Non RT spin/rwlocks disable preemption and eventually interrupts.
|
| 15 | * Disabling preemption has the side effect of disabling migration and
|
| 16 | * preventing RCU grace periods.
|
| 17 | *
|
| 18 | * The RT substitutions explicitly disable migration and take
|
| 19 | * rcu_read_lock() across the lock held section.
|
| 20 |
|
| 21 | #include <linux/spinlock.h>
|
| 22 | #include <linux/export.h>
|
| 23 |
|
| 24 | #define RT_MUTEX_BUILD_SPINLOCKS
|
| 25 | #include "rtmutex.c"
|
| 26 |
|
| 27 | |
| 28 | * __might_resched() skips the state check as rtlocks are state
|
| 29 | * preserving. Take RCU nesting into account as spin/read/write_lock() can
|
| 30 | * legitimately nest into an RCU read side critical section.
|
| 31 |
|
| 32 | #define RTLOCK_RESCHED_OFFSETS \
|
| 33 | (rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT)
|
| 34 |
|
| 35 | #define rtlock_might_resched() \
|
| 36 | __might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS)
|
| 37 |
|
| 38 | static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
|
| 39 | {
|
| 40 | if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
|
| 41 | rtlock_slowlock(rtm);
|
| 42 | }
|
| 43 |
|
| 44 | static __always_inline void __rt_spin_lock(spinlock_t *lock)
|
| 45 | {
|
| 46 | rtlock_might_resched();
|
| 47 | rtlock_lock(&lock->lock);
|
| 48 | rcu_read_lock();
|
| 49 | migrate_disable();
|
| 50 | }
|
| 51 |
|
| 52 | void __sched rt_spin_lock(spinlock_t *lock)
|
| 53 | {
|
| 54 | spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
|
| 55 | __rt_spin_lock(lock);
|
| 56 | }
|
| 57 | EXPORT_SYMBOL(rt_spin_lock);
|
| 58 |
|
| 59 | #ifdef CONFIG_DEBUG_LOCK_ALLOC
|
| 60 | void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass)
|
| 61 | {
|
| 62 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
|
| 63 | __rt_spin_lock(lock);
|
| 64 | }
|
| 65 | EXPORT_SYMBOL(rt_spin_lock_nested);
|
| 66 |
|
| 67 | void __sched rt_spin_lock_nest_lock(spinlock_t *lock,
|
| 68 | struct lockdep_map *nest_lock)
|
| 69 | {
|
| 70 | spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
|
| 71 | __rt_spin_lock(lock);
|
| 72 | }
|
| 73 | EXPORT_SYMBOL(rt_spin_lock_nest_lock);
|
| 74 | #endif
|
| 75 |
|
| 76 | void __sched rt_spin_unlock(spinlock_t *lock)
|
| 77 | {
|
| 78 | spin_release(&lock->dep_map, _RET_IP_);
|
| 79 | migrate_enable();
|
| 80 | rcu_read_unlock();
|
| 81 |
|
| 82 | if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL)))
|
| 83 | rt_mutex_slowunlock(&lock->lock);
|
| 84 | }
|
| 85 | EXPORT_SYMBOL(rt_spin_unlock);
|
| 86 |
|
| 87 | |
| 88 | * Wait for the lock to get unlocked: instead of polling for an unlock
|
| 89 | * (like raw spinlocks do), lock and unlock, to force the kernel to
|
| 90 | * schedule if there's contention:
|
| 91 |
|
| 92 | void __sched rt_spin_lock_unlock(spinlock_t *lock)
|
| 93 | {
|
| 94 | spin_lock(lock);
|
| 95 | spin_unlock(lock);
|
| 96 | }
|
| 97 | EXPORT_SYMBOL(rt_spin_lock_unlock);
|
| 98 |
|
| 99 | static __always_inline int __rt_spin_trylock(spinlock_t *lock)
|
| 100 | {
|
| 101 | int ret = 1;
|
| 102 |
|
| 103 | if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current)))
|
| 104 | ret = rt_mutex_slowtrylock(&lock->lock);
|
| 105 |
|
| 106 | if (ret) {
|
| 107 | spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
|
| 108 | rcu_read_lock();
|
| 109 | migrate_disable();
|
| 110 | }
|
| 111 | return ret;
|
| 112 | }
|
| 113 |
|
| 114 | int __sched rt_spin_trylock(spinlock_t *lock)
|
| 115 | {
|
| 116 | return __rt_spin_trylock(lock);
|
| 117 | }
|
| 118 | EXPORT_SYMBOL(rt_spin_trylock);
|
| 119 |
|
| 120 | int __sched rt_spin_trylock_bh(spinlock_t *lock)
|
| 121 | {
|
| 122 | int ret;
|
| 123 |
|
| 124 | local_bh_disable();
|
| 125 | ret = __rt_spin_trylock(lock);
|
| 126 | if (!ret)
|
| 127 | local_bh_enable();
|
| 128 | return ret;
|
| 129 | }
|
| 130 | EXPORT_SYMBOL(rt_spin_trylock_bh);
|
| 131 |
|
| 132 | #ifdef CONFIG_DEBUG_LOCK_ALLOC
|
| 133 | void __rt_spin_lock_init(spinlock_t *lock, const char *name,
|
| 134 | struct lock_class_key *key, bool percpu)
|
| 135 | {
|
| 136 | u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL;
|
| 137 |
|
| 138 | debug_check_no_locks_freed((void *)lock, sizeof(*lock));
|
| 139 | lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG,
|
| 140 | LD_WAIT_INV, type);
|
| 141 | }
|
| 142 | EXPORT_SYMBOL(__rt_spin_lock_init);
|
| 143 | #endif
|
| 144 |
|
| 145 | |
| 146 | * RT-specific reader/writer locks
|
| 147 |
|
| 148 | #define rwbase_set_and_save_current_state(state) \
|
| 149 | current_save_and_set_rtlock_wait_state()
|
| 150 |
|
| 151 | #define rwbase_restore_current_state() \
|
| 152 | current_restore_rtlock_saved_state()
|
| 153 |
|
| 154 | static __always_inline int
|
| 155 | rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state)
|
| 156 | {
|
| 157 | if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
|
| 158 | rtlock_slowlock(rtm);
|
| 159 | return 0;
|
| 160 | }
|
| 161 |
|
| 162 | static __always_inline int
|
| 163 | rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state)
|
| 164 | {
|
| 165 | rtlock_slowlock_locked(rtm);
|
| 166 | return 0;
|
| 167 | }
|
| 168 |
|
| 169 | static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm)
|
| 170 | {
|
| 171 | if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL)))
|
| 172 | return;
|
| 173 |
|
| 174 | rt_mutex_slowunlock(rtm);
|
| 175 | }
|
| 176 |
|
| 177 | static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm)
|
| 178 | {
|
| 179 | if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
|
| 180 | return 1;
|
| 181 |
|
| 182 | return rt_mutex_slowtrylock(rtm);
|
| 183 | }
|
| 184 |
|
| 185 | #define rwbase_signal_pending_state(state, current) (0)
|
| 186 |
|
| 187 | #define rwbase_schedule() \
|
| 188 | schedule_rtlock()
|
| 189 |
|
| 190 | #include "rwbase_rt.c"
|
| 191 | |
| 192 | * The common functions which get wrapped into the rwlock API.
|
| 193 |
|
| 194 | int __sched rt_read_trylock(rwlock_t *rwlock)
|
| 195 | {
|
| 196 | int ret;
|
| 197 |
|
| 198 | ret = rwbase_read_trylock(&rwlock->rwbase);
|
| 199 | if (ret) {
|
| 200 | rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_);
|
| 201 | rcu_read_lock();
|
| 202 | migrate_disable();
|
| 203 | }
|
| 204 | return ret;
|
| 205 | }
|
| 206 | EXPORT_SYMBOL(rt_read_trylock);
|
| 207 |
|
| 208 | int __sched rt_write_trylock(rwlock_t *rwlock)
|
| 209 | {
|
| 210 | int ret;
|
| 211 |
|
| 212 | ret = rwbase_write_trylock(&rwlock->rwbase);
|
| 213 | if (ret) {
|
| 214 | rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
|
| 215 | rcu_read_lock();
|
| 216 | migrate_disable();
|
| 217 | }
|
| 218 | return ret;
|
| 219 | }
|
| 220 | EXPORT_SYMBOL(rt_write_trylock);
|
| 221 |
|
| 222 | void __sched rt_read_lock(rwlock_t *rwlock)
|
| 223 | {
|
| 224 | rtlock_might_resched();
|
| 225 | rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_);
|
| 226 | rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
|
| 227 | rcu_read_lock();
|
| 228 | migrate_disable();
|
| 229 | }
|
| 230 | EXPORT_SYMBOL(rt_read_lock);
|
| 231 |
|
| 232 | void __sched rt_write_lock(rwlock_t *rwlock)
|
| 233 | {
|
| 234 | rtlock_might_resched();
|
| 235 | rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
|
| 236 | rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
|
| 237 | rcu_read_lock();
|
| 238 | migrate_disable();
|
| 239 | }
|
| 240 | EXPORT_SYMBOL(rt_write_lock);
|
| 241 |
|
| 242 | #ifdef CONFIG_DEBUG_LOCK_ALLOC
|
| 243 | void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass)
|
| 244 | {
|
| 245 | rtlock_might_resched();
|
| 246 | rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_);
|
| 247 | rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
|
| 248 | rcu_read_lock();
|
| 249 | migrate_disable();
|
| 250 | }
|
| 251 | EXPORT_SYMBOL(rt_write_lock_nested);
|
| 252 | #endif
|
| 253 |
|
| 254 | void __sched rt_read_unlock(rwlock_t *rwlock)
|
| 255 | {
|
| 256 | rwlock_release(&rwlock->dep_map, _RET_IP_);
|
| 257 | migrate_enable();
|
| 258 | rcu_read_unlock();
|
| 259 | rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
|
| 260 | }
|
| 261 | EXPORT_SYMBOL(rt_read_unlock);
|
| 262 |
|
| 263 | void __sched rt_write_unlock(rwlock_t *rwlock)
|
| 264 | {
|
| 265 | rwlock_release(&rwlock->dep_map, _RET_IP_);
|
| 266 | rcu_read_unlock();
|
| 267 | migrate_enable();
|
| 268 | rwbase_write_unlock(&rwlock->rwbase);
|
| 269 | }
|
| 270 | EXPORT_SYMBOL(rt_write_unlock);
|
| 271 |
|
| 272 | #ifdef CONFIG_DEBUG_LOCK_ALLOC
|
| 273 | void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
|
| 274 | struct lock_class_key *key)
|
| 275 | {
|
| 276 | debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock));
|
| 277 | lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG);
|
| 278 | }
|
| 279 | EXPORT_SYMBOL(__rt_rwlock_init);
|
| 280 | #endif
|