| 1 | #include <linux/atomic.h>
|
| 2 | #include <linux/rwsem.h>
|
| 3 | #include <linux/percpu.h>
|
| 4 | #include <linux/wait.h>
|
| 5 | #include <linux/lockdep.h>
|
| 6 | #include <linux/percpu-rwsem.h>
|
| 7 | #include <linux/rcupdate.h>
|
| 8 | #include <linux/sched.h>
|
| 9 | #include <linux/errno.h>
|
| 10 |
|
| 11 | int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
|
| 12 | const char *name, struct lock_class_key *rwsem_key)
|
| 13 | {
|
| 14 | sem->read_count = alloc_percpu(int);
|
| 15 | if (unlikely(!sem->read_count))
|
| 16 | return -ENOMEM;
|
| 17 |
|
| 18 |
|
| 19 | rcu_sync_init(&sem->rss, RCU_SCHED_SYNC);
|
| 20 | __init_rwsem(&sem->rw_sem, name, rwsem_key);
|
| 21 | init_waitqueue_head(&sem->writer);
|
| 22 | sem->readers_block = 0;
|
| 23 | return 0;
|
| 24 | }
|
| 25 | EXPORT_SYMBOL_GPL(__percpu_init_rwsem);
|
| 26 |
|
| 27 | void percpu_free_rwsem(struct percpu_rw_semaphore *sem)
|
| 28 | {
|
| 29 | |
| 30 | * XXX: temporary kludge. The error path in alloc_super()
|
| 31 | * assumes that percpu_free_rwsem() is safe after kzalloc().
|
| 32 |
|
| 33 | if (!sem->read_count)
|
| 34 | return;
|
| 35 |
|
| 36 | rcu_sync_dtor(&sem->rss);
|
| 37 | free_percpu(sem->read_count);
|
| 38 | sem->read_count = NULL;
|
| 39 | }
|
| 40 | EXPORT_SYMBOL_GPL(percpu_free_rwsem);
|
| 41 |
|
| 42 | int __percpu_down_read(struct percpu_rw_semaphore *sem, int try)
|
| 43 | {
|
| 44 | |
| 45 | * Due to having preemption disabled the decrement happens on
|
| 46 | * the same CPU as the increment, avoiding the
|
| 47 | * increment-on-one-CPU-and-decrement-on-another problem.
|
| 48 | *
|
| 49 | * If the reader misses the writer's assignment of readers_block, then
|
| 50 | * the writer is guaranteed to see the reader's increment.
|
| 51 | *
|
| 52 | * Conversely, any readers that increment their sem->read_count after
|
| 53 | * the writer looks are guaranteed to see the readers_block value,
|
| 54 | * which in turn means that they are guaranteed to immediately
|
| 55 | * decrement their sem->read_count, so that it doesn't matter that the
|
| 56 | * writer missed them.
|
| 57 |
|
| 58 |
|
| 59 | smp_mb();
|
| 60 |
|
| 61 | |
| 62 | * If !readers_block the critical section starts here, matched by the
|
| 63 | * release in percpu_up_write().
|
| 64 |
|
| 65 | if (likely(!smp_load_acquire(&sem->readers_block)))
|
| 66 | return 1;
|
| 67 |
|
| 68 | |
| 69 | * Per the above comment; we still have preemption disabled and
|
| 70 | * will thus decrement on the same CPU as we incremented.
|
| 71 |
|
| 72 | __percpu_up_read(sem);
|
| 73 |
|
| 74 | if (try)
|
| 75 | return 0;
|
| 76 |
|
| 77 | |
| 78 | * We either call schedule() in the wait, or we'll fall through
|
| 79 | * and reschedule on the preempt_enable() in percpu_down_read().
|
| 80 |
|
| 81 | preempt_enable_no_resched();
|
| 82 |
|
| 83 | |
| 84 | * Avoid lockdep for the down/up_read() we already have them.
|
| 85 |
|
| 86 | __down_read(&sem->rw_sem);
|
| 87 | this_cpu_inc(*sem->read_count);
|
| 88 | __up_read(&sem->rw_sem);
|
| 89 |
|
| 90 | preempt_disable();
|
| 91 | return 1;
|
| 92 | }
|
| 93 | EXPORT_SYMBOL_GPL(__percpu_down_read);
|
| 94 |
|
| 95 | void __percpu_up_read(struct percpu_rw_semaphore *sem)
|
| 96 | {
|
| 97 | smp_mb();
|
| 98 | |
| 99 | * In other words, if they see our decrement (presumably to aggregate
|
| 100 | * zero, as that is the only time it matters) they will also see our
|
| 101 | * critical section.
|
| 102 |
|
| 103 | __this_cpu_dec(*sem->read_count);
|
| 104 |
|
| 105 |
|
| 106 | wake_up(&sem->writer);
|
| 107 | }
|
| 108 | EXPORT_SYMBOL_GPL(__percpu_up_read);
|
| 109 |
|
| 110 | #define per_cpu_sum(var) \
|
| 111 | ({ \
|
| 112 | typeof(var) __sum = 0; \
|
| 113 | int cpu; \
|
| 114 | compiletime_assert_atomic_type(__sum); \
|
| 115 | for_each_possible_cpu(cpu) \
|
| 116 | __sum += per_cpu(var, cpu); \
|
| 117 | __sum; \
|
| 118 | })
|
| 119 |
|
| 120 | |
| 121 | * Return true if the modular sum of the sem->read_count per-CPU variable is
|
| 122 | * zero. If this sum is zero, then it is stable due to the fact that if any
|
| 123 | * newly arriving readers increment a given counter, they will immediately
|
| 124 | * decrement that same counter.
|
| 125 |
|
| 126 | static bool readers_active_check(struct percpu_rw_semaphore *sem)
|
| 127 | {
|
| 128 | if (per_cpu_sum(*sem->read_count) != 0)
|
| 129 | return false;
|
| 130 |
|
| 131 | |
| 132 | * If we observed the decrement; ensure we see the entire critical
|
| 133 | * section.
|
| 134 |
|
| 135 |
|
| 136 | smp_mb();
|
| 137 |
|
| 138 | return true;
|
| 139 | }
|
| 140 |
|
| 141 | void percpu_down_write(struct percpu_rw_semaphore *sem)
|
| 142 | {
|
| 143 |
|
| 144 | rcu_sync_enter(&sem->rss);
|
| 145 |
|
| 146 | down_write(&sem->rw_sem);
|
| 147 |
|
| 148 | |
| 149 | * Notify new readers to block; up until now, and thus throughout the
|
| 150 | * longish rcu_sync_enter() above, new readers could still come in.
|
| 151 |
|
| 152 | WRITE_ONCE(sem->readers_block, 1);
|
| 153 |
|
| 154 | smp_mb();
|
| 155 |
|
| 156 | |
| 157 | * If they don't see our writer of readers_block, then we are
|
| 158 | * guaranteed to see their sem->read_count increment, and therefore
|
| 159 | * will wait for them.
|
| 160 |
|
| 161 |
|
| 162 |
|
| 163 | wait_event(sem->writer, readers_active_check(sem));
|
| 164 | }
|
| 165 | EXPORT_SYMBOL_GPL(percpu_down_write);
|
| 166 |
|
| 167 | void percpu_up_write(struct percpu_rw_semaphore *sem)
|
| 168 | {
|
| 169 | |
| 170 | * Signal the writer is done, no fast path yet.
|
| 171 | *
|
| 172 | * One reason that we cannot just immediately flip to readers_fast is
|
| 173 | * that new readers might fail to see the results of this writer's
|
| 174 | * critical section.
|
| 175 | *
|
| 176 | * Therefore we force it through the slow path which guarantees an
|
| 177 | * acquire and thereby guarantees the critical section's consistency.
|
| 178 |
|
| 179 | smp_store_release(&sem->readers_block, 0);
|
| 180 |
|
| 181 | |
| 182 | * Release the write lock, this will allow readers back in the game.
|
| 183 |
|
| 184 | up_write(&sem->rw_sem);
|
| 185 |
|
| 186 | |
| 187 | * Once this completes (at least one RCU-sched grace period hence) the
|
| 188 | * reader fast path will be available again. Safe to use outside the
|
| 189 | * exclusive write lock because its counting.
|
| 190 |
|
| 191 | rcu_sync_exit(&sem->rss);
|
| 192 | }
|
| 193 | EXPORT_SYMBOL_GPL(percpu_up_write);
|