| 1 | // SPDX-License-Identifier: GPL-2.0+ |
| 2 | // |
| 3 | // An earlier version of this file appeared in the companion webpage for |
| 4 | // "Frightening small children and disconcerting grown-ups: Concurrency |
| 5 | // in the Linux kernel" by Alglave, Maranget, McKenney, Parri, and Stern, |
| 6 | // which appeared in ASPLOS 2018. |
| 7 | |
| 8 | // ONCE |
| 9 | READ_ONCE(X) __load{ONCE}(X) |
| 10 | WRITE_ONCE(X,V) { __store{ONCE}(X,V); } |
| 11 | |
| 12 | // Release Acquire and friends |
| 13 | smp_store_release(X,V) { __store{RELEASE}(*X,V); } |
| 14 | smp_load_acquire(X) __load{ACQUIRE}(*X) |
| 15 | rcu_assign_pointer(X,V) { __store{RELEASE}(X,V); } |
| 16 | rcu_dereference(X) __load{ONCE}(X) |
| 17 | smp_store_mb(X,V) { __store{ONCE}(X,V); __fence{MB}; } |
| 18 | |
| 19 | // Fences |
| 20 | smp_mb() { __fence{MB}; } |
| 21 | smp_rmb() { __fence{rmb}; } |
| 22 | smp_wmb() { __fence{wmb}; } |
| 23 | smp_mb__before_atomic() { __fence{before-atomic}; } |
| 24 | smp_mb__after_atomic() { __fence{after-atomic}; } |
| 25 | smp_mb__after_spinlock() { __fence{after-spinlock}; } |
| 26 | smp_mb__after_unlock_lock() { __fence{after-unlock-lock}; } |
| 27 | smp_mb__after_srcu_read_unlock() { __fence{after-srcu-read-unlock}; } |
| 28 | barrier() { __fence{barrier}; } |
| 29 | |
| 30 | // Exchange |
| 31 | xchg(X,V) __xchg{MB}(X,V) |
| 32 | xchg_relaxed(X,V) __xchg{ONCE}(X,V) |
| 33 | xchg_release(X,V) __xchg{RELEASE}(X,V) |
| 34 | xchg_acquire(X,V) __xchg{ACQUIRE}(X,V) |
| 35 | cmpxchg(X,V,W) __cmpxchg{MB}(X,V,W) |
| 36 | cmpxchg_relaxed(X,V,W) __cmpxchg{ONCE}(X,V,W) |
| 37 | cmpxchg_acquire(X,V,W) __cmpxchg{ACQUIRE}(X,V,W) |
| 38 | cmpxchg_release(X,V,W) __cmpxchg{RELEASE}(X,V,W) |
| 39 | |
| 40 | // Spinlocks |
| 41 | spin_lock(X) { __lock(X); } |
| 42 | spin_unlock(X) { __unlock(X); } |
| 43 | spin_trylock(X) __trylock(X) |
| 44 | spin_is_locked(X) __islocked(X) |
| 45 | |
| 46 | // RCU |
| 47 | rcu_read_lock() { __fence{rcu-lock}; } |
| 48 | rcu_read_unlock() { __fence{rcu-unlock}; } |
| 49 | synchronize_rcu() { __fence{sync-rcu}; } |
| 50 | synchronize_rcu_expedited() { __fence{sync-rcu}; } |
| 51 | |
| 52 | // SRCU |
| 53 | srcu_read_lock(X) __load{srcu-lock}(*X) |
| 54 | srcu_read_unlock(X,Y) { __store{srcu-unlock}(*X,Y); } |
| 55 | srcu_down_read(X) __load{srcu-lock}(*X) |
| 56 | srcu_up_read(X,Y) { __store{srcu-unlock}(*X,Y); } |
| 57 | synchronize_srcu(X) { __srcu{sync-srcu}(X); } |
| 58 | synchronize_srcu_expedited(X) { __srcu{sync-srcu}(X); } |
| 59 | |
| 60 | // Atomic |
| 61 | atomic_read(X) READ_ONCE(*X) |
| 62 | atomic_set(X,V) { WRITE_ONCE(*X,V); } |
| 63 | atomic_read_acquire(X) smp_load_acquire(X) |
| 64 | atomic_set_release(X,V) { smp_store_release(X,V); } |
| 65 | |
| 66 | atomic_add(V,X) { __atomic_op{NORETURN}(X,+,V); } |
| 67 | atomic_sub(V,X) { __atomic_op{NORETURN}(X,-,V); } |
| 68 | atomic_and(V,X) { __atomic_op{NORETURN}(X,&,V); } |
| 69 | atomic_or(V,X) { __atomic_op{NORETURN}(X,|,V); } |
| 70 | atomic_xor(V,X) { __atomic_op{NORETURN}(X,^,V); } |
| 71 | atomic_inc(X) { __atomic_op{NORETURN}(X,+,1); } |
| 72 | atomic_dec(X) { __atomic_op{NORETURN}(X,-,1); } |
| 73 | atomic_andnot(V,X) { __atomic_op{NORETURN}(X,&~,V); } |
| 74 | |
| 75 | atomic_add_return(V,X) __atomic_op_return{MB}(X,+,V) |
| 76 | atomic_add_return_relaxed(V,X) __atomic_op_return{ONCE}(X,+,V) |
| 77 | atomic_add_return_acquire(V,X) __atomic_op_return{ACQUIRE}(X,+,V) |
| 78 | atomic_add_return_release(V,X) __atomic_op_return{RELEASE}(X,+,V) |
| 79 | atomic_fetch_add(V,X) __atomic_fetch_op{MB}(X,+,V) |
| 80 | atomic_fetch_add_relaxed(V,X) __atomic_fetch_op{ONCE}(X,+,V) |
| 81 | atomic_fetch_add_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,+,V) |
| 82 | atomic_fetch_add_release(V,X) __atomic_fetch_op{RELEASE}(X,+,V) |
| 83 | |
| 84 | atomic_fetch_and(V,X) __atomic_fetch_op{MB}(X,&,V) |
| 85 | atomic_fetch_and_relaxed(V,X) __atomic_fetch_op{ONCE}(X,&,V) |
| 86 | atomic_fetch_and_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,&,V) |
| 87 | atomic_fetch_and_release(V,X) __atomic_fetch_op{RELEASE}(X,&,V) |
| 88 | |
| 89 | atomic_fetch_or(V,X) __atomic_fetch_op{MB}(X,|,V) |
| 90 | atomic_fetch_or_relaxed(V,X) __atomic_fetch_op{ONCE}(X,|,V) |
| 91 | atomic_fetch_or_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,|,V) |
| 92 | atomic_fetch_or_release(V,X) __atomic_fetch_op{RELEASE}(X,|,V) |
| 93 | |
| 94 | atomic_fetch_xor(V,X) __atomic_fetch_op{MB}(X,^,V) |
| 95 | atomic_fetch_xor_relaxed(V,X) __atomic_fetch_op{ONCE}(X,^,V) |
| 96 | atomic_fetch_xor_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,^,V) |
| 97 | atomic_fetch_xor_release(V,X) __atomic_fetch_op{RELEASE}(X,^,V) |
| 98 | |
| 99 | atomic_inc_return(X) __atomic_op_return{MB}(X,+,1) |
| 100 | atomic_inc_return_relaxed(X) __atomic_op_return{ONCE}(X,+,1) |
| 101 | atomic_inc_return_acquire(X) __atomic_op_return{ACQUIRE}(X,+,1) |
| 102 | atomic_inc_return_release(X) __atomic_op_return{RELEASE}(X,+,1) |
| 103 | atomic_fetch_inc(X) __atomic_fetch_op{MB}(X,+,1) |
| 104 | atomic_fetch_inc_relaxed(X) __atomic_fetch_op{ONCE}(X,+,1) |
| 105 | atomic_fetch_inc_acquire(X) __atomic_fetch_op{ACQUIRE}(X,+,1) |
| 106 | atomic_fetch_inc_release(X) __atomic_fetch_op{RELEASE}(X,+,1) |
| 107 | |
| 108 | atomic_sub_return(V,X) __atomic_op_return{MB}(X,-,V) |
| 109 | atomic_sub_return_relaxed(V,X) __atomic_op_return{ONCE}(X,-,V) |
| 110 | atomic_sub_return_acquire(V,X) __atomic_op_return{ACQUIRE}(X,-,V) |
| 111 | atomic_sub_return_release(V,X) __atomic_op_return{RELEASE}(X,-,V) |
| 112 | atomic_fetch_sub(V,X) __atomic_fetch_op{MB}(X,-,V) |
| 113 | atomic_fetch_sub_relaxed(V,X) __atomic_fetch_op{ONCE}(X,-,V) |
| 114 | atomic_fetch_sub_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,-,V) |
| 115 | atomic_fetch_sub_release(V,X) __atomic_fetch_op{RELEASE}(X,-,V) |
| 116 | |
| 117 | atomic_dec_return(X) __atomic_op_return{MB}(X,-,1) |
| 118 | atomic_dec_return_relaxed(X) __atomic_op_return{ONCE}(X,-,1) |
| 119 | atomic_dec_return_acquire(X) __atomic_op_return{ACQUIRE}(X,-,1) |
| 120 | atomic_dec_return_release(X) __atomic_op_return{RELEASE}(X,-,1) |
| 121 | atomic_fetch_dec(X) __atomic_fetch_op{MB}(X,-,1) |
| 122 | atomic_fetch_dec_relaxed(X) __atomic_fetch_op{ONCE}(X,-,1) |
| 123 | atomic_fetch_dec_acquire(X) __atomic_fetch_op{ACQUIRE}(X,-,1) |
| 124 | atomic_fetch_dec_release(X) __atomic_fetch_op{RELEASE}(X,-,1) |
| 125 | |
| 126 | atomic_xchg(X,V) __xchg{MB}(X,V) |
| 127 | atomic_xchg_relaxed(X,V) __xchg{ONCE}(X,V) |
| 128 | atomic_xchg_release(X,V) __xchg{RELEASE}(X,V) |
| 129 | atomic_xchg_acquire(X,V) __xchg{ACQUIRE}(X,V) |
| 130 | atomic_cmpxchg(X,V,W) __cmpxchg{MB}(X,V,W) |
| 131 | atomic_cmpxchg_relaxed(X,V,W) __cmpxchg{ONCE}(X,V,W) |
| 132 | atomic_cmpxchg_acquire(X,V,W) __cmpxchg{ACQUIRE}(X,V,W) |
| 133 | atomic_cmpxchg_release(X,V,W) __cmpxchg{RELEASE}(X,V,W) |
| 134 | |
| 135 | atomic_sub_and_test(V,X) __atomic_op_return{MB}(X,-,V) == 0 |
| 136 | atomic_dec_and_test(X) __atomic_op_return{MB}(X,-,1) == 0 |
| 137 | atomic_inc_and_test(X) __atomic_op_return{MB}(X,+,1) == 0 |
| 138 | atomic_add_negative(V,X) __atomic_op_return{MB}(X,+,V) < 0 |
| 139 | atomic_add_negative_relaxed(V,X) __atomic_op_return{ONCE}(X,+,V) < 0 |
| 140 | atomic_add_negative_acquire(V,X) __atomic_op_return{ACQUIRE}(X,+,V) < 0 |
| 141 | atomic_add_negative_release(V,X) __atomic_op_return{RELEASE}(X,+,V) < 0 |
| 142 | |
| 143 | atomic_fetch_andnot(V,X) __atomic_fetch_op{MB}(X,&~,V) |
| 144 | atomic_fetch_andnot_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,&~,V) |
| 145 | atomic_fetch_andnot_release(V,X) __atomic_fetch_op{RELEASE}(X,&~,V) |
| 146 | atomic_fetch_andnot_relaxed(V,X) __atomic_fetch_op{ONCE}(X,&~,V) |
| 147 | |
| 148 | atomic_add_unless(X,V,W) __atomic_add_unless{MB}(X,V,W) |