| 1 |
|
| 2 | #ifndef _ALPHA_ATOMIC_H
|
| 3 | #define _ALPHA_ATOMIC_H
|
| 4 |
|
| 5 | #include <linux/types.h>
|
| 6 | #include <asm/barrier.h>
|
| 7 | #include <asm/cmpxchg.h>
|
| 8 |
|
| 9 | |
| 10 | * Atomic operations that C can't guarantee us. Useful for
|
| 11 | * resource counting etc...
|
| 12 | *
|
| 13 | * But use these as seldom as possible since they are much slower
|
| 14 | * than regular operations.
|
| 15 |
|
| 16 |
|
| 17 | |
| 18 | * To ensure dependency ordering is preserved for the _relaxed and
|
| 19 | * _release atomics, an smp_mb() is unconditionally inserted into the
|
| 20 | * _relaxed variants, which are used to build the barriered versions.
|
| 21 | * Avoid redundant back-to-back fences in the _acquire and _fence
|
| 22 | * versions.
|
| 23 |
|
| 24 | #define __atomic_acquire_fence()
|
| 25 | #define __atomic_post_full_fence()
|
| 26 |
|
| 27 | #define ATOMIC64_INIT(i) { (i) }
|
| 28 |
|
| 29 | #define arch_atomic_read(v) READ_ONCE((v)->counter)
|
| 30 | #define arch_atomic64_read(v) READ_ONCE((v)->counter)
|
| 31 |
|
| 32 | #define arch_atomic_set(v,i) WRITE_ONCE((v)->counter, (i))
|
| 33 | #define arch_atomic64_set(v,i) WRITE_ONCE((v)->counter, (i))
|
| 34 |
|
| 35 | |
| 36 | * To get proper branch prediction for the main line, we must branch
|
| 37 | * forward to code at the end of this object's .text section, then
|
| 38 | * branch back to restart the operation.
|
| 39 |
|
| 40 |
|
| 41 | #define ATOMIC_OP(op, asm_op) \
|
| 42 | static __inline__ void arch_atomic_##op(int i, atomic_t * v) \
|
| 43 | { \
|
| 44 | unsigned long temp; \
|
| 45 | __asm__ __volatile__( \
|
| 46 | "1: ldl_l %0,%1\n" \
|
| 47 | " " #asm_op " %0,%2,%0\n" \
|
| 48 | " stl_c %0,%1\n" \
|
| 49 | " beq %0,2f\n" \
|
| 50 | ".subsection 2\n" \
|
| 51 | "2: br 1b\n" \
|
| 52 | ".previous" \
|
| 53 | :"=&r" (temp), "=m" (v->counter) \
|
| 54 | :"Ir" (i), "m" (v->counter)); \
|
| 55 | } \
|
| 56 |
|
| 57 | #define ATOMIC_OP_RETURN(op, asm_op) \
|
| 58 | static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \
|
| 59 | { \
|
| 60 | long temp, result; \
|
| 61 | __asm__ __volatile__( \
|
| 62 | "1: ldl_l %0,%1\n" \
|
| 63 | " " #asm_op " %0,%3,%2\n" \
|
| 64 | " " #asm_op " %0,%3,%0\n" \
|
| 65 | " stl_c %0,%1\n" \
|
| 66 | " beq %0,2f\n" \
|
| 67 | ".subsection 2\n" \
|
| 68 | "2: br 1b\n" \
|
| 69 | ".previous" \
|
| 70 | :"=&r" (temp), "=m" (v->counter), "=&r" (result) \
|
| 71 | :"Ir" (i), "m" (v->counter) : "memory"); \
|
| 72 | smp_mb(); \
|
| 73 | return result; \
|
| 74 | }
|
| 75 |
|
| 76 | #define ATOMIC_FETCH_OP(op, asm_op) \
|
| 77 | static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \
|
| 78 | { \
|
| 79 | long temp, result; \
|
| 80 | __asm__ __volatile__( \
|
| 81 | "1: ldl_l %2,%1\n" \
|
| 82 | " " #asm_op " %2,%3,%0\n" \
|
| 83 | " stl_c %0,%1\n" \
|
| 84 | " beq %0,2f\n" \
|
| 85 | ".subsection 2\n" \
|
| 86 | "2: br 1b\n" \
|
| 87 | ".previous" \
|
| 88 | :"=&r" (temp), "=m" (v->counter), "=&r" (result) \
|
| 89 | :"Ir" (i), "m" (v->counter) : "memory"); \
|
| 90 | smp_mb(); \
|
| 91 | return result; \
|
| 92 | }
|
| 93 |
|
| 94 | #define ATOMIC64_OP(op, asm_op) \
|
| 95 | static __inline__ void arch_atomic64_##op(s64 i, atomic64_t * v) \
|
| 96 | { \
|
| 97 | s64 temp; \
|
| 98 | __asm__ __volatile__( \
|
| 99 | "1: ldq_l %0,%1\n" \
|
| 100 | " " #asm_op " %0,%2,%0\n" \
|
| 101 | " stq_c %0,%1\n" \
|
| 102 | " beq %0,2f\n" \
|
| 103 | ".subsection 2\n" \
|
| 104 | "2: br 1b\n" \
|
| 105 | ".previous" \
|
| 106 | :"=&r" (temp), "=m" (v->counter) \
|
| 107 | :"Ir" (i), "m" (v->counter)); \
|
| 108 | } \
|
| 109 |
|
| 110 | #define ATOMIC64_OP_RETURN(op, asm_op) \
|
| 111 | static __inline__ s64 \
|
| 112 | arch_atomic64_##op##_return_relaxed(s64 i, atomic64_t * v) \
|
| 113 | { \
|
| 114 | s64 temp, result; \
|
| 115 | __asm__ __volatile__( \
|
| 116 | "1: ldq_l %0,%1\n" \
|
| 117 | " " #asm_op " %0,%3,%2\n" \
|
| 118 | " " #asm_op " %0,%3,%0\n" \
|
| 119 | " stq_c %0,%1\n" \
|
| 120 | " beq %0,2f\n" \
|
| 121 | ".subsection 2\n" \
|
| 122 | "2: br 1b\n" \
|
| 123 | ".previous" \
|
| 124 | :"=&r" (temp), "=m" (v->counter), "=&r" (result) \
|
| 125 | :"Ir" (i), "m" (v->counter) : "memory"); \
|
| 126 | smp_mb(); \
|
| 127 | return result; \
|
| 128 | }
|
| 129 |
|
| 130 | #define ATOMIC64_FETCH_OP(op, asm_op) \
|
| 131 | static __inline__ s64 \
|
| 132 | arch_atomic64_fetch_##op##_relaxed(s64 i, atomic64_t * v) \
|
| 133 | { \
|
| 134 | s64 temp, result; \
|
| 135 | __asm__ __volatile__( \
|
| 136 | "1: ldq_l %2,%1\n" \
|
| 137 | " " #asm_op " %2,%3,%0\n" \
|
| 138 | " stq_c %0,%1\n" \
|
| 139 | " beq %0,2f\n" \
|
| 140 | ".subsection 2\n" \
|
| 141 | "2: br 1b\n" \
|
| 142 | ".previous" \
|
| 143 | :"=&r" (temp), "=m" (v->counter), "=&r" (result) \
|
| 144 | :"Ir" (i), "m" (v->counter) : "memory"); \
|
| 145 | smp_mb(); \
|
| 146 | return result; \
|
| 147 | }
|
| 148 |
|
| 149 | #define ATOMIC_OPS(op) \
|
| 150 | ATOMIC_OP(op, op##l) \
|
| 151 | ATOMIC_OP_RETURN(op, op##l) \
|
| 152 | ATOMIC_FETCH_OP(op, op##l) \
|
| 153 | ATOMIC64_OP(op, op##q) \
|
| 154 | ATOMIC64_OP_RETURN(op, op##q) \
|
| 155 | ATOMIC64_FETCH_OP(op, op##q)
|
| 156 |
|
| 157 | ATOMIC_OPS(add)
|
| 158 | ATOMIC_OPS(sub)
|
| 159 |
|
| 160 | #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
|
| 161 | #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
|
| 162 | #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
|
| 163 | #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
|
| 164 |
|
| 165 | #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
|
| 166 | #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
|
| 167 | #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
|
| 168 | #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
|
| 169 |
|
| 170 | #define arch_atomic_andnot arch_atomic_andnot
|
| 171 | #define arch_atomic64_andnot arch_atomic64_andnot
|
| 172 |
|
| 173 | #undef ATOMIC_OPS
|
| 174 | #define ATOMIC_OPS(op, asm) \
|
| 175 | ATOMIC_OP(op, asm) \
|
| 176 | ATOMIC_FETCH_OP(op, asm) \
|
| 177 | ATOMIC64_OP(op, asm) \
|
| 178 | ATOMIC64_FETCH_OP(op, asm)
|
| 179 |
|
| 180 | ATOMIC_OPS(and, and)
|
| 181 | ATOMIC_OPS(andnot, bic)
|
| 182 | ATOMIC_OPS(or, bis)
|
| 183 | ATOMIC_OPS(xor, xor)
|
| 184 |
|
| 185 | #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
|
| 186 | #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
|
| 187 | #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
|
| 188 | #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
|
| 189 |
|
| 190 | #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
|
| 191 | #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
|
| 192 | #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
|
| 193 | #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
|
| 194 |
|
| 195 | #undef ATOMIC_OPS
|
| 196 | #undef ATOMIC64_FETCH_OP
|
| 197 | #undef ATOMIC64_OP_RETURN
|
| 198 | #undef ATOMIC64_OP
|
| 199 | #undef ATOMIC_FETCH_OP
|
| 200 | #undef ATOMIC_OP_RETURN
|
| 201 | #undef ATOMIC_OP
|
| 202 |
|
| 203 | static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
|
| 204 | {
|
| 205 | int c, new, old;
|
| 206 | smp_mb();
|
| 207 | __asm__ __volatile__(
|
| 208 | "1: ldl_l %[old],%[mem]\n"
|
| 209 | " cmpeq %[old],%[u],%[c]\n"
|
| 210 | " addl %[old],%[a],%[new]\n"
|
| 211 | " bne %[c],2f\n"
|
| 212 | " stl_c %[new],%[mem]\n"
|
| 213 | " beq %[new],3f\n"
|
| 214 | "2:\n"
|
| 215 | ".subsection 2\n"
|
| 216 | "3: br 1b\n"
|
| 217 | ".previous"
|
| 218 | : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c)
|
| 219 | : [mem] "m"(*v), [a] "rI"(a), [u] "rI"((long)u)
|
| 220 | : "memory");
|
| 221 | smp_mb();
|
| 222 | return old;
|
| 223 | }
|
| 224 | #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
|
| 225 |
|
| 226 | static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
|
| 227 | {
|
| 228 | s64 c, new, old;
|
| 229 | smp_mb();
|
| 230 | __asm__ __volatile__(
|
| 231 | "1: ldq_l %[old],%[mem]\n"
|
| 232 | " cmpeq %[old],%[u],%[c]\n"
|
| 233 | " addq %[old],%[a],%[new]\n"
|
| 234 | " bne %[c],2f\n"
|
| 235 | " stq_c %[new],%[mem]\n"
|
| 236 | " beq %[new],3f\n"
|
| 237 | "2:\n"
|
| 238 | ".subsection 2\n"
|
| 239 | "3: br 1b\n"
|
| 240 | ".previous"
|
| 241 | : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c)
|
| 242 | : [mem] "m"(*v), [a] "rI"(a), [u] "rI"(u)
|
| 243 | : "memory");
|
| 244 | smp_mb();
|
| 245 | return old;
|
| 246 | }
|
| 247 | #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
|
| 248 |
|
| 249 | static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
|
| 250 | {
|
| 251 | s64 old, tmp;
|
| 252 | smp_mb();
|
| 253 | __asm__ __volatile__(
|
| 254 | "1: ldq_l %[old],%[mem]\n"
|
| 255 | " subq %[old],1,%[tmp]\n"
|
| 256 | " ble %[old],2f\n"
|
| 257 | " stq_c %[tmp],%[mem]\n"
|
| 258 | " beq %[tmp],3f\n"
|
| 259 | "2:\n"
|
| 260 | ".subsection 2\n"
|
| 261 | "3: br 1b\n"
|
| 262 | ".previous"
|
| 263 | : [old] "=&r"(old), [tmp] "=&r"(tmp)
|
| 264 | : [mem] "m"(*v)
|
| 265 | : "memory");
|
| 266 | smp_mb();
|
| 267 | return old - 1;
|
| 268 | }
|
| 269 | #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
|
| 270 |
|
| 271 | #endif
|