| 1 |
|
| 2 |
|
| 3 | #ifndef _LINUX_ATOMIC_H
|
| 4 | #define _LINUX_ATOMIC_H
|
| 5 | #include <linux/types.h>
|
| 6 |
|
| 7 | #include <asm/atomic.h>
|
| 8 | #include <asm/barrier.h>
|
| 9 |
|
| 10 | |
| 11 | * Relaxed variants of xchg, cmpxchg and some atomic operations.
|
| 12 | *
|
| 13 | * We support four variants:
|
| 14 | *
|
| 15 | * - Fully ordered: The default implementation, no suffix required.
|
| 16 | * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
|
| 17 | * - Release: Provides RELEASE semantics, _release suffix.
|
| 18 | * - Relaxed: No ordering guarantees, _relaxed suffix.
|
| 19 | *
|
| 20 | * For compound atomics performing both a load and a store, ACQUIRE
|
| 21 | * semantics apply only to the load and RELEASE semantics only to the
|
| 22 | * store portion of the operation. Note that a failed cmpxchg_acquire
|
| 23 | * does -not- imply any memory ordering constraints.
|
| 24 | *
|
| 25 | * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
|
| 26 |
|
| 27 |
|
| 28 | #ifndef atomic_read_acquire
|
| 29 | #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
|
| 30 | #endif
|
| 31 |
|
| 32 | #ifndef atomic_set_release
|
| 33 | #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
|
| 34 | #endif
|
| 35 |
|
| 36 | |
| 37 | * The idea here is to build acquire/release variants by adding explicit
|
| 38 | * barriers on top of the relaxed variant. In the case where the relaxed
|
| 39 | * variant is already fully ordered, no additional barriers are needed.
|
| 40 | *
|
| 41 | * If an architecture overrides __atomic_acquire_fence() it will probably
|
| 42 | * want to define smp_mb__after_spinlock().
|
| 43 |
|
| 44 | #ifndef __atomic_acquire_fence
|
| 45 | #define __atomic_acquire_fence smp_mb__after_atomic
|
| 46 | #endif
|
| 47 |
|
| 48 | #ifndef __atomic_release_fence
|
| 49 | #define __atomic_release_fence smp_mb__before_atomic
|
| 50 | #endif
|
| 51 |
|
| 52 | #ifndef __atomic_pre_full_fence
|
| 53 | #define __atomic_pre_full_fence smp_mb__before_atomic
|
| 54 | #endif
|
| 55 |
|
| 56 | #ifndef __atomic_post_full_fence
|
| 57 | #define __atomic_post_full_fence smp_mb__after_atomic
|
| 58 | #endif
|
| 59 |
|
| 60 | #define __atomic_op_acquire(op, args...) \
|
| 61 | ({ \
|
| 62 | typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
|
| 63 | __atomic_acquire_fence(); \
|
| 64 | __ret; \
|
| 65 | })
|
| 66 |
|
| 67 | #define __atomic_op_release(op, args...) \
|
| 68 | ({ \
|
| 69 | __atomic_release_fence(); \
|
| 70 | op##_relaxed(args); \
|
| 71 | })
|
| 72 |
|
| 73 | #define __atomic_op_fence(op, args...) \
|
| 74 | ({ \
|
| 75 | typeof(op##_relaxed(args)) __ret; \
|
| 76 | __atomic_pre_full_fence(); \
|
| 77 | __ret = op##_relaxed(args); \
|
| 78 | __atomic_post_full_fence(); \
|
| 79 | __ret; \
|
| 80 | })
|
| 81 |
|
| 82 |
|
| 83 | #ifndef atomic_add_return_relaxed
|
| 84 | #define atomic_add_return_relaxed atomic_add_return
|
| 85 | #define atomic_add_return_acquire atomic_add_return
|
| 86 | #define atomic_add_return_release atomic_add_return
|
| 87 |
|
| 88 | #else
|
| 89 |
|
| 90 | #ifndef atomic_add_return_acquire
|
| 91 | #define atomic_add_return_acquire(...) \
|
| 92 | __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
|
| 93 | #endif
|
| 94 |
|
| 95 | #ifndef atomic_add_return_release
|
| 96 | #define atomic_add_return_release(...) \
|
| 97 | __atomic_op_release(atomic_add_return, __VA_ARGS__)
|
| 98 | #endif
|
| 99 |
|
| 100 | #ifndef atomic_add_return
|
| 101 | #define atomic_add_return(...) \
|
| 102 | __atomic_op_fence(atomic_add_return, __VA_ARGS__)
|
| 103 | #endif
|
| 104 | #endif
|
| 105 |
|
| 106 | #ifndef atomic_inc
|
| 107 | #define atomic_inc(v) atomic_add(1, (v))
|
| 108 | #endif
|
| 109 |
|
| 110 |
|
| 111 | #ifndef atomic_inc_return_relaxed
|
| 112 |
|
| 113 | #ifndef atomic_inc_return
|
| 114 | #define atomic_inc_return(v) atomic_add_return(1, (v))
|
| 115 | #define atomic_inc_return_relaxed(v) atomic_add_return_relaxed(1, (v))
|
| 116 | #define atomic_inc_return_acquire(v) atomic_add_return_acquire(1, (v))
|
| 117 | #define atomic_inc_return_release(v) atomic_add_return_release(1, (v))
|
| 118 | #else
|
| 119 | #define atomic_inc_return_relaxed atomic_inc_return
|
| 120 | #define atomic_inc_return_acquire atomic_inc_return
|
| 121 | #define atomic_inc_return_release atomic_inc_return
|
| 122 | #endif
|
| 123 |
|
| 124 | #else
|
| 125 |
|
| 126 | #ifndef atomic_inc_return_acquire
|
| 127 | #define atomic_inc_return_acquire(...) \
|
| 128 | __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
|
| 129 | #endif
|
| 130 |
|
| 131 | #ifndef atomic_inc_return_release
|
| 132 | #define atomic_inc_return_release(...) \
|
| 133 | __atomic_op_release(atomic_inc_return, __VA_ARGS__)
|
| 134 | #endif
|
| 135 |
|
| 136 | #ifndef atomic_inc_return
|
| 137 | #define atomic_inc_return(...) \
|
| 138 | __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
|
| 139 | #endif
|
| 140 | #endif
|
| 141 |
|
| 142 |
|
| 143 | #ifndef atomic_sub_return_relaxed
|
| 144 | #define atomic_sub_return_relaxed atomic_sub_return
|
| 145 | #define atomic_sub_return_acquire atomic_sub_return
|
| 146 | #define atomic_sub_return_release atomic_sub_return
|
| 147 |
|
| 148 | #else
|
| 149 |
|
| 150 | #ifndef atomic_sub_return_acquire
|
| 151 | #define atomic_sub_return_acquire(...) \
|
| 152 | __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
|
| 153 | #endif
|
| 154 |
|
| 155 | #ifndef atomic_sub_return_release
|
| 156 | #define atomic_sub_return_release(...) \
|
| 157 | __atomic_op_release(atomic_sub_return, __VA_ARGS__)
|
| 158 | #endif
|
| 159 |
|
| 160 | #ifndef atomic_sub_return
|
| 161 | #define atomic_sub_return(...) \
|
| 162 | __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
|
| 163 | #endif
|
| 164 | #endif
|
| 165 |
|
| 166 | #ifndef atomic_dec
|
| 167 | #define atomic_dec(v) atomic_sub(1, (v))
|
| 168 | #endif
|
| 169 |
|
| 170 |
|
| 171 | #ifndef atomic_dec_return_relaxed
|
| 172 |
|
| 173 | #ifndef atomic_dec_return
|
| 174 | #define atomic_dec_return(v) atomic_sub_return(1, (v))
|
| 175 | #define atomic_dec_return_relaxed(v) atomic_sub_return_relaxed(1, (v))
|
| 176 | #define atomic_dec_return_acquire(v) atomic_sub_return_acquire(1, (v))
|
| 177 | #define atomic_dec_return_release(v) atomic_sub_return_release(1, (v))
|
| 178 | #else
|
| 179 | #define atomic_dec_return_relaxed atomic_dec_return
|
| 180 | #define atomic_dec_return_acquire atomic_dec_return
|
| 181 | #define atomic_dec_return_release atomic_dec_return
|
| 182 | #endif
|
| 183 |
|
| 184 | #else
|
| 185 |
|
| 186 | #ifndef atomic_dec_return_acquire
|
| 187 | #define atomic_dec_return_acquire(...) \
|
| 188 | __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
|
| 189 | #endif
|
| 190 |
|
| 191 | #ifndef atomic_dec_return_release
|
| 192 | #define atomic_dec_return_release(...) \
|
| 193 | __atomic_op_release(atomic_dec_return, __VA_ARGS__)
|
| 194 | #endif
|
| 195 |
|
| 196 | #ifndef atomic_dec_return
|
| 197 | #define atomic_dec_return(...) \
|
| 198 | __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
|
| 199 | #endif
|
| 200 | #endif
|
| 201 |
|
| 202 |
|
| 203 |
|
| 204 | #ifndef atomic_fetch_add_relaxed
|
| 205 | #define atomic_fetch_add_relaxed atomic_fetch_add
|
| 206 | #define atomic_fetch_add_acquire atomic_fetch_add
|
| 207 | #define atomic_fetch_add_release atomic_fetch_add
|
| 208 |
|
| 209 | #else
|
| 210 |
|
| 211 | #ifndef atomic_fetch_add_acquire
|
| 212 | #define atomic_fetch_add_acquire(...) \
|
| 213 | __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
|
| 214 | #endif
|
| 215 |
|
| 216 | #ifndef atomic_fetch_add_release
|
| 217 | #define atomic_fetch_add_release(...) \
|
| 218 | __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
|
| 219 | #endif
|
| 220 |
|
| 221 | #ifndef atomic_fetch_add
|
| 222 | #define atomic_fetch_add(...) \
|
| 223 | __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
|
| 224 | #endif
|
| 225 | #endif
|
| 226 |
|
| 227 |
|
| 228 | #ifndef atomic_fetch_inc_relaxed
|
| 229 |
|
| 230 | #ifndef atomic_fetch_inc
|
| 231 | #define atomic_fetch_inc(v) atomic_fetch_add(1, (v))
|
| 232 | #define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v))
|
| 233 | #define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v))
|
| 234 | #define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v))
|
| 235 | #else
|
| 236 | #define atomic_fetch_inc_relaxed atomic_fetch_inc
|
| 237 | #define atomic_fetch_inc_acquire atomic_fetch_inc
|
| 238 | #define atomic_fetch_inc_release atomic_fetch_inc
|
| 239 | #endif
|
| 240 |
|
| 241 | #else
|
| 242 |
|
| 243 | #ifndef atomic_fetch_inc_acquire
|
| 244 | #define atomic_fetch_inc_acquire(...) \
|
| 245 | __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
|
| 246 | #endif
|
| 247 |
|
| 248 | #ifndef atomic_fetch_inc_release
|
| 249 | #define atomic_fetch_inc_release(...) \
|
| 250 | __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
|
| 251 | #endif
|
| 252 |
|
| 253 | #ifndef atomic_fetch_inc
|
| 254 | #define atomic_fetch_inc(...) \
|
| 255 | __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
|
| 256 | #endif
|
| 257 | #endif
|
| 258 |
|
| 259 |
|
| 260 | #ifndef atomic_fetch_sub_relaxed
|
| 261 | #define atomic_fetch_sub_relaxed atomic_fetch_sub
|
| 262 | #define atomic_fetch_sub_acquire atomic_fetch_sub
|
| 263 | #define atomic_fetch_sub_release atomic_fetch_sub
|
| 264 |
|
| 265 | #else
|
| 266 |
|
| 267 | #ifndef atomic_fetch_sub_acquire
|
| 268 | #define atomic_fetch_sub_acquire(...) \
|
| 269 | __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
|
| 270 | #endif
|
| 271 |
|
| 272 | #ifndef atomic_fetch_sub_release
|
| 273 | #define atomic_fetch_sub_release(...) \
|
| 274 | __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
|
| 275 | #endif
|
| 276 |
|
| 277 | #ifndef atomic_fetch_sub
|
| 278 | #define atomic_fetch_sub(...) \
|
| 279 | __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
|
| 280 | #endif
|
| 281 | #endif
|
| 282 |
|
| 283 |
|
| 284 | #ifndef atomic_fetch_dec_relaxed
|
| 285 |
|
| 286 | #ifndef atomic_fetch_dec
|
| 287 | #define atomic_fetch_dec(v) atomic_fetch_sub(1, (v))
|
| 288 | #define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v))
|
| 289 | #define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v))
|
| 290 | #define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v))
|
| 291 | #else
|
| 292 | #define atomic_fetch_dec_relaxed atomic_fetch_dec
|
| 293 | #define atomic_fetch_dec_acquire atomic_fetch_dec
|
| 294 | #define atomic_fetch_dec_release atomic_fetch_dec
|
| 295 | #endif
|
| 296 |
|
| 297 | #else
|
| 298 |
|
| 299 | #ifndef atomic_fetch_dec_acquire
|
| 300 | #define atomic_fetch_dec_acquire(...) \
|
| 301 | __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
|
| 302 | #endif
|
| 303 |
|
| 304 | #ifndef atomic_fetch_dec_release
|
| 305 | #define atomic_fetch_dec_release(...) \
|
| 306 | __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
|
| 307 | #endif
|
| 308 |
|
| 309 | #ifndef atomic_fetch_dec
|
| 310 | #define atomic_fetch_dec(...) \
|
| 311 | __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
|
| 312 | #endif
|
| 313 | #endif
|
| 314 |
|
| 315 |
|
| 316 | #ifndef atomic_fetch_or_relaxed
|
| 317 | #define atomic_fetch_or_relaxed atomic_fetch_or
|
| 318 | #define atomic_fetch_or_acquire atomic_fetch_or
|
| 319 | #define atomic_fetch_or_release atomic_fetch_or
|
| 320 |
|
| 321 | #else
|
| 322 |
|
| 323 | #ifndef atomic_fetch_or_acquire
|
| 324 | #define atomic_fetch_or_acquire(...) \
|
| 325 | __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
|
| 326 | #endif
|
| 327 |
|
| 328 | #ifndef atomic_fetch_or_release
|
| 329 | #define atomic_fetch_or_release(...) \
|
| 330 | __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
|
| 331 | #endif
|
| 332 |
|
| 333 | #ifndef atomic_fetch_or
|
| 334 | #define atomic_fetch_or(...) \
|
| 335 | __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
|
| 336 | #endif
|
| 337 | #endif
|
| 338 |
|
| 339 |
|
| 340 | #ifndef atomic_fetch_and_relaxed
|
| 341 | #define atomic_fetch_and_relaxed atomic_fetch_and
|
| 342 | #define atomic_fetch_and_acquire atomic_fetch_and
|
| 343 | #define atomic_fetch_and_release atomic_fetch_and
|
| 344 |
|
| 345 | #else
|
| 346 |
|
| 347 | #ifndef atomic_fetch_and_acquire
|
| 348 | #define atomic_fetch_and_acquire(...) \
|
| 349 | __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
|
| 350 | #endif
|
| 351 |
|
| 352 | #ifndef atomic_fetch_and_release
|
| 353 | #define atomic_fetch_and_release(...) \
|
| 354 | __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
|
| 355 | #endif
|
| 356 |
|
| 357 | #ifndef atomic_fetch_and
|
| 358 | #define atomic_fetch_and(...) \
|
| 359 | __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
|
| 360 | #endif
|
| 361 | #endif
|
| 362 |
|
| 363 | #ifndef atomic_andnot
|
| 364 | #define atomic_andnot(i, v) atomic_and(~(int)(i), (v))
|
| 365 | #endif
|
| 366 |
|
| 367 | #ifndef atomic_fetch_andnot_relaxed
|
| 368 |
|
| 369 | #ifndef atomic_fetch_andnot
|
| 370 | #define atomic_fetch_andnot(i, v) atomic_fetch_and(~(int)(i), (v))
|
| 371 | #define atomic_fetch_andnot_relaxed(i, v) atomic_fetch_and_relaxed(~(int)(i), (v))
|
| 372 | #define atomic_fetch_andnot_acquire(i, v) atomic_fetch_and_acquire(~(int)(i), (v))
|
| 373 | #define atomic_fetch_andnot_release(i, v) atomic_fetch_and_release(~(int)(i), (v))
|
| 374 | #else
|
| 375 | #define atomic_fetch_andnot_relaxed atomic_fetch_andnot
|
| 376 | #define atomic_fetch_andnot_acquire atomic_fetch_andnot
|
| 377 | #define atomic_fetch_andnot_release atomic_fetch_andnot
|
| 378 | #endif
|
| 379 |
|
| 380 | #else
|
| 381 |
|
| 382 | #ifndef atomic_fetch_andnot_acquire
|
| 383 | #define atomic_fetch_andnot_acquire(...) \
|
| 384 | __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
|
| 385 | #endif
|
| 386 |
|
| 387 | #ifndef atomic_fetch_andnot_release
|
| 388 | #define atomic_fetch_andnot_release(...) \
|
| 389 | __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
|
| 390 | #endif
|
| 391 |
|
| 392 | #ifndef atomic_fetch_andnot
|
| 393 | #define atomic_fetch_andnot(...) \
|
| 394 | __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
|
| 395 | #endif
|
| 396 | #endif
|
| 397 |
|
| 398 |
|
| 399 | #ifndef atomic_fetch_xor_relaxed
|
| 400 | #define atomic_fetch_xor_relaxed atomic_fetch_xor
|
| 401 | #define atomic_fetch_xor_acquire atomic_fetch_xor
|
| 402 | #define atomic_fetch_xor_release atomic_fetch_xor
|
| 403 |
|
| 404 | #else
|
| 405 |
|
| 406 | #ifndef atomic_fetch_xor_acquire
|
| 407 | #define atomic_fetch_xor_acquire(...) \
|
| 408 | __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
|
| 409 | #endif
|
| 410 |
|
| 411 | #ifndef atomic_fetch_xor_release
|
| 412 | #define atomic_fetch_xor_release(...) \
|
| 413 | __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
|
| 414 | #endif
|
| 415 |
|
| 416 | #ifndef atomic_fetch_xor
|
| 417 | #define atomic_fetch_xor(...) \
|
| 418 | __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
|
| 419 | #endif
|
| 420 | #endif
|
| 421 |
|
| 422 |
|
| 423 |
|
| 424 | #ifndef atomic_xchg_relaxed
|
| 425 | #define atomic_xchg_relaxed atomic_xchg
|
| 426 | #define atomic_xchg_acquire atomic_xchg
|
| 427 | #define atomic_xchg_release atomic_xchg
|
| 428 |
|
| 429 | #else
|
| 430 |
|
| 431 | #ifndef atomic_xchg_acquire
|
| 432 | #define atomic_xchg_acquire(...) \
|
| 433 | __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
|
| 434 | #endif
|
| 435 |
|
| 436 | #ifndef atomic_xchg_release
|
| 437 | #define atomic_xchg_release(...) \
|
| 438 | __atomic_op_release(atomic_xchg, __VA_ARGS__)
|
| 439 | #endif
|
| 440 |
|
| 441 | #ifndef atomic_xchg
|
| 442 | #define atomic_xchg(...) \
|
| 443 | __atomic_op_fence(atomic_xchg, __VA_ARGS__)
|
| 444 | #endif
|
| 445 | #endif
|
| 446 |
|
| 447 |
|
| 448 | #ifndef atomic_cmpxchg_relaxed
|
| 449 | #define atomic_cmpxchg_relaxed atomic_cmpxchg
|
| 450 | #define atomic_cmpxchg_acquire atomic_cmpxchg
|
| 451 | #define atomic_cmpxchg_release atomic_cmpxchg
|
| 452 |
|
| 453 | #else
|
| 454 |
|
| 455 | #ifndef atomic_cmpxchg_acquire
|
| 456 | #define atomic_cmpxchg_acquire(...) \
|
| 457 | __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
|
| 458 | #endif
|
| 459 |
|
| 460 | #ifndef atomic_cmpxchg_release
|
| 461 | #define atomic_cmpxchg_release(...) \
|
| 462 | __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
|
| 463 | #endif
|
| 464 |
|
| 465 | #ifndef atomic_cmpxchg
|
| 466 | #define atomic_cmpxchg(...) \
|
| 467 | __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
|
| 468 | #endif
|
| 469 | #endif
|
| 470 |
|
| 471 | #ifndef atomic_try_cmpxchg
|
| 472 |
|
| 473 | #define __atomic_try_cmpxchg(type, _p, _po, _n) \
|
| 474 | ({ \
|
| 475 | typeof(_po) __po = (_po); \
|
| 476 | typeof(*(_po)) __r, __o = *__po; \
|
| 477 | __r = atomic_cmpxchg##type((_p), __o, (_n)); \
|
| 478 | if (unlikely(__r != __o)) \
|
| 479 | *__po = __r; \
|
| 480 | likely(__r == __o); \
|
| 481 | })
|
| 482 |
|
| 483 | #define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n)
|
| 484 | #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n)
|
| 485 | #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n)
|
| 486 | #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n)
|
| 487 |
|
| 488 | #else
|
| 489 | #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg
|
| 490 | #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg
|
| 491 | #define atomic_try_cmpxchg_release atomic_try_cmpxchg
|
| 492 | #endif
|
| 493 |
|
| 494 |
|
| 495 | #ifndef cmpxchg_relaxed
|
| 496 | #define cmpxchg_relaxed cmpxchg
|
| 497 | #define cmpxchg_acquire cmpxchg
|
| 498 | #define cmpxchg_release cmpxchg
|
| 499 |
|
| 500 | #else
|
| 501 |
|
| 502 | #ifndef cmpxchg_acquire
|
| 503 | #define cmpxchg_acquire(...) \
|
| 504 | __atomic_op_acquire(cmpxchg, __VA_ARGS__)
|
| 505 | #endif
|
| 506 |
|
| 507 | #ifndef cmpxchg_release
|
| 508 | #define cmpxchg_release(...) \
|
| 509 | __atomic_op_release(cmpxchg, __VA_ARGS__)
|
| 510 | #endif
|
| 511 |
|
| 512 | #ifndef cmpxchg
|
| 513 | #define cmpxchg(...) \
|
| 514 | __atomic_op_fence(cmpxchg, __VA_ARGS__)
|
| 515 | #endif
|
| 516 | #endif
|
| 517 |
|
| 518 |
|
| 519 | #ifndef cmpxchg64_relaxed
|
| 520 | #define cmpxchg64_relaxed cmpxchg64
|
| 521 | #define cmpxchg64_acquire cmpxchg64
|
| 522 | #define cmpxchg64_release cmpxchg64
|
| 523 |
|
| 524 | #else
|
| 525 |
|
| 526 | #ifndef cmpxchg64_acquire
|
| 527 | #define cmpxchg64_acquire(...) \
|
| 528 | __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
|
| 529 | #endif
|
| 530 |
|
| 531 | #ifndef cmpxchg64_release
|
| 532 | #define cmpxchg64_release(...) \
|
| 533 | __atomic_op_release(cmpxchg64, __VA_ARGS__)
|
| 534 | #endif
|
| 535 |
|
| 536 | #ifndef cmpxchg64
|
| 537 | #define cmpxchg64(...) \
|
| 538 | __atomic_op_fence(cmpxchg64, __VA_ARGS__)
|
| 539 | #endif
|
| 540 | #endif
|
| 541 |
|
| 542 |
|
| 543 | #ifndef xchg_relaxed
|
| 544 | #define xchg_relaxed xchg
|
| 545 | #define xchg_acquire xchg
|
| 546 | #define xchg_release xchg
|
| 547 |
|
| 548 | #else
|
| 549 |
|
| 550 | #ifndef xchg_acquire
|
| 551 | #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
|
| 552 | #endif
|
| 553 |
|
| 554 | #ifndef xchg_release
|
| 555 | #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
|
| 556 | #endif
|
| 557 |
|
| 558 | #ifndef xchg
|
| 559 | #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
|
| 560 | #endif
|
| 561 | #endif
|
| 562 |
|
| 563 | |
| 564 | * atomic_fetch_add_unless - add unless the number is already a given value
|
| 565 | * @v: pointer of type atomic_t
|
| 566 | * @a: the amount to add to v...
|
| 567 | * @u: ...unless v is equal to u.
|
| 568 | *
|
| 569 | * Atomically adds @a to @v, if @v was not already @u.
|
| 570 | * Returns the original value of @v.
|
| 571 |
|
| 572 | #ifndef atomic_fetch_add_unless
|
| 573 | static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)
|
| 574 | {
|
| 575 | int c = atomic_read(v);
|
| 576 |
|
| 577 | do {
|
| 578 | if (unlikely(c == u))
|
| 579 | break;
|
| 580 | } while (!atomic_try_cmpxchg(v, &c, c + a));
|
| 581 |
|
| 582 | return c;
|
| 583 | }
|
| 584 | #endif
|
| 585 |
|
| 586 | |
| 587 | * atomic_add_unless - add unless the number is already a given value
|
| 588 | * @v: pointer of type atomic_t
|
| 589 | * @a: the amount to add to v...
|
| 590 | * @u: ...unless v is equal to u.
|
| 591 | *
|
| 592 | * Atomically adds @a to @v, if @v was not already @u.
|
| 593 | * Returns true if the addition was done.
|
| 594 |
|
| 595 | static inline bool atomic_add_unless(atomic_t *v, int a, int u)
|
| 596 | {
|
| 597 | return atomic_fetch_add_unless(v, a, u) != u;
|
| 598 | }
|
| 599 |
|
| 600 | |
| 601 | * atomic_inc_not_zero - increment unless the number is zero
|
| 602 | * @v: pointer of type atomic_t
|
| 603 | *
|
| 604 | * Atomically increments @v by 1, if @v is non-zero.
|
| 605 | * Returns true if the increment was done.
|
| 606 |
|
| 607 | #ifndef atomic_inc_not_zero
|
| 608 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
|
| 609 | #endif
|
| 610 |
|
| 611 | |
| 612 | * atomic_inc_and_test - increment and test
|
| 613 | * @v: pointer of type atomic_t
|
| 614 | *
|
| 615 | * Atomically increments @v by 1
|
| 616 | * and returns true if the result is zero, or false for all
|
| 617 | * other cases.
|
| 618 |
|
| 619 | #ifndef atomic_inc_and_test
|
| 620 | static inline bool atomic_inc_and_test(atomic_t *v)
|
| 621 | {
|
| 622 | return atomic_inc_return(v) == 0;
|
| 623 | }
|
| 624 | #endif
|
| 625 |
|
| 626 | |
| 627 | * atomic_dec_and_test - decrement and test
|
| 628 | * @v: pointer of type atomic_t
|
| 629 | *
|
| 630 | * Atomically decrements @v by 1 and
|
| 631 | * returns true if the result is 0, or false for all other
|
| 632 | * cases.
|
| 633 |
|
| 634 | #ifndef atomic_dec_and_test
|
| 635 | static inline bool atomic_dec_and_test(atomic_t *v)
|
| 636 | {
|
| 637 | return atomic_dec_return(v) == 0;
|
| 638 | }
|
| 639 | #endif
|
| 640 |
|
| 641 | |
| 642 | * atomic_sub_and_test - subtract value from variable and test result
|
| 643 | * @i: integer value to subtract
|
| 644 | * @v: pointer of type atomic_t
|
| 645 | *
|
| 646 | * Atomically subtracts @i from @v and returns
|
| 647 | * true if the result is zero, or false for all
|
| 648 | * other cases.
|
| 649 |
|
| 650 | #ifndef atomic_sub_and_test
|
| 651 | static inline bool atomic_sub_and_test(int i, atomic_t *v)
|
| 652 | {
|
| 653 | return atomic_sub_return(i, v) == 0;
|
| 654 | }
|
| 655 | #endif
|
| 656 |
|
| 657 | |
| 658 | * atomic_add_negative - add and test if negative
|
| 659 | * @i: integer value to add
|
| 660 | * @v: pointer of type atomic_t
|
| 661 | *
|
| 662 | * Atomically adds @i to @v and returns true
|
| 663 | * if the result is negative, or false when
|
| 664 | * result is greater than or equal to zero.
|
| 665 |
|
| 666 | #ifndef atomic_add_negative
|
| 667 | static inline bool atomic_add_negative(int i, atomic_t *v)
|
| 668 | {
|
| 669 | return atomic_add_return(i, v) < 0;
|
| 670 | }
|
| 671 | #endif
|
| 672 |
|
| 673 | #ifndef atomic_inc_unless_negative
|
| 674 | static inline bool atomic_inc_unless_negative(atomic_t *v)
|
| 675 | {
|
| 676 | int c = atomic_read(v);
|
| 677 |
|
| 678 | do {
|
| 679 | if (unlikely(c < 0))
|
| 680 | return false;
|
| 681 | } while (!atomic_try_cmpxchg(v, &c, c + 1));
|
| 682 |
|
| 683 | return true;
|
| 684 | }
|
| 685 | #endif
|
| 686 |
|
| 687 | #ifndef atomic_dec_unless_positive
|
| 688 | static inline bool atomic_dec_unless_positive(atomic_t *v)
|
| 689 | {
|
| 690 | int c = atomic_read(v);
|
| 691 |
|
| 692 | do {
|
| 693 | if (unlikely(c > 0))
|
| 694 | return false;
|
| 695 | } while (!atomic_try_cmpxchg(v, &c, c - 1));
|
| 696 |
|
| 697 | return true;
|
| 698 | }
|
| 699 | #endif
|
| 700 |
|
| 701 | |
| 702 | * atomic_dec_if_positive - decrement by 1 if old value positive
|
| 703 | * @v: pointer of type atomic_t
|
| 704 | *
|
| 705 | * The function returns the old value of *v minus 1, even if
|
| 706 | * the atomic variable, v, was not decremented.
|
| 707 |
|
| 708 | #ifndef atomic_dec_if_positive
|
| 709 | static inline int atomic_dec_if_positive(atomic_t *v)
|
| 710 | {
|
| 711 | int dec, c = atomic_read(v);
|
| 712 |
|
| 713 | do {
|
| 714 | dec = c - 1;
|
| 715 | if (unlikely(dec < 0))
|
| 716 | break;
|
| 717 | } while (!atomic_try_cmpxchg(v, &c, dec));
|
| 718 |
|
| 719 | return dec;
|
| 720 | }
|
| 721 | #endif
|
| 722 |
|
| 723 | #define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
|
| 724 | #define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
|
| 725 |
|
| 726 | #ifdef CONFIG_GENERIC_ATOMIC64
|
| 727 | #include <asm-generic/atomic64.h>
|
| 728 | #endif
|
| 729 |
|
| 730 | #ifndef atomic64_read_acquire
|
| 731 | #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
|
| 732 | #endif
|
| 733 |
|
| 734 | #ifndef atomic64_set_release
|
| 735 | #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
|
| 736 | #endif
|
| 737 |
|
| 738 |
|
| 739 | #ifndef atomic64_add_return_relaxed
|
| 740 | #define atomic64_add_return_relaxed atomic64_add_return
|
| 741 | #define atomic64_add_return_acquire atomic64_add_return
|
| 742 | #define atomic64_add_return_release atomic64_add_return
|
| 743 |
|
| 744 | #else
|
| 745 |
|
| 746 | #ifndef atomic64_add_return_acquire
|
| 747 | #define atomic64_add_return_acquire(...) \
|
| 748 | __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
|
| 749 | #endif
|
| 750 |
|
| 751 | #ifndef atomic64_add_return_release
|
| 752 | #define atomic64_add_return_release(...) \
|
| 753 | __atomic_op_release(atomic64_add_return, __VA_ARGS__)
|
| 754 | #endif
|
| 755 |
|
| 756 | #ifndef atomic64_add_return
|
| 757 | #define atomic64_add_return(...) \
|
| 758 | __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
|
| 759 | #endif
|
| 760 | #endif
|
| 761 |
|
| 762 | #ifndef atomic64_inc
|
| 763 | #define atomic64_inc(v) atomic64_add(1, (v))
|
| 764 | #endif
|
| 765 |
|
| 766 |
|
| 767 | #ifndef atomic64_inc_return_relaxed
|
| 768 |
|
| 769 | #ifndef atomic64_inc_return
|
| 770 | #define atomic64_inc_return(v) atomic64_add_return(1, (v))
|
| 771 | #define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1, (v))
|
| 772 | #define atomic64_inc_return_acquire(v) atomic64_add_return_acquire(1, (v))
|
| 773 | #define atomic64_inc_return_release(v) atomic64_add_return_release(1, (v))
|
| 774 | #else
|
| 775 | #define atomic64_inc_return_relaxed atomic64_inc_return
|
| 776 | #define atomic64_inc_return_acquire atomic64_inc_return
|
| 777 | #define atomic64_inc_return_release atomic64_inc_return
|
| 778 | #endif
|
| 779 |
|
| 780 | #else
|
| 781 |
|
| 782 | #ifndef atomic64_inc_return_acquire
|
| 783 | #define atomic64_inc_return_acquire(...) \
|
| 784 | __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
|
| 785 | #endif
|
| 786 |
|
| 787 | #ifndef atomic64_inc_return_release
|
| 788 | #define atomic64_inc_return_release(...) \
|
| 789 | __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
|
| 790 | #endif
|
| 791 |
|
| 792 | #ifndef atomic64_inc_return
|
| 793 | #define atomic64_inc_return(...) \
|
| 794 | __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
|
| 795 | #endif
|
| 796 | #endif
|
| 797 |
|
| 798 |
|
| 799 |
|
| 800 | #ifndef atomic64_sub_return_relaxed
|
| 801 | #define atomic64_sub_return_relaxed atomic64_sub_return
|
| 802 | #define atomic64_sub_return_acquire atomic64_sub_return
|
| 803 | #define atomic64_sub_return_release atomic64_sub_return
|
| 804 |
|
| 805 | #else
|
| 806 |
|
| 807 | #ifndef atomic64_sub_return_acquire
|
| 808 | #define atomic64_sub_return_acquire(...) \
|
| 809 | __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
|
| 810 | #endif
|
| 811 |
|
| 812 | #ifndef atomic64_sub_return_release
|
| 813 | #define atomic64_sub_return_release(...) \
|
| 814 | __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
|
| 815 | #endif
|
| 816 |
|
| 817 | #ifndef atomic64_sub_return
|
| 818 | #define atomic64_sub_return(...) \
|
| 819 | __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
|
| 820 | #endif
|
| 821 | #endif
|
| 822 |
|
| 823 | #ifndef atomic64_dec
|
| 824 | #define atomic64_dec(v) atomic64_sub(1, (v))
|
| 825 | #endif
|
| 826 |
|
| 827 |
|
| 828 | #ifndef atomic64_dec_return_relaxed
|
| 829 |
|
| 830 | #ifndef atomic64_dec_return
|
| 831 | #define atomic64_dec_return(v) atomic64_sub_return(1, (v))
|
| 832 | #define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1, (v))
|
| 833 | #define atomic64_dec_return_acquire(v) atomic64_sub_return_acquire(1, (v))
|
| 834 | #define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v))
|
| 835 | #else
|
| 836 | #define atomic64_dec_return_relaxed atomic64_dec_return
|
| 837 | #define atomic64_dec_return_acquire atomic64_dec_return
|
| 838 | #define atomic64_dec_return_release atomic64_dec_return
|
| 839 | #endif
|
| 840 |
|
| 841 | #else
|
| 842 |
|
| 843 | #ifndef atomic64_dec_return_acquire
|
| 844 | #define atomic64_dec_return_acquire(...) \
|
| 845 | __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
|
| 846 | #endif
|
| 847 |
|
| 848 | #ifndef atomic64_dec_return_release
|
| 849 | #define atomic64_dec_return_release(...) \
|
| 850 | __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
|
| 851 | #endif
|
| 852 |
|
| 853 | #ifndef atomic64_dec_return
|
| 854 | #define atomic64_dec_return(...) \
|
| 855 | __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
|
| 856 | #endif
|
| 857 | #endif
|
| 858 |
|
| 859 |
|
| 860 |
|
| 861 | #ifndef atomic64_fetch_add_relaxed
|
| 862 | #define atomic64_fetch_add_relaxed atomic64_fetch_add
|
| 863 | #define atomic64_fetch_add_acquire atomic64_fetch_add
|
| 864 | #define atomic64_fetch_add_release atomic64_fetch_add
|
| 865 |
|
| 866 | #else
|
| 867 |
|
| 868 | #ifndef atomic64_fetch_add_acquire
|
| 869 | #define atomic64_fetch_add_acquire(...) \
|
| 870 | __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
|
| 871 | #endif
|
| 872 |
|
| 873 | #ifndef atomic64_fetch_add_release
|
| 874 | #define atomic64_fetch_add_release(...) \
|
| 875 | __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
|
| 876 | #endif
|
| 877 |
|
| 878 | #ifndef atomic64_fetch_add
|
| 879 | #define atomic64_fetch_add(...) \
|
| 880 | __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
|
| 881 | #endif
|
| 882 | #endif
|
| 883 |
|
| 884 |
|
| 885 | #ifndef atomic64_fetch_inc_relaxed
|
| 886 |
|
| 887 | #ifndef atomic64_fetch_inc
|
| 888 | #define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v))
|
| 889 | #define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v))
|
| 890 | #define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v))
|
| 891 | #define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v))
|
| 892 | #else
|
| 893 | #define atomic64_fetch_inc_relaxed atomic64_fetch_inc
|
| 894 | #define atomic64_fetch_inc_acquire atomic64_fetch_inc
|
| 895 | #define atomic64_fetch_inc_release atomic64_fetch_inc
|
| 896 | #endif
|
| 897 |
|
| 898 | #else
|
| 899 |
|
| 900 | #ifndef atomic64_fetch_inc_acquire
|
| 901 | #define atomic64_fetch_inc_acquire(...) \
|
| 902 | __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
|
| 903 | #endif
|
| 904 |
|
| 905 | #ifndef atomic64_fetch_inc_release
|
| 906 | #define atomic64_fetch_inc_release(...) \
|
| 907 | __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
|
| 908 | #endif
|
| 909 |
|
| 910 | #ifndef atomic64_fetch_inc
|
| 911 | #define atomic64_fetch_inc(...) \
|
| 912 | __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
|
| 913 | #endif
|
| 914 | #endif
|
| 915 |
|
| 916 |
|
| 917 | #ifndef atomic64_fetch_sub_relaxed
|
| 918 | #define atomic64_fetch_sub_relaxed atomic64_fetch_sub
|
| 919 | #define atomic64_fetch_sub_acquire atomic64_fetch_sub
|
| 920 | #define atomic64_fetch_sub_release atomic64_fetch_sub
|
| 921 |
|
| 922 | #else
|
| 923 |
|
| 924 | #ifndef atomic64_fetch_sub_acquire
|
| 925 | #define atomic64_fetch_sub_acquire(...) \
|
| 926 | __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
|
| 927 | #endif
|
| 928 |
|
| 929 | #ifndef atomic64_fetch_sub_release
|
| 930 | #define atomic64_fetch_sub_release(...) \
|
| 931 | __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
|
| 932 | #endif
|
| 933 |
|
| 934 | #ifndef atomic64_fetch_sub
|
| 935 | #define atomic64_fetch_sub(...) \
|
| 936 | __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
|
| 937 | #endif
|
| 938 | #endif
|
| 939 |
|
| 940 |
|
| 941 | #ifndef atomic64_fetch_dec_relaxed
|
| 942 |
|
| 943 | #ifndef atomic64_fetch_dec
|
| 944 | #define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v))
|
| 945 | #define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v))
|
| 946 | #define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v))
|
| 947 | #define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v))
|
| 948 | #else
|
| 949 | #define atomic64_fetch_dec_relaxed atomic64_fetch_dec
|
| 950 | #define atomic64_fetch_dec_acquire atomic64_fetch_dec
|
| 951 | #define atomic64_fetch_dec_release atomic64_fetch_dec
|
| 952 | #endif
|
| 953 |
|
| 954 | #else
|
| 955 |
|
| 956 | #ifndef atomic64_fetch_dec_acquire
|
| 957 | #define atomic64_fetch_dec_acquire(...) \
|
| 958 | __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
|
| 959 | #endif
|
| 960 |
|
| 961 | #ifndef atomic64_fetch_dec_release
|
| 962 | #define atomic64_fetch_dec_release(...) \
|
| 963 | __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
|
| 964 | #endif
|
| 965 |
|
| 966 | #ifndef atomic64_fetch_dec
|
| 967 | #define atomic64_fetch_dec(...) \
|
| 968 | __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
|
| 969 | #endif
|
| 970 | #endif
|
| 971 |
|
| 972 |
|
| 973 | #ifndef atomic64_fetch_or_relaxed
|
| 974 | #define atomic64_fetch_or_relaxed atomic64_fetch_or
|
| 975 | #define atomic64_fetch_or_acquire atomic64_fetch_or
|
| 976 | #define atomic64_fetch_or_release atomic64_fetch_or
|
| 977 |
|
| 978 | #else
|
| 979 |
|
| 980 | #ifndef atomic64_fetch_or_acquire
|
| 981 | #define atomic64_fetch_or_acquire(...) \
|
| 982 | __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
|
| 983 | #endif
|
| 984 |
|
| 985 | #ifndef atomic64_fetch_or_release
|
| 986 | #define atomic64_fetch_or_release(...) \
|
| 987 | __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
|
| 988 | #endif
|
| 989 |
|
| 990 | #ifndef atomic64_fetch_or
|
| 991 | #define atomic64_fetch_or(...) \
|
| 992 | __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
|
| 993 | #endif
|
| 994 | #endif
|
| 995 |
|
| 996 |
|
| 997 | #ifndef atomic64_fetch_and_relaxed
|
| 998 | #define atomic64_fetch_and_relaxed atomic64_fetch_and
|
| 999 | #define atomic64_fetch_and_acquire atomic64_fetch_and
|
| 1000 | #define atomic64_fetch_and_release atomic64_fetch_and
|
| 1001 |
|
| 1002 | #else
|
| 1003 |
|
| 1004 | #ifndef atomic64_fetch_and_acquire
|
| 1005 | #define atomic64_fetch_and_acquire(...) \
|
| 1006 | __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
|
| 1007 | #endif
|
| 1008 |
|
| 1009 | #ifndef atomic64_fetch_and_release
|
| 1010 | #define atomic64_fetch_and_release(...) \
|
| 1011 | __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
|
| 1012 | #endif
|
| 1013 |
|
| 1014 | #ifndef atomic64_fetch_and
|
| 1015 | #define atomic64_fetch_and(...) \
|
| 1016 | __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
|
| 1017 | #endif
|
| 1018 | #endif
|
| 1019 |
|
| 1020 | #ifndef atomic64_andnot
|
| 1021 | #define atomic64_andnot(i, v) atomic64_and(~(long long)(i), (v))
|
| 1022 | #endif
|
| 1023 |
|
| 1024 | #ifndef atomic64_fetch_andnot_relaxed
|
| 1025 |
|
| 1026 | #ifndef atomic64_fetch_andnot
|
| 1027 | #define atomic64_fetch_andnot(i, v) atomic64_fetch_and(~(long long)(i), (v))
|
| 1028 | #define atomic64_fetch_andnot_relaxed(i, v) atomic64_fetch_and_relaxed(~(long long)(i), (v))
|
| 1029 | #define atomic64_fetch_andnot_acquire(i, v) atomic64_fetch_and_acquire(~(long long)(i), (v))
|
| 1030 | #define atomic64_fetch_andnot_release(i, v) atomic64_fetch_and_release(~(long long)(i), (v))
|
| 1031 | #else
|
| 1032 | #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
|
| 1033 | #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
|
| 1034 | #define atomic64_fetch_andnot_release atomic64_fetch_andnot
|
| 1035 | #endif
|
| 1036 |
|
| 1037 | #else
|
| 1038 |
|
| 1039 | #ifndef atomic64_fetch_andnot_acquire
|
| 1040 | #define atomic64_fetch_andnot_acquire(...) \
|
| 1041 | __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
|
| 1042 | #endif
|
| 1043 |
|
| 1044 | #ifndef atomic64_fetch_andnot_release
|
| 1045 | #define atomic64_fetch_andnot_release(...) \
|
| 1046 | __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
|
| 1047 | #endif
|
| 1048 |
|
| 1049 | #ifndef atomic64_fetch_andnot
|
| 1050 | #define atomic64_fetch_andnot(...) \
|
| 1051 | __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
|
| 1052 | #endif
|
| 1053 | #endif
|
| 1054 |
|
| 1055 |
|
| 1056 | #ifndef atomic64_fetch_xor_relaxed
|
| 1057 | #define atomic64_fetch_xor_relaxed atomic64_fetch_xor
|
| 1058 | #define atomic64_fetch_xor_acquire atomic64_fetch_xor
|
| 1059 | #define atomic64_fetch_xor_release atomic64_fetch_xor
|
| 1060 |
|
| 1061 | #else
|
| 1062 |
|
| 1063 | #ifndef atomic64_fetch_xor_acquire
|
| 1064 | #define atomic64_fetch_xor_acquire(...) \
|
| 1065 | __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
|
| 1066 | #endif
|
| 1067 |
|
| 1068 | #ifndef atomic64_fetch_xor_release
|
| 1069 | #define atomic64_fetch_xor_release(...) \
|
| 1070 | __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
|
| 1071 | #endif
|
| 1072 |
|
| 1073 | #ifndef atomic64_fetch_xor
|
| 1074 | #define atomic64_fetch_xor(...) \
|
| 1075 | __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
|
| 1076 | #endif
|
| 1077 | #endif
|
| 1078 |
|
| 1079 |
|
| 1080 |
|
| 1081 | #ifndef atomic64_xchg_relaxed
|
| 1082 | #define atomic64_xchg_relaxed atomic64_xchg
|
| 1083 | #define atomic64_xchg_acquire atomic64_xchg
|
| 1084 | #define atomic64_xchg_release atomic64_xchg
|
| 1085 |
|
| 1086 | #else
|
| 1087 |
|
| 1088 | #ifndef atomic64_xchg_acquire
|
| 1089 | #define atomic64_xchg_acquire(...) \
|
| 1090 | __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
|
| 1091 | #endif
|
| 1092 |
|
| 1093 | #ifndef atomic64_xchg_release
|
| 1094 | #define atomic64_xchg_release(...) \
|
| 1095 | __atomic_op_release(atomic64_xchg, __VA_ARGS__)
|
| 1096 | #endif
|
| 1097 |
|
| 1098 | #ifndef atomic64_xchg
|
| 1099 | #define atomic64_xchg(...) \
|
| 1100 | __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
|
| 1101 | #endif
|
| 1102 | #endif
|
| 1103 |
|
| 1104 |
|
| 1105 | #ifndef atomic64_cmpxchg_relaxed
|
| 1106 | #define atomic64_cmpxchg_relaxed atomic64_cmpxchg
|
| 1107 | #define atomic64_cmpxchg_acquire atomic64_cmpxchg
|
| 1108 | #define atomic64_cmpxchg_release atomic64_cmpxchg
|
| 1109 |
|
| 1110 | #else
|
| 1111 |
|
| 1112 | #ifndef atomic64_cmpxchg_acquire
|
| 1113 | #define atomic64_cmpxchg_acquire(...) \
|
| 1114 | __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
|
| 1115 | #endif
|
| 1116 |
|
| 1117 | #ifndef atomic64_cmpxchg_release
|
| 1118 | #define atomic64_cmpxchg_release(...) \
|
| 1119 | __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
|
| 1120 | #endif
|
| 1121 |
|
| 1122 | #ifndef atomic64_cmpxchg
|
| 1123 | #define atomic64_cmpxchg(...) \
|
| 1124 | __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
|
| 1125 | #endif
|
| 1126 | #endif
|
| 1127 |
|
| 1128 | #ifndef atomic64_try_cmpxchg
|
| 1129 |
|
| 1130 | #define __atomic64_try_cmpxchg(type, _p, _po, _n) \
|
| 1131 | ({ \
|
| 1132 | typeof(_po) __po = (_po); \
|
| 1133 | typeof(*(_po)) __r, __o = *__po; \
|
| 1134 | __r = atomic64_cmpxchg##type((_p), __o, (_n)); \
|
| 1135 | if (unlikely(__r != __o)) \
|
| 1136 | *__po = __r; \
|
| 1137 | likely(__r == __o); \
|
| 1138 | })
|
| 1139 |
|
| 1140 | #define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n)
|
| 1141 | #define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
|
| 1142 | #define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n)
|
| 1143 | #define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n)
|
| 1144 |
|
| 1145 | #else
|
| 1146 | #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg
|
| 1147 | #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg
|
| 1148 | #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg
|
| 1149 | #endif
|
| 1150 |
|
| 1151 | |
| 1152 | * atomic64_fetch_add_unless - add unless the number is already a given value
|
| 1153 | * @v: pointer of type atomic64_t
|
| 1154 | * @a: the amount to add to v...
|
| 1155 | * @u: ...unless v is equal to u.
|
| 1156 | *
|
| 1157 | * Atomically adds @a to @v, if @v was not already @u.
|
| 1158 | * Returns the original value of @v.
|
| 1159 |
|
| 1160 | #ifndef atomic64_fetch_add_unless
|
| 1161 | static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a,
|
| 1162 | long long u)
|
| 1163 | {
|
| 1164 | long long c = atomic64_read(v);
|
| 1165 |
|
| 1166 | do {
|
| 1167 | if (unlikely(c == u))
|
| 1168 | break;
|
| 1169 | } while (!atomic64_try_cmpxchg(v, &c, c + a));
|
| 1170 |
|
| 1171 | return c;
|
| 1172 | }
|
| 1173 | #endif
|
| 1174 |
|
| 1175 | |
| 1176 | * atomic64_add_unless - add unless the number is already a given value
|
| 1177 | * @v: pointer of type atomic_t
|
| 1178 | * @a: the amount to add to v...
|
| 1179 | * @u: ...unless v is equal to u.
|
| 1180 | *
|
| 1181 | * Atomically adds @a to @v, if @v was not already @u.
|
| 1182 | * Returns true if the addition was done.
|
| 1183 |
|
| 1184 | static inline bool atomic64_add_unless(atomic64_t *v, long long a, long long u)
|
| 1185 | {
|
| 1186 | return atomic64_fetch_add_unless(v, a, u) != u;
|
| 1187 | }
|
| 1188 |
|
| 1189 | |
| 1190 | * atomic64_inc_not_zero - increment unless the number is zero
|
| 1191 | * @v: pointer of type atomic64_t
|
| 1192 | *
|
| 1193 | * Atomically increments @v by 1, if @v is non-zero.
|
| 1194 | * Returns true if the increment was done.
|
| 1195 |
|
| 1196 | #ifndef atomic64_inc_not_zero
|
| 1197 | #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
|
| 1198 | #endif
|
| 1199 |
|
| 1200 | |
| 1201 | * atomic64_inc_and_test - increment and test
|
| 1202 | * @v: pointer of type atomic64_t
|
| 1203 | *
|
| 1204 | * Atomically increments @v by 1
|
| 1205 | * and returns true if the result is zero, or false for all
|
| 1206 | * other cases.
|
| 1207 |
|
| 1208 | #ifndef atomic64_inc_and_test
|
| 1209 | static inline bool atomic64_inc_and_test(atomic64_t *v)
|
| 1210 | {
|
| 1211 | return atomic64_inc_return(v) == 0;
|
| 1212 | }
|
| 1213 | #endif
|
| 1214 |
|
| 1215 | |
| 1216 | * atomic64_dec_and_test - decrement and test
|
| 1217 | * @v: pointer of type atomic64_t
|
| 1218 | *
|
| 1219 | * Atomically decrements @v by 1 and
|
| 1220 | * returns true if the result is 0, or false for all other
|
| 1221 | * cases.
|
| 1222 |
|
| 1223 | #ifndef atomic64_dec_and_test
|
| 1224 | static inline bool atomic64_dec_and_test(atomic64_t *v)
|
| 1225 | {
|
| 1226 | return atomic64_dec_return(v) == 0;
|
| 1227 | }
|
| 1228 | #endif
|
| 1229 |
|
| 1230 | |
| 1231 | * atomic64_sub_and_test - subtract value from variable and test result
|
| 1232 | * @i: integer value to subtract
|
| 1233 | * @v: pointer of type atomic64_t
|
| 1234 | *
|
| 1235 | * Atomically subtracts @i from @v and returns
|
| 1236 | * true if the result is zero, or false for all
|
| 1237 | * other cases.
|
| 1238 |
|
| 1239 | #ifndef atomic64_sub_and_test
|
| 1240 | static inline bool atomic64_sub_and_test(long long i, atomic64_t *v)
|
| 1241 | {
|
| 1242 | return atomic64_sub_return(i, v) == 0;
|
| 1243 | }
|
| 1244 | #endif
|
| 1245 |
|
| 1246 | |
| 1247 | * atomic64_add_negative - add and test if negative
|
| 1248 | * @i: integer value to add
|
| 1249 | * @v: pointer of type atomic64_t
|
| 1250 | *
|
| 1251 | * Atomically adds @i to @v and returns true
|
| 1252 | * if the result is negative, or false when
|
| 1253 | * result is greater than or equal to zero.
|
| 1254 |
|
| 1255 | #ifndef atomic64_add_negative
|
| 1256 | static inline bool atomic64_add_negative(long long i, atomic64_t *v)
|
| 1257 | {
|
| 1258 | return atomic64_add_return(i, v) < 0;
|
| 1259 | }
|
| 1260 | #endif
|
| 1261 |
|
| 1262 | #ifndef atomic64_inc_unless_negative
|
| 1263 | static inline bool atomic64_inc_unless_negative(atomic64_t *v)
|
| 1264 | {
|
| 1265 | long long c = atomic64_read(v);
|
| 1266 |
|
| 1267 | do {
|
| 1268 | if (unlikely(c < 0))
|
| 1269 | return false;
|
| 1270 | } while (!atomic64_try_cmpxchg(v, &c, c + 1));
|
| 1271 |
|
| 1272 | return true;
|
| 1273 | }
|
| 1274 | #endif
|
| 1275 |
|
| 1276 | #ifndef atomic64_dec_unless_positive
|
| 1277 | static inline bool atomic64_dec_unless_positive(atomic64_t *v)
|
| 1278 | {
|
| 1279 | long long c = atomic64_read(v);
|
| 1280 |
|
| 1281 | do {
|
| 1282 | if (unlikely(c > 0))
|
| 1283 | return false;
|
| 1284 | } while (!atomic64_try_cmpxchg(v, &c, c - 1));
|
| 1285 |
|
| 1286 | return true;
|
| 1287 | }
|
| 1288 | #endif
|
| 1289 |
|
| 1290 | |
| 1291 | * atomic64_dec_if_positive - decrement by 1 if old value positive
|
| 1292 | * @v: pointer of type atomic64_t
|
| 1293 | *
|
| 1294 | * The function returns the old value of *v minus 1, even if
|
| 1295 | * the atomic64 variable, v, was not decremented.
|
| 1296 |
|
| 1297 | #ifndef atomic64_dec_if_positive
|
| 1298 | static inline long long atomic64_dec_if_positive(atomic64_t *v)
|
| 1299 | {
|
| 1300 | long long dec, c = atomic64_read(v);
|
| 1301 |
|
| 1302 | do {
|
| 1303 | dec = c - 1;
|
| 1304 | if (unlikely(dec < 0))
|
| 1305 | break;
|
| 1306 | } while (!atomic64_try_cmpxchg(v, &c, dec));
|
| 1307 |
|
| 1308 | return dec;
|
| 1309 | }
|
| 1310 | #endif
|
| 1311 |
|
| 1312 | #define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
|
| 1313 | #define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
|
| 1314 |
|
| 1315 | #include <asm-generic/atomic-long.h>
|
| 1316 |
|
| 1317 | #endif
|