| 1 | |
| 2 | * Atomic operations that C can't guarantee us. Useful for
|
| 3 | * resource counting etc..
|
| 4 | *
|
| 5 | * But use these as seldom as possible since they are much more slower
|
| 6 | * than regular operations.
|
| 7 | *
|
| 8 | * This file is subject to the terms and conditions of the GNU General Public
|
| 9 | * License. See the file "COPYING" in the main directory of this archive
|
| 10 | * for more details.
|
| 11 | *
|
| 12 | * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle
|
| 13 |
|
| 14 | #ifndef _ASM_ATOMIC_H
|
| 15 | #define _ASM_ATOMIC_H
|
| 16 |
|
| 17 | #include <linux/irqflags.h>
|
| 18 | #include <asm/barrier.h>
|
| 19 | #include <asm/cpu-features.h>
|
| 20 | #include <asm/war.h>
|
| 21 |
|
| 22 | typedef struct { volatile int counter; } atomic_t;
|
| 23 |
|
| 24 | #define ATOMIC_INIT(i) { (i) }
|
| 25 |
|
| 26 | |
| 27 | * atomic_read - read atomic variable
|
| 28 | * @v: pointer of type atomic_t
|
| 29 | *
|
| 30 | * Atomically reads the value of @v.
|
| 31 |
|
| 32 | #define atomic_read(v) ((v)->counter)
|
| 33 |
|
| 34 | |
| 35 | * atomic_set - set atomic variable
|
| 36 | * @v: pointer of type atomic_t
|
| 37 | * @i: required value
|
| 38 | *
|
| 39 | * Atomically sets the value of @v to @i.
|
| 40 |
|
| 41 | #define atomic_set(v,i) ((v)->counter = (i))
|
| 42 |
|
| 43 | |
| 44 | * atomic_add - add integer to atomic variable
|
| 45 | * @i: integer value to add
|
| 46 | * @v: pointer of type atomic_t
|
| 47 | *
|
| 48 | * Atomically adds @i to @v.
|
| 49 |
|
| 50 | static __inline__ void atomic_add(int i, atomic_t * v)
|
| 51 | {
|
| 52 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 53 | unsigned long temp;
|
| 54 |
|
| 55 | __asm__ __volatile__(
|
| 56 | " .set mips3 \n"
|
| 57 | "1: ll %0, %1 # atomic_add \n"
|
| 58 | " addu %0, %2 \n"
|
| 59 | " sc %0, %1 \n"
|
| 60 | " beqzl %0, 1b \n"
|
| 61 | " .set mips0 \n"
|
| 62 | : "=&r" (temp), "=m" (v->counter)
|
| 63 | : "Ir" (i), "m" (v->counter));
|
| 64 | } else if (cpu_has_llsc) {
|
| 65 | unsigned long temp;
|
| 66 |
|
| 67 | __asm__ __volatile__(
|
| 68 | " .set mips3 \n"
|
| 69 | "1: ll %0, %1 # atomic_add \n"
|
| 70 | " addu %0, %2 \n"
|
| 71 | " sc %0, %1 \n"
|
| 72 | " beqz %0, 2f \n"
|
| 73 | " .subsection 2 \n"
|
| 74 | "2: b 1b \n"
|
| 75 | " .previous \n"
|
| 76 | " .set mips0 \n"
|
| 77 | : "=&r" (temp), "=m" (v->counter)
|
| 78 | : "Ir" (i), "m" (v->counter));
|
| 79 | } else {
|
| 80 | unsigned long flags;
|
| 81 |
|
| 82 | raw_local_irq_save(flags);
|
| 83 | v->counter += i;
|
| 84 | raw_local_irq_restore(flags);
|
| 85 | }
|
| 86 | }
|
| 87 |
|
| 88 | |
| 89 | * atomic_sub - subtract the atomic variable
|
| 90 | * @i: integer value to subtract
|
| 91 | * @v: pointer of type atomic_t
|
| 92 | *
|
| 93 | * Atomically subtracts @i from @v.
|
| 94 |
|
| 95 | static __inline__ void atomic_sub(int i, atomic_t * v)
|
| 96 | {
|
| 97 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 98 | unsigned long temp;
|
| 99 |
|
| 100 | __asm__ __volatile__(
|
| 101 | " .set mips3 \n"
|
| 102 | "1: ll %0, %1 # atomic_sub \n"
|
| 103 | " subu %0, %2 \n"
|
| 104 | " sc %0, %1 \n"
|
| 105 | " beqzl %0, 1b \n"
|
| 106 | " .set mips0 \n"
|
| 107 | : "=&r" (temp), "=m" (v->counter)
|
| 108 | : "Ir" (i), "m" (v->counter));
|
| 109 | } else if (cpu_has_llsc) {
|
| 110 | unsigned long temp;
|
| 111 |
|
| 112 | __asm__ __volatile__(
|
| 113 | " .set mips3 \n"
|
| 114 | "1: ll %0, %1 # atomic_sub \n"
|
| 115 | " subu %0, %2 \n"
|
| 116 | " sc %0, %1 \n"
|
| 117 | " beqz %0, 2f \n"
|
| 118 | " .subsection 2 \n"
|
| 119 | "2: b 1b \n"
|
| 120 | " .previous \n"
|
| 121 | " .set mips0 \n"
|
| 122 | : "=&r" (temp), "=m" (v->counter)
|
| 123 | : "Ir" (i), "m" (v->counter));
|
| 124 | } else {
|
| 125 | unsigned long flags;
|
| 126 |
|
| 127 | raw_local_irq_save(flags);
|
| 128 | v->counter -= i;
|
| 129 | raw_local_irq_restore(flags);
|
| 130 | }
|
| 131 | }
|
| 132 |
|
| 133 | |
| 134 | * Same as above, but return the result value
|
| 135 |
|
| 136 | static __inline__ int atomic_add_return(int i, atomic_t * v)
|
| 137 | {
|
| 138 | unsigned long result;
|
| 139 |
|
| 140 | smp_mb();
|
| 141 |
|
| 142 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 143 | unsigned long temp;
|
| 144 |
|
| 145 | __asm__ __volatile__(
|
| 146 | " .set mips3 \n"
|
| 147 | "1: ll %1, %2 # atomic_add_return \n"
|
| 148 | " addu %0, %1, %3 \n"
|
| 149 | " sc %0, %2 \n"
|
| 150 | " beqzl %0, 1b \n"
|
| 151 | " addu %0, %1, %3 \n"
|
| 152 | " .set mips0 \n"
|
| 153 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 154 | : "Ir" (i), "m" (v->counter)
|
| 155 | : "memory");
|
| 156 | } else if (cpu_has_llsc) {
|
| 157 | unsigned long temp;
|
| 158 |
|
| 159 | __asm__ __volatile__(
|
| 160 | " .set mips3 \n"
|
| 161 | "1: ll %1, %2 # atomic_add_return \n"
|
| 162 | " addu %0, %1, %3 \n"
|
| 163 | " sc %0, %2 \n"
|
| 164 | " beqz %0, 2f \n"
|
| 165 | " addu %0, %1, %3 \n"
|
| 166 | " .subsection 2 \n"
|
| 167 | "2: b 1b \n"
|
| 168 | " .previous \n"
|
| 169 | " .set mips0 \n"
|
| 170 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 171 | : "Ir" (i), "m" (v->counter)
|
| 172 | : "memory");
|
| 173 | } else {
|
| 174 | unsigned long flags;
|
| 175 |
|
| 176 | raw_local_irq_save(flags);
|
| 177 | result = v->counter;
|
| 178 | result += i;
|
| 179 | v->counter = result;
|
| 180 | raw_local_irq_restore(flags);
|
| 181 | }
|
| 182 |
|
| 183 | smp_mb();
|
| 184 |
|
| 185 | return result;
|
| 186 | }
|
| 187 |
|
| 188 | static __inline__ int atomic_sub_return(int i, atomic_t * v)
|
| 189 | {
|
| 190 | unsigned long result;
|
| 191 |
|
| 192 | smp_mb();
|
| 193 |
|
| 194 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 195 | unsigned long temp;
|
| 196 |
|
| 197 | __asm__ __volatile__(
|
| 198 | " .set mips3 \n"
|
| 199 | "1: ll %1, %2 # atomic_sub_return \n"
|
| 200 | " subu %0, %1, %3 \n"
|
| 201 | " sc %0, %2 \n"
|
| 202 | " beqzl %0, 1b \n"
|
| 203 | " subu %0, %1, %3 \n"
|
| 204 | " .set mips0 \n"
|
| 205 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 206 | : "Ir" (i), "m" (v->counter)
|
| 207 | : "memory");
|
| 208 | } else if (cpu_has_llsc) {
|
| 209 | unsigned long temp;
|
| 210 |
|
| 211 | __asm__ __volatile__(
|
| 212 | " .set mips3 \n"
|
| 213 | "1: ll %1, %2 # atomic_sub_return \n"
|
| 214 | " subu %0, %1, %3 \n"
|
| 215 | " sc %0, %2 \n"
|
| 216 | " beqz %0, 2f \n"
|
| 217 | " subu %0, %1, %3 \n"
|
| 218 | " .subsection 2 \n"
|
| 219 | "2: b 1b \n"
|
| 220 | " .previous \n"
|
| 221 | " .set mips0 \n"
|
| 222 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 223 | : "Ir" (i), "m" (v->counter)
|
| 224 | : "memory");
|
| 225 | } else {
|
| 226 | unsigned long flags;
|
| 227 |
|
| 228 | raw_local_irq_save(flags);
|
| 229 | result = v->counter;
|
| 230 | result -= i;
|
| 231 | v->counter = result;
|
| 232 | raw_local_irq_restore(flags);
|
| 233 | }
|
| 234 |
|
| 235 | smp_mb();
|
| 236 |
|
| 237 | return result;
|
| 238 | }
|
| 239 |
|
| 240 | |
| 241 | * atomic_sub_if_positive - conditionally subtract integer from atomic variable
|
| 242 | * @i: integer value to subtract
|
| 243 | * @v: pointer of type atomic_t
|
| 244 | *
|
| 245 | * Atomically test @v and subtract @i if @v is greater or equal than @i.
|
| 246 | * The function returns the old value of @v minus @i.
|
| 247 |
|
| 248 | static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
|
| 249 | {
|
| 250 | unsigned long result;
|
| 251 |
|
| 252 | smp_mb();
|
| 253 |
|
| 254 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 255 | unsigned long temp;
|
| 256 |
|
| 257 | __asm__ __volatile__(
|
| 258 | " .set mips3 \n"
|
| 259 | "1: ll %1, %2 # atomic_sub_if_positive\n"
|
| 260 | " subu %0, %1, %3 \n"
|
| 261 | " bltz %0, 1f \n"
|
| 262 | " sc %0, %2 \n"
|
| 263 | " .set noreorder \n"
|
| 264 | " beqzl %0, 1b \n"
|
| 265 | " subu %0, %1, %3 \n"
|
| 266 | " .set reorder \n"
|
| 267 | "1: \n"
|
| 268 | " .set mips0 \n"
|
| 269 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 270 | : "Ir" (i), "m" (v->counter)
|
| 271 | : "memory");
|
| 272 | } else if (cpu_has_llsc) {
|
| 273 | unsigned long temp;
|
| 274 |
|
| 275 | __asm__ __volatile__(
|
| 276 | " .set mips3 \n"
|
| 277 | "1: ll %1, %2 # atomic_sub_if_positive\n"
|
| 278 | " subu %0, %1, %3 \n"
|
| 279 | " bltz %0, 1f \n"
|
| 280 | " sc %0, %2 \n"
|
| 281 | " .set noreorder \n"
|
| 282 | " beqz %0, 2f \n"
|
| 283 | " subu %0, %1, %3 \n"
|
| 284 | " .set reorder \n"
|
| 285 | "1: \n"
|
| 286 | " .subsection 2 \n"
|
| 287 | "2: b 1b \n"
|
| 288 | " .previous \n"
|
| 289 | " .set mips0 \n"
|
| 290 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 291 | : "Ir" (i), "m" (v->counter)
|
| 292 | : "memory");
|
| 293 | } else {
|
| 294 | unsigned long flags;
|
| 295 |
|
| 296 | raw_local_irq_save(flags);
|
| 297 | result = v->counter;
|
| 298 | result -= i;
|
| 299 | if (result >= 0)
|
| 300 | v->counter = result;
|
| 301 | raw_local_irq_restore(flags);
|
| 302 | }
|
| 303 |
|
| 304 | smp_mb();
|
| 305 |
|
| 306 | return result;
|
| 307 | }
|
| 308 |
|
| 309 | #define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
|
| 310 | #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
|
| 311 |
|
| 312 | |
| 313 | * atomic_add_unless - add unless the number is a given value
|
| 314 | * @v: pointer of type atomic_t
|
| 315 | * @a: the amount to add to v...
|
| 316 | * @u: ...unless v is equal to u.
|
| 317 | *
|
| 318 | * Atomically adds @a to @v, so long as it was not @u.
|
| 319 | * Returns non-zero if @v was not @u, and zero otherwise.
|
| 320 |
|
| 321 | #define atomic_add_unless(v, a, u) \
|
| 322 | ({ \
|
| 323 | int c, old; \
|
| 324 | c = atomic_read(v); \
|
| 325 | while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
|
| 326 | c = old; \
|
| 327 | c != (u); \
|
| 328 | })
|
| 329 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
|
| 330 |
|
| 331 | #define atomic_dec_return(v) atomic_sub_return(1,(v))
|
| 332 | #define atomic_inc_return(v) atomic_add_return(1,(v))
|
| 333 |
|
| 334 | |
| 335 | * atomic_sub_and_test - subtract value from variable and test result
|
| 336 | * @i: integer value to subtract
|
| 337 | * @v: pointer of type atomic_t
|
| 338 | *
|
| 339 | * Atomically subtracts @i from @v and returns
|
| 340 | * true if the result is zero, or false for all
|
| 341 | * other cases.
|
| 342 |
|
| 343 | #define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0)
|
| 344 |
|
| 345 | |
| 346 | * atomic_inc_and_test - increment and test
|
| 347 | * @v: pointer of type atomic_t
|
| 348 | *
|
| 349 | * Atomically increments @v by 1
|
| 350 | * and returns true if the result is zero, or false for all
|
| 351 | * other cases.
|
| 352 |
|
| 353 | #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
|
| 354 |
|
| 355 | |
| 356 | * atomic_dec_and_test - decrement by 1 and test
|
| 357 | * @v: pointer of type atomic_t
|
| 358 | *
|
| 359 | * Atomically decrements @v by 1 and
|
| 360 | * returns true if the result is 0, or false for all other
|
| 361 | * cases.
|
| 362 |
|
| 363 | #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
|
| 364 |
|
| 365 | |
| 366 | * atomic_dec_if_positive - decrement by 1 if old value positive
|
| 367 | * @v: pointer of type atomic_t
|
| 368 |
|
| 369 | #define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v)
|
| 370 |
|
| 371 | |
| 372 | * atomic_inc - increment atomic variable
|
| 373 | * @v: pointer of type atomic_t
|
| 374 | *
|
| 375 | * Atomically increments @v by 1.
|
| 376 |
|
| 377 | #define atomic_inc(v) atomic_add(1,(v))
|
| 378 |
|
| 379 | |
| 380 | * atomic_dec - decrement and test
|
| 381 | * @v: pointer of type atomic_t
|
| 382 | *
|
| 383 | * Atomically decrements @v by 1.
|
| 384 |
|
| 385 | #define atomic_dec(v) atomic_sub(1,(v))
|
| 386 |
|
| 387 | |
| 388 | * atomic_add_negative - add and test if negative
|
| 389 | * @v: pointer of type atomic_t
|
| 390 | * @i: integer value to add
|
| 391 | *
|
| 392 | * Atomically adds @i to @v and returns true
|
| 393 | * if the result is negative, or false when
|
| 394 | * result is greater than or equal to zero.
|
| 395 |
|
| 396 | #define atomic_add_negative(i,v) (atomic_add_return(i, (v)) < 0)
|
| 397 |
|
| 398 | #ifdef CONFIG_64BIT
|
| 399 |
|
| 400 | typedef struct { volatile long counter; } atomic64_t;
|
| 401 |
|
| 402 | #define ATOMIC64_INIT(i) { (i) }
|
| 403 |
|
| 404 | |
| 405 | * atomic64_read - read atomic variable
|
| 406 | * @v: pointer of type atomic64_t
|
| 407 | *
|
| 408 |
|
| 409 | #define atomic64_read(v) ((v)->counter)
|
| 410 |
|
| 411 | |
| 412 | * atomic64_set - set atomic variable
|
| 413 | * @v: pointer of type atomic64_t
|
| 414 | * @i: required value
|
| 415 |
|
| 416 | #define atomic64_set(v,i) ((v)->counter = (i))
|
| 417 |
|
| 418 | |
| 419 | * atomic64_add - add integer to atomic variable
|
| 420 | * @i: integer value to add
|
| 421 | * @v: pointer of type atomic64_t
|
| 422 | *
|
| 423 | * Atomically adds @i to @v.
|
| 424 |
|
| 425 | static __inline__ void atomic64_add(long i, atomic64_t * v)
|
| 426 | {
|
| 427 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 428 | unsigned long temp;
|
| 429 |
|
| 430 | __asm__ __volatile__(
|
| 431 | " .set mips3 \n"
|
| 432 | "1: lld %0, %1 # atomic64_add \n"
|
| 433 | " addu %0, %2 \n"
|
| 434 | " scd %0, %1 \n"
|
| 435 | " beqzl %0, 1b \n"
|
| 436 | " .set mips0 \n"
|
| 437 | : "=&r" (temp), "=m" (v->counter)
|
| 438 | : "Ir" (i), "m" (v->counter));
|
| 439 | } else if (cpu_has_llsc) {
|
| 440 | unsigned long temp;
|
| 441 |
|
| 442 | __asm__ __volatile__(
|
| 443 | " .set mips3 \n"
|
| 444 | "1: lld %0, %1 # atomic64_add \n"
|
| 445 | " addu %0, %2 \n"
|
| 446 | " scd %0, %1 \n"
|
| 447 | " beqz %0, 2f \n"
|
| 448 | " .subsection 2 \n"
|
| 449 | "2: b 1b \n"
|
| 450 | " .previous \n"
|
| 451 | " .set mips0 \n"
|
| 452 | : "=&r" (temp), "=m" (v->counter)
|
| 453 | : "Ir" (i), "m" (v->counter));
|
| 454 | } else {
|
| 455 | unsigned long flags;
|
| 456 |
|
| 457 | raw_local_irq_save(flags);
|
| 458 | v->counter += i;
|
| 459 | raw_local_irq_restore(flags);
|
| 460 | }
|
| 461 | }
|
| 462 |
|
| 463 | |
| 464 | * atomic64_sub - subtract the atomic variable
|
| 465 | * @i: integer value to subtract
|
| 466 | * @v: pointer of type atomic64_t
|
| 467 | *
|
| 468 | * Atomically subtracts @i from @v.
|
| 469 |
|
| 470 | static __inline__ void atomic64_sub(long i, atomic64_t * v)
|
| 471 | {
|
| 472 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 473 | unsigned long temp;
|
| 474 |
|
| 475 | __asm__ __volatile__(
|
| 476 | " .set mips3 \n"
|
| 477 | "1: lld %0, %1 # atomic64_sub \n"
|
| 478 | " subu %0, %2 \n"
|
| 479 | " scd %0, %1 \n"
|
| 480 | " beqzl %0, 1b \n"
|
| 481 | " .set mips0 \n"
|
| 482 | : "=&r" (temp), "=m" (v->counter)
|
| 483 | : "Ir" (i), "m" (v->counter));
|
| 484 | } else if (cpu_has_llsc) {
|
| 485 | unsigned long temp;
|
| 486 |
|
| 487 | __asm__ __volatile__(
|
| 488 | " .set mips3 \n"
|
| 489 | "1: lld %0, %1 # atomic64_sub \n"
|
| 490 | " subu %0, %2 \n"
|
| 491 | " scd %0, %1 \n"
|
| 492 | " beqz %0, 2f \n"
|
| 493 | " .subsection 2 \n"
|
| 494 | "2: b 1b \n"
|
| 495 | " .previous \n"
|
| 496 | " .set mips0 \n"
|
| 497 | : "=&r" (temp), "=m" (v->counter)
|
| 498 | : "Ir" (i), "m" (v->counter));
|
| 499 | } else {
|
| 500 | unsigned long flags;
|
| 501 |
|
| 502 | raw_local_irq_save(flags);
|
| 503 | v->counter -= i;
|
| 504 | raw_local_irq_restore(flags);
|
| 505 | }
|
| 506 | }
|
| 507 |
|
| 508 | |
| 509 | * Same as above, but return the result value
|
| 510 |
|
| 511 | static __inline__ long atomic64_add_return(long i, atomic64_t * v)
|
| 512 | {
|
| 513 | unsigned long result;
|
| 514 |
|
| 515 | smp_mb();
|
| 516 |
|
| 517 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 518 | unsigned long temp;
|
| 519 |
|
| 520 | __asm__ __volatile__(
|
| 521 | " .set mips3 \n"
|
| 522 | "1: lld %1, %2 # atomic64_add_return \n"
|
| 523 | " addu %0, %1, %3 \n"
|
| 524 | " scd %0, %2 \n"
|
| 525 | " beqzl %0, 1b \n"
|
| 526 | " addu %0, %1, %3 \n"
|
| 527 | " .set mips0 \n"
|
| 528 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 529 | : "Ir" (i), "m" (v->counter)
|
| 530 | : "memory");
|
| 531 | } else if (cpu_has_llsc) {
|
| 532 | unsigned long temp;
|
| 533 |
|
| 534 | __asm__ __volatile__(
|
| 535 | " .set mips3 \n"
|
| 536 | "1: lld %1, %2 # atomic64_add_return \n"
|
| 537 | " addu %0, %1, %3 \n"
|
| 538 | " scd %0, %2 \n"
|
| 539 | " beqz %0, 2f \n"
|
| 540 | " addu %0, %1, %3 \n"
|
| 541 | " .subsection 2 \n"
|
| 542 | "2: b 1b \n"
|
| 543 | " .previous \n"
|
| 544 | " .set mips0 \n"
|
| 545 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 546 | : "Ir" (i), "m" (v->counter)
|
| 547 | : "memory");
|
| 548 | } else {
|
| 549 | unsigned long flags;
|
| 550 |
|
| 551 | raw_local_irq_save(flags);
|
| 552 | result = v->counter;
|
| 553 | result += i;
|
| 554 | v->counter = result;
|
| 555 | raw_local_irq_restore(flags);
|
| 556 | }
|
| 557 |
|
| 558 | smp_mb();
|
| 559 |
|
| 560 | return result;
|
| 561 | }
|
| 562 |
|
| 563 | static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
|
| 564 | {
|
| 565 | unsigned long result;
|
| 566 |
|
| 567 | smp_mb();
|
| 568 |
|
| 569 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 570 | unsigned long temp;
|
| 571 |
|
| 572 | __asm__ __volatile__(
|
| 573 | " .set mips3 \n"
|
| 574 | "1: lld %1, %2 # atomic64_sub_return \n"
|
| 575 | " subu %0, %1, %3 \n"
|
| 576 | " scd %0, %2 \n"
|
| 577 | " beqzl %0, 1b \n"
|
| 578 | " subu %0, %1, %3 \n"
|
| 579 | " .set mips0 \n"
|
| 580 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 581 | : "Ir" (i), "m" (v->counter)
|
| 582 | : "memory");
|
| 583 | } else if (cpu_has_llsc) {
|
| 584 | unsigned long temp;
|
| 585 |
|
| 586 | __asm__ __volatile__(
|
| 587 | " .set mips3 \n"
|
| 588 | "1: lld %1, %2 # atomic64_sub_return \n"
|
| 589 | " subu %0, %1, %3 \n"
|
| 590 | " scd %0, %2 \n"
|
| 591 | " beqz %0, 2f \n"
|
| 592 | " subu %0, %1, %3 \n"
|
| 593 | " .subsection 2 \n"
|
| 594 | "2: b 1b \n"
|
| 595 | " .previous \n"
|
| 596 | " .set mips0 \n"
|
| 597 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 598 | : "Ir" (i), "m" (v->counter)
|
| 599 | : "memory");
|
| 600 | } else {
|
| 601 | unsigned long flags;
|
| 602 |
|
| 603 | raw_local_irq_save(flags);
|
| 604 | result = v->counter;
|
| 605 | result -= i;
|
| 606 | v->counter = result;
|
| 607 | raw_local_irq_restore(flags);
|
| 608 | }
|
| 609 |
|
| 610 | smp_mb();
|
| 611 |
|
| 612 | return result;
|
| 613 | }
|
| 614 |
|
| 615 | |
| 616 | * atomic64_sub_if_positive - conditionally subtract integer from atomic variable
|
| 617 | * @i: integer value to subtract
|
| 618 | * @v: pointer of type atomic64_t
|
| 619 | *
|
| 620 | * Atomically test @v and subtract @i if @v is greater or equal than @i.
|
| 621 | * The function returns the old value of @v minus @i.
|
| 622 |
|
| 623 | static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
|
| 624 | {
|
| 625 | unsigned long result;
|
| 626 |
|
| 627 | smp_mb();
|
| 628 |
|
| 629 | if (cpu_has_llsc && R10000_LLSC_WAR) {
|
| 630 | unsigned long temp;
|
| 631 |
|
| 632 | __asm__ __volatile__(
|
| 633 | " .set mips3 \n"
|
| 634 | "1: lld %1, %2 # atomic64_sub_if_positive\n"
|
| 635 | " dsubu %0, %1, %3 \n"
|
| 636 | " bltz %0, 1f \n"
|
| 637 | " scd %0, %2 \n"
|
| 638 | " .set noreorder \n"
|
| 639 | " beqzl %0, 1b \n"
|
| 640 | " dsubu %0, %1, %3 \n"
|
| 641 | " .set reorder \n"
|
| 642 | "1: \n"
|
| 643 | " .set mips0 \n"
|
| 644 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 645 | : "Ir" (i), "m" (v->counter)
|
| 646 | : "memory");
|
| 647 | } else if (cpu_has_llsc) {
|
| 648 | unsigned long temp;
|
| 649 |
|
| 650 | __asm__ __volatile__(
|
| 651 | " .set mips3 \n"
|
| 652 | "1: lld %1, %2 # atomic64_sub_if_positive\n"
|
| 653 | " dsubu %0, %1, %3 \n"
|
| 654 | " bltz %0, 1f \n"
|
| 655 | " scd %0, %2 \n"
|
| 656 | " .set noreorder \n"
|
| 657 | " beqz %0, 2f \n"
|
| 658 | " dsubu %0, %1, %3 \n"
|
| 659 | " .set reorder \n"
|
| 660 | "1: \n"
|
| 661 | " .subsection 2 \n"
|
| 662 | "2: b 1b \n"
|
| 663 | " .previous \n"
|
| 664 | " .set mips0 \n"
|
| 665 | : "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
| 666 | : "Ir" (i), "m" (v->counter)
|
| 667 | : "memory");
|
| 668 | } else {
|
| 669 | unsigned long flags;
|
| 670 |
|
| 671 | raw_local_irq_save(flags);
|
| 672 | result = v->counter;
|
| 673 | result -= i;
|
| 674 | if (result >= 0)
|
| 675 | v->counter = result;
|
| 676 | raw_local_irq_restore(flags);
|
| 677 | }
|
| 678 |
|
| 679 | smp_mb();
|
| 680 |
|
| 681 | return result;
|
| 682 | }
|
| 683 |
|
| 684 | #define atomic64_dec_return(v) atomic64_sub_return(1,(v))
|
| 685 | #define atomic64_inc_return(v) atomic64_add_return(1,(v))
|
| 686 |
|
| 687 | |
| 688 | * atomic64_sub_and_test - subtract value from variable and test result
|
| 689 | * @i: integer value to subtract
|
| 690 | * @v: pointer of type atomic64_t
|
| 691 | *
|
| 692 | * Atomically subtracts @i from @v and returns
|
| 693 | * true if the result is zero, or false for all
|
| 694 | * other cases.
|
| 695 |
|
| 696 | #define atomic64_sub_and_test(i,v) (atomic64_sub_return((i), (v)) == 0)
|
| 697 |
|
| 698 | |
| 699 | * atomic64_inc_and_test - increment and test
|
| 700 | * @v: pointer of type atomic64_t
|
| 701 | *
|
| 702 | * Atomically increments @v by 1
|
| 703 | * and returns true if the result is zero, or false for all
|
| 704 | * other cases.
|
| 705 |
|
| 706 | #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
|
| 707 |
|
| 708 | |
| 709 | * atomic64_dec_and_test - decrement by 1 and test
|
| 710 | * @v: pointer of type atomic64_t
|
| 711 | *
|
| 712 | * Atomically decrements @v by 1 and
|
| 713 | * returns true if the result is 0, or false for all other
|
| 714 | * cases.
|
| 715 |
|
| 716 | #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
|
| 717 |
|
| 718 | |
| 719 | * atomic64_dec_if_positive - decrement by 1 if old value positive
|
| 720 | * @v: pointer of type atomic64_t
|
| 721 |
|
| 722 | #define atomic64_dec_if_positive(v) atomic64_sub_if_positive(1, v)
|
| 723 |
|
| 724 | |
| 725 | * atomic64_inc - increment atomic variable
|
| 726 | * @v: pointer of type atomic64_t
|
| 727 | *
|
| 728 | * Atomically increments @v by 1.
|
| 729 |
|
| 730 | #define atomic64_inc(v) atomic64_add(1,(v))
|
| 731 |
|
| 732 | |
| 733 | * atomic64_dec - decrement and test
|
| 734 | * @v: pointer of type atomic64_t
|
| 735 | *
|
| 736 | * Atomically decrements @v by 1.
|
| 737 |
|
| 738 | #define atomic64_dec(v) atomic64_sub(1,(v))
|
| 739 |
|
| 740 | |
| 741 | * atomic64_add_negative - add and test if negative
|
| 742 | * @v: pointer of type atomic64_t
|
| 743 | * @i: integer value to add
|
| 744 | *
|
| 745 | * Atomically adds @i to @v and returns true
|
| 746 | * if the result is negative, or false when
|
| 747 | * result is greater than or equal to zero.
|
| 748 |
|
| 749 | #define atomic64_add_negative(i,v) (atomic64_add_return(i, (v)) < 0)
|
| 750 |
|
| 751 | #endif
|
| 752 |
|
| 753 | |
| 754 | * atomic*_return operations are serializing but not the non-*_return
|
| 755 | * versions.
|
| 756 |
|
| 757 | #define smp_mb__before_atomic_dec() smp_mb()
|
| 758 | #define smp_mb__after_atomic_dec() smp_mb()
|
| 759 | #define smp_mb__before_atomic_inc() smp_mb()
|
| 760 | #define smp_mb__after_atomic_inc() smp_mb()
|
| 761 |
|
| 762 | #include <asm-generic/atomic.h>
|
| 763 | #endif
|