include/linux/atomic.h

   1 /* Atomic operations usable in machine independent code */
   2 #ifndef _LINUX_ATOMIC_H
   3 #define _LINUX_ATOMIC_H
   4 #include <asm/atomic.h>
   5 #include <asm/barrier.h>
   6
   7 /*
   8  * Relaxed variants of xchg, cmpxchg and some atomic operations.
   9  *
  10  * We support four variants:
  11  *
  12  * - Fully ordered: The default implementation, no suffix required.
  13  * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
  14  * - Release: Provides RELEASE semantics, _release suffix.
  15  * - Relaxed: No ordering guarantees, _relaxed suffix.
  16  *
  17  * For compound atomics performing both a load and a store, ACQUIRE
  18  * semantics apply only to the load and RELEASE semantics only to the
  19  * store portion of the operation. Note that a failed cmpxchg_acquire
  20  * does -not- imply any memory ordering constraints.
  21  *
  22  * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
  23  */
  24
  25 #ifndef atomic_read_acquire
  26 #define  atomic_read_acquire(v)         smp_load_acquire(&(v)->counter)
  27 #endif
  28
  29 #ifndef atomic_set_release
  30 #define  atomic_set_release(v, i)       smp_store_release(&(v)->counter, (i))
  31 #endif
  32
  33 /*
  34  * The idea here is to build acquire/release variants by adding explicit
  35  * barriers on top of the relaxed variant. In the case where the relaxed
  36  * variant is already fully ordered, no additional barriers are needed.
  37  */
  38 #define __atomic_op_acquire(op, args...)                                \
  39 ({                                                                      \
  40         typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
  41         smp_mb__after_atomic();                                         \
  42         __ret;                                                          \
  43 })
  44
  45 #define __atomic_op_release(op, args...)                                \
  46 ({                                                                      \
  47         smp_mb__before_atomic();                                        \
  48         op##_relaxed(args);                                             \
  49 })
  50
  51 #define __atomic_op_fence(op, args...)                                  \
  52 ({                                                                      \
  53         typeof(op##_relaxed(args)) __ret;                               \
  54         smp_mb__before_atomic();                                        \
  55         __ret = op##_relaxed(args);                                     \
  56         smp_mb__after_atomic();                                         \
  57         __ret;                                                          \
  58 })
  59
  60 /* atomic_add_return_relaxed */
  61 #ifndef atomic_add_return_relaxed
  62 #define  atomic_add_return_relaxed      atomic_add_return
  63 #define  atomic_add_return_acquire      atomic_add_return
  64 #define  atomic_add_return_release      atomic_add_return
  65
  66 #else /* atomic_add_return_relaxed */
  67
  68 #ifndef atomic_add_return_acquire
  69 #define  atomic_add_return_acquire(...)                                 \
  70         __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
  71 #endif
  72
  73 #ifndef atomic_add_return_release
  74 #define  atomic_add_return_release(...)                                 \
  75         __atomic_op_release(atomic_add_return, __VA_ARGS__)
  76 #endif
  77
  78 #ifndef atomic_add_return
  79 #define  atomic_add_return(...)                                         \
  80         __atomic_op_fence(atomic_add_return, __VA_ARGS__)
  81 #endif
  82 #endif /* atomic_add_return_relaxed */
  83
  84 /* atomic_inc_return_relaxed */
  85 #ifndef atomic_inc_return_relaxed
  86 #define  atomic_inc_return_relaxed      atomic_inc_return
  87 #define  atomic_inc_return_acquire      atomic_inc_return
  88 #define  atomic_inc_return_release      atomic_inc_return
  89
  90 #else /* atomic_inc_return_relaxed */
  91
  92 #ifndef atomic_inc_return_acquire
  93 #define  atomic_inc_return_acquire(...)                                 \
  94         __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
  95 #endif
  96
  97 #ifndef atomic_inc_return_release
  98 #define  atomic_inc_return_release(...)                                 \
  99         __atomic_op_release(atomic_inc_return, __VA_ARGS__)
 100 #endif
 101
 102 #ifndef atomic_inc_return
 103 #define  atomic_inc_return(...)                                         \
 104         __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
 105 #endif
 106 #endif /* atomic_inc_return_relaxed */
 107
 108 /* atomic_sub_return_relaxed */
 109 #ifndef atomic_sub_return_relaxed
 110 #define  atomic_sub_return_relaxed      atomic_sub_return
 111 #define  atomic_sub_return_acquire      atomic_sub_return
 112 #define  atomic_sub_return_release      atomic_sub_return
 113
 114 #else /* atomic_sub_return_relaxed */
 115
 116 #ifndef atomic_sub_return_acquire
 117 #define  atomic_sub_return_acquire(...)                                 \
 118         __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
 119 #endif
 120
 121 #ifndef atomic_sub_return_release
 122 #define  atomic_sub_return_release(...)                                 \
 123         __atomic_op_release(atomic_sub_return, __VA_ARGS__)
 124 #endif
 125
 126 #ifndef atomic_sub_return
 127 #define  atomic_sub_return(...)                                         \
 128         __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
 129 #endif
 130 #endif /* atomic_sub_return_relaxed */
 131
 132 /* atomic_dec_return_relaxed */
 133 #ifndef atomic_dec_return_relaxed
 134 #define  atomic_dec_return_relaxed      atomic_dec_return
 135 #define  atomic_dec_return_acquire      atomic_dec_return
 136 #define  atomic_dec_return_release      atomic_dec_return
 137
 138 #else /* atomic_dec_return_relaxed */
 139
 140 #ifndef atomic_dec_return_acquire
 141 #define  atomic_dec_return_acquire(...)                                 \
 142         __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
 143 #endif
 144
 145 #ifndef atomic_dec_return_release
 146 #define  atomic_dec_return_release(...)                                 \
 147         __atomic_op_release(atomic_dec_return, __VA_ARGS__)
 148 #endif
 149
 150 #ifndef atomic_dec_return
 151 #define  atomic_dec_return(...)                                         \
 152         __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
 153 #endif
 154 #endif /* atomic_dec_return_relaxed */
 155
 156 /* atomic_xchg_relaxed */
 157 #ifndef atomic_xchg_relaxed
 158 #define  atomic_xchg_relaxed            atomic_xchg
 159 #define  atomic_xchg_acquire            atomic_xchg
 160 #define  atomic_xchg_release            atomic_xchg
 161
 162 #else /* atomic_xchg_relaxed */
 163
 164 #ifndef atomic_xchg_acquire
 165 #define  atomic_xchg_acquire(...)                                       \
 166         __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
 167 #endif
 168
 169 #ifndef atomic_xchg_release
 170 #define  atomic_xchg_release(...)                                       \
 171         __atomic_op_release(atomic_xchg, __VA_ARGS__)
 172 #endif
 173
 174 #ifndef atomic_xchg
 175 #define  atomic_xchg(...)                                               \
 176         __atomic_op_fence(atomic_xchg, __VA_ARGS__)
 177 #endif
 178 #endif /* atomic_xchg_relaxed */
 179
 180 /* atomic_cmpxchg_relaxed */
 181 #ifndef atomic_cmpxchg_relaxed
 182 #define  atomic_cmpxchg_relaxed         atomic_cmpxchg
 183 #define  atomic_cmpxchg_acquire         atomic_cmpxchg
 184 #define  atomic_cmpxchg_release         atomic_cmpxchg
 185
 186 #else /* atomic_cmpxchg_relaxed */
 187
 188 #ifndef atomic_cmpxchg_acquire
 189 #define  atomic_cmpxchg_acquire(...)                                    \
 190         __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
 191 #endif
 192
 193 #ifndef atomic_cmpxchg_release
 194 #define  atomic_cmpxchg_release(...)                                    \
 195         __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
 196 #endif
 197
 198 #ifndef atomic_cmpxchg
 199 #define  atomic_cmpxchg(...)                                            \
 200         __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
 201 #endif
 202 #endif /* atomic_cmpxchg_relaxed */
 203
 204 #ifndef atomic64_read_acquire
 205 #define  atomic64_read_acquire(v)       smp_load_acquire(&(v)->counter)
 206 #endif
 207
 208 #ifndef atomic64_set_release
 209 #define  atomic64_set_release(v, i)     smp_store_release(&(v)->counter, (i))
 210 #endif
 211
 212 /* atomic64_add_return_relaxed */
 213 #ifndef atomic64_add_return_relaxed
 214 #define  atomic64_add_return_relaxed    atomic64_add_return
 215 #define  atomic64_add_return_acquire    atomic64_add_return
 216 #define  atomic64_add_return_release    atomic64_add_return
 217
 218 #else /* atomic64_add_return_relaxed */
 219
 220 #ifndef atomic64_add_return_acquire
 221 #define  atomic64_add_return_acquire(...)                               \
 222         __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
 223 #endif
 224
 225 #ifndef atomic64_add_return_release
 226 #define  atomic64_add_return_release(...)                               \
 227         __atomic_op_release(atomic64_add_return, __VA_ARGS__)
 228 #endif
 229
 230 #ifndef atomic64_add_return
 231 #define  atomic64_add_return(...)                                       \
 232         __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
 233 #endif
 234 #endif /* atomic64_add_return_relaxed */
 235
 236 /* atomic64_inc_return_relaxed */
 237 #ifndef atomic64_inc_return_relaxed
 238 #define  atomic64_inc_return_relaxed    atomic64_inc_return
 239 #define  atomic64_inc_return_acquire    atomic64_inc_return
 240 #define  atomic64_inc_return_release    atomic64_inc_return
 241
 242 #else /* atomic64_inc_return_relaxed */
 243
 244 #ifndef atomic64_inc_return_acquire
 245 #define  atomic64_inc_return_acquire(...)                               \
 246         __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
 247 #endif
 248
 249 #ifndef atomic64_inc_return_release
 250 #define  atomic64_inc_return_release(...)                               \
 251         __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
 252 #endif
 253
 254 #ifndef atomic64_inc_return
 255 #define  atomic64_inc_return(...)                                       \
 256         __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
 257 #endif
 258 #endif /* atomic64_inc_return_relaxed */
 259
 260
 261 /* atomic64_sub_return_relaxed */
 262 #ifndef atomic64_sub_return_relaxed
 263 #define  atomic64_sub_return_relaxed    atomic64_sub_return
 264 #define  atomic64_sub_return_acquire    atomic64_sub_return
 265 #define  atomic64_sub_return_release    atomic64_sub_return
 266
 267 #else /* atomic64_sub_return_relaxed */
 268
 269 #ifndef atomic64_sub_return_acquire
 270 #define  atomic64_sub_return_acquire(...)                               \
 271         __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
 272 #endif
 273
 274 #ifndef atomic64_sub_return_release
 275 #define  atomic64_sub_return_release(...)                               \
 276         __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
 277 #endif
 278
 279 #ifndef atomic64_sub_return
 280 #define  atomic64_sub_return(...)                                       \
 281         __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
 282 #endif
 283 #endif /* atomic64_sub_return_relaxed */
 284
 285 /* atomic64_dec_return_relaxed */
 286 #ifndef atomic64_dec_return_relaxed
 287 #define  atomic64_dec_return_relaxed    atomic64_dec_return
 288 #define  atomic64_dec_return_acquire    atomic64_dec_return
 289 #define  atomic64_dec_return_release    atomic64_dec_return
 290
 291 #else /* atomic64_dec_return_relaxed */
 292
 293 #ifndef atomic64_dec_return_acquire
 294 #define  atomic64_dec_return_acquire(...)                               \
 295         __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
 296 #endif
 297
 298 #ifndef atomic64_dec_return_release
 299 #define  atomic64_dec_return_release(...)                               \
 300         __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
 301 #endif
 302
 303 #ifndef atomic64_dec_return
 304 #define  atomic64_dec_return(...)                                       \
 305         __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
 306 #endif
 307 #endif /* atomic64_dec_return_relaxed */
 308
 309 /* atomic64_xchg_relaxed */
 310 #ifndef atomic64_xchg_relaxed
 311 #define  atomic64_xchg_relaxed          atomic64_xchg
 312 #define  atomic64_xchg_acquire          atomic64_xchg
 313 #define  atomic64_xchg_release          atomic64_xchg
 314
 315 #else /* atomic64_xchg_relaxed */
 316
 317 #ifndef atomic64_xchg_acquire
 318 #define  atomic64_xchg_acquire(...)                                     \
 319         __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
 320 #endif
 321
 322 #ifndef atomic64_xchg_release
 323 #define  atomic64_xchg_release(...)                                     \
 324         __atomic_op_release(atomic64_xchg, __VA_ARGS__)
 325 #endif
 326
 327 #ifndef atomic64_xchg
 328 #define  atomic64_xchg(...)                                             \
 329         __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
 330 #endif
 331 #endif /* atomic64_xchg_relaxed */
 332
 333 /* atomic64_cmpxchg_relaxed */
 334 #ifndef atomic64_cmpxchg_relaxed
 335 #define  atomic64_cmpxchg_relaxed       atomic64_cmpxchg
 336 #define  atomic64_cmpxchg_acquire       atomic64_cmpxchg
 337 #define  atomic64_cmpxchg_release       atomic64_cmpxchg
 338
 339 #else /* atomic64_cmpxchg_relaxed */
 340
 341 #ifndef atomic64_cmpxchg_acquire
 342 #define  atomic64_cmpxchg_acquire(...)                                  \
 343         __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
 344 #endif
 345
 346 #ifndef atomic64_cmpxchg_release
 347 #define  atomic64_cmpxchg_release(...)                                  \
 348         __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
 349 #endif
 350
 351 #ifndef atomic64_cmpxchg
 352 #define  atomic64_cmpxchg(...)                                          \
 353         __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
 354 #endif
 355 #endif /* atomic64_cmpxchg_relaxed */
 356
 357 /* cmpxchg_relaxed */
 358 #ifndef cmpxchg_relaxed
 359 #define  cmpxchg_relaxed                cmpxchg
 360 #define  cmpxchg_acquire                cmpxchg
 361 #define  cmpxchg_release                cmpxchg
 362
 363 #else /* cmpxchg_relaxed */
 364
 365 #ifndef cmpxchg_acquire
 366 #define  cmpxchg_acquire(...)                                           \
 367         __atomic_op_acquire(cmpxchg, __VA_ARGS__)
 368 #endif
 369
 370 #ifndef cmpxchg_release
 371 #define  cmpxchg_release(...)                                           \
 372         __atomic_op_release(cmpxchg, __VA_ARGS__)
 373 #endif
 374
 375 #ifndef cmpxchg
 376 #define  cmpxchg(...)                                                   \
 377         __atomic_op_fence(cmpxchg, __VA_ARGS__)
 378 #endif
 379 #endif /* cmpxchg_relaxed */
 380
 381 /* cmpxchg64_relaxed */
 382 #ifndef cmpxchg64_relaxed
 383 #define  cmpxchg64_relaxed              cmpxchg64
 384 #define  cmpxchg64_acquire              cmpxchg64
 385 #define  cmpxchg64_release              cmpxchg64
 386
 387 #else /* cmpxchg64_relaxed */
 388
 389 #ifndef cmpxchg64_acquire
 390 #define  cmpxchg64_acquire(...)                                         \
 391         __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
 392 #endif
 393
 394 #ifndef cmpxchg64_release
 395 #define  cmpxchg64_release(...)                                         \
 396         __atomic_op_release(cmpxchg64, __VA_ARGS__)
 397 #endif
 398
 399 #ifndef cmpxchg64
 400 #define  cmpxchg64(...)                                                 \
 401         __atomic_op_fence(cmpxchg64, __VA_ARGS__)
 402 #endif
 403 #endif /* cmpxchg64_relaxed */
 404
 405 /* xchg_relaxed */
 406 #ifndef xchg_relaxed
 407 #define  xchg_relaxed                   xchg
 408 #define  xchg_acquire                   xchg
 409 #define  xchg_release                   xchg
 410
 411 #else /* xchg_relaxed */
 412
 413 #ifndef xchg_acquire
 414 #define  xchg_acquire(...)              __atomic_op_acquire(xchg, __VA_ARGS__)
 415 #endif
 416
 417 #ifndef xchg_release
 418 #define  xchg_release(...)              __atomic_op_release(xchg, __VA_ARGS__)
 419 #endif
 420
 421 #ifndef xchg
 422 #define  xchg(...)                      __atomic_op_fence(xchg, __VA_ARGS__)
 423 #endif
 424 #endif /* xchg_relaxed */
 425
 426 /**
 427  * atomic_add_unless - add unless the number is already a given value
 428  * @v: pointer of type atomic_t
 429  * @a: the amount to add to v...
 430  * @u: ...unless v is equal to u.
 431  *
 432  * Atomically adds @a to @v, so long as @v was not already @u.
 433  * Returns non-zero if @v was not @u, and zero otherwise.
 434  */
 435 static inline int atomic_add_unless(atomic_t *v, int a, int u)
 436 {
 437         return __atomic_add_unless(v, a, u) != u;
 438 }
 439
 440 /**
 441  * atomic_inc_not_zero - increment unless the number is zero
 442  * @v: pointer of type atomic_t
 443  *
 444  * Atomically increments @v by 1, so long as @v is non-zero.
 445  * Returns non-zero if @v was non-zero, and zero otherwise.
 446  */
 447 #ifndef atomic_inc_not_zero
 448 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 449 #endif
 450
 451 #ifndef atomic_andnot
 452 static inline void atomic_andnot(int i, atomic_t *v)
 453 {
 454         atomic_and(~i, v);
 455 }
 456 #endif
 457
 458 static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
 459 {
 460         atomic_andnot(mask, v);
 461 }
 462
 463 static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
 464 {
 465         atomic_or(mask, v);
 466 }
 467
 468 /**
 469  * atomic_inc_not_zero_hint - increment if not null
 470  * @v: pointer of type atomic_t
 471  * @hint: probable value of the atomic before the increment
 472  *
 473  * This version of atomic_inc_not_zero() gives a hint of probable
 474  * value of the atomic. This helps processor to not read the memory
 475  * before doing the atomic read/modify/write cycle, lowering
 476  * number of bus transactions on some arches.
 477  *
 478  * Returns: 0 if increment was not done, 1 otherwise.
 479  */
 480 #ifndef atomic_inc_not_zero_hint
 481 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
 482 {
 483         int val, c = hint;
 484
 485         /* sanity test, should be removed by compiler if hint is a constant */
 486         if (!hint)
 487                 return atomic_inc_not_zero(v);
 488
 489         do {
 490                 val = atomic_cmpxchg(v, c, c + 1);
 491                 if (val == c)
 492                         return 1;
 493                 c = val;
 494         } while (c);
 495
 496         return 0;
 497 }
 498 #endif
 499
 500 #ifndef atomic_inc_unless_negative
 501 static inline int atomic_inc_unless_negative(atomic_t *p)
 502 {
 503         int v, v1;
 504         for (v = 0; v >= 0; v = v1) {
 505                 v1 = atomic_cmpxchg(p, v, v + 1);
 506                 if (likely(v1 == v))
 507                         return 1;
 508         }
 509         return 0;
 510 }
 511 #endif
 512
 513 #ifndef atomic_dec_unless_positive
 514 static inline int atomic_dec_unless_positive(atomic_t *p)
 515 {
 516         int v, v1;
 517         for (v = 0; v <= 0; v = v1) {
 518                 v1 = atomic_cmpxchg(p, v, v - 1);
 519                 if (likely(v1 == v))
 520                         return 1;
 521         }
 522         return 0;
 523 }
 524 #endif
 525
 526 /*
 527  * atomic_dec_if_positive - decrement by 1 if old value positive
 528  * @v: pointer of type atomic_t
 529  *
 530  * The function returns the old value of *v minus 1, even if
 531  * the atomic variable, v, was not decremented.
 532  */
 533 #ifndef atomic_dec_if_positive
 534 static inline int atomic_dec_if_positive(atomic_t *v)
 535 {
 536         int c, old, dec;
 537         c = atomic_read(v);
 538         for (;;) {
 539                 dec = c - 1;
 540                 if (unlikely(dec < 0))
 541                         break;
 542                 old = atomic_cmpxchg((v), c, dec);
 543                 if (likely(old == c))
 544                         break;
 545                 c = old;
 546         }
 547         return dec;
 548 }
 549 #endif
 550
 551 #ifdef CONFIG_GENERIC_ATOMIC64
 552 #include <asm-generic/atomic64.h>
 553 #endif
 554
 555 #ifndef atomic64_andnot
 556 static inline void atomic64_andnot(long long i, atomic64_t *v)
 557 {
 558         atomic64_and(~i, v);
 559 }
 560 #endif
 561
 562 #include <asm-generic/atomic-long.h>
 563
 564 #endif /* _LINUX_ATOMIC_H */