arch/arm64/include/asm/spinlock.h

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15  */
  16 #ifndef __ASM_SPINLOCK_H
  17 #define __ASM_SPINLOCK_H
  18
  19 #include <asm/lse.h>
  20 #include <asm/spinlock_types.h>
  21 #include <asm/processor.h>
  22
  23 /*
  24  * Spinlock implementation.
  25  *
  26  * The memory barriers are implicit with the load-acquire and store-release
  27  * instructions.
  28  */
  29 static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
  30 {
  31         unsigned int tmp;
  32         arch_spinlock_t lockval;
  33         u32 owner;
  34
  35         /*
  36          * Ensure prior spin_lock operations to other locks have completed
  37          * on this CPU before we test whether "lock" is locked.
  38          */
  39         smp_mb();
  40         owner = READ_ONCE(lock->owner) << 16;
  41
  42         asm volatile(
  43 "       sevl\n"
  44 "1:     wfe\n"
  45 "2:     ldaxr   %w0, %2\n"
  46         /* Is the lock free? */
  47 "       eor     %w1, %w0, %w0, ror #16\n"
  48 "       cbz     %w1, 3f\n"
  49         /* Lock taken -- has there been a subsequent unlock->lock transition? */
  50 "       eor     %w1, %w3, %w0, lsl #16\n"
  51 "       cbz     %w1, 1b\n"
  52         /*
  53          * The owner has been updated, so there was an unlock->lock
  54          * transition that we missed. That means we can rely on the
  55          * store-release of the unlock operation paired with the
  56          * load-acquire of the lock operation to publish any of our
  57          * previous stores to the new lock owner and therefore don't
  58          * need to bother with the writeback below.
  59          */
  60 "       b       4f\n"
  61 "3:\n"
  62         /*
  63          * Serialise against any concurrent lockers by writing back the
  64          * unlocked lock value
  65          */
  66         ARM64_LSE_ATOMIC_INSN(
  67         /* LL/SC */
  68 "       stxr    %w1, %w0, %2\n"
  69         __nops(2),
  70         /* LSE atomics */
  71 "       mov     %w1, %w0\n"
  72 "       cas     %w0, %w0, %2\n"
  73 "       eor     %w1, %w1, %w0\n")
  74         /* Somebody else wrote to the lock, GOTO 10 and reload the value */
  75 "       cbnz    %w1, 2b\n"
  76 "4:"
  77         : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
  78         : "r" (owner)
  79         : "memory");
  80 }
  81
  82 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
  83
  84 static inline void arch_spin_lock(arch_spinlock_t *lock)
  85 {
  86         unsigned int tmp;
  87         arch_spinlock_t lockval, newval;
  88
  89         asm volatile(
  90         /* Atomically increment the next ticket. */
  91         ARM64_LSE_ATOMIC_INSN(
  92         /* LL/SC */
  93 "       prfm    pstl1strm, %3\n"
  94 "1:     ldaxr   %w0, %3\n"
  95 "       add     %w1, %w0, %w5\n"
  96 "       stxr    %w2, %w1, %3\n"
  97 "       cbnz    %w2, 1b\n",
  98         /* LSE atomics */
  99 "       mov     %w2, %w5\n"
 100 "       ldadda  %w2, %w0, %3\n"
 101         __nops(3)
 102         )
 103
 104         /* Did we get the lock? */
 105 "       eor     %w1, %w0, %w0, ror #16\n"
 106 "       cbz     %w1, 3f\n"
 107         /*
 108          * No: spin on the owner. Send a local event to avoid missing an
 109          * unlock before the exclusive load.
 110          */
 111 "       sevl\n"
 112 "2:     wfe\n"
 113 "       ldaxrh  %w2, %4\n"
 114 "       eor     %w1, %w2, %w0, lsr #16\n"
 115 "       cbnz    %w1, 2b\n"
 116         /* We got the lock. Critical section starts here. */
 117 "3:"
 118         : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
 119         : "Q" (lock->owner), "I" (1 << TICKET_SHIFT)
 120         : "memory");
 121 }
 122
 123 static inline int arch_spin_trylock(arch_spinlock_t *lock)
 124 {
 125         unsigned int tmp;
 126         arch_spinlock_t lockval;
 127
 128         asm volatile(ARM64_LSE_ATOMIC_INSN(
 129         /* LL/SC */
 130         "       prfm    pstl1strm, %2\n"
 131         "1:     ldaxr   %w0, %2\n"
 132         "       eor     %w1, %w0, %w0, ror #16\n"
 133         "       cbnz    %w1, 2f\n"
 134         "       add     %w0, %w0, %3\n"
 135         "       stxr    %w1, %w0, %2\n"
 136         "       cbnz    %w1, 1b\n"
 137         "2:",
 138         /* LSE atomics */
 139         "       ldr     %w0, %2\n"
 140         "       eor     %w1, %w0, %w0, ror #16\n"
 141         "       cbnz    %w1, 1f\n"
 142         "       add     %w1, %w0, %3\n"
 143         "       casa    %w0, %w1, %2\n"
 144         "       and     %w1, %w1, #0xffff\n"
 145         "       eor     %w1, %w1, %w0, lsr #16\n"
 146         "1:")
 147         : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
 148         : "I" (1 << TICKET_SHIFT)
 149         : "memory");
 150
 151         return !tmp;
 152 }
 153
 154 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 155 {
 156         unsigned long tmp;
 157
 158         asm volatile(ARM64_LSE_ATOMIC_INSN(
 159         /* LL/SC */
 160         "       ldrh    %w1, %0\n"
 161         "       add     %w1, %w1, #1\n"
 162         "       stlrh   %w1, %0",
 163         /* LSE atomics */
 164         "       mov     %w1, #1\n"
 165         "       staddlh %w1, %0\n"
 166         __nops(1))
 167         : "=Q" (lock->owner), "=&r" (tmp)
 168         :
 169         : "memory");
 170 }
 171
 172 static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
 173 {
 174         return lock.owner == lock.next;
 175 }
 176
 177 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
 178 {
 179         smp_mb(); /* See arch_spin_unlock_wait */
 180         return !arch_spin_value_unlocked(READ_ONCE(*lock));
 181 }
 182
 183 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
 184 {
 185         arch_spinlock_t lockval = READ_ONCE(*lock);
 186         return (lockval.next - lockval.owner) > 1;
 187 }
 188 #define arch_spin_is_contended  arch_spin_is_contended
 189
 190 /*
 191  * Write lock implementation.
 192  *
 193  * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is
 194  * exclusively held.
 195  *
 196  * The memory barriers are implicit with the load-acquire and store-release
 197  * instructions.
 198  */
 199
 200 static inline void arch_write_lock(arch_rwlock_t *rw)
 201 {
 202         unsigned int tmp;
 203
 204         asm volatile(ARM64_LSE_ATOMIC_INSN(
 205         /* LL/SC */
 206         "       sevl\n"
 207         "1:     wfe\n"
 208         "2:     ldaxr   %w0, %1\n"
 209         "       cbnz    %w0, 1b\n"
 210         "       stxr    %w0, %w2, %1\n"
 211         "       cbnz    %w0, 2b\n"
 212         __nops(1),
 213         /* LSE atomics */
 214         "1:     mov     %w0, wzr\n"
 215         "2:     casa    %w0, %w2, %1\n"
 216         "       cbz     %w0, 3f\n"
 217         "       ldxr    %w0, %1\n"
 218         "       cbz     %w0, 2b\n"
 219         "       wfe\n"
 220         "       b       1b\n"
 221         "3:")
 222         : "=&r" (tmp), "+Q" (rw->lock)
 223         : "r" (0x80000000)
 224         : "memory");
 225 }
 226
 227 static inline int arch_write_trylock(arch_rwlock_t *rw)
 228 {
 229         unsigned int tmp;
 230
 231         asm volatile(ARM64_LSE_ATOMIC_INSN(
 232         /* LL/SC */
 233         "1:     ldaxr   %w0, %1\n"
 234         "       cbnz    %w0, 2f\n"
 235         "       stxr    %w0, %w2, %1\n"
 236         "       cbnz    %w0, 1b\n"
 237         "2:",
 238         /* LSE atomics */
 239         "       mov     %w0, wzr\n"
 240         "       casa    %w0, %w2, %1\n"
 241         __nops(2))
 242         : "=&r" (tmp), "+Q" (rw->lock)
 243         : "r" (0x80000000)
 244         : "memory");
 245
 246         return !tmp;
 247 }
 248
 249 static inline void arch_write_unlock(arch_rwlock_t *rw)
 250 {
 251         asm volatile(ARM64_LSE_ATOMIC_INSN(
 252         "       stlr    wzr, %0",
 253         "       swpl    wzr, wzr, %0")
 254         : "=Q" (rw->lock) :: "memory");
 255 }
 256
 257 /* write_can_lock - would write_trylock() succeed? */
 258 #define arch_write_can_lock(x)          ((x)->lock == 0)
 259
 260 /*
 261  * Read lock implementation.
 262  *
 263  * It exclusively loads the lock value, increments it and stores the new value
 264  * back if positive and the CPU still exclusively owns the location. If the
 265  * value is negative, the lock is already held.
 266  *
 267  * During unlocking there may be multiple active read locks but no write lock.
 268  *
 269  * The memory barriers are implicit with the load-acquire and store-release
 270  * instructions.
 271  *
 272  * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC
 273  * and LSE implementations may exhibit different behaviour (although this
 274  * will have no effect on lockdep).
 275  */
 276 static inline void arch_read_lock(arch_rwlock_t *rw)
 277 {
 278         unsigned int tmp, tmp2;
 279
 280         asm volatile(
 281         "       sevl\n"
 282         ARM64_LSE_ATOMIC_INSN(
 283         /* LL/SC */
 284         "1:     wfe\n"
 285         "2:     ldaxr   %w0, %2\n"
 286         "       add     %w0, %w0, #1\n"
 287         "       tbnz    %w0, #31, 1b\n"
 288         "       stxr    %w1, %w0, %2\n"
 289         "       cbnz    %w1, 2b\n"
 290         __nops(1),
 291         /* LSE atomics */
 292         "1:     wfe\n"
 293         "2:     ldxr    %w0, %2\n"
 294         "       adds    %w1, %w0, #1\n"
 295         "       tbnz    %w1, #31, 1b\n"
 296         "       casa    %w0, %w1, %2\n"
 297         "       sbc     %w0, %w1, %w0\n"
 298         "       cbnz    %w0, 2b")
 299         : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
 300         :
 301         : "cc", "memory");
 302 }
 303
 304 static inline void arch_read_unlock(arch_rwlock_t *rw)
 305 {
 306         unsigned int tmp, tmp2;
 307
 308         asm volatile(ARM64_LSE_ATOMIC_INSN(
 309         /* LL/SC */
 310         "1:     ldxr    %w0, %2\n"
 311         "       sub     %w0, %w0, #1\n"
 312         "       stlxr   %w1, %w0, %2\n"
 313         "       cbnz    %w1, 1b",
 314         /* LSE atomics */
 315         "       movn    %w0, #0\n"
 316         "       staddl  %w0, %2\n"
 317         __nops(2))
 318         : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
 319         :
 320         : "memory");
 321 }
 322
 323 static inline int arch_read_trylock(arch_rwlock_t *rw)
 324 {
 325         unsigned int tmp, tmp2;
 326
 327         asm volatile(ARM64_LSE_ATOMIC_INSN(
 328         /* LL/SC */
 329         "       mov     %w1, #1\n"
 330         "1:     ldaxr   %w0, %2\n"
 331         "       add     %w0, %w0, #1\n"
 332         "       tbnz    %w0, #31, 2f\n"
 333         "       stxr    %w1, %w0, %2\n"
 334         "       cbnz    %w1, 1b\n"
 335         "2:",
 336         /* LSE atomics */
 337         "       ldr     %w0, %2\n"
 338         "       adds    %w1, %w0, #1\n"
 339         "       tbnz    %w1, #31, 1f\n"
 340         "       casa    %w0, %w1, %2\n"
 341         "       sbc     %w1, %w1, %w0\n"
 342         __nops(1)
 343         "1:")
 344         : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
 345         :
 346         : "cc", "memory");
 347
 348         return !tmp2;
 349 }
 350
 351 /* read_can_lock - would read_trylock() succeed? */
 352 #define arch_read_can_lock(x)           ((x)->lock < 0x80000000)
 353
 354 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
 355 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
 356
 357 #define arch_spin_relax(lock)   cpu_relax()
 358 #define arch_read_relax(lock)   cpu_relax()
 359 #define arch_write_relax(lock)  cpu_relax()
 360
 361 /*
 362  * Accesses appearing in program order before a spin_lock() operation
 363  * can be reordered with accesses inside the critical section, by virtue
 364  * of arch_spin_lock being constructed using acquire semantics.
 365  *
 366  * In cases where this is problematic (e.g. try_to_wake_up), an
 367  * smp_mb__before_spinlock() can restore the required ordering.
 368  */
 369 #define smp_mb__before_spinlock()       smp_mb()
 370
 371 #endif /* __ASM_SPINLOCK_H */