Commit | Line | Data |
---|---|---|
08e875c1 CM |
1 | /* |
2 | * Copyright (C) 2012 ARM Ltd. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License | |
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | */ | |
16 | #ifndef __ASM_SPINLOCK_H | |
17 | #define __ASM_SPINLOCK_H | |
18 | ||
81bb5c64 | 19 | #include <asm/lse.h> |
08e875c1 CM |
20 | #include <asm/spinlock_types.h> |
21 | #include <asm/processor.h> | |
22 | ||
23 | /* | |
24 | * Spinlock implementation. | |
25 | * | |
08e875c1 CM |
26 | * The memory barriers are implicit with the load-acquire and store-release |
27 | * instructions. | |
08e875c1 | 28 | */ |
d86b8da0 WD |
29 | static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) |
30 | { | |
31 | unsigned int tmp; | |
32 | arch_spinlock_t lockval; | |
08e875c1 | 33 | |
38b850a7 WD |
34 | /* |
35 | * Ensure prior spin_lock operations to other locks have completed | |
36 | * on this CPU before we test whether "lock" is locked. | |
37 | */ | |
38 | smp_mb(); | |
39 | ||
d86b8da0 WD |
40 | asm volatile( |
41 | " sevl\n" | |
42 | "1: wfe\n" | |
43 | "2: ldaxr %w0, %2\n" | |
44 | " eor %w1, %w0, %w0, ror #16\n" | |
45 | " cbnz %w1, 1b\n" | |
3a5facd0 | 46 | /* Serialise against any concurrent lockers */ |
d86b8da0 WD |
47 | ARM64_LSE_ATOMIC_INSN( |
48 | /* LL/SC */ | |
49 | " stxr %w1, %w0, %2\n" | |
d86b8da0 | 50 | " nop\n" |
3a5facd0 WD |
51 | " nop\n", |
52 | /* LSE atomics */ | |
53 | " mov %w1, %w0\n" | |
54 | " cas %w0, %w0, %2\n" | |
55 | " eor %w1, %w1, %w0\n") | |
56 | " cbnz %w1, 2b\n" | |
d86b8da0 WD |
57 | : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) |
58 | : | |
59 | : "memory"); | |
60 | } | |
08e875c1 CM |
61 | |
62 | #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) | |
63 | ||
64 | static inline void arch_spin_lock(arch_spinlock_t *lock) | |
65 | { | |
66 | unsigned int tmp; | |
52ea2a56 | 67 | arch_spinlock_t lockval, newval; |
08e875c1 CM |
68 | |
69 | asm volatile( | |
52ea2a56 | 70 | /* Atomically increment the next ticket. */ |
81bb5c64 WD |
71 | ARM64_LSE_ATOMIC_INSN( |
72 | /* LL/SC */ | |
52ea2a56 WD |
73 | " prfm pstl1strm, %3\n" |
74 | "1: ldaxr %w0, %3\n" | |
75 | " add %w1, %w0, %w5\n" | |
76 | " stxr %w2, %w1, %3\n" | |
81bb5c64 WD |
77 | " cbnz %w2, 1b\n", |
78 | /* LSE atomics */ | |
79 | " mov %w2, %w5\n" | |
80 | " ldadda %w2, %w0, %3\n" | |
81 | " nop\n" | |
82 | " nop\n" | |
83 | " nop\n" | |
84 | ) | |
85 | ||
52ea2a56 WD |
86 | /* Did we get the lock? */ |
87 | " eor %w1, %w0, %w0, ror #16\n" | |
88 | " cbz %w1, 3f\n" | |
89 | /* | |
90 | * No: spin on the owner. Send a local event to avoid missing an | |
91 | * unlock before the exclusive load. | |
92 | */ | |
93 | " sevl\n" | |
94 | "2: wfe\n" | |
95 | " ldaxrh %w2, %4\n" | |
96 | " eor %w1, %w2, %w0, lsr #16\n" | |
97 | " cbnz %w1, 2b\n" | |
98 | /* We got the lock. Critical section starts here. */ | |
99 | "3:" | |
100 | : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) | |
101 | : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) | |
102 | : "memory"); | |
08e875c1 CM |
103 | } |
104 | ||
105 | static inline int arch_spin_trylock(arch_spinlock_t *lock) | |
106 | { | |
107 | unsigned int tmp; | |
52ea2a56 | 108 | arch_spinlock_t lockval; |
08e875c1 | 109 | |
81bb5c64 WD |
110 | asm volatile(ARM64_LSE_ATOMIC_INSN( |
111 | /* LL/SC */ | |
112 | " prfm pstl1strm, %2\n" | |
113 | "1: ldaxr %w0, %2\n" | |
114 | " eor %w1, %w0, %w0, ror #16\n" | |
115 | " cbnz %w1, 2f\n" | |
116 | " add %w0, %w0, %3\n" | |
117 | " stxr %w1, %w0, %2\n" | |
118 | " cbnz %w1, 1b\n" | |
119 | "2:", | |
120 | /* LSE atomics */ | |
121 | " ldr %w0, %2\n" | |
122 | " eor %w1, %w0, %w0, ror #16\n" | |
123 | " cbnz %w1, 1f\n" | |
124 | " add %w1, %w0, %3\n" | |
125 | " casa %w0, %w1, %2\n" | |
126 | " and %w1, %w1, #0xffff\n" | |
127 | " eor %w1, %w1, %w0, lsr #16\n" | |
128 | "1:") | |
52ea2a56 WD |
129 | : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) |
130 | : "I" (1 << TICKET_SHIFT) | |
131 | : "memory"); | |
08e875c1 CM |
132 | |
133 | return !tmp; | |
134 | } | |
135 | ||
136 | static inline void arch_spin_unlock(arch_spinlock_t *lock) | |
137 | { | |
81bb5c64 WD |
138 | unsigned long tmp; |
139 | ||
140 | asm volatile(ARM64_LSE_ATOMIC_INSN( | |
141 | /* LL/SC */ | |
c1d7cd22 | 142 | " ldrh %w1, %0\n" |
81bb5c64 WD |
143 | " add %w1, %w1, #1\n" |
144 | " stlrh %w1, %0", | |
145 | /* LSE atomics */ | |
146 | " mov %w1, #1\n" | |
147 | " nop\n" | |
148 | " staddlh %w1, %0") | |
149 | : "=Q" (lock->owner), "=&r" (tmp) | |
150 | : | |
52ea2a56 WD |
151 | : "memory"); |
152 | } | |
153 | ||
5686b06c WD |
154 | static inline int arch_spin_value_unlocked(arch_spinlock_t lock) |
155 | { | |
156 | return lock.owner == lock.next; | |
157 | } | |
158 | ||
52ea2a56 WD |
159 | static inline int arch_spin_is_locked(arch_spinlock_t *lock) |
160 | { | |
38b850a7 | 161 | smp_mb(); /* See arch_spin_unlock_wait */ |
af2e7aae | 162 | return !arch_spin_value_unlocked(READ_ONCE(*lock)); |
52ea2a56 WD |
163 | } |
164 | ||
165 | static inline int arch_spin_is_contended(arch_spinlock_t *lock) | |
166 | { | |
af2e7aae | 167 | arch_spinlock_t lockval = READ_ONCE(*lock); |
52ea2a56 | 168 | return (lockval.next - lockval.owner) > 1; |
08e875c1 | 169 | } |
52ea2a56 | 170 | #define arch_spin_is_contended arch_spin_is_contended |
08e875c1 CM |
171 | |
172 | /* | |
173 | * Write lock implementation. | |
174 | * | |
175 | * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is | |
176 | * exclusively held. | |
177 | * | |
178 | * The memory barriers are implicit with the load-acquire and store-release | |
179 | * instructions. | |
180 | */ | |
181 | ||
182 | static inline void arch_write_lock(arch_rwlock_t *rw) | |
183 | { | |
184 | unsigned int tmp; | |
185 | ||
81bb5c64 WD |
186 | asm volatile(ARM64_LSE_ATOMIC_INSN( |
187 | /* LL/SC */ | |
08e875c1 CM |
188 | " sevl\n" |
189 | "1: wfe\n" | |
3a0310eb | 190 | "2: ldaxr %w0, %1\n" |
08e875c1 | 191 | " cbnz %w0, 1b\n" |
3a0310eb | 192 | " stxr %w0, %w2, %1\n" |
08e875c1 | 193 | " cbnz %w0, 2b\n" |
81bb5c64 WD |
194 | " nop", |
195 | /* LSE atomics */ | |
196 | "1: mov %w0, wzr\n" | |
197 | "2: casa %w0, %w2, %1\n" | |
198 | " cbz %w0, 3f\n" | |
199 | " ldxr %w0, %1\n" | |
200 | " cbz %w0, 2b\n" | |
201 | " wfe\n" | |
202 | " b 1b\n" | |
203 | "3:") | |
3a0310eb WD |
204 | : "=&r" (tmp), "+Q" (rw->lock) |
205 | : "r" (0x80000000) | |
95c41896 | 206 | : "memory"); |
08e875c1 CM |
207 | } |
208 | ||
209 | static inline int arch_write_trylock(arch_rwlock_t *rw) | |
210 | { | |
211 | unsigned int tmp; | |
212 | ||
81bb5c64 WD |
213 | asm volatile(ARM64_LSE_ATOMIC_INSN( |
214 | /* LL/SC */ | |
9511ca19 WD |
215 | "1: ldaxr %w0, %1\n" |
216 | " cbnz %w0, 2f\n" | |
3a0310eb | 217 | " stxr %w0, %w2, %1\n" |
9511ca19 | 218 | " cbnz %w0, 1b\n" |
81bb5c64 WD |
219 | "2:", |
220 | /* LSE atomics */ | |
221 | " mov %w0, wzr\n" | |
222 | " casa %w0, %w2, %1\n" | |
223 | " nop\n" | |
224 | " nop") | |
3a0310eb WD |
225 | : "=&r" (tmp), "+Q" (rw->lock) |
226 | : "r" (0x80000000) | |
95c41896 | 227 | : "memory"); |
08e875c1 CM |
228 | |
229 | return !tmp; | |
230 | } | |
231 | ||
232 | static inline void arch_write_unlock(arch_rwlock_t *rw) | |
233 | { | |
81bb5c64 WD |
234 | asm volatile(ARM64_LSE_ATOMIC_INSN( |
235 | " stlr wzr, %0", | |
236 | " swpl wzr, wzr, %0") | |
237 | : "=Q" (rw->lock) :: "memory"); | |
08e875c1 CM |
238 | } |
239 | ||
240 | /* write_can_lock - would write_trylock() succeed? */ | |
241 | #define arch_write_can_lock(x) ((x)->lock == 0) | |
242 | ||
243 | /* | |
244 | * Read lock implementation. | |
245 | * | |
246 | * It exclusively loads the lock value, increments it and stores the new value | |
247 | * back if positive and the CPU still exclusively owns the location. If the | |
248 | * value is negative, the lock is already held. | |
249 | * | |
250 | * During unlocking there may be multiple active read locks but no write lock. | |
251 | * | |
252 | * The memory barriers are implicit with the load-acquire and store-release | |
253 | * instructions. | |
81bb5c64 WD |
254 | * |
255 | * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC | |
256 | * and LSE implementations may exhibit different behaviour (although this | |
257 | * will have no effect on lockdep). | |
08e875c1 CM |
258 | */ |
259 | static inline void arch_read_lock(arch_rwlock_t *rw) | |
260 | { | |
261 | unsigned int tmp, tmp2; | |
262 | ||
263 | asm volatile( | |
264 | " sevl\n" | |
81bb5c64 WD |
265 | ARM64_LSE_ATOMIC_INSN( |
266 | /* LL/SC */ | |
08e875c1 | 267 | "1: wfe\n" |
3a0310eb | 268 | "2: ldaxr %w0, %2\n" |
08e875c1 CM |
269 | " add %w0, %w0, #1\n" |
270 | " tbnz %w0, #31, 1b\n" | |
3a0310eb | 271 | " stxr %w1, %w0, %2\n" |
81bb5c64 WD |
272 | " nop\n" |
273 | " cbnz %w1, 2b", | |
274 | /* LSE atomics */ | |
275 | "1: wfe\n" | |
276 | "2: ldxr %w0, %2\n" | |
277 | " adds %w1, %w0, #1\n" | |
278 | " tbnz %w1, #31, 1b\n" | |
279 | " casa %w0, %w1, %2\n" | |
280 | " sbc %w0, %w1, %w0\n" | |
281 | " cbnz %w0, 2b") | |
3a0310eb WD |
282 | : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) |
283 | : | |
81bb5c64 | 284 | : "cc", "memory"); |
08e875c1 CM |
285 | } |
286 | ||
287 | static inline void arch_read_unlock(arch_rwlock_t *rw) | |
288 | { | |
289 | unsigned int tmp, tmp2; | |
290 | ||
81bb5c64 WD |
291 | asm volatile(ARM64_LSE_ATOMIC_INSN( |
292 | /* LL/SC */ | |
3a0310eb | 293 | "1: ldxr %w0, %2\n" |
08e875c1 | 294 | " sub %w0, %w0, #1\n" |
3a0310eb | 295 | " stlxr %w1, %w0, %2\n" |
81bb5c64 WD |
296 | " cbnz %w1, 1b", |
297 | /* LSE atomics */ | |
298 | " movn %w0, #0\n" | |
299 | " nop\n" | |
300 | " nop\n" | |
301 | " staddl %w0, %2") | |
3a0310eb WD |
302 | : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) |
303 | : | |
95c41896 | 304 | : "memory"); |
08e875c1 CM |
305 | } |
306 | ||
307 | static inline int arch_read_trylock(arch_rwlock_t *rw) | |
308 | { | |
81bb5c64 | 309 | unsigned int tmp, tmp2; |
08e875c1 | 310 | |
81bb5c64 WD |
311 | asm volatile(ARM64_LSE_ATOMIC_INSN( |
312 | /* LL/SC */ | |
313 | " mov %w1, #1\n" | |
9511ca19 | 314 | "1: ldaxr %w0, %2\n" |
08e875c1 | 315 | " add %w0, %w0, #1\n" |
9511ca19 | 316 | " tbnz %w0, #31, 2f\n" |
3a0310eb | 317 | " stxr %w1, %w0, %2\n" |
9511ca19 | 318 | " cbnz %w1, 1b\n" |
81bb5c64 WD |
319 | "2:", |
320 | /* LSE atomics */ | |
321 | " ldr %w0, %2\n" | |
322 | " adds %w1, %w0, #1\n" | |
323 | " tbnz %w1, #31, 1f\n" | |
324 | " casa %w0, %w1, %2\n" | |
325 | " sbc %w1, %w1, %w0\n" | |
326 | " nop\n" | |
327 | "1:") | |
328 | : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) | |
3a0310eb | 329 | : |
81bb5c64 | 330 | : "cc", "memory"); |
08e875c1 CM |
331 | |
332 | return !tmp2; | |
333 | } | |
334 | ||
335 | /* read_can_lock - would read_trylock() succeed? */ | |
336 | #define arch_read_can_lock(x) ((x)->lock < 0x80000000) | |
337 | ||
338 | #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) | |
339 | #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) | |
340 | ||
341 | #define arch_spin_relax(lock) cpu_relax() | |
342 | #define arch_read_relax(lock) cpu_relax() | |
343 | #define arch_write_relax(lock) cpu_relax() | |
344 | ||
345 | #endif /* __ASM_SPINLOCK_H */ |