projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
arm64: atomics: implement native {relaxed, acquire, release} atomics
[deliverable/linux.git] / arch / arm64 / include / asm / atomic_lse.h
1 /*
2 * Based on arch/arm/include/asm/atomic.h
3 *
4 * Copyright (C) 1996 Russell King.
5 * Copyright (C) 2002 Deep Blue Solutions Ltd.
6 * Copyright (C) 2012 ARM Ltd.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #ifndef __ASM_ATOMIC_LSE_H
22 #define __ASM_ATOMIC_LSE_H
23
24 #ifndef __ARM64_IN_ATOMIC_IMPL
25 #error "please don't include this file directly"
26 #endif
27
28 #define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op)
29
30 static inline void atomic_andnot(int i, atomic_t *v)
31 {
32 register int w0 asm ("w0") = i;
33 register atomic_t *x1 asm ("x1") = v;
34
35 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot),
36 " stclr %w[i], %[v]\n")
37 : [i] "+r" (w0), [v] "+Q" (v->counter)
38 : "r" (x1)
39 : "x30");
40 }
41
42 static inline void atomic_or(int i, atomic_t *v)
43 {
44 register int w0 asm ("w0") = i;
45 register atomic_t *x1 asm ("x1") = v;
46
47 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or),
48 " stset %w[i], %[v]\n")
49 : [i] "+r" (w0), [v] "+Q" (v->counter)
50 : "r" (x1)
51 : "x30");
52 }
53
54 static inline void atomic_xor(int i, atomic_t *v)
55 {
56 register int w0 asm ("w0") = i;
57 register atomic_t *x1 asm ("x1") = v;
58
59 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor),
60 " steor %w[i], %[v]\n")
61 : [i] "+r" (w0), [v] "+Q" (v->counter)
62 : "r" (x1)
63 : "x30");
64 }
65
66 static inline void atomic_add(int i, atomic_t *v)
67 {
68 register int w0 asm ("w0") = i;
69 register atomic_t *x1 asm ("x1") = v;
70
71 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add),
72 " stadd %w[i], %[v]\n")
73 : [i] "+r" (w0), [v] "+Q" (v->counter)
74 : "r" (x1)
75 : "x30");
76 }
77
78 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
79 static inline int atomic_add_return##name(int i, atomic_t *v) \
80 { \
81 register int w0 asm ("w0") = i; \
82 register atomic_t *x1 asm ("x1") = v; \
83 \
84 asm volatile(ARM64_LSE_ATOMIC_INSN( \
85 /* LL/SC */ \
86 " nop\n" \
87 __LL_SC_ATOMIC(add_return##name), \
88 /* LSE atomics */ \
89 " ldadd" #mb " %w[i], w30, %[v]\n" \
90 " add %w[i], %w[i], w30") \
91 : [i] "+r" (w0), [v] "+Q" (v->counter) \
92 : "r" (x1) \
93 : "x30" , ##cl); \
94 \
95 return w0; \
96 }
97
98 ATOMIC_OP_ADD_RETURN(_relaxed, )
99 ATOMIC_OP_ADD_RETURN(_acquire, a, "memory")
100 ATOMIC_OP_ADD_RETURN(_release, l, "memory")
101 ATOMIC_OP_ADD_RETURN( , al, "memory")
102
103 #undef ATOMIC_OP_ADD_RETURN
104
105 static inline void atomic_and(int i, atomic_t *v)
106 {
107 register int w0 asm ("w0") = i;
108 register atomic_t *x1 asm ("x1") = v;
109
110 asm volatile(ARM64_LSE_ATOMIC_INSN(
111 /* LL/SC */
112 " nop\n"
113 __LL_SC_ATOMIC(and),
114 /* LSE atomics */
115 " mvn %w[i], %w[i]\n"
116 " stclr %w[i], %[v]")
117 : [i] "+r" (w0), [v] "+Q" (v->counter)
118 : "r" (x1)
119 : "x30");
120 }
121
122 static inline void atomic_sub(int i, atomic_t *v)
123 {
124 register int w0 asm ("w0") = i;
125 register atomic_t *x1 asm ("x1") = v;
126
127 asm volatile(ARM64_LSE_ATOMIC_INSN(
128 /* LL/SC */
129 " nop\n"
130 __LL_SC_ATOMIC(sub),
131 /* LSE atomics */
132 " neg %w[i], %w[i]\n"
133 " stadd %w[i], %[v]")
134 : [i] "+r" (w0), [v] "+Q" (v->counter)
135 : "r" (x1)
136 : "x30");
137 }
138
139 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
140 static inline int atomic_sub_return##name(int i, atomic_t *v) \
141 { \
142 register int w0 asm ("w0") = i; \
143 register atomic_t *x1 asm ("x1") = v; \
144 \
145 asm volatile(ARM64_LSE_ATOMIC_INSN( \
146 /* LL/SC */ \
147 " nop\n" \
148 __LL_SC_ATOMIC(sub_return##name) \
149 " nop", \
150 /* LSE atomics */ \
151 " neg %w[i], %w[i]\n" \
152 " ldadd" #mb " %w[i], w30, %[v]\n" \
153 " add %w[i], %w[i], w30") \
154 : [i] "+r" (w0), [v] "+Q" (v->counter) \
155 : "r" (x1) \
156 : "x30" , ##cl); \
157 \
158 return w0; \
159 }
160
161 ATOMIC_OP_SUB_RETURN(_relaxed, )
162 ATOMIC_OP_SUB_RETURN(_acquire, a, "memory")
163 ATOMIC_OP_SUB_RETURN(_release, l, "memory")
164 ATOMIC_OP_SUB_RETURN( , al, "memory")
165
166 #undef ATOMIC_OP_SUB_RETURN
167 #undef __LL_SC_ATOMIC
168
169 #define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
170
171 static inline void atomic64_andnot(long i, atomic64_t *v)
172 {
173 register long x0 asm ("x0") = i;
174 register atomic64_t *x1 asm ("x1") = v;
175
176 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot),
177 " stclr %[i], %[v]\n")
178 : [i] "+r" (x0), [v] "+Q" (v->counter)
179 : "r" (x1)
180 : "x30");
181 }
182
183 static inline void atomic64_or(long i, atomic64_t *v)
184 {
185 register long x0 asm ("x0") = i;
186 register atomic64_t *x1 asm ("x1") = v;
187
188 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or),
189 " stset %[i], %[v]\n")
190 : [i] "+r" (x0), [v] "+Q" (v->counter)
191 : "r" (x1)
192 : "x30");
193 }
194
195 static inline void atomic64_xor(long i, atomic64_t *v)
196 {
197 register long x0 asm ("x0") = i;
198 register atomic64_t *x1 asm ("x1") = v;
199
200 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor),
201 " steor %[i], %[v]\n")
202 : [i] "+r" (x0), [v] "+Q" (v->counter)
203 : "r" (x1)
204 : "x30");
205 }
206
207 static inline void atomic64_add(long i, atomic64_t *v)
208 {
209 register long x0 asm ("x0") = i;
210 register atomic64_t *x1 asm ("x1") = v;
211
212 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add),
213 " stadd %[i], %[v]\n")
214 : [i] "+r" (x0), [v] "+Q" (v->counter)
215 : "r" (x1)
216 : "x30");
217 }
218
219 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
220 static inline long atomic64_add_return##name(long i, atomic64_t *v) \
221 { \
222 register long x0 asm ("x0") = i; \
223 register atomic64_t *x1 asm ("x1") = v; \
224 \
225 asm volatile(ARM64_LSE_ATOMIC_INSN( \
226 /* LL/SC */ \
227 " nop\n" \
228 __LL_SC_ATOMIC64(add_return##name), \
229 /* LSE atomics */ \
230 " ldadd" #mb " %[i], x30, %[v]\n" \
231 " add %[i], %[i], x30") \
232 : [i] "+r" (x0), [v] "+Q" (v->counter) \
233 : "r" (x1) \
234 : "x30" , ##cl); \
235 \
236 return x0; \
237 }
238
239 ATOMIC64_OP_ADD_RETURN(_relaxed, )
240 ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory")
241 ATOMIC64_OP_ADD_RETURN(_release, l, "memory")
242 ATOMIC64_OP_ADD_RETURN( , al, "memory")
243
244 #undef ATOMIC64_OP_ADD_RETURN
245
246 static inline void atomic64_and(long i, atomic64_t *v)
247 {
248 register long x0 asm ("x0") = i;
249 register atomic64_t *x1 asm ("x1") = v;
250
251 asm volatile(ARM64_LSE_ATOMIC_INSN(
252 /* LL/SC */
253 " nop\n"
254 __LL_SC_ATOMIC64(and),
255 /* LSE atomics */
256 " mvn %[i], %[i]\n"
257 " stclr %[i], %[v]")
258 : [i] "+r" (x0), [v] "+Q" (v->counter)
259 : "r" (x1)
260 : "x30");
261 }
262
263 static inline void atomic64_sub(long i, atomic64_t *v)
264 {
265 register long x0 asm ("x0") = i;
266 register atomic64_t *x1 asm ("x1") = v;
267
268 asm volatile(ARM64_LSE_ATOMIC_INSN(
269 /* LL/SC */
270 " nop\n"
271 __LL_SC_ATOMIC64(sub),
272 /* LSE atomics */
273 " neg %[i], %[i]\n"
274 " stadd %[i], %[v]")
275 : [i] "+r" (x0), [v] "+Q" (v->counter)
276 : "r" (x1)
277 : "x30");
278 }
279
280 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
281 static inline long atomic64_sub_return##name(long i, atomic64_t *v) \
282 { \
283 register long x0 asm ("x0") = i; \
284 register atomic64_t *x1 asm ("x1") = v; \
285 \
286 asm volatile(ARM64_LSE_ATOMIC_INSN( \
287 /* LL/SC */ \
288 " nop\n" \
289 __LL_SC_ATOMIC64(sub_return##name) \
290 " nop", \
291 /* LSE atomics */ \
292 " neg %[i], %[i]\n" \
293 " ldadd" #mb " %[i], x30, %[v]\n" \
294 " add %[i], %[i], x30") \
295 : [i] "+r" (x0), [v] "+Q" (v->counter) \
296 : "r" (x1) \
297 : "x30" , ##cl); \
298 \
299 return x0; \
300 }
301
302 ATOMIC64_OP_SUB_RETURN(_relaxed, )
303 ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory")
304 ATOMIC64_OP_SUB_RETURN(_release, l, "memory")
305 ATOMIC64_OP_SUB_RETURN( , al, "memory")
306
307 #undef ATOMIC64_OP_SUB_RETURN
308
309 static inline long atomic64_dec_if_positive(atomic64_t *v)
310 {
311 register long x0 asm ("x0") = (long)v;
312
313 asm volatile(ARM64_LSE_ATOMIC_INSN(
314 /* LL/SC */
315 " nop\n"
316 __LL_SC_ATOMIC64(dec_if_positive)
317 " nop\n"
318 " nop\n"
319 " nop\n"
320 " nop\n"
321 " nop",
322 /* LSE atomics */
323 "1: ldr x30, %[v]\n"
324 " subs %[ret], x30, #1\n"
325 " b.lt 2f\n"
326 " casal x30, %[ret], %[v]\n"
327 " sub x30, x30, #1\n"
328 " sub x30, x30, %[ret]\n"
329 " cbnz x30, 1b\n"
330 "2:")
331 : [ret] "+&r" (x0), [v] "+Q" (v->counter)
332 :
333 : "x30", "cc", "memory");
334
335 return x0;
336 }
337
338 #undef __LL_SC_ATOMIC64
339
340 #define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
341
342 #define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
343 static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
344 unsigned long old, \
345 unsigned long new) \
346 { \
347 register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
348 register unsigned long x1 asm ("x1") = old; \
349 register unsigned long x2 asm ("x2") = new; \
350 \
351 asm volatile(ARM64_LSE_ATOMIC_INSN( \
352 /* LL/SC */ \
353 " nop\n" \
354 __LL_SC_CMPXCHG(name) \
355 " nop", \
356 /* LSE atomics */ \
357 " mov " #w "30, %" #w "[old]\n" \
358 " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
359 " mov %" #w "[ret], " #w "30") \
360 : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
361 : [old] "r" (x1), [new] "r" (x2) \
362 : "x30" , ##cl); \
363 \
364 return x0; \
365 }
366
367 __CMPXCHG_CASE(w, b, 1, )
368 __CMPXCHG_CASE(w, h, 2, )
369 __CMPXCHG_CASE(w, , 4, )
370 __CMPXCHG_CASE(x, , 8, )
371 __CMPXCHG_CASE(w, b, acq_1, a, "memory")
372 __CMPXCHG_CASE(w, h, acq_2, a, "memory")
373 __CMPXCHG_CASE(w, , acq_4, a, "memory")
374 __CMPXCHG_CASE(x, , acq_8, a, "memory")
375 __CMPXCHG_CASE(w, b, rel_1, l, "memory")
376 __CMPXCHG_CASE(w, h, rel_2, l, "memory")
377 __CMPXCHG_CASE(w, , rel_4, l, "memory")
378 __CMPXCHG_CASE(x, , rel_8, l, "memory")
379 __CMPXCHG_CASE(w, b, mb_1, al, "memory")
380 __CMPXCHG_CASE(w, h, mb_2, al, "memory")
381 __CMPXCHG_CASE(w, , mb_4, al, "memory")
382 __CMPXCHG_CASE(x, , mb_8, al, "memory")
383
384 #undef __LL_SC_CMPXCHG
385 #undef __CMPXCHG_CASE
386
387 #define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
388
389 #define __CMPXCHG_DBL(name, mb, cl...) \
390 static inline int __cmpxchg_double##name(unsigned long old1, \
391 unsigned long old2, \
392 unsigned long new1, \
393 unsigned long new2, \
394 volatile void *ptr) \
395 { \
396 unsigned long oldval1 = old1; \
397 unsigned long oldval2 = old2; \
398 register unsigned long x0 asm ("x0") = old1; \
399 register unsigned long x1 asm ("x1") = old2; \
400 register unsigned long x2 asm ("x2") = new1; \
401 register unsigned long x3 asm ("x3") = new2; \
402 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
403 \
404 asm volatile(ARM64_LSE_ATOMIC_INSN( \
405 /* LL/SC */ \
406 " nop\n" \
407 " nop\n" \
408 " nop\n" \
409 __LL_SC_CMPXCHG_DBL(name), \
410 /* LSE atomics */ \
411 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
412 " eor %[old1], %[old1], %[oldval1]\n" \
413 " eor %[old2], %[old2], %[oldval2]\n" \
414 " orr %[old1], %[old1], %[old2]") \
415 : [old1] "+r" (x0), [old2] "+r" (x1), \
416 [v] "+Q" (*(unsigned long *)ptr) \
417 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
418 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
419 : "x30" , ##cl); \
420 \
421 return x0; \
422 }
423
424 __CMPXCHG_DBL( , )
425 __CMPXCHG_DBL(_mb, al, "memory")
426
427 #undef __LL_SC_CMPXCHG_DBL
428 #undef __CMPXCHG_DBL
429
430 #endif /* __ASM_ATOMIC_LSE_H */
Linux kernel - Deliverables
This page took 0.059014 seconds and 6 git commands to generate.