[deliverable/linux.git] / arch / sparc64 / lib / atomic.S

/* atomic.S: These things are too big to do inline.
 *
 * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net)
 */

#include <asm/asi.h>
#include <asm/backoff.h>

	.text

	/* Two versions of the atomic routines, one that
	 * does not return a value and does not perform
	 * memory barriers, and a second which returns
	 * a value and does the barriers.
	 */
	.globl	atomic_add
	.type	atomic_add,#function
atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
	BACKOFF_SETUP(%o2)
1:	lduw	[%o1], %g1
	add	%g1, %o0, %g7
	cas	[%o1], %g1, %g7
	cmp	%g1, %g7
	bne,pn	%icc, 2f
	 nop
	retl
	 nop
2:	BACKOFF_SPIN(%o2, %o3, 1b)
	.size	atomic_add, .-atomic_add

	.globl	atomic_sub
	.type	atomic_sub,#function
atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
	BACKOFF_SETUP(%o2)
1:	lduw	[%o1], %g1
	sub	%g1, %o0, %g7
	cas	[%o1], %g1, %g7
	cmp	%g1, %g7
	bne,pn	%icc, 2f
	 nop
	retl
	 nop
2:	BACKOFF_SPIN(%o2, %o3, 1b)
	.size	atomic_sub, .-atomic_sub

	/* On SMP we need to use memory barriers to ensure
	 * correct memory operation ordering, nop these out
	 * for uniprocessor.
	 */
#ifdef CONFIG_SMP

#define ATOMIC_PRE_BARRIER	membar #StoreLoad | #LoadLoad;
#define ATOMIC_POST_BARRIER	\
	ba,pt %xcc, 80b;	\
	membar #StoreLoad | #StoreStore

80:	retl
	 nop
#else
#define ATOMIC_PRE_BARRIER
#define ATOMIC_POST_BARRIER
#endif

	.globl	atomic_add_ret
	.type	atomic_add_ret,#function
atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
	BACKOFF_SETUP(%o2)
	ATOMIC_PRE_BARRIER
1:	lduw	[%o1], %g1
	add	%g1, %o0, %g7
	cas	[%o1], %g1, %g7
	cmp	%g1, %g7
	bne,pn	%icc, 2f
	 add	%g7, %o0, %g7
	sra	%g7, 0, %o0
	ATOMIC_POST_BARRIER
	retl
	 nop
2:	BACKOFF_SPIN(%o2, %o3, 1b)
	.size	atomic_add_ret, .-atomic_add_ret

	.globl	atomic_sub_ret
	.type	atomic_sub_ret,#function
atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
	BACKOFF_SETUP(%o2)
	ATOMIC_PRE_BARRIER
1:	lduw	[%o1], %g1
	sub	%g1, %o0, %g7
	cas	[%o1], %g1, %g7
	cmp	%g1, %g7
	bne,pn	%icc, 2f
	 sub	%g7, %o0, %g7
	sra	%g7, 0, %o0
	ATOMIC_POST_BARRIER
	retl
	 nop
2:	BACKOFF_SPIN(%o2, %o3, 1b)
	.size	atomic_sub_ret, .-atomic_sub_ret

	.globl	atomic64_add
	.type	atomic64_add,#function
atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
	BACKOFF_SETUP(%o2)
1:	ldx	[%o1], %g1
	add	%g1, %o0, %g7
	casx	[%o1], %g1, %g7
	cmp	%g1, %g7
	bne,pn	%xcc, 2f
	 nop
	retl
	 nop
2:	BACKOFF_SPIN(%o2, %o3, 1b)
	.size	atomic64_add, .-atomic64_add

	.globl	atomic64_sub
	.type	atomic64_sub,#function
atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
	BACKOFF_SETUP(%o2)
1:	ldx	[%o1], %g1
	sub	%g1, %o0, %g7
	casx	[%o1], %g1, %g7
	cmp	%g1, %g7
	bne,pn	%xcc, 2f
	 nop
	retl
	 nop
2:	BACKOFF_SPIN(%o2, %o3, 1b)
	.size	atomic64_sub, .-atomic64_sub

	.globl	atomic64_add_ret
	.type	atomic64_add_ret,#function
atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
	BACKOFF_SETUP(%o2)
	ATOMIC_PRE_BARRIER
1:	ldx	[%o1], %g1
	add	%g1, %o0, %g7
	casx	[%o1], %g1, %g7
	cmp	%g1, %g7
	bne,pn	%xcc, 2f
	 add	%g7, %o0, %g7
	mov	%g7, %o0
	ATOMIC_POST_BARRIER
	retl
	 nop
2:	BACKOFF_SPIN(%o2, %o3, 1b)
	.size	atomic64_add_ret, .-atomic64_add_ret

	.globl	atomic64_sub_ret
	.type	atomic64_sub_ret,#function
atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
	BACKOFF_SETUP(%o2)
	ATOMIC_PRE_BARRIER
1:	ldx	[%o1], %g1
	sub	%g1, %o0, %g7
	casx	[%o1], %g1, %g7
	cmp	%g1, %g7
	bne,pn	%xcc, 2f
	 sub	%g7, %o0, %g7
	mov	%g7, %o0
	ATOMIC_POST_BARRIER
	retl
	 nop
2:	BACKOFF_SPIN(%o2, %o3, 1b)
	.size	atomic64_sub_ret, .-atomic64_sub_ret
Commit	Line	Data
24f287e4	1	/* atomic.S: These things are too big to do inline.
1da177e4	2	*
24f287e4	3	* Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net)
1da177e4 LT	4	*/
1da177e4 LT	5
1da177e4	6	#include <asm/asi.h>
24f287e4	7	#include <asm/backoff.h>
1da177e4	8
1da177e4 LT	9	.text
	10
	11	/* Two versions of the atomic routines, one that
	12	* does not return a value and does not perform
	13	* memory barriers, and a second which returns
	14	* a value and does the barriers.
	15	*/
	16	.globl atomic_add
	17	.type atomic_add,#function
	18	atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
24f287e4	19	BACKOFF_SETUP(%o2)
1da177e4 LT	20	1: lduw [%o1], %g1
	21	add %g1, %o0, %g7
	22	cas [%o1], %g1, %g7
	23	cmp %g1, %g7
24f287e4	24	bne,pn %icc, 2f
1da177e4 LT	25	nop
	26	retl
	27	nop
24f287e4	28	2: BACKOFF_SPIN(%o2, %o3, 1b)
1da177e4 LT	29	.size atomic_add, .-atomic_add
	30
	31	.globl atomic_sub
	32	.type atomic_sub,#function
	33	atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
24f287e4	34	BACKOFF_SETUP(%o2)
1da177e4 LT	35	1: lduw [%o1], %g1
	36	sub %g1, %o0, %g7
	37	cas [%o1], %g1, %g7
	38	cmp %g1, %g7
24f287e4	39	bne,pn %icc, 2f
1da177e4 LT	40	nop
	41	retl
	42	nop
24f287e4	43	2: BACKOFF_SPIN(%o2, %o3, 1b)
1da177e4 LT	44	.size atomic_sub, .-atomic_sub
1da177e4 LT	45
b445e26c DM	46	/* On SMP we need to use memory barriers to ensure
	47	* correct memory operation ordering, nop these out
	48	* for uniprocessor.
	49	*/
	50	#ifdef CONFIG_SMP
	51
	52	#define ATOMIC_PRE_BARRIER membar #StoreLoad \| #LoadLoad;
	53	#define ATOMIC_POST_BARRIER \
	54	ba,pt %xcc, 80b; \
	55	membar #StoreLoad \| #StoreStore
	56
	57	80: retl
	58	nop
	59	#else
	60	#define ATOMIC_PRE_BARRIER
	61	#define ATOMIC_POST_BARRIER
	62	#endif
	63
1da177e4 LT	64	.globl atomic_add_ret
	65	.type atomic_add_ret,#function
	66	atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
24f287e4	67	BACKOFF_SETUP(%o2)
1da177e4 LT	68	ATOMIC_PRE_BARRIER
	69	1: lduw [%o1], %g1
	70	add %g1, %o0, %g7
	71	cas [%o1], %g1, %g7
	72	cmp %g1, %g7
24f287e4	73	bne,pn %icc, 2f
1da177e4	74	add %g7, %o0, %g7
b445e26c	75	sra %g7, 0, %o0
1da177e4 LT	76	ATOMIC_POST_BARRIER
1da177e4 LT	77	retl
b445e26c	78	nop
24f287e4	79	2: BACKOFF_SPIN(%o2, %o3, 1b)
1da177e4 LT	80	.size atomic_add_ret, .-atomic_add_ret
	81
	82	.globl atomic_sub_ret
	83	.type atomic_sub_ret,#function
	84	atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
24f287e4	85	BACKOFF_SETUP(%o2)
1da177e4 LT	86	ATOMIC_PRE_BARRIER
	87	1: lduw [%o1], %g1
	88	sub %g1, %o0, %g7
	89	cas [%o1], %g1, %g7
	90	cmp %g1, %g7
24f287e4	91	bne,pn %icc, 2f
1da177e4	92	sub %g7, %o0, %g7
b445e26c	93	sra %g7, 0, %o0
1da177e4 LT	94	ATOMIC_POST_BARRIER
1da177e4 LT	95	retl
b445e26c	96	nop
24f287e4	97	2: BACKOFF_SPIN(%o2, %o3, 1b)
1da177e4 LT	98	.size atomic_sub_ret, .-atomic_sub_ret
	99
	100	.globl atomic64_add
	101	.type atomic64_add,#function
	102	atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
24f287e4	103	BACKOFF_SETUP(%o2)
1da177e4 LT	104	1: ldx [%o1], %g1
	105	add %g1, %o0, %g7
	106	casx [%o1], %g1, %g7
	107	cmp %g1, %g7
24f287e4	108	bne,pn %xcc, 2f
1da177e4 LT	109	nop
	110	retl
	111	nop
24f287e4	112	2: BACKOFF_SPIN(%o2, %o3, 1b)
1da177e4 LT	113	.size atomic64_add, .-atomic64_add
	114
	115	.globl atomic64_sub
	116	.type atomic64_sub,#function
	117	atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
24f287e4	118	BACKOFF_SETUP(%o2)
1da177e4 LT	119	1: ldx [%o1], %g1
	120	sub %g1, %o0, %g7
	121	casx [%o1], %g1, %g7
	122	cmp %g1, %g7
24f287e4	123	bne,pn %xcc, 2f
1da177e4 LT	124	nop
	125	retl
	126	nop
24f287e4	127	2: BACKOFF_SPIN(%o2, %o3, 1b)
1da177e4 LT	128	.size atomic64_sub, .-atomic64_sub
	129
	130	.globl atomic64_add_ret
	131	.type atomic64_add_ret,#function
	132	atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
24f287e4	133	BACKOFF_SETUP(%o2)
1da177e4 LT	134	ATOMIC_PRE_BARRIER
	135	1: ldx [%o1], %g1
	136	add %g1, %o0, %g7
	137	casx [%o1], %g1, %g7
	138	cmp %g1, %g7
24f287e4	139	bne,pn %xcc, 2f
1da177e4	140	add %g7, %o0, %g7
b445e26c	141	mov %g7, %o0
1da177e4 LT	142	ATOMIC_POST_BARRIER
1da177e4 LT	143	retl
b445e26c	144	nop
24f287e4	145	2: BACKOFF_SPIN(%o2, %o3, 1b)
1da177e4 LT	146	.size atomic64_add_ret, .-atomic64_add_ret
	147
	148	.globl atomic64_sub_ret
	149	.type atomic64_sub_ret,#function
	150	atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
24f287e4	151	BACKOFF_SETUP(%o2)
1da177e4 LT	152	ATOMIC_PRE_BARRIER
	153	1: ldx [%o1], %g1
	154	sub %g1, %o0, %g7
	155	casx [%o1], %g1, %g7
	156	cmp %g1, %g7
24f287e4	157	bne,pn %xcc, 2f
1da177e4	158	sub %g7, %o0, %g7
b445e26c	159	mov %g7, %o0
1da177e4 LT	160	ATOMIC_POST_BARRIER
1da177e4 LT	161	retl
b445e26c	162	nop
24f287e4	163	2: BACKOFF_SPIN(%o2, %o3, 1b)
1da177e4	164	.size atomic64_sub_ret, .-atomic64_sub_ret