[deliverable/linux.git] / arch / mips / include / asm / spinlock.h

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 */
#ifndef _ASM_SPINLOCK_H
#define _ASM_SPINLOCK_H

#include <linux/compiler.h>

#include <asm/barrier.h>
#include <asm/processor.h>
#include <asm/compiler.h>
#include <asm/war.h>

/*
 * Your basic SMP spinlocks, allowing only a single CPU anywhere
 *
 * Simple spin lock operations.	 There are two variants, one clears IRQ's
 * on the local processor, one does not.
 *
 * These are fair FIFO ticket locks
 *
 * (the type definitions are in asm/spinlock_types.h)
 */


/*
 * Ticket locks are conceptually two parts, one indicating the current head of
 * the queue, and the other indicating the current tail. The lock is acquired
 * by atomically noting the tail and incrementing it by one (thus adding
 * ourself to the queue and noting our position), then waiting until the head
 * becomes equal to the the initial value of the tail.
 */

static inline int arch_spin_is_locked(arch_spinlock_t *lock)
{
	u32 counters = ACCESS_ONCE(lock->lock);

	return ((counters >> 16) ^ counters) & 0xffff;
}

static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
{
	return lock.h.serving_now == lock.h.ticket;
}

#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)

static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
{
	u16 owner = READ_ONCE(lock->h.serving_now);
	smp_rmb();
	for (;;) {
		arch_spinlock_t tmp = READ_ONCE(*lock);

		if (tmp.h.serving_now == tmp.h.ticket ||
		    tmp.h.serving_now != owner)
			break;

		cpu_relax();
	}
	smp_acquire__after_ctrl_dep();
}

static inline int arch_spin_is_contended(arch_spinlock_t *lock)
{
	u32 counters = ACCESS_ONCE(lock->lock);

	return (((counters >> 16) - counters) & 0xffff) > 1;
}
#define arch_spin_is_contended	arch_spin_is_contended

static inline void arch_spin_lock(arch_spinlock_t *lock)
{
	int my_ticket;
	int tmp;
	int inc = 0x10000;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_lock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
		"	sc	%[my_ticket], %[ticket_ptr]		\n"
		"	beqzl	%[my_ticket], 1b			\n"
		"	 nop						\n"
		"	srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[ticket], %[ticket], 0xffff		\n"
		"	bne	%[ticket], %[my_ticket], 4f		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"2:							\n"
		"	.subsection 2					\n"
		"4:	andi	%[ticket], %[ticket], 0xffff		\n"
		"	sll	%[ticket], 5				\n"
		"							\n"
		"6:	bnez	%[ticket], 6b				\n"
		"	 subu	%[ticket], 1				\n"
		"							\n"
		"	lhu	%[ticket], %[serving_now_ptr]		\n"
		"	beq	%[ticket], %[my_ticket], 2b		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"	b	4b					\n"
		"	 subu	%[ticket], %[ticket], 1			\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
		  [serving_now_ptr] "+m" (lock->h.serving_now),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (my_ticket)
		: [inc] "r" (inc));
	} else {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_lock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
		"	sc	%[my_ticket], %[ticket_ptr]		\n"
		"	beqz	%[my_ticket], 1b			\n"
		"	 srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[ticket], %[ticket], 0xffff		\n"
		"	bne	%[ticket], %[my_ticket], 4f		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"2:							\n"
		"	.subsection 2					\n"
		"4:	andi	%[ticket], %[ticket], 0xffff		\n"
		"	sll	%[ticket], 5				\n"
		"							\n"
		"6:	bnez	%[ticket], 6b				\n"
		"	 subu	%[ticket], 1				\n"
		"							\n"
		"	lhu	%[ticket], %[serving_now_ptr]		\n"
		"	beq	%[ticket], %[my_ticket], 2b		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"	b	4b					\n"
		"	 subu	%[ticket], %[ticket], 1			\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
		  [serving_now_ptr] "+m" (lock->h.serving_now),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (my_ticket)
		: [inc] "r" (inc));
	}

	smp_llsc_mb();
}

static inline void arch_spin_unlock(arch_spinlock_t *lock)
{
	unsigned int serving_now = lock->h.serving_now + 1;
	wmb();
	lock->h.serving_now = (u16)serving_now;
	nudge_writes();
}

static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
{
	int tmp, tmp2, tmp3;
	int inc = 0x10000;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_trylock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[now_serving], %[ticket], 0xffff	\n"
		"	bne	%[my_ticket], %[now_serving], 3f	\n"
		"	 addu	%[ticket], %[ticket], %[inc]		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
		"	beqzl	%[ticket], 1b				\n"
		"	 li	%[ticket], 1				\n"
		"2:							\n"
		"	.subsection 2					\n"
		"3:	b	2b					\n"
		"	 li	%[ticket], 0				\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (tmp2),
		  [now_serving] "=&r" (tmp3)
		: [inc] "r" (inc));
	} else {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_trylock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[now_serving], %[ticket], 0xffff	\n"
		"	bne	%[my_ticket], %[now_serving], 3f	\n"
		"	 addu	%[ticket], %[ticket], %[inc]		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
		"	beqz	%[ticket], 1b				\n"
		"	 li	%[ticket], 1				\n"
		"2:							\n"
		"	.subsection 2					\n"
		"3:	b	2b					\n"
		"	 li	%[ticket], 0				\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (tmp2),
		  [now_serving] "=&r" (tmp3)
		: [inc] "r" (inc));
	}

	smp_llsc_mb();

	return tmp;
}

/*
 * Read-write spinlocks, allowing multiple readers but only one writer.
 *
 * NOTE! it is quite common to have readers in interrupts but no interrupt
 * writers. For those circumstances we can "mix" irq-safe locks - any writer
 * needs to get a irq-safe write-lock, but readers can get non-irqsafe
 * read-locks.
 */

/*
 * read_can_lock - would read_trylock() succeed?
 * @lock: the rwlock in question.
 */
#define arch_read_can_lock(rw)	((rw)->lock >= 0)

/*
 * write_can_lock - would write_trylock() succeed?
 * @lock: the rwlock in question.
 */
#define arch_write_can_lock(rw) (!(rw)->lock)

static inline void arch_read_lock(arch_rwlock_t *rw)
{
	unsigned int tmp;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_read_lock	\n"
		"1:	ll	%1, %2					\n"
		"	bltz	%1, 1b					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
		: GCC_OFF_SMALL_ASM() (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"1:	ll	%1, %2	# arch_read_lock	\n"
			"	bltz	%1, 1b				\n"
			"	 addu	%1, 1				\n"
			"2:	sc	%1, %0				\n"
			: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
			: GCC_OFF_SMALL_ASM() (rw->lock)
			: "memory");
		} while (unlikely(!tmp));
	}

	smp_llsc_mb();
}

static inline void arch_read_unlock(arch_rwlock_t *rw)
{
	unsigned int tmp;

	smp_mb__before_llsc();

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"1:	ll	%1, %2		# arch_read_unlock	\n"
		"	addiu	%1, -1					\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
		: GCC_OFF_SMALL_ASM() (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"1:	ll	%1, %2	# arch_read_unlock	\n"
			"	addiu	%1, -1				\n"
			"	sc	%1, %0				\n"
			: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
			: GCC_OFF_SMALL_ASM() (rw->lock)
			: "memory");
		} while (unlikely(!tmp));
	}
}

static inline void arch_write_lock(arch_rwlock_t *rw)
{
	unsigned int tmp;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_write_lock	\n"
		"1:	ll	%1, %2					\n"
		"	bnez	%1, 1b					\n"
		"	 lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
		: GCC_OFF_SMALL_ASM() (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"1:	ll	%1, %2	# arch_write_lock	\n"
			"	bnez	%1, 1b				\n"
			"	 lui	%1, 0x8000			\n"
			"2:	sc	%1, %0				\n"
			: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
			: GCC_OFF_SMALL_ASM() (rw->lock)
			: "memory");
		} while (unlikely(!tmp));
	}

	smp_llsc_mb();
}

static inline void arch_write_unlock(arch_rwlock_t *rw)
{
	smp_mb__before_llsc();

	__asm__ __volatile__(
	"				# arch_write_unlock	\n"
	"	sw	$0, %0					\n"
	: "=m" (rw->lock)
	: "m" (rw->lock)
	: "memory");
}

static inline int arch_read_trylock(arch_rwlock_t *rw)
{
	unsigned int tmp;
	int ret;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_read_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bltz	%1, 2f					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	.set	reorder					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		__WEAK_LLSC_MB
		"	li	%2, 1					\n"
		"2:							\n"
		: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
		: GCC_OFF_SMALL_ASM() (rw->lock)
		: "memory");
	} else {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_read_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bltz	%1, 2f					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqz	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		__WEAK_LLSC_MB
		"	li	%2, 1					\n"
		"2:							\n"
		: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
		: GCC_OFF_SMALL_ASM() (rw->lock)
		: "memory");
	}

	return ret;
}

static inline int arch_write_trylock(arch_rwlock_t *rw)
{
	unsigned int tmp;
	int ret;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_write_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bnez	%1, 2f					\n"
		"	 lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		__WEAK_LLSC_MB
		"	li	%2, 1					\n"
		"	.set	reorder					\n"
		"2:							\n"
		: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
		: GCC_OFF_SMALL_ASM() (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"	ll	%1, %3	# arch_write_trylock	\n"
			"	li	%2, 0				\n"
			"	bnez	%1, 2f				\n"
			"	lui	%1, 0x8000			\n"
			"	sc	%1, %0				\n"
			"	li	%2, 1				\n"
			"2:						\n"
			: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp),
			  "=&r" (ret)
			: GCC_OFF_SMALL_ASM() (rw->lock)
			: "memory");
		} while (unlikely(!tmp));

		smp_llsc_mb();
	}

	return ret;
}

#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)

#define arch_spin_relax(lock)	cpu_relax()
#define arch_read_relax(lock)	cpu_relax()
#define arch_write_relax(lock)	cpu_relax()

#endif /* _ASM_SPINLOCK_H */
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
f65e4fa8	6	* Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
1da177e4 LT	7	* Copyright (C) 1999, 2000 Silicon Graphics, Inc.
	8	*/
	9	#ifndef _ASM_SPINLOCK_H
	10	#define _ASM_SPINLOCK_H
	11
2a31b033 RB	12	#include <linux/compiler.h>
2a31b033 RB	13
0004a9df	14	#include <asm/barrier.h>
726328d9	15	#include <asm/processor.h>
b0984c43	16	#include <asm/compiler.h>
1da177e4 LT	17	#include <asm/war.h>
	18
	19	/*
	20	* Your basic SMP spinlocks, allowing only a single CPU anywhere
2a31b033	21	*
70342287	22	* Simple spin lock operations. There are two variants, one clears IRQ's
2a31b033 RB	23	* on the local processor, one does not.
	24	*
	25	* These are fair FIFO ticket locks
	26	*
	27	* (the type definitions are in asm/spinlock_types.h)
1da177e4 LT	28	*/
1da177e4 LT	29
1da177e4 LT	30
1da177e4 LT	31	/*
2a31b033 RB	32	* Ticket locks are conceptually two parts, one indicating the current head of
	33	* the queue, and the other indicating the current tail. The lock is acquired
	34	* by atomically noting the tail and incrementing it by one (thus adding
	35	* ourself to the queue and noting our position), then waiting until the head
	36	* becomes equal to the the initial value of the tail.
1da177e4 LT	37	*/
1da177e4 LT	38
0199c4e6	39	static inline int arch_spin_is_locked(arch_spinlock_t *lock)
2a31b033	40	{
500c2e1f	41	u32 counters = ACCESS_ONCE(lock->lock);
2a31b033	42
500c2e1f	43	return ((counters >> 16) ^ counters) & 0xffff;
2a31b033 RB	44	}
2a31b033 RB	45
5fac4f7a PB	46	static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
	47	{
	48	return lock.h.serving_now == lock.h.ticket;
	49	}
	50
0199c4e6	51	#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
726328d9 PZ	52
	53	static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
	54	{
	55	u16 owner = READ_ONCE(lock->h.serving_now);
	56	smp_rmb();
	57	for (;;) {
	58	arch_spinlock_t tmp = READ_ONCE(*lock);
	59
	60	if (tmp.h.serving_now == tmp.h.ticket \|\|
	61	tmp.h.serving_now != owner)
	62	break;
	63
	64	cpu_relax();
	65	}
	66	smp_acquire__after_ctrl_dep();
	67	}
2a31b033	68
0199c4e6	69	static inline int arch_spin_is_contended(arch_spinlock_t *lock)
2a31b033	70	{
500c2e1f	71	u32 counters = ACCESS_ONCE(lock->lock);
2a31b033	72
500c2e1f	73	return (((counters >> 16) - counters) & 0xffff) > 1;
2a31b033	74	}
0199c4e6	75	#define arch_spin_is_contended arch_spin_is_contended
2a31b033	76
0199c4e6	77	static inline void arch_spin_lock(arch_spinlock_t *lock)
1da177e4	78	{
2a31b033 RB	79	int my_ticket;
2a31b033 RB	80	int tmp;
500c2e1f	81	int inc = 0x10000;
1da177e4 LT	82
1da177e4 LT	83	if (R10000_LLSC_WAR) {
2a31b033	84	__asm__ __volatile__ (
0199c4e6	85	" .set push # arch_spin_lock \n"
2a31b033 RB	86	" .set noreorder \n"
	87	" \n"
	88	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f	89	" addu %[my_ticket], %[ticket], %[inc] \n"
2a31b033 RB	90	" sc %[my_ticket], %[ticket_ptr] \n"
2a31b033 RB	91	" beqzl %[my_ticket], 1b \n"
1da177e4	92	" nop \n"
500c2e1f DD	93	" srl %[my_ticket], %[ticket], 16 \n"
500c2e1f DD	94	" andi %[ticket], %[ticket], 0xffff \n"
2a31b033 RB	95	" bne %[ticket], %[my_ticket], 4f \n"
	96	" subu %[ticket], %[my_ticket], %[ticket] \n"
	97	"2: \n"
	98	" .subsection 2 \n"
500c2e1f	99	"4: andi %[ticket], %[ticket], 0xffff \n"
0e6826c7	100	" sll %[ticket], 5 \n"
2a31b033 RB	101	" \n"
	102	"6: bnez %[ticket], 6b \n"
	103	" subu %[ticket], 1 \n"
	104	" \n"
500c2e1f	105	" lhu %[ticket], %[serving_now_ptr] \n"
2a31b033 RB	106	" beq %[ticket], %[my_ticket], 2b \n"
2a31b033 RB	107	" subu %[ticket], %[my_ticket], %[ticket] \n"
0e6826c7	108	" b 4b \n"
2a31b033 RB	109	" subu %[ticket], %[ticket], 1 \n"
	110	" .previous \n"
	111	" .set pop \n"
94bfb75a	112	: [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
500c2e1f	113	[serving_now_ptr] "+m" (lock->h.serving_now),
2a31b033	114	[ticket] "=&r" (tmp),
500c2e1f DD	115	[my_ticket] "=&r" (my_ticket)
500c2e1f DD	116	: [inc] "r" (inc));
1da177e4	117	} else {
2a31b033	118	__asm__ __volatile__ (
0199c4e6	119	" .set push # arch_spin_lock \n"
2a31b033 RB	120	" .set noreorder \n"
2a31b033 RB	121	" \n"
500c2e1f DD	122	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f DD	123	" addu %[my_ticket], %[ticket], %[inc] \n"
2a31b033	124	" sc %[my_ticket], %[ticket_ptr] \n"
500c2e1f DD	125	" beqz %[my_ticket], 1b \n"
	126	" srl %[my_ticket], %[ticket], 16 \n"
	127	" andi %[ticket], %[ticket], 0xffff \n"
2a31b033 RB	128	" bne %[ticket], %[my_ticket], 4f \n"
	129	" subu %[ticket], %[my_ticket], %[ticket] \n"
	130	"2: \n"
f65e4fa8	131	" .subsection 2 \n"
9ff897c4	132	"4: andi %[ticket], %[ticket], 0xffff \n"
0e6826c7	133	" sll %[ticket], 5 \n"
2a31b033 RB	134	" \n"
	135	"6: bnez %[ticket], 6b \n"
	136	" subu %[ticket], 1 \n"
	137	" \n"
500c2e1f	138	" lhu %[ticket], %[serving_now_ptr] \n"
2a31b033 RB	139	" beq %[ticket], %[my_ticket], 2b \n"
2a31b033 RB	140	" subu %[ticket], %[my_ticket], %[ticket] \n"
0e6826c7	141	" b 4b \n"
2a31b033	142	" subu %[ticket], %[ticket], 1 \n"
f65e4fa8	143	" .previous \n"
2a31b033	144	" .set pop \n"
94bfb75a	145	: [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
500c2e1f	146	[serving_now_ptr] "+m" (lock->h.serving_now),
2a31b033	147	[ticket] "=&r" (tmp),
500c2e1f DD	148	[my_ticket] "=&r" (my_ticket)
500c2e1f DD	149	: [inc] "r" (inc));
1da177e4	150	}
0004a9df	151
17099b11	152	smp_llsc_mb();
1da177e4 LT	153	}
1da177e4 LT	154
0199c4e6	155	static inline void arch_spin_unlock(arch_spinlock_t *lock)
1da177e4	156	{
500c2e1f DD	157	unsigned int serving_now = lock->h.serving_now + 1;
	158	wmb();
	159	lock->h.serving_now = (u16)serving_now;
	160	nudge_writes();
1da177e4 LT	161	}
1da177e4 LT	162
0199c4e6	163	static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
1da177e4	164	{
2a31b033	165	int tmp, tmp2, tmp3;
500c2e1f	166	int inc = 0x10000;
1da177e4 LT	167
1da177e4 LT	168	if (R10000_LLSC_WAR) {
2a31b033	169	__asm__ __volatile__ (
0199c4e6	170	" .set push # arch_spin_trylock \n"
2a31b033 RB	171	" .set noreorder \n"
	172	" \n"
	173	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f	174	" srl %[my_ticket], %[ticket], 16 \n"
500c2e1f	175	" andi %[now_serving], %[ticket], 0xffff \n"
2a31b033	176	" bne %[my_ticket], %[now_serving], 3f \n"
500c2e1f	177	" addu %[ticket], %[ticket], %[inc] \n"
2a31b033 RB	178	" sc %[ticket], %[ticket_ptr] \n"
	179	" beqzl %[ticket], 1b \n"
	180	" li %[ticket], 1 \n"
	181	"2: \n"
	182	" .subsection 2 \n"
	183	"3: b 2b \n"
	184	" li %[ticket], 0 \n"
	185	" .previous \n"
	186	" .set pop \n"
94bfb75a	187	: [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
2a31b033 RB	188	[ticket] "=&r" (tmp),
2a31b033 RB	189	[my_ticket] "=&r" (tmp2),
500c2e1f DD	190	[now_serving] "=&r" (tmp3)
500c2e1f DD	191	: [inc] "r" (inc));
1da177e4	192	} else {
2a31b033	193	__asm__ __volatile__ (
0199c4e6	194	" .set push # arch_spin_trylock \n"
2a31b033 RB	195	" .set noreorder \n"
2a31b033 RB	196	" \n"
500c2e1f DD	197	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f DD	198	" srl %[my_ticket], %[ticket], 16 \n"
500c2e1f	199	" andi %[now_serving], %[ticket], 0xffff \n"
2a31b033	200	" bne %[my_ticket], %[now_serving], 3f \n"
500c2e1f	201	" addu %[ticket], %[ticket], %[inc] \n"
2a31b033	202	" sc %[ticket], %[ticket_ptr] \n"
500c2e1f	203	" beqz %[ticket], 1b \n"
2a31b033 RB	204	" li %[ticket], 1 \n"
2a31b033 RB	205	"2: \n"
f65e4fa8	206	" .subsection 2 \n"
2a31b033 RB	207	"3: b 2b \n"
2a31b033 RB	208	" li %[ticket], 0 \n"
f65e4fa8	209	" .previous \n"
2a31b033	210	" .set pop \n"
94bfb75a	211	: [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
2a31b033 RB	212	[ticket] "=&r" (tmp),
2a31b033 RB	213	[my_ticket] "=&r" (tmp2),
500c2e1f DD	214	[now_serving] "=&r" (tmp3)
500c2e1f DD	215	: [inc] "r" (inc));
1da177e4 LT	216	}
1da177e4 LT	217
17099b11	218	smp_llsc_mb();
0004a9df	219
2a31b033	220	return tmp;
1da177e4 LT	221	}
	222
	223	/*
	224	* Read-write spinlocks, allowing multiple readers but only one writer.
	225	*
	226	* NOTE! it is quite common to have readers in interrupts but no interrupt
	227	* writers. For those circumstances we can "mix" irq-safe locks - any writer
	228	* needs to get a irq-safe write-lock, but readers can get non-irqsafe
	229	* read-locks.
	230	*/
	231
e3c48078 RB	232	/*
	233	* read_can_lock - would read_trylock() succeed?
	234	* @lock: the rwlock in question.
	235	*/
e5931943	236	#define arch_read_can_lock(rw) ((rw)->lock >= 0)
e3c48078 RB	237
	238	/*
	239	* write_can_lock - would write_trylock() succeed?
	240	* @lock: the rwlock in question.
	241	*/
70342287	242	#define arch_write_can_lock(rw) (!(rw)->lock)
e3c48078	243
e5931943	244	static inline void arch_read_lock(arch_rwlock_t *rw)
1da177e4 LT	245	{
	246	unsigned int tmp;
	247
	248	if (R10000_LLSC_WAR) {
	249	__asm__ __volatile__(
e5931943	250	" .set noreorder # arch_read_lock \n"
1da177e4 LT	251	"1: ll %1, %2 \n"
	252	" bltz %1, 1b \n"
	253	" addu %1, 1 \n"
	254	" sc %1, %0 \n"
	255	" beqzl %1, 1b \n"
	256	" nop \n"
1da177e4	257	" .set reorder \n"
94bfb75a MC	258	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
94bfb75a MC	259	: GCC_OFF_SMALL_ASM() (rw->lock)
1da177e4 LT	260	: "memory");
1da177e4 LT	261	} else {
e01961ce RB	262	do {
	263	__asm__ __volatile__(
	264	"1: ll %1, %2 # arch_read_lock \n"
	265	" bltz %1, 1b \n"
	266	" addu %1, 1 \n"
	267	"2: sc %1, %0 \n"
94bfb75a MC	268	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
94bfb75a MC	269	: GCC_OFF_SMALL_ASM() (rw->lock)
e01961ce RB	270	: "memory");
e01961ce RB	271	} while (unlikely(!tmp));
1da177e4	272	}
0004a9df	273
17099b11	274	smp_llsc_mb();
1da177e4 LT	275	}
1da177e4 LT	276
e5931943	277	static inline void arch_read_unlock(arch_rwlock_t *rw)
1da177e4 LT	278	{
	279	unsigned int tmp;
	280
f252ffd5	281	smp_mb__before_llsc();
0004a9df	282
1da177e4 LT	283	if (R10000_LLSC_WAR) {
1da177e4 LT	284	__asm__ __volatile__(
e5931943	285	"1: ll %1, %2 # arch_read_unlock \n"
51822216	286	" addiu %1, -1 \n"
1da177e4 LT	287	" sc %1, %0 \n"
1da177e4 LT	288	" beqzl %1, 1b \n"
94bfb75a MC	289	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
94bfb75a MC	290	: GCC_OFF_SMALL_ASM() (rw->lock)
1da177e4 LT	291	: "memory");
1da177e4 LT	292	} else {
e01961ce RB	293	do {
	294	__asm__ __volatile__(
	295	"1: ll %1, %2 # arch_read_unlock \n"
5753762c	296	" addiu %1, -1 \n"
e01961ce	297	" sc %1, %0 \n"
94bfb75a MC	298	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
94bfb75a MC	299	: GCC_OFF_SMALL_ASM() (rw->lock)
e01961ce RB	300	: "memory");
e01961ce RB	301	} while (unlikely(!tmp));
1da177e4 LT	302	}
	303	}
	304
e5931943	305	static inline void arch_write_lock(arch_rwlock_t *rw)
1da177e4 LT	306	{
	307	unsigned int tmp;
	308
	309	if (R10000_LLSC_WAR) {
	310	__asm__ __volatile__(
e5931943	311	" .set noreorder # arch_write_lock \n"
1da177e4 LT	312	"1: ll %1, %2 \n"
	313	" bnez %1, 1b \n"
	314	" lui %1, 0x8000 \n"
	315	" sc %1, %0 \n"
	316	" beqzl %1, 1b \n"
0004a9df	317	" nop \n"
1da177e4	318	" .set reorder \n"
94bfb75a MC	319	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
94bfb75a MC	320	: GCC_OFF_SMALL_ASM() (rw->lock)
1da177e4 LT	321	: "memory");
1da177e4 LT	322	} else {
e01961ce RB	323	do {
	324	__asm__ __volatile__(
	325	"1: ll %1, %2 # arch_write_lock \n"
	326	" bnez %1, 1b \n"
	327	" lui %1, 0x8000 \n"
	328	"2: sc %1, %0 \n"
94bfb75a MC	329	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
94bfb75a MC	330	: GCC_OFF_SMALL_ASM() (rw->lock)
e01961ce RB	331	: "memory");
e01961ce RB	332	} while (unlikely(!tmp));
1da177e4	333	}
0004a9df	334
17099b11	335	smp_llsc_mb();
1da177e4 LT	336	}
1da177e4 LT	337
e5931943	338	static inline void arch_write_unlock(arch_rwlock_t *rw)
1da177e4	339	{
6f6ed482	340	smp_mb__before_llsc();
0004a9df	341
1da177e4	342	__asm__ __volatile__(
e5931943	343	" # arch_write_unlock \n"
1da177e4 LT	344	" sw $0, %0 \n"
	345	: "=m" (rw->lock)
	346	: "m" (rw->lock)
	347	: "memory");
	348	}
	349
e5931943	350	static inline int arch_read_trylock(arch_rwlock_t *rw)
65316fd1 RB	351	{
	352	unsigned int tmp;
	353	int ret;
	354
	355	if (R10000_LLSC_WAR) {
	356	__asm__ __volatile__(
e5931943	357	" .set noreorder # arch_read_trylock \n"
65316fd1 RB	358	" li %2, 0 \n"
65316fd1 RB	359	"1: ll %1, %3 \n"
d52c2d5a	360	" bltz %1, 2f \n"
65316fd1 RB	361	" addu %1, 1 \n"
65316fd1 RB	362	" sc %1, %0 \n"
65316fd1	363	" .set reorder \n"
0004a9df RB	364	" beqzl %1, 1b \n"
0004a9df RB	365	" nop \n"
17099b11	366	__WEAK_LLSC_MB
65316fd1 RB	367	" li %2, 1 \n"
65316fd1 RB	368	"2: \n"
94bfb75a MC	369	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
94bfb75a MC	370	: GCC_OFF_SMALL_ASM() (rw->lock)
65316fd1 RB	371	: "memory");
	372	} else {
	373	__asm__ __volatile__(
e5931943	374	" .set noreorder # arch_read_trylock \n"
65316fd1 RB	375	" li %2, 0 \n"
65316fd1 RB	376	"1: ll %1, %3 \n"
d52c2d5a	377	" bltz %1, 2f \n"
65316fd1 RB	378	" addu %1, 1 \n"
	379	" sc %1, %0 \n"
	380	" beqz %1, 1b \n"
0004a9df	381	" nop \n"
65316fd1	382	" .set reorder \n"
17099b11	383	__WEAK_LLSC_MB
65316fd1 RB	384	" li %2, 1 \n"
65316fd1 RB	385	"2: \n"
94bfb75a MC	386	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
94bfb75a MC	387	: GCC_OFF_SMALL_ASM() (rw->lock)
65316fd1 RB	388	: "memory");
	389	}
	390
	391	return ret;
	392	}
1da177e4	393
e5931943	394	static inline int arch_write_trylock(arch_rwlock_t *rw)
1da177e4 LT	395	{
	396	unsigned int tmp;
	397	int ret;
	398
	399	if (R10000_LLSC_WAR) {
	400	__asm__ __volatile__(
e5931943	401	" .set noreorder # arch_write_trylock \n"
1da177e4 LT	402	" li %2, 0 \n"
	403	"1: ll %1, %3 \n"
	404	" bnez %1, 2f \n"
	405	" lui %1, 0x8000 \n"
	406	" sc %1, %0 \n"
	407	" beqzl %1, 1b \n"
0004a9df	408	" nop \n"
17099b11	409	__WEAK_LLSC_MB
1da177e4 LT	410	" li %2, 1 \n"
	411	" .set reorder \n"
	412	"2: \n"
94bfb75a MC	413	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
94bfb75a MC	414	: GCC_OFF_SMALL_ASM() (rw->lock)
1da177e4 LT	415	: "memory");
1da177e4 LT	416	} else {
e01961ce RB	417	do {
	418	__asm__ __volatile__(
	419	" ll %1, %3 # arch_write_trylock \n"
	420	" li %2, 0 \n"
	421	" bnez %1, 2f \n"
	422	" lui %1, 0x8000 \n"
	423	" sc %1, %0 \n"
	424	" li %2, 1 \n"
	425	"2: \n"
94bfb75a	426	: "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp),
b0984c43	427	"=&r" (ret)
94bfb75a	428	: GCC_OFF_SMALL_ASM() (rw->lock)
e01961ce RB	429	: "memory");
	430	} while (unlikely(!tmp));
	431
	432	smp_llsc_mb();
1da177e4 LT	433	}
	434
	435	return ret;
	436	}
	437
e5931943 TG	438	#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
e5931943 TG	439	#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
65316fd1	440
0199c4e6 TG	441	#define arch_spin_relax(lock) cpu_relax()
	442	#define arch_read_relax(lock) cpu_relax()
	443	#define arch_write_relax(lock) cpu_relax()
ef6edc97	444
1da177e4	445	#endif /* _ASM_SPINLOCK_H */