[deliverable/linux.git] / arch / s390 / lib / spinlock.c

/*
 *    Out of line spinlock code.
 *
 *    Copyright IBM Corp. 2004, 2006
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <asm/io.h>

int spin_retry = -1;

static int __init spin_retry_init(void)
{
	if (spin_retry < 0)
		spin_retry = MACHINE_HAS_CAD ? 10 : 1000;
	return 0;
}
early_initcall(spin_retry_init);

/**
 * spin_retry= parameter
 */
static int __init spin_retry_setup(char *str)
{
	spin_retry = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("spin_retry=", spin_retry_setup);

static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old)
{
	asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock));
}

static inline int cpu_is_preempted(int cpu)
{
	if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu))
		return 0;
	if (smp_vcpu_scheduled(cpu))
		return 0;
	return 1;
}

void arch_spin_lock_wait(arch_spinlock_t *lp)
{
	unsigned int cpu = SPINLOCK_LOCKVAL;
	unsigned int owner;
	int count, first_diag;

	first_diag = 1;
	while (1) {
		owner = ACCESS_ONCE(lp->lock);
		/* Try to get the lock if it is free. */
		if (!owner) {
			if (_raw_compare_and_swap(&lp->lock, 0, cpu))
				return;
			continue;
		}
		/* First iteration: check if the lock owner is running. */
		if (first_diag && cpu_is_preempted(~owner)) {
			smp_yield_cpu(~owner);
			first_diag = 0;
			continue;
		}
		/* Loop for a while on the lock value. */
		count = spin_retry;
		do {
			if (MACHINE_HAS_CAD)
				_raw_compare_and_delay(&lp->lock, owner);
			owner = ACCESS_ONCE(lp->lock);
		} while (owner && count-- > 0);
		if (!owner)
			continue;
		/*
		 * For multiple layers of hypervisors, e.g. z/VM + LPAR
		 * yield the CPU unconditionally. For LPAR rely on the
		 * sense running status.
		 */
		if (!MACHINE_IS_LPAR || cpu_is_preempted(~owner)) {
			smp_yield_cpu(~owner);
			first_diag = 0;
		}
	}
}
EXPORT_SYMBOL(arch_spin_lock_wait);

void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
{
	unsigned int cpu = SPINLOCK_LOCKVAL;
	unsigned int owner;
	int count, first_diag;

	local_irq_restore(flags);
	first_diag = 1;
	while (1) {
		owner = ACCESS_ONCE(lp->lock);
		/* Try to get the lock if it is free. */
		if (!owner) {
			local_irq_disable();
			if (_raw_compare_and_swap(&lp->lock, 0, cpu))
				return;
			local_irq_restore(flags);
			continue;
		}
		/* Check if the lock owner is running. */
		if (first_diag && cpu_is_preempted(~owner)) {
			smp_yield_cpu(~owner);
			first_diag = 0;
			continue;
		}
		/* Loop for a while on the lock value. */
		count = spin_retry;
		do {
			if (MACHINE_HAS_CAD)
				_raw_compare_and_delay(&lp->lock, owner);
			owner = ACCESS_ONCE(lp->lock);
		} while (owner && count-- > 0);
		if (!owner)
			continue;
		/*
		 * For multiple layers of hypervisors, e.g. z/VM + LPAR
		 * yield the CPU unconditionally. For LPAR rely on the
		 * sense running status.
		 */
		if (!MACHINE_IS_LPAR || cpu_is_preempted(~owner)) {
			smp_yield_cpu(~owner);
			first_diag = 0;
		}
	}
}
EXPORT_SYMBOL(arch_spin_lock_wait_flags);

int arch_spin_trylock_retry(arch_spinlock_t *lp)
{
	unsigned int cpu = SPINLOCK_LOCKVAL;
	unsigned int owner;
	int count;

	for (count = spin_retry; count > 0; count--) {
		owner = ACCESS_ONCE(lp->lock);
		/* Try to get the lock if it is free. */
		if (!owner) {
			if (_raw_compare_and_swap(&lp->lock, 0, cpu))
				return 1;
		} else if (MACHINE_HAS_CAD)
			_raw_compare_and_delay(&lp->lock, owner);
	}
	return 0;
}
EXPORT_SYMBOL(arch_spin_trylock_retry);

void _raw_read_lock_wait(arch_rwlock_t *rw)
{
	unsigned int owner, old;
	int count = spin_retry;

#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	__RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD);
#endif
	owner = 0;
	while (1) {
		if (count-- <= 0) {
			if (owner && cpu_is_preempted(~owner))
				smp_yield_cpu(~owner);
			count = spin_retry;
		}
		old = ACCESS_ONCE(rw->lock);
		owner = ACCESS_ONCE(rw->owner);
		if ((int) old < 0) {
			if (MACHINE_HAS_CAD)
				_raw_compare_and_delay(&rw->lock, old);
			continue;
		}
		if (_raw_compare_and_swap(&rw->lock, old, old + 1))
			return;
	}
}
EXPORT_SYMBOL(_raw_read_lock_wait);

int _raw_read_trylock_retry(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (count-- > 0) {
		old = ACCESS_ONCE(rw->lock);
		if ((int) old < 0) {
			if (MACHINE_HAS_CAD)
				_raw_compare_and_delay(&rw->lock, old);
			continue;
		}
		if (_raw_compare_and_swap(&rw->lock, old, old + 1))
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(_raw_read_trylock_retry);

#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES

void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev)
{
	unsigned int owner, old;
	int count = spin_retry;

	owner = 0;
	while (1) {
		if (count-- <= 0) {
			if (owner && cpu_is_preempted(~owner))
				smp_yield_cpu(~owner);
			count = spin_retry;
		}
		old = ACCESS_ONCE(rw->lock);
		owner = ACCESS_ONCE(rw->owner);
		smp_mb();
		if ((int) old >= 0) {
			prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);
			old = prev;
		}
		if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
			break;
		if (MACHINE_HAS_CAD)
			_raw_compare_and_delay(&rw->lock, old);
	}
}
EXPORT_SYMBOL(_raw_write_lock_wait);

#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */

void _raw_write_lock_wait(arch_rwlock_t *rw)
{
	unsigned int owner, old, prev;
	int count = spin_retry;

	prev = 0x80000000;
	owner = 0;
	while (1) {
		if (count-- <= 0) {
			if (owner && cpu_is_preempted(~owner))
				smp_yield_cpu(~owner);
			count = spin_retry;
		}
		old = ACCESS_ONCE(rw->lock);
		owner = ACCESS_ONCE(rw->owner);
		if ((int) old >= 0 &&
		    _raw_compare_and_swap(&rw->lock, old, old | 0x80000000))
			prev = old;
		else
			smp_mb();
		if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
			break;
		if (MACHINE_HAS_CAD)
			_raw_compare_and_delay(&rw->lock, old);
	}
}
EXPORT_SYMBOL(_raw_write_lock_wait);

#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */

int _raw_write_trylock_retry(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (count-- > 0) {
		old = ACCESS_ONCE(rw->lock);
		if (old) {
			if (MACHINE_HAS_CAD)
				_raw_compare_and_delay(&rw->lock, old);
			continue;
		}
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(_raw_write_trylock_retry);

void arch_lock_relax(unsigned int cpu)
{
	if (!cpu)
		return;
	if (MACHINE_IS_LPAR && !cpu_is_preempted(~cpu))
		return;
	smp_yield_cpu(~cpu);
}
EXPORT_SYMBOL(arch_lock_relax);
Commit	Line	Data
951f22d5	1	/*
951f22d5 MS	2	* Out of line spinlock code.
951f22d5 MS	3	*
a53c8fab	4	* Copyright IBM Corp. 2004, 2006
951f22d5 MS	5	* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
	6	*/
	7
	8	#include <linux/types.h>
	9	#include <linux/module.h>
	10	#include <linux/spinlock.h>
	11	#include <linux/init.h>
8b646bd7	12	#include <linux/smp.h>
951f22d5 MS	13	#include <asm/io.h>
951f22d5 MS	14
2c72a44e MS	15	int spin_retry = -1;
	16
	17	static int __init spin_retry_init(void)
	18	{
	19	if (spin_retry < 0)
	20	spin_retry = MACHINE_HAS_CAD ? 10 : 1000;
	21	return 0;
	22	}
	23	early_initcall(spin_retry_init);
951f22d5 MS	24
	25	/**
	26	* spin_retry= parameter
	27	*/
	28	static int __init spin_retry_setup(char *str)
	29	{
	30	spin_retry = simple_strtoul(str, &str, 0);
	31	return 1;
	32	}
	33	__setup("spin_retry=", spin_retry_setup);
	34
2c72a44e MS	35	static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old)
	36	{
	37	asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock));
	38	}
	39
419123f9 MS	40	static inline int cpu_is_preempted(int cpu)
	41	{
	42	if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu))
	43	return 0;
	44	if (smp_vcpu_scheduled(cpu))
	45	return 0;
	46	return 1;
	47	}
	48
0199c4e6	49	void arch_spin_lock_wait(arch_spinlock_t *lp)
951f22d5	50	{
6c8cd5bb	51	unsigned int cpu = SPINLOCK_LOCKVAL;
59b69787	52	unsigned int owner;
db1c4515	53	int count, first_diag;
951f22d5	54
db1c4515	55	first_diag = 1;
951f22d5	56	while (1) {
470ada6b MS	57	owner = ACCESS_ONCE(lp->lock);
	58	/* Try to get the lock if it is free. */
	59	if (!owner) {
	60	if (_raw_compare_and_swap(&lp->lock, 0, cpu))
	61	return;
	62	continue;
951f22d5	63	}
db1c4515	64	/* First iteration: check if the lock owner is running. */
419123f9	65	if (first_diag && cpu_is_preempted(~owner)) {
470ada6b	66	smp_yield_cpu(~owner);
db1c4515	67	first_diag = 0;
470ada6b MS	68	continue;
	69	}
	70	/* Loop for a while on the lock value. */
	71	count = spin_retry;
	72	do {
2c72a44e MS	73	if (MACHINE_HAS_CAD)
2c72a44e MS	74	_raw_compare_and_delay(&lp->lock, owner);
470ada6b MS	75	owner = ACCESS_ONCE(lp->lock);
	76	} while (owner && count-- > 0);
	77	if (!owner)
	78	continue;
	79	/*
	80	* For multiple layers of hypervisors, e.g. z/VM + LPAR
db1c4515 MS	81	* yield the CPU unconditionally. For LPAR rely on the
db1c4515 MS	82	* sense running status.
470ada6b	83	*/
419123f9	84	if (!MACHINE_IS_LPAR \|\| cpu_is_preempted(~owner)) {
8b646bd7	85	smp_yield_cpu(~owner);
db1c4515 MS	86	first_diag = 0;
db1c4515 MS	87	}
951f22d5 MS	88	}
951f22d5 MS	89	}
0199c4e6	90	EXPORT_SYMBOL(arch_spin_lock_wait);
951f22d5	91
0199c4e6	92	void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
894cdde2	93	{
6c8cd5bb	94	unsigned int cpu = SPINLOCK_LOCKVAL;
59b69787	95	unsigned int owner;
db1c4515	96	int count, first_diag;
894cdde2 HH	97
894cdde2 HH	98	local_irq_restore(flags);
db1c4515	99	first_diag = 1;
894cdde2	100	while (1) {
470ada6b MS	101	owner = ACCESS_ONCE(lp->lock);
	102	/* Try to get the lock if it is free. */
	103	if (!owner) {
	104	local_irq_disable();
	105	if (_raw_compare_and_swap(&lp->lock, 0, cpu))
	106	return;
	107	local_irq_restore(flags);
84976952	108	continue;
470ada6b MS	109	}
470ada6b MS	110	/* Check if the lock owner is running. */
419123f9	111	if (first_diag && cpu_is_preempted(~owner)) {
470ada6b	112	smp_yield_cpu(~owner);
db1c4515	113	first_diag = 0;
470ada6b	114	continue;
894cdde2	115	}
470ada6b MS	116	/* Loop for a while on the lock value. */
	117	count = spin_retry;
	118	do {
2c72a44e MS	119	if (MACHINE_HAS_CAD)
2c72a44e MS	120	_raw_compare_and_delay(&lp->lock, owner);
470ada6b MS	121	owner = ACCESS_ONCE(lp->lock);
	122	} while (owner && count-- > 0);
	123	if (!owner)
	124	continue;
	125	/*
	126	* For multiple layers of hypervisors, e.g. z/VM + LPAR
db1c4515 MS	127	* yield the CPU unconditionally. For LPAR rely on the
db1c4515 MS	128	* sense running status.
470ada6b	129	*/
419123f9	130	if (!MACHINE_IS_LPAR \|\| cpu_is_preempted(~owner)) {
8b646bd7	131	smp_yield_cpu(~owner);
db1c4515 MS	132	first_diag = 0;
db1c4515 MS	133	}
894cdde2 HH	134	}
894cdde2 HH	135	}
0199c4e6	136	EXPORT_SYMBOL(arch_spin_lock_wait_flags);
894cdde2	137
0199c4e6	138	int arch_spin_trylock_retry(arch_spinlock_t *lp)
951f22d5	139	{
2c72a44e MS	140	unsigned int cpu = SPINLOCK_LOCKVAL;
2c72a44e MS	141	unsigned int owner;
3c1fcfe2	142	int count;
951f22d5	143
2c72a44e MS	144	for (count = spin_retry; count > 0; count--) {
	145	owner = ACCESS_ONCE(lp->lock);
	146	/* Try to get the lock if it is free. */
	147	if (!owner) {
	148	if (_raw_compare_and_swap(&lp->lock, 0, cpu))
	149	return 1;
	150	} else if (MACHINE_HAS_CAD)
	151	_raw_compare_and_delay(&lp->lock, owner);
	152	}
951f22d5 MS	153	return 0;
951f22d5 MS	154	}
0199c4e6	155	EXPORT_SYMBOL(arch_spin_trylock_retry);
951f22d5	156
fb3a6bbc	157	void _raw_read_lock_wait(arch_rwlock_t *rw)
951f22d5	158	{
d59b93da	159	unsigned int owner, old;
951f22d5 MS	160	int count = spin_retry;
951f22d5 MS	161
bbae71bf MS	162	#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	163	__RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD);
	164	#endif
d59b93da	165	owner = 0;
951f22d5 MS	166	while (1) {
951f22d5 MS	167	if (count-- <= 0) {
419123f9	168	if (owner && cpu_is_preempted(~owner))
d59b93da	169	smp_yield_cpu(~owner);
951f22d5 MS	170	count = spin_retry;
951f22d5 MS	171	}
bae8f567	172	old = ACCESS_ONCE(rw->lock);
d59b93da	173	owner = ACCESS_ONCE(rw->owner);
2c72a44e MS	174	if ((int) old < 0) {
	175	if (MACHINE_HAS_CAD)
	176	_raw_compare_and_delay(&rw->lock, old);
96567161	177	continue;
2c72a44e	178	}
5b3f683e	179	if (_raw_compare_and_swap(&rw->lock, old, old + 1))
951f22d5 MS	180	return;
	181	}
	182	}
	183	EXPORT_SYMBOL(_raw_read_lock_wait);
	184
fb3a6bbc	185	int _raw_read_trylock_retry(arch_rwlock_t *rw)
951f22d5 MS	186	{
	187	unsigned int old;
	188	int count = spin_retry;
	189
	190	while (count-- > 0) {
bae8f567	191	old = ACCESS_ONCE(rw->lock);
2c72a44e MS	192	if ((int) old < 0) {
	193	if (MACHINE_HAS_CAD)
	194	_raw_compare_and_delay(&rw->lock, old);
96567161	195	continue;
2c72a44e	196	}
5b3f683e	197	if (_raw_compare_and_swap(&rw->lock, old, old + 1))
951f22d5 MS	198	return 1;
	199	}
	200	return 0;
	201	}
	202	EXPORT_SYMBOL(_raw_read_trylock_retry);
	203
bbae71bf MS	204	#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	205
	206	void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev)
	207	{
	208	unsigned int owner, old;
	209	int count = spin_retry;
	210
	211	owner = 0;
	212	while (1) {
	213	if (count-- <= 0) {
419123f9	214	if (owner && cpu_is_preempted(~owner))
bbae71bf MS	215	smp_yield_cpu(~owner);
	216	count = spin_retry;
	217	}
	218	old = ACCESS_ONCE(rw->lock);
	219	owner = ACCESS_ONCE(rw->owner);
e0af21c5	220	smp_mb();
bbae71bf MS	221	if ((int) old >= 0) {
	222	prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);
	223	old = prev;
	224	}
	225	if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
	226	break;
2c72a44e MS	227	if (MACHINE_HAS_CAD)
2c72a44e MS	228	_raw_compare_and_delay(&rw->lock, old);
bbae71bf MS	229	}
	230	}
	231	EXPORT_SYMBOL(_raw_write_lock_wait);
	232
	233	#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
	234
fb3a6bbc	235	void _raw_write_lock_wait(arch_rwlock_t *rw)
951f22d5	236	{
94232a43	237	unsigned int owner, old, prev;
951f22d5 MS	238	int count = spin_retry;
951f22d5 MS	239
94232a43	240	prev = 0x80000000;
d59b93da	241	owner = 0;
951f22d5 MS	242	while (1) {
951f22d5 MS	243	if (count-- <= 0) {
419123f9	244	if (owner && cpu_is_preempted(~owner))
d59b93da	245	smp_yield_cpu(~owner);
951f22d5 MS	246	count = spin_retry;
951f22d5 MS	247	}
bae8f567	248	old = ACCESS_ONCE(rw->lock);
d59b93da	249	owner = ACCESS_ONCE(rw->owner);
94232a43 MS	250	if ((int) old >= 0 &&
	251	_raw_compare_and_swap(&rw->lock, old, old \| 0x80000000))
	252	prev = old;
	253	else
e0af21c5	254	smp_mb();
94232a43 MS	255	if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
94232a43 MS	256	break;
2c72a44e MS	257	if (MACHINE_HAS_CAD)
2c72a44e MS	258	_raw_compare_and_delay(&rw->lock, old);
951f22d5 MS	259	}
	260	}
	261	EXPORT_SYMBOL(_raw_write_lock_wait);
	262
bbae71bf MS	263	#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
bbae71bf MS	264
fb3a6bbc	265	int _raw_write_trylock_retry(arch_rwlock_t *rw)
951f22d5	266	{
bae8f567	267	unsigned int old;
951f22d5 MS	268	int count = spin_retry;
	269
	270	while (count-- > 0) {
bae8f567	271	old = ACCESS_ONCE(rw->lock);
2c72a44e MS	272	if (old) {
	273	if (MACHINE_HAS_CAD)
	274	_raw_compare_and_delay(&rw->lock, old);
96567161	275	continue;
2c72a44e	276	}
5b3f683e	277	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
951f22d5 MS	278	return 1;
	279	}
	280	return 0;
	281	}
	282	EXPORT_SYMBOL(_raw_write_trylock_retry);
d59b93da MS	283
	284	void arch_lock_relax(unsigned int cpu)
	285	{
	286	if (!cpu)
	287	return;
419123f9	288	if (MACHINE_IS_LPAR && !cpu_is_preempted(~cpu))
d59b93da MS	289	return;
	290	smp_yield_cpu(~cpu);
	291	}
	292	EXPORT_SYMBOL(arch_lock_relax);