[deliverable/linux.git] / arch / s390 / lib / spinlock.c

/*
 *  arch/s390/lib/spinlock.c
 *    Out of line spinlock code.
 *
 *    Copyright (C) IBM Corp. 2004, 2006
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <asm/io.h>

int spin_retry = 1000;

/**
 * spin_retry= parameter
 */
static int __init spin_retry_setup(char *str)
{
	spin_retry = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("spin_retry=", spin_retry_setup);

void arch_spin_lock_wait(arch_spinlock_t *lp)
{
	int count = spin_retry;
	unsigned int cpu = ~smp_processor_id();
	unsigned int owner;

	while (1) {
		owner = lp->owner_cpu;
		if (!owner || smp_vcpu_scheduled(~owner)) {
			for (count = spin_retry; count > 0; count--) {
				if (arch_spin_is_locked(lp))
					continue;
				if (_raw_compare_and_swap(&lp->owner_cpu, 0,
							  cpu) == 0)
					return;
			}
			if (MACHINE_IS_LPAR)
				continue;
		}
		owner = lp->owner_cpu;
		if (owner)
			smp_yield_cpu(~owner);
		if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
			return;
	}
}
EXPORT_SYMBOL(arch_spin_lock_wait);

void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
{
	int count = spin_retry;
	unsigned int cpu = ~smp_processor_id();
	unsigned int owner;

	local_irq_restore(flags);
	while (1) {
		owner = lp->owner_cpu;
		if (!owner || smp_vcpu_scheduled(~owner)) {
			for (count = spin_retry; count > 0; count--) {
				if (arch_spin_is_locked(lp))
					continue;
				local_irq_disable();
				if (_raw_compare_and_swap(&lp->owner_cpu, 0,
							  cpu) == 0)
					return;
				local_irq_restore(flags);
			}
			if (MACHINE_IS_LPAR)
				continue;
		}
		owner = lp->owner_cpu;
		if (owner)
			smp_yield_cpu(~owner);
		local_irq_disable();
		if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
			return;
		local_irq_restore(flags);
	}
}
EXPORT_SYMBOL(arch_spin_lock_wait_flags);

int arch_spin_trylock_retry(arch_spinlock_t *lp)
{
	unsigned int cpu = ~smp_processor_id();
	int count;

	for (count = spin_retry; count > 0; count--) {
		if (arch_spin_is_locked(lp))
			continue;
		if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(arch_spin_trylock_retry);

void arch_spin_relax(arch_spinlock_t *lock)
{
	unsigned int cpu = lock->owner_cpu;
	if (cpu != 0) {
		if (MACHINE_IS_VM || MACHINE_IS_KVM ||
		    !smp_vcpu_scheduled(~cpu))
			smp_yield_cpu(~cpu);
	}
}
EXPORT_SYMBOL(arch_spin_relax);

void _raw_read_lock_wait(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		if (!arch_read_can_lock(rw))
			continue;
		old = rw->lock & 0x7fffffffU;
		if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
			return;
	}
}
EXPORT_SYMBOL(_raw_read_lock_wait);

void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
{
	unsigned int old;
	int count = spin_retry;

	local_irq_restore(flags);
	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		if (!arch_read_can_lock(rw))
			continue;
		old = rw->lock & 0x7fffffffU;
		local_irq_disable();
		if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
			return;
	}
}
EXPORT_SYMBOL(_raw_read_lock_wait_flags);

int _raw_read_trylock_retry(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (count-- > 0) {
		if (!arch_read_can_lock(rw))
			continue;
		old = rw->lock & 0x7fffffffU;
		if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(_raw_read_trylock_retry);

void _raw_write_lock_wait(arch_rwlock_t *rw)
{
	int count = spin_retry;

	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		if (!arch_write_can_lock(rw))
			continue;
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
			return;
	}
}
EXPORT_SYMBOL(_raw_write_lock_wait);

void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
{
	int count = spin_retry;

	local_irq_restore(flags);
	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		if (!arch_write_can_lock(rw))
			continue;
		local_irq_disable();
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
			return;
	}
}
EXPORT_SYMBOL(_raw_write_lock_wait_flags);

int _raw_write_trylock_retry(arch_rwlock_t *rw)
{
	int count = spin_retry;

	while (count-- > 0) {
		if (!arch_write_can_lock(rw))
			continue;
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(_raw_write_trylock_retry);
Commit	Line	Data
951f22d5 MS	1	/*
	2	* arch/s390/lib/spinlock.c
	3	* Out of line spinlock code.
	4	*
96567161	5	* Copyright (C) IBM Corp. 2004, 2006
951f22d5 MS	6	* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
	7	*/
	8
	9	#include <linux/types.h>
	10	#include <linux/module.h>
	11	#include <linux/spinlock.h>
	12	#include <linux/init.h>
8b646bd7	13	#include <linux/smp.h>
951f22d5 MS	14	#include <asm/io.h>
951f22d5 MS	15
951f22d5 MS	16	int spin_retry = 1000;
	17
	18	/**
	19	* spin_retry= parameter
	20	*/
	21	static int __init spin_retry_setup(char *str)
	22	{
	23	spin_retry = simple_strtoul(str, &str, 0);
	24	return 1;
	25	}
	26	__setup("spin_retry=", spin_retry_setup);
	27
0199c4e6	28	void arch_spin_lock_wait(arch_spinlock_t *lp)
951f22d5 MS	29	{
951f22d5 MS	30	int count = spin_retry;
3c1fcfe2	31	unsigned int cpu = ~smp_processor_id();
59b69787	32	unsigned int owner;
951f22d5 MS	33
951f22d5 MS	34	while (1) {
59b69787 GS	35	owner = lp->owner_cpu;
	36	if (!owner \|\| smp_vcpu_scheduled(~owner)) {
	37	for (count = spin_retry; count > 0; count--) {
	38	if (arch_spin_is_locked(lp))
	39	continue;
	40	if (_raw_compare_and_swap(&lp->owner_cpu, 0,
	41	cpu) == 0)
	42	return;
	43	}
	44	if (MACHINE_IS_LPAR)
	45	continue;
951f22d5	46	}
59b69787 GS	47	owner = lp->owner_cpu;
59b69787 GS	48	if (owner)
8b646bd7	49	smp_yield_cpu(~owner);
3b4beb31	50	if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
951f22d5 MS	51	return;
	52	}
	53	}
0199c4e6	54	EXPORT_SYMBOL(arch_spin_lock_wait);
951f22d5	55
0199c4e6	56	void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
894cdde2 HH	57	{
	58	int count = spin_retry;
	59	unsigned int cpu = ~smp_processor_id();
59b69787	60	unsigned int owner;
894cdde2 HH	61
	62	local_irq_restore(flags);
	63	while (1) {
59b69787 GS	64	owner = lp->owner_cpu;
	65	if (!owner \|\| smp_vcpu_scheduled(~owner)) {
	66	for (count = spin_retry; count > 0; count--) {
	67	if (arch_spin_is_locked(lp))
	68	continue;
	69	local_irq_disable();
	70	if (_raw_compare_and_swap(&lp->owner_cpu, 0,
	71	cpu) == 0)
	72	return;
	73	local_irq_restore(flags);
	74	}
	75	if (MACHINE_IS_LPAR)
	76	continue;
894cdde2	77	}
59b69787 GS	78	owner = lp->owner_cpu;
59b69787 GS	79	if (owner)
8b646bd7	80	smp_yield_cpu(~owner);
894cdde2 HH	81	local_irq_disable();
	82	if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
	83	return;
	84	local_irq_restore(flags);
	85	}
	86	}
0199c4e6	87	EXPORT_SYMBOL(arch_spin_lock_wait_flags);
894cdde2	88
0199c4e6	89	int arch_spin_trylock_retry(arch_spinlock_t *lp)
951f22d5	90	{
3c1fcfe2 MS	91	unsigned int cpu = ~smp_processor_id();
3c1fcfe2 MS	92	int count;
951f22d5	93
3c1fcfe2	94	for (count = spin_retry; count > 0; count--) {
0199c4e6	95	if (arch_spin_is_locked(lp))
96567161	96	continue;
3b4beb31	97	if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
951f22d5 MS	98	return 1;
	99	}
	100	return 0;
	101	}
0199c4e6	102	EXPORT_SYMBOL(arch_spin_trylock_retry);
951f22d5	103
0199c4e6	104	void arch_spin_relax(arch_spinlock_t *lock)
3c1fcfe2 MS	105	{
3c1fcfe2 MS	106	unsigned int cpu = lock->owner_cpu;
59b69787 GS	107	if (cpu != 0) {
	108	if (MACHINE_IS_VM \|\| MACHINE_IS_KVM \|\|
	109	!smp_vcpu_scheduled(~cpu))
8b646bd7	110	smp_yield_cpu(~cpu);
59b69787	111	}
3c1fcfe2	112	}
0199c4e6	113	EXPORT_SYMBOL(arch_spin_relax);
3c1fcfe2	114
fb3a6bbc	115	void _raw_read_lock_wait(arch_rwlock_t *rw)
951f22d5 MS	116	{
	117	unsigned int old;
	118	int count = spin_retry;
	119
	120	while (1) {
	121	if (count-- <= 0) {
8b646bd7	122	smp_yield();
951f22d5 MS	123	count = spin_retry;
951f22d5 MS	124	}
e5931943	125	if (!arch_read_can_lock(rw))
96567161	126	continue;
951f22d5 MS	127	old = rw->lock & 0x7fffffffU;
	128	if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
	129	return;
	130	}
	131	}
	132	EXPORT_SYMBOL(_raw_read_lock_wait);
	133
fb3a6bbc	134	void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
ce58ae6f HC	135	{
	136	unsigned int old;
	137	int count = spin_retry;
	138
	139	local_irq_restore(flags);
	140	while (1) {
	141	if (count-- <= 0) {
8b646bd7	142	smp_yield();
ce58ae6f HC	143	count = spin_retry;
ce58ae6f HC	144	}
e5931943	145	if (!arch_read_can_lock(rw))
ce58ae6f HC	146	continue;
	147	old = rw->lock & 0x7fffffffU;
	148	local_irq_disable();
	149	if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
	150	return;
	151	}
	152	}
	153	EXPORT_SYMBOL(_raw_read_lock_wait_flags);
	154
fb3a6bbc	155	int _raw_read_trylock_retry(arch_rwlock_t *rw)
951f22d5 MS	156	{
	157	unsigned int old;
	158	int count = spin_retry;
	159
	160	while (count-- > 0) {
e5931943	161	if (!arch_read_can_lock(rw))
96567161	162	continue;
951f22d5 MS	163	old = rw->lock & 0x7fffffffU;
	164	if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
	165	return 1;
	166	}
	167	return 0;
	168	}
	169	EXPORT_SYMBOL(_raw_read_trylock_retry);
	170
fb3a6bbc	171	void _raw_write_lock_wait(arch_rwlock_t *rw)
951f22d5 MS	172	{
	173	int count = spin_retry;
	174
	175	while (1) {
	176	if (count-- <= 0) {
8b646bd7	177	smp_yield();
951f22d5 MS	178	count = spin_retry;
951f22d5 MS	179	}
e5931943	180	if (!arch_write_can_lock(rw))
96567161	181	continue;
951f22d5 MS	182	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
	183	return;
	184	}
	185	}
	186	EXPORT_SYMBOL(_raw_write_lock_wait);
	187
fb3a6bbc	188	void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
ce58ae6f HC	189	{
	190	int count = spin_retry;
	191
	192	local_irq_restore(flags);
	193	while (1) {
	194	if (count-- <= 0) {
8b646bd7	195	smp_yield();
ce58ae6f HC	196	count = spin_retry;
ce58ae6f HC	197	}
e5931943	198	if (!arch_write_can_lock(rw))
ce58ae6f HC	199	continue;
	200	local_irq_disable();
	201	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
	202	return;
	203	}
	204	}
	205	EXPORT_SYMBOL(_raw_write_lock_wait_flags);
	206
fb3a6bbc	207	int _raw_write_trylock_retry(arch_rwlock_t *rw)
951f22d5 MS	208	{
	209	int count = spin_retry;
	210
	211	while (count-- > 0) {
e5931943	212	if (!arch_write_can_lock(rw))
96567161	213	continue;
951f22d5 MS	214	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
	215	return 1;
	216	}
	217	return 0;
	218	}
	219	EXPORT_SYMBOL(_raw_write_trylock_retry);