[deliverable/linux.git] / arch / s390 / lib / uaccess_pt.c

/*
 *  User access functions based on page table walks for enhanced
 *  system layout without hardware support.
 *
 *    Copyright IBM Corp. 2006, 2012
 *    Author(s): Gerald Schaefer (gerald.schaefer@de.ibm.com)
 */

#include <linux/errno.h>
#include <linux/hardirq.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <asm/uaccess.h>
#include <asm/futex.h>
#include "uaccess.h"


/*
 * Returns kernel address for user virtual address. If the returned address is
 * >= -4095 (IS_ERR_VALUE(x) returns true), a fault has occured and the address
 * contains the (negative) exception code.
 */
static __always_inline unsigned long follow_table(struct mm_struct *mm,
						  unsigned long addr, int write)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *ptep;

	pgd = pgd_offset(mm, addr);
	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
		return -0x3aUL;

	pud = pud_offset(pgd, addr);
	if (pud_none(*pud) || unlikely(pud_bad(*pud)))
		return -0x3bUL;

	pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd))
		return -0x10UL;
	if (pmd_large(*pmd)) {
		if (write && (pmd_val(*pmd) & _SEGMENT_ENTRY_RO))
			return -0x04UL;
		return (pmd_val(*pmd) & HPAGE_MASK) + (addr & ~HPAGE_MASK);
	}
	if (unlikely(pmd_bad(*pmd)))
		return -0x10UL;

	ptep = pte_offset_map(pmd, addr);
	if (!pte_present(*ptep))
		return -0x11UL;
	if (write && (!pte_write(*ptep) || !pte_dirty(*ptep)))
		return -0x04UL;

	return (pte_val(*ptep) & PAGE_MASK) + (addr & ~PAGE_MASK);
}

static __always_inline size_t __user_copy_pt(unsigned long uaddr, void *kptr,
					     size_t n, int write_user)
{
	struct mm_struct *mm = current->mm;
	unsigned long offset, done, size, kaddr;
	void *from, *to;

	done = 0;
retry:
	spin_lock(&mm->page_table_lock);
	do {
		kaddr = follow_table(mm, uaddr, write_user);
		if (IS_ERR_VALUE(kaddr))
			goto fault;

		offset = uaddr & ~PAGE_MASK;
		size = min(n - done, PAGE_SIZE - offset);
		if (write_user) {
			to = (void *) kaddr;
			from = kptr + done;
		} else {
			from = (void *) kaddr;
			to = kptr + done;
		}
		memcpy(to, from, size);
		done += size;
		uaddr += size;
	} while (done < n);
	spin_unlock(&mm->page_table_lock);
	return n - done;
fault:
	spin_unlock(&mm->page_table_lock);
	if (__handle_fault(uaddr, -kaddr, write_user))
		return n - done;
	goto retry;
}

/*
 * Do DAT for user address by page table walk, return kernel address.
 * This function needs to be called with current->mm->page_table_lock held.
 */
static __always_inline unsigned long __dat_user_addr(unsigned long uaddr,
						     int write)
{
	struct mm_struct *mm = current->mm;
	unsigned long kaddr;
	int rc;

retry:
	kaddr = follow_table(mm, uaddr, write);
	if (IS_ERR_VALUE(kaddr))
		goto fault;

	return kaddr;
fault:
	spin_unlock(&mm->page_table_lock);
	rc = __handle_fault(uaddr, -kaddr, write);
	spin_lock(&mm->page_table_lock);
	if (!rc)
		goto retry;
	return 0;
}

size_t copy_from_user_pt(size_t n, const void __user *from, void *to)
{
	size_t rc;

	if (segment_eq(get_fs(), KERNEL_DS)) {
		memcpy(to, (void __kernel __force *) from, n);
		return 0;
	}
	rc = __user_copy_pt((unsigned long) from, to, n, 0);
	if (unlikely(rc))
		memset(to + n - rc, 0, rc);
	return rc;
}

size_t copy_to_user_pt(size_t n, void __user *to, const void *from)
{
	if (segment_eq(get_fs(), KERNEL_DS)) {
		memcpy((void __kernel __force *) to, from, n);
		return 0;
	}
	return __user_copy_pt((unsigned long) to, (void *) from, n, 1);
}

static size_t clear_user_pt(size_t n, void __user *to)
{
	long done, size, ret;

	if (segment_eq(get_fs(), KERNEL_DS)) {
		memset((void __kernel __force *) to, 0, n);
		return 0;
	}
	done = 0;
	do {
		if (n - done > PAGE_SIZE)
			size = PAGE_SIZE;
		else
			size = n - done;
		ret = __user_copy_pt((unsigned long) to + done,
				      &empty_zero_page, size, 1);
		done += size;
		if (ret)
			return ret + n - done;
	} while (done < n);
	return 0;
}

static size_t strnlen_user_pt(size_t count, const char __user *src)
{
	unsigned long uaddr = (unsigned long) src;
	struct mm_struct *mm = current->mm;
	unsigned long offset, done, len, kaddr;
	size_t len_str;

	if (segment_eq(get_fs(), KERNEL_DS))
		return strnlen((const char __kernel __force *) src, count) + 1;
	done = 0;
retry:
	spin_lock(&mm->page_table_lock);
	do {
		kaddr = follow_table(mm, uaddr, 0);
		if (IS_ERR_VALUE(kaddr))
			goto fault;

		offset = uaddr & ~PAGE_MASK;
		len = min(count - done, PAGE_SIZE - offset);
		len_str = strnlen((char *) kaddr, len);
		done += len_str;
		uaddr += len_str;
	} while ((len_str == len) && (done < count));
	spin_unlock(&mm->page_table_lock);
	return done + 1;
fault:
	spin_unlock(&mm->page_table_lock);
	if (__handle_fault(uaddr, -kaddr, 0))
		return 0;
	goto retry;
}

static size_t strncpy_from_user_pt(size_t count, const char __user *src,
				   char *dst)
{
	size_t n = strnlen_user_pt(count, src);

	if (!n)
		return -EFAULT;
	if (n > count)
		n = count;
	if (segment_eq(get_fs(), KERNEL_DS)) {
		memcpy(dst, (const char __kernel __force *) src, n);
		if (dst[n-1] == '\0')
			return n-1;
		else
			return n;
	}
	if (__user_copy_pt((unsigned long) src, dst, n, 0))
		return -EFAULT;
	if (dst[n-1] == '\0')
		return n-1;
	else
		return n;
}

static size_t copy_in_user_pt(size_t n, void __user *to,
			      const void __user *from)
{
	struct mm_struct *mm = current->mm;
	unsigned long offset_max, uaddr, done, size, error_code;
	unsigned long uaddr_from = (unsigned long) from;
	unsigned long uaddr_to = (unsigned long) to;
	unsigned long kaddr_to, kaddr_from;
	int write_user;

	if (segment_eq(get_fs(), KERNEL_DS)) {
		memcpy((void __force *) to, (void __force *) from, n);
		return 0;
	}
	done = 0;
retry:
	spin_lock(&mm->page_table_lock);
	do {
		write_user = 0;
		uaddr = uaddr_from;
		kaddr_from = follow_table(mm, uaddr_from, 0);
		error_code = kaddr_from;
		if (IS_ERR_VALUE(error_code))
			goto fault;

		write_user = 1;
		uaddr = uaddr_to;
		kaddr_to = follow_table(mm, uaddr_to, 1);
		error_code = (unsigned long) kaddr_to;
		if (IS_ERR_VALUE(error_code))
			goto fault;

		offset_max = max(uaddr_from & ~PAGE_MASK,
				 uaddr_to & ~PAGE_MASK);
		size = min(n - done, PAGE_SIZE - offset_max);

		memcpy((void *) kaddr_to, (void *) kaddr_from, size);
		done += size;
		uaddr_from += size;
		uaddr_to += size;
	} while (done < n);
	spin_unlock(&mm->page_table_lock);
	return n - done;
fault:
	spin_unlock(&mm->page_table_lock);
	if (__handle_fault(uaddr, -error_code, write_user))
		return n - done;
	goto retry;
}

#define __futex_atomic_op(insn, ret, oldval, newval, uaddr, oparg)	\
	asm volatile("0: l   %1,0(%6)\n"				\
		     "1: " insn						\
		     "2: cs  %1,%2,0(%6)\n"				\
		     "3: jl  1b\n"					\
		     "   lhi %0,0\n"					\
		     "4:\n"						\
		     EX_TABLE(0b,4b) EX_TABLE(2b,4b) EX_TABLE(3b,4b)	\
		     : "=d" (ret), "=&d" (oldval), "=&d" (newval),	\
		       "=m" (*uaddr)					\
		     : "0" (-EFAULT), "d" (oparg), "a" (uaddr),		\
		       "m" (*uaddr) : "cc" );

static int __futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old)
{
	int oldval = 0, newval, ret;

	switch (op) {
	case FUTEX_OP_SET:
		__futex_atomic_op("lr %2,%5\n",
				  ret, oldval, newval, uaddr, oparg);
		break;
	case FUTEX_OP_ADD:
		__futex_atomic_op("lr %2,%1\nar %2,%5\n",
				  ret, oldval, newval, uaddr, oparg);
		break;
	case FUTEX_OP_OR:
		__futex_atomic_op("lr %2,%1\nor %2,%5\n",
				  ret, oldval, newval, uaddr, oparg);
		break;
	case FUTEX_OP_ANDN:
		__futex_atomic_op("lr %2,%1\nnr %2,%5\n",
				  ret, oldval, newval, uaddr, oparg);
		break;
	case FUTEX_OP_XOR:
		__futex_atomic_op("lr %2,%1\nxr %2,%5\n",
				  ret, oldval, newval, uaddr, oparg);
		break;
	default:
		ret = -ENOSYS;
	}
	if (ret == 0)
		*old = oldval;
	return ret;
}

int futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old)
{
	int ret;

	if (segment_eq(get_fs(), KERNEL_DS))
		return __futex_atomic_op_pt(op, uaddr, oparg, old);
	spin_lock(&current->mm->page_table_lock);
	uaddr = (u32 __force __user *)
		__dat_user_addr((__force unsigned long) uaddr, 1);
	if (!uaddr) {
		spin_unlock(&current->mm->page_table_lock);
		return -EFAULT;
	}
	get_page(virt_to_page(uaddr));
	spin_unlock(&current->mm->page_table_lock);
	ret = __futex_atomic_op_pt(op, uaddr, oparg, old);
	put_page(virt_to_page(uaddr));
	return ret;
}

static int __futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
				     u32 oldval, u32 newval)
{
	int ret;

	asm volatile("0: cs   %1,%4,0(%5)\n"
		     "1: la   %0,0\n"
		     "2:\n"
		     EX_TABLE(0b,2b) EX_TABLE(1b,2b)
		     : "=d" (ret), "+d" (oldval), "=m" (*uaddr)
		     : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr)
		     : "cc", "memory" );
	*uval = oldval;
	return ret;
}

int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
			    u32 oldval, u32 newval)
{
	int ret;

	if (segment_eq(get_fs(), KERNEL_DS))
		return __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval);
	spin_lock(&current->mm->page_table_lock);
	uaddr = (u32 __force __user *)
		__dat_user_addr((__force unsigned long) uaddr, 1);
	if (!uaddr) {
		spin_unlock(&current->mm->page_table_lock);
		return -EFAULT;
	}
	get_page(virt_to_page(uaddr));
	spin_unlock(&current->mm->page_table_lock);
	ret = __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval);
	put_page(virt_to_page(uaddr));
	return ret;
}

struct uaccess_ops uaccess_pt = {
	.copy_from_user		= copy_from_user_pt,
	.copy_from_user_small	= copy_from_user_pt,
	.copy_to_user		= copy_to_user_pt,
	.copy_to_user_small	= copy_to_user_pt,
	.copy_in_user		= copy_in_user_pt,
	.clear_user		= clear_user_pt,
	.strnlen_user		= strnlen_user_pt,
	.strncpy_from_user	= strncpy_from_user_pt,
	.futex_atomic_op	= futex_atomic_op_pt,
	.futex_atomic_cmpxchg	= futex_atomic_cmpxchg_pt,
};
Commit	Line	Data
59f35d53	1	/*
c1821c2e GS	2	* User access functions based on page table walks for enhanced
c1821c2e GS	3	* system layout without hardware support.
59f35d53	4	*
4db84d4f	5	* Copyright IBM Corp. 2006, 2012
59f35d53 GS	6	* Author(s): Gerald Schaefer (gerald.schaefer@de.ibm.com)
	7	*/
	8
	9	#include <linux/errno.h>
d8ad075e	10	#include <linux/hardirq.h>
59f35d53	11	#include <linux/mm.h>
4db84d4f	12	#include <linux/hugetlb.h>
22155914	13	#include <asm/uaccess.h>
59f35d53	14	#include <asm/futex.h>
2b67fc46	15	#include "uaccess.h"
59f35d53	16
4db84d4f GS	17
	18	/*
	19	* Returns kernel address for user virtual address. If the returned address is
	20	* >= -4095 (IS_ERR_VALUE(x) returns true), a fault has occured and the address
	21	* contains the (negative) exception code.
	22	*/
	23	static __always_inline unsigned long follow_table(struct mm_struct *mm,
	24	unsigned long addr, int write)
e4aa402e MS	25	{
e4aa402e MS	26	pgd_t *pgd;
190a1d72	27	pud_t *pud;
e4aa402e	28	pmd_t *pmd;
4db84d4f	29	pte_t *ptep;
e4aa402e MS	30
	31	pgd = pgd_offset(mm, addr);
	32	if (pgd_none(pgd) \|\| unlikely(pgd_bad(pgd)))
4db84d4f	33	return -0x3aUL;
e4aa402e	34
190a1d72 MS	35	pud = pud_offset(pgd, addr);
190a1d72 MS	36	if (pud_none(pud) \|\| unlikely(pud_bad(pud)))
4db84d4f	37	return -0x3bUL;
190a1d72 MS	38
190a1d72 MS	39	pmd = pmd_offset(pud, addr);
4db84d4f GS	40	if (pmd_none(*pmd))
4db84d4f GS	41	return -0x10UL;
156152f8	42	if (pmd_large(*pmd)) {
4db84d4f GS	43	if (write && (pmd_val(*pmd) & _SEGMENT_ENTRY_RO))
	44	return -0x04UL;
	45	return (pmd_val(*pmd) & HPAGE_MASK) + (addr & ~HPAGE_MASK);
	46	}
	47	if (unlikely(pmd_bad(*pmd)))
	48	return -0x10UL;
	49
	50	ptep = pte_offset_map(pmd, addr);
	51	if (!pte_present(*ptep))
	52	return -0x11UL;
abf09bed	53	if (write && (!pte_write(ptep) \|\| !pte_dirty(ptep)))
4db84d4f	54	return -0x04UL;
e4aa402e	55
4db84d4f	56	return (pte_val(*ptep) & PAGE_MASK) + (addr & ~PAGE_MASK);
e4aa402e MS	57	}
e4aa402e MS	58
6c1e3e79 GS	59	static __always_inline size_t __user_copy_pt(unsigned long uaddr, void *kptr,
6c1e3e79 GS	60	size_t n, int write_user)
59f35d53 GS	61	{
59f35d53 GS	62	struct mm_struct *mm = current->mm;
4db84d4f	63	unsigned long offset, done, size, kaddr;
59f35d53 GS	64	void from, to;
	65
	66	done = 0;
	67	retry:
	68	spin_lock(&mm->page_table_lock);
	69	do {
4db84d4f GS	70	kaddr = follow_table(mm, uaddr, write_user);
4db84d4f GS	71	if (IS_ERR_VALUE(kaddr))
6c1e3e79	72	goto fault;
59f35d53	73
4db84d4f	74	offset = uaddr & ~PAGE_MASK;
59f35d53 GS	75	size = min(n - done, PAGE_SIZE - offset);
59f35d53 GS	76	if (write_user) {
4db84d4f	77	to = (void *) kaddr;
59f35d53 GS	78	from = kptr + done;
59f35d53 GS	79	} else {
4db84d4f	80	from = (void *) kaddr;
59f35d53 GS	81	to = kptr + done;
	82	}
	83	memcpy(to, from, size);
	84	done += size;
	85	uaddr += size;
	86	} while (done < n);
59f35d53 GS	87	spin_unlock(&mm->page_table_lock);
	88	return n - done;
	89	fault:
	90	spin_unlock(&mm->page_table_lock);
4db84d4f	91	if (__handle_fault(uaddr, -kaddr, write_user))
59f35d53 GS	92	return n - done;
	93	goto retry;
	94	}
	95
c1821c2e GS	96	/*
	97	* Do DAT for user address by page table walk, return kernel address.
	98	* This function needs to be called with current->mm->page_table_lock held.
	99	*/
4db84d4f GS	100	static __always_inline unsigned long __dat_user_addr(unsigned long uaddr,
4db84d4f GS	101	int write)
c1821c2e GS	102	{
c1821c2e GS	103	struct mm_struct *mm = current->mm;
4db84d4f	104	unsigned long kaddr;
c1821c2e GS	105	int rc;
c1821c2e GS	106
c1821c2e	107	retry:
4db84d4f GS	108	kaddr = follow_table(mm, uaddr, write);
4db84d4f GS	109	if (IS_ERR_VALUE(kaddr))
c1821c2e GS	110	goto fault;
c1821c2e GS	111
4db84d4f	112	return kaddr;
c1821c2e GS	113	fault:
c1821c2e GS	114	spin_unlock(&mm->page_table_lock);
4db84d4f	115	rc = __handle_fault(uaddr, -kaddr, write);
c1821c2e	116	spin_lock(&mm->page_table_lock);
6c1e3e79 GS	117	if (!rc)
	118	goto retry;
	119	return 0;
c1821c2e GS	120	}
c1821c2e GS	121
59f35d53 GS	122	size_t copy_from_user_pt(size_t n, const void __user from, void to)
	123	{
	124	size_t rc;
	125
	126	if (segment_eq(get_fs(), KERNEL_DS)) {
	127	memcpy(to, (void __kernel __force *) from, n);
	128	return 0;
	129	}
	130	rc = __user_copy_pt((unsigned long) from, to, n, 0);
	131	if (unlikely(rc))
	132	memset(to + n - rc, 0, rc);
	133	return rc;
	134	}
	135
	136	size_t copy_to_user_pt(size_t n, void __user to, const void from)
	137	{
	138	if (segment_eq(get_fs(), KERNEL_DS)) {
	139	memcpy((void __kernel __force *) to, from, n);
	140	return 0;
	141	}
	142	return __user_copy_pt((unsigned long) to, (void *) from, n, 1);
	143	}
c1821c2e GS	144
	145	static size_t clear_user_pt(size_t n, void __user *to)
	146	{
	147	long done, size, ret;
	148
	149	if (segment_eq(get_fs(), KERNEL_DS)) {
	150	memset((void __kernel __force *) to, 0, n);
	151	return 0;
	152	}
	153	done = 0;
	154	do {
	155	if (n - done > PAGE_SIZE)
	156	size = PAGE_SIZE;
	157	else
	158	size = n - done;
	159	ret = __user_copy_pt((unsigned long) to + done,
	160	&empty_zero_page, size, 1);
	161	done += size;
	162	if (ret)
	163	return ret + n - done;
	164	} while (done < n);
	165	return 0;
	166	}
	167
	168	static size_t strnlen_user_pt(size_t count, const char __user *src)
	169	{
c1821c2e GS	170	unsigned long uaddr = (unsigned long) src;
c1821c2e GS	171	struct mm_struct *mm = current->mm;
4db84d4f	172	unsigned long offset, done, len, kaddr;
c1821c2e GS	173	size_t len_str;
	174
	175	if (segment_eq(get_fs(), KERNEL_DS))
	176	return strnlen((const char __kernel __force *) src, count) + 1;
	177	done = 0;
	178	retry:
	179	spin_lock(&mm->page_table_lock);
	180	do {
4db84d4f GS	181	kaddr = follow_table(mm, uaddr, 0);
4db84d4f GS	182	if (IS_ERR_VALUE(kaddr))
6c1e3e79	183	goto fault;
c1821c2e	184
4db84d4f	185	offset = uaddr & ~PAGE_MASK;
c1821c2e	186	len = min(count - done, PAGE_SIZE - offset);
4db84d4f	187	len_str = strnlen((char *) kaddr, len);
c1821c2e GS	188	done += len_str;
	189	uaddr += len_str;
	190	} while ((len_str == len) && (done < count));
c1821c2e GS	191	spin_unlock(&mm->page_table_lock);
	192	return done + 1;
	193	fault:
	194	spin_unlock(&mm->page_table_lock);
4db84d4f	195	if (__handle_fault(uaddr, -kaddr, 0))
c1821c2e	196	return 0;
c1821c2e GS	197	goto retry;
	198	}
	199
	200	static size_t strncpy_from_user_pt(size_t count, const char __user *src,
	201	char *dst)
	202	{
	203	size_t n = strnlen_user_pt(count, src);
	204
	205	if (!n)
	206	return -EFAULT;
	207	if (n > count)
	208	n = count;
	209	if (segment_eq(get_fs(), KERNEL_DS)) {
	210	memcpy(dst, (const char __kernel __force *) src, n);
	211	if (dst[n-1] == '\0')
	212	return n-1;
	213	else
	214	return n;
	215	}
	216	if (__user_copy_pt((unsigned long) src, dst, n, 0))
	217	return -EFAULT;
	218	if (dst[n-1] == '\0')
	219	return n-1;
	220	else
	221	return n;
	222	}
	223
	224	static size_t copy_in_user_pt(size_t n, void __user *to,
	225	const void __user *from)
	226	{
	227	struct mm_struct *mm = current->mm;
4db84d4f	228	unsigned long offset_max, uaddr, done, size, error_code;
c1821c2e GS	229	unsigned long uaddr_from = (unsigned long) from;
c1821c2e GS	230	unsigned long uaddr_to = (unsigned long) to;
4db84d4f	231	unsigned long kaddr_to, kaddr_from;
c1821c2e GS	232	int write_user;
c1821c2e GS	233
3f12ebce HC	234	if (segment_eq(get_fs(), KERNEL_DS)) {
	235	memcpy((void __force ) to, (void __force ) from, n);
	236	return 0;
	237	}
c1821c2e GS	238	done = 0;
	239	retry:
	240	spin_lock(&mm->page_table_lock);
	241	do {
6c1e3e79 GS	242	write_user = 0;
6c1e3e79 GS	243	uaddr = uaddr_from;
4db84d4f GS	244	kaddr_from = follow_table(mm, uaddr_from, 0);
	245	error_code = kaddr_from;
	246	if (IS_ERR_VALUE(error_code))
c1821c2e	247	goto fault;
e4aa402e	248
6c1e3e79 GS	249	write_user = 1;
6c1e3e79 GS	250	uaddr = uaddr_to;
4db84d4f GS	251	kaddr_to = follow_table(mm, uaddr_to, 1);
	252	error_code = (unsigned long) kaddr_to;
	253	if (IS_ERR_VALUE(error_code))
6c1e3e79	254	goto fault;
c1821c2e	255
4db84d4f GS	256	offset_max = max(uaddr_from & ~PAGE_MASK,
4db84d4f GS	257	uaddr_to & ~PAGE_MASK);
c1821c2e GS	258	size = min(n - done, PAGE_SIZE - offset_max);
c1821c2e GS	259
4db84d4f	260	memcpy((void ) kaddr_to, (void ) kaddr_from, size);
c1821c2e GS	261	done += size;
	262	uaddr_from += size;
	263	uaddr_to += size;
	264	} while (done < n);
c1821c2e GS	265	spin_unlock(&mm->page_table_lock);
	266	return n - done;
	267	fault:
	268	spin_unlock(&mm->page_table_lock);
4db84d4f	269	if (__handle_fault(uaddr, -error_code, write_user))
c1821c2e GS	270	return n - done;
	271	goto retry;
	272	}
	273
	274	#define __futex_atomic_op(insn, ret, oldval, newval, uaddr, oparg) \
	275	asm volatile("0: l %1,0(%6)\n" \
	276	"1: " insn \
	277	"2: cs %1,%2,0(%6)\n" \
	278	"3: jl 1b\n" \
	279	" lhi %0,0\n" \
	280	"4:\n" \
	281	EX_TABLE(0b,4b) EX_TABLE(2b,4b) EX_TABLE(3b,4b) \
	282	: "=d" (ret), "=&d" (oldval), "=&d" (newval), \
	283	"=m" (*uaddr) \
	284	: "0" (-EFAULT), "d" (oparg), "a" (uaddr), \
	285	"m" (*uaddr) : "cc" );
	286
8d7718aa	287	static int __futex_atomic_op_pt(int op, u32 __user uaddr, int oparg, int old)
c1821c2e GS	288	{
	289	int oldval = 0, newval, ret;
	290
c1821c2e GS	291	switch (op) {
	292	case FUTEX_OP_SET:
	293	__futex_atomic_op("lr %2,%5\n",
	294	ret, oldval, newval, uaddr, oparg);
	295	break;
	296	case FUTEX_OP_ADD:
	297	__futex_atomic_op("lr %2,%1\nar %2,%5\n",
	298	ret, oldval, newval, uaddr, oparg);
	299	break;
	300	case FUTEX_OP_OR:
	301	__futex_atomic_op("lr %2,%1\nor %2,%5\n",
	302	ret, oldval, newval, uaddr, oparg);
	303	break;
	304	case FUTEX_OP_ANDN:
	305	__futex_atomic_op("lr %2,%1\nnr %2,%5\n",
	306	ret, oldval, newval, uaddr, oparg);
	307	break;
	308	case FUTEX_OP_XOR:
	309	__futex_atomic_op("lr %2,%1\nxr %2,%5\n",
	310	ret, oldval, newval, uaddr, oparg);
	311	break;
	312	default:
	313	ret = -ENOSYS;
	314	}
3f12ebce HC	315	if (ret == 0)
3f12ebce HC	316	*old = oldval;
c1821c2e GS	317	return ret;
	318	}
	319
8d7718aa	320	int futex_atomic_op_pt(int op, u32 __user uaddr, int oparg, int old)
c1821c2e GS	321	{
	322	int ret;
	323
3f12ebce HC	324	if (segment_eq(get_fs(), KERNEL_DS))
3f12ebce HC	325	return __futex_atomic_op_pt(op, uaddr, oparg, old);
c1821c2e	326	spin_lock(&current->mm->page_table_lock);
3c52e49d	327	uaddr = (u32 __force __user *)
4db84d4f	328	__dat_user_addr((__force unsigned long) uaddr, 1);
c1821c2e GS	329	if (!uaddr) {
	330	spin_unlock(&current->mm->page_table_lock);
	331	return -EFAULT;
	332	}
	333	get_page(virt_to_page(uaddr));
	334	spin_unlock(&current->mm->page_table_lock);
3f12ebce HC	335	ret = __futex_atomic_op_pt(op, uaddr, oparg, old);
	336	put_page(virt_to_page(uaddr));
	337	return ret;
	338	}
	339
8d7718aa ML	340	static int __futex_atomic_cmpxchg_pt(u32 uval, u32 __user uaddr,
8d7718aa ML	341	u32 oldval, u32 newval)
3f12ebce HC	342	{
	343	int ret;
	344
	345	asm volatile("0: cs %1,%4,0(%5)\n"
37a9d912	346	"1: la %0,0\n"
3f12ebce HC	347	"2:\n"
3f12ebce HC	348	EX_TABLE(0b,2b) EX_TABLE(1b,2b)
c1821c2e GS	349	: "=d" (ret), "+d" (oldval), "=m" (*uaddr)
	350	: "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr)
	351	: "cc", "memory" );
37a9d912	352	*uval = oldval;
3f12ebce HC	353	return ret;
	354	}
	355
8d7718aa ML	356	int futex_atomic_cmpxchg_pt(u32 uval, u32 __user uaddr,
8d7718aa ML	357	u32 oldval, u32 newval)
3f12ebce HC	358	{
	359	int ret;
	360
	361	if (segment_eq(get_fs(), KERNEL_DS))
37a9d912	362	return __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval);
3f12ebce	363	spin_lock(&current->mm->page_table_lock);
3c52e49d	364	uaddr = (u32 __force __user *)
4db84d4f	365	__dat_user_addr((__force unsigned long) uaddr, 1);
3f12ebce HC	366	if (!uaddr) {
	367	spin_unlock(&current->mm->page_table_lock);
	368	return -EFAULT;
	369	}
	370	get_page(virt_to_page(uaddr));
	371	spin_unlock(&current->mm->page_table_lock);
37a9d912	372	ret = __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval);
c1821c2e GS	373	put_page(virt_to_page(uaddr));
	374	return ret;
	375	}
	376
	377	struct uaccess_ops uaccess_pt = {
	378	.copy_from_user = copy_from_user_pt,
	379	.copy_from_user_small = copy_from_user_pt,
	380	.copy_to_user = copy_to_user_pt,
	381	.copy_to_user_small = copy_to_user_pt,
	382	.copy_in_user = copy_in_user_pt,
	383	.clear_user = clear_user_pt,
	384	.strnlen_user = strnlen_user_pt,
	385	.strncpy_from_user = strncpy_from_user_pt,
	386	.futex_atomic_op = futex_atomic_op_pt,
	387	.futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt,
	388	};