[deliverable/linux.git] / arch / x86 / include / asm / uaccess_64.h

#ifndef _ASM_X86_UACCESS_64_H
#define _ASM_X86_UACCESS_64_H

/*
 * User space memory access functions
 */
#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/lockdep.h>
#include <linux/kasan-checks.h>
#include <asm/alternative.h>
#include <asm/cpufeatures.h>
#include <asm/page.h>

/*
 * Copy To/From Userspace
 */

/* Handles exceptions in both to and from, but doesn't do access_ok */
__must_check unsigned long
copy_user_enhanced_fast_string(void *to, const void *from, unsigned len);
__must_check unsigned long
copy_user_generic_string(void *to, const void *from, unsigned len);
__must_check unsigned long
copy_user_generic_unrolled(void *to, const void *from, unsigned len);

static __always_inline __must_check unsigned long
copy_user_generic(void *to, const void *from, unsigned len)
{
	unsigned ret;

	/*
	 * If CPU has ERMS feature, use copy_user_enhanced_fast_string.
	 * Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
	 * Otherwise, use copy_user_generic_unrolled.
	 */
	alternative_call_2(copy_user_generic_unrolled,
			 copy_user_generic_string,
			 X86_FEATURE_REP_GOOD,
			 copy_user_enhanced_fast_string,
			 X86_FEATURE_ERMS,
			 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
				     "=d" (len)),
			 "1" (to), "2" (from), "3" (len)
			 : "memory", "rcx", "r8", "r9", "r10", "r11");
	return ret;
}

__must_check unsigned long
copy_in_user(void __user *to, const void __user *from, unsigned len);

static __always_inline __must_check
int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size)
{
	int ret = 0;

	check_object_size(dst, size, false);
	if (!__builtin_constant_p(size))
		return copy_user_generic(dst, (__force void *)src, size);
	switch (size) {
	case 1:
		__uaccess_begin();
		__get_user_asm(*(u8 *)dst, (u8 __user *)src,
			      ret, "b", "b", "=q", 1);
		__uaccess_end();
		return ret;
	case 2:
		__uaccess_begin();
		__get_user_asm(*(u16 *)dst, (u16 __user *)src,
			      ret, "w", "w", "=r", 2);
		__uaccess_end();
		return ret;
	case 4:
		__uaccess_begin();
		__get_user_asm(*(u32 *)dst, (u32 __user *)src,
			      ret, "l", "k", "=r", 4);
		__uaccess_end();
		return ret;
	case 8:
		__uaccess_begin();
		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
			      ret, "q", "", "=r", 8);
		__uaccess_end();
		return ret;
	case 10:
		__uaccess_begin();
		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
			       ret, "q", "", "=r", 10);
		if (likely(!ret))
			__get_user_asm(*(u16 *)(8 + (char *)dst),
				       (u16 __user *)(8 + (char __user *)src),
				       ret, "w", "w", "=r", 2);
		__uaccess_end();
		return ret;
	case 16:
		__uaccess_begin();
		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
			       ret, "q", "", "=r", 16);
		if (likely(!ret))
			__get_user_asm(*(u64 *)(8 + (char *)dst),
				       (u64 __user *)(8 + (char __user *)src),
				       ret, "q", "", "=r", 8);
		__uaccess_end();
		return ret;
	default:
		return copy_user_generic(dst, (__force void *)src, size);
	}
}

static __always_inline __must_check
int __copy_from_user(void *dst, const void __user *src, unsigned size)
{
	might_fault();
	kasan_check_write(dst, size);
	return __copy_from_user_nocheck(dst, src, size);
}

static __always_inline __must_check
int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size)
{
	int ret = 0;

	check_object_size(src, size, true);
	if (!__builtin_constant_p(size))
		return copy_user_generic((__force void *)dst, src, size);
	switch (size) {
	case 1:
		__uaccess_begin();
		__put_user_asm(*(u8 *)src, (u8 __user *)dst,
			      ret, "b", "b", "iq", 1);
		__uaccess_end();
		return ret;
	case 2:
		__uaccess_begin();
		__put_user_asm(*(u16 *)src, (u16 __user *)dst,
			      ret, "w", "w", "ir", 2);
		__uaccess_end();
		return ret;
	case 4:
		__uaccess_begin();
		__put_user_asm(*(u32 *)src, (u32 __user *)dst,
			      ret, "l", "k", "ir", 4);
		__uaccess_end();
		return ret;
	case 8:
		__uaccess_begin();
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
			      ret, "q", "", "er", 8);
		__uaccess_end();
		return ret;
	case 10:
		__uaccess_begin();
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
			       ret, "q", "", "er", 10);
		if (likely(!ret)) {
			asm("":::"memory");
			__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
				       ret, "w", "w", "ir", 2);
		}
		__uaccess_end();
		return ret;
	case 16:
		__uaccess_begin();
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
			       ret, "q", "", "er", 16);
		if (likely(!ret)) {
			asm("":::"memory");
			__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
				       ret, "q", "", "er", 8);
		}
		__uaccess_end();
		return ret;
	default:
		return copy_user_generic((__force void *)dst, src, size);
	}
}

static __always_inline __must_check
int __copy_to_user(void __user *dst, const void *src, unsigned size)
{
	might_fault();
	kasan_check_read(src, size);
	return __copy_to_user_nocheck(dst, src, size);
}

static __always_inline __must_check
int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
{
	int ret = 0;

	might_fault();
	if (!__builtin_constant_p(size))
		return copy_user_generic((__force void *)dst,
					 (__force void *)src, size);
	switch (size) {
	case 1: {
		u8 tmp;
		__uaccess_begin();
		__get_user_asm(tmp, (u8 __user *)src,
			       ret, "b", "b", "=q", 1);
		if (likely(!ret))
			__put_user_asm(tmp, (u8 __user *)dst,
				       ret, "b", "b", "iq", 1);
		__uaccess_end();
		return ret;
	}
	case 2: {
		u16 tmp;
		__uaccess_begin();
		__get_user_asm(tmp, (u16 __user *)src,
			       ret, "w", "w", "=r", 2);
		if (likely(!ret))
			__put_user_asm(tmp, (u16 __user *)dst,
				       ret, "w", "w", "ir", 2);
		__uaccess_end();
		return ret;
	}

	case 4: {
		u32 tmp;
		__uaccess_begin();
		__get_user_asm(tmp, (u32 __user *)src,
			       ret, "l", "k", "=r", 4);
		if (likely(!ret))
			__put_user_asm(tmp, (u32 __user *)dst,
				       ret, "l", "k", "ir", 4);
		__uaccess_end();
		return ret;
	}
	case 8: {
		u64 tmp;
		__uaccess_begin();
		__get_user_asm(tmp, (u64 __user *)src,
			       ret, "q", "", "=r", 8);
		if (likely(!ret))
			__put_user_asm(tmp, (u64 __user *)dst,
				       ret, "q", "", "er", 8);
		__uaccess_end();
		return ret;
	}
	default:
		return copy_user_generic((__force void *)dst,
					 (__force void *)src, size);
	}
}

static __must_check __always_inline int
__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
{
	kasan_check_write(dst, size);
	return __copy_from_user_nocheck(dst, src, size);
}

static __must_check __always_inline int
__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
{
	kasan_check_read(src, size);
	return __copy_to_user_nocheck(dst, src, size);
}

extern long __copy_user_nocache(void *dst, const void __user *src,
				unsigned size, int zerorest);

static inline int
__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
{
	might_fault();
	kasan_check_write(dst, size);
	return __copy_user_nocache(dst, src, size, 1);
}

static inline int
__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
				  unsigned size)
{
	kasan_check_write(dst, size);
	return __copy_user_nocache(dst, src, size, 0);
}

unsigned long
copy_user_handle_tail(char *to, char *from, unsigned len);

#endif /* _ASM_X86_UACCESS_64_H */
Commit	Line	Data
1965aae3 PA	1	#ifndef _ASM_X86_UACCESS_64_H
1965aae3 PA	2	#define _ASM_X86_UACCESS_64_H
1da177e4 LT	3
	4	/*
	5	* User space memory access functions
	6	*/
1da177e4 LT	7	#include <linux/compiler.h>
1da177e4 LT	8	#include <linux/errno.h>
16dbc6c9	9	#include <linux/lockdep.h>
1771c6e1	10	#include <linux/kasan-checks.h>
1b1d9258	11	#include <asm/alternative.h>
cd4d09ec	12	#include <asm/cpufeatures.h>
1da177e4 LT	13	#include <asm/page.h>
1da177e4 LT	14
1da177e4 LT	15	/*
	16	* Copy To/From Userspace
	17	*/
	18
	19	/* Handles exceptions in both to and from, but doesn't do access_ok */
95912008	20	__must_check unsigned long
954e482b FY	21	copy_user_enhanced_fast_string(void to, const void from, unsigned len);
954e482b FY	22	__must_check unsigned long
1b1d9258 JB	23	copy_user_generic_string(void to, const void from, unsigned len);
	24	__must_check unsigned long
	25	copy_user_generic_unrolled(void to, const void from, unsigned len);
	26
	27	static __always_inline __must_check unsigned long
	28	copy_user_generic(void to, const void from, unsigned len)
	29	{
	30	unsigned ret;
	31
954e482b FY	32	/*
	33	* If CPU has ERMS feature, use copy_user_enhanced_fast_string.
	34	* Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
	35	* Otherwise, use copy_user_generic_unrolled.
	36	*/
	37	alternative_call_2(copy_user_generic_unrolled,
1b1d9258 JB	38	copy_user_generic_string,
1b1d9258 JB	39	X86_FEATURE_REP_GOOD,
954e482b FY	40	copy_user_enhanced_fast_string,
954e482b FY	41	X86_FEATURE_ERMS,
1b1d9258 JB	42	ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
	43	"=d" (len)),
	44	"1" (to), "2" (from), "3" (len)
	45	: "memory", "rcx", "r8", "r9", "r10", "r11");
	46	return ret;
	47	}
95912008	48
95912008	49	__must_check unsigned long
95912008 AK	50	copy_in_user(void __user to, const void __user from, unsigned len);
	51
	52	static __always_inline __must_check
ff47ab4f	53	int __copy_from_user_nocheck(void dst, const void __user src, unsigned size)
b896313e	54	{
383d079b	55	int ret = 0;
c10d38dd	56
5b710f34	57	check_object_size(dst, size, false);
1da177e4	58	if (!__builtin_constant_p(size))
b896313e JP	59	return copy_user_generic(dst, (__force void *)src, size);
b896313e JP	60	switch (size) {
11f1a4b9 LT	61	case 1:
	62	__uaccess_begin();
	63	__get_user_asm((u8 )dst, (u8 __user *)src,
b896313e	64	ret, "b", "b", "=q", 1);
11f1a4b9	65	__uaccess_end();
1da177e4	66	return ret;
11f1a4b9 LT	67	case 2:
	68	__uaccess_begin();
	69	__get_user_asm((u16 )dst, (u16 __user *)src,
b896313e	70	ret, "w", "w", "=r", 2);
11f1a4b9	71	__uaccess_end();
1da177e4	72	return ret;
11f1a4b9 LT	73	case 4:
	74	__uaccess_begin();
	75	__get_user_asm((u32 )dst, (u32 __user *)src,
b896313e	76	ret, "l", "k", "=r", 4);
11f1a4b9	77	__uaccess_end();
b896313e	78	return ret;
11f1a4b9 LT	79	case 8:
	80	__uaccess_begin();
	81	__get_user_asm((u64 )dst, (u64 __user *)src,
b896313e	82	ret, "q", "", "=r", 8);
11f1a4b9	83	__uaccess_end();
1da177e4	84	return ret;
1da177e4	85	case 10:
11f1a4b9	86	__uaccess_begin();
b896313e	87	__get_user_asm((u64 )dst, (u64 __user *)src,
20a4a236	88	ret, "q", "", "=r", 10);
11f1a4b9 LT	89	if (likely(!ret))
	90	__get_user_asm((u16 )(8 + (char *)dst),
	91	(u16 __user )(8 + (char __user )src),
	92	ret, "w", "w", "=r", 2);
	93	__uaccess_end();
b896313e	94	return ret;
1da177e4	95	case 16:
11f1a4b9	96	__uaccess_begin();
b896313e JP	97	__get_user_asm((u64 )dst, (u64 __user *)src,
b896313e JP	98	ret, "q", "", "=r", 16);
11f1a4b9 LT	99	if (likely(!ret))
	100	__get_user_asm((u64 )(8 + (char *)dst),
	101	(u64 __user )(8 + (char __user )src),
	102	ret, "q", "", "=r", 8);
	103	__uaccess_end();
b896313e	104	return ret;
1da177e4	105	default:
b896313e	106	return copy_user_generic(dst, (__force void *)src, size);
1da177e4	107	}
b896313e	108	}
1da177e4	109
95912008	110	static __always_inline __must_check
ff47ab4f AK	111	int __copy_from_user(void dst, const void __user src, unsigned size)
	112	{
	113	might_fault();
1771c6e1	114	kasan_check_write(dst, size);
ff47ab4f AK	115	return __copy_from_user_nocheck(dst, src, size);
	116	}
	117
	118	static __always_inline __must_check
	119	int __copy_to_user_nocheck(void __user dst, const void src, unsigned size)
b896313e	120	{
383d079b	121	int ret = 0;
c10d38dd	122
5b710f34	123	check_object_size(src, size, true);
1da177e4	124	if (!__builtin_constant_p(size))
b896313e JP	125	return copy_user_generic((__force void *)dst, src, size);
b896313e JP	126	switch (size) {
11f1a4b9 LT	127	case 1:
	128	__uaccess_begin();
	129	__put_user_asm((u8 )src, (u8 __user *)dst,
b896313e	130	ret, "b", "b", "iq", 1);
11f1a4b9	131	__uaccess_end();
1da177e4	132	return ret;
11f1a4b9 LT	133	case 2:
	134	__uaccess_begin();
	135	__put_user_asm((u16 )src, (u16 __user *)dst,
b896313e	136	ret, "w", "w", "ir", 2);
11f1a4b9	137	__uaccess_end();
1da177e4	138	return ret;
11f1a4b9 LT	139	case 4:
	140	__uaccess_begin();
	141	__put_user_asm((u32 )src, (u32 __user *)dst,
b896313e	142	ret, "l", "k", "ir", 4);
11f1a4b9	143	__uaccess_end();
b896313e	144	return ret;
11f1a4b9 LT	145	case 8:
	146	__uaccess_begin();
	147	__put_user_asm((u64 )src, (u64 __user *)dst,
155b7352	148	ret, "q", "", "er", 8);
11f1a4b9	149	__uaccess_end();
1da177e4	150	return ret;
1da177e4	151	case 10:
11f1a4b9	152	__uaccess_begin();
b896313e	153	__put_user_asm((u64 )src, (u64 __user *)dst,
155b7352	154	ret, "q", "", "er", 10);
11f1a4b9 LT	155	if (likely(!ret)) {
	156	asm("":::"memory");
	157	__put_user_asm(4[(u16 )src], 4 + (u16 __user )dst,
	158	ret, "w", "w", "ir", 2);
	159	}
	160	__uaccess_end();
b896313e	161	return ret;
1da177e4	162	case 16:
11f1a4b9	163	__uaccess_begin();
b896313e	164	__put_user_asm((u64 )src, (u64 __user *)dst,
155b7352	165	ret, "q", "", "er", 16);
11f1a4b9 LT	166	if (likely(!ret)) {
	167	asm("":::"memory");
	168	__put_user_asm(1[(u64 )src], 1 + (u64 __user )dst,
	169	ret, "q", "", "er", 8);
	170	}
	171	__uaccess_end();
b896313e	172	return ret;
1da177e4	173	default:
b896313e	174	return copy_user_generic((__force void *)dst, src, size);
1da177e4	175	}
b896313e	176	}
1da177e4	177
ff47ab4f AK	178	static __always_inline __must_check
	179	int __copy_to_user(void __user dst, const void src, unsigned size)
	180	{
	181	might_fault();
1771c6e1	182	kasan_check_read(src, size);
ff47ab4f AK	183	return __copy_to_user_nocheck(dst, src, size);
	184	}
	185
95912008 AK	186	static __always_inline __must_check
95912008 AK	187	int __copy_in_user(void __user dst, const void __user src, unsigned size)
b896313e	188	{
383d079b	189	int ret = 0;
c10d38dd	190
3ee1afa3	191	might_fault();
1da177e4	192	if (!__builtin_constant_p(size))
b896313e JP	193	return copy_user_generic((__force void *)dst,
	194	(__force void *)src, size);
	195	switch (size) {
	196	case 1: {
1da177e4	197	u8 tmp;
11f1a4b9	198	__uaccess_begin();
b896313e JP	199	__get_user_asm(tmp, (u8 __user *)src,
b896313e JP	200	ret, "b", "b", "=q", 1);
1da177e4	201	if (likely(!ret))
b896313e JP	202	__put_user_asm(tmp, (u8 __user *)dst,
b896313e JP	203	ret, "b", "b", "iq", 1);
11f1a4b9	204	__uaccess_end();
1da177e4 LT	205	return ret;
1da177e4 LT	206	}
b896313e	207	case 2: {
1da177e4	208	u16 tmp;
11f1a4b9	209	__uaccess_begin();
b896313e JP	210	__get_user_asm(tmp, (u16 __user *)src,
b896313e JP	211	ret, "w", "w", "=r", 2);
1da177e4	212	if (likely(!ret))
b896313e JP	213	__put_user_asm(tmp, (u16 __user *)dst,
b896313e JP	214	ret, "w", "w", "ir", 2);
11f1a4b9	215	__uaccess_end();
1da177e4 LT	216	return ret;
	217	}
	218
b896313e	219	case 4: {
1da177e4	220	u32 tmp;
11f1a4b9	221	__uaccess_begin();
b896313e JP	222	__get_user_asm(tmp, (u32 __user *)src,
b896313e JP	223	ret, "l", "k", "=r", 4);
1da177e4	224	if (likely(!ret))
b896313e JP	225	__put_user_asm(tmp, (u32 __user *)dst,
b896313e JP	226	ret, "l", "k", "ir", 4);
11f1a4b9	227	__uaccess_end();
1da177e4 LT	228	return ret;
1da177e4 LT	229	}
b896313e	230	case 8: {
1da177e4	231	u64 tmp;
11f1a4b9	232	__uaccess_begin();
b896313e JP	233	__get_user_asm(tmp, (u64 __user *)src,
b896313e JP	234	ret, "q", "", "=r", 8);
1da177e4	235	if (likely(!ret))
b896313e	236	__put_user_asm(tmp, (u64 __user *)dst,
155b7352	237	ret, "q", "", "er", 8);
11f1a4b9	238	__uaccess_end();
1da177e4 LT	239	return ret;
	240	}
	241	default:
b896313e JP	242	return copy_user_generic((__force void *)dst,
b896313e JP	243	(__force void *)src, size);
1da177e4	244	}
b896313e	245	}
1da177e4	246
14722485 JB	247	static __must_check __always_inline int
	248	__copy_from_user_inatomic(void dst, const void __user src, unsigned size)
	249	{
1771c6e1	250	kasan_check_write(dst, size);
df90ca96	251	return __copy_from_user_nocheck(dst, src, size);
14722485	252	}
b885808e AK	253
	254	static __must_check __always_inline int
	255	__copy_to_user_inatomic(void __user dst, const void src, unsigned size)
	256	{
1771c6e1	257	kasan_check_read(src, size);
df90ca96	258	return __copy_to_user_nocheck(dst, src, size);
b885808e	259	}
1da177e4	260
b896313e JP	261	extern long __copy_user_nocache(void dst, const void __user src,
b896313e JP	262	unsigned size, int zerorest);
0812a579	263
f1800536 IM	264	static inline int
f1800536 IM	265	__copy_from_user_nocache(void dst, const void __user src, unsigned size)
0812a579	266	{
016be2e5	267	might_fault();
1771c6e1	268	kasan_check_write(dst, size);
f1800536	269	return __copy_user_nocache(dst, src, size, 1);
0812a579 AK	270	}
0812a579 AK	271
f1800536 IM	272	static inline int
	273	__copy_from_user_inatomic_nocache(void dst, const void __user src,
	274	unsigned size)
0812a579	275	{
1771c6e1	276	kasan_check_write(dst, size);
f1800536	277	return __copy_user_nocache(dst, src, size, 0);
0812a579 AK	278	}
0812a579 AK	279
1129585a	280	unsigned long
cae2a173	281	copy_user_handle_tail(char to, char from, unsigned len);
1129585a	282
1965aae3	283	#endif /* _ASM_X86_UACCESS_64_H */