2 * User address space access functions.
3 * The non inlined parts of asm-i386/uaccess.h are here.
5 * Copyright 1997 Andi Kleen <ak@muc.de>
6 * Copyright 1997 Linus Torvalds
9 #include <linux/highmem.h>
10 #include <linux/blkdev.h>
11 #include <linux/module.h>
12 #include <linux/backing-dev.h>
13 #include <linux/interrupt.h>
14 #include <asm/uaccess.h>
17 static inline int __movsl_is_ok(unsigned long a1
, unsigned long a2
, unsigned long n
)
19 #ifdef CONFIG_X86_INTEL_USERCOPY
20 if (n
>= 64 && ((a1
^ a2
) & movsl_mask
.mask
))
25 #define movsl_is_ok(a1, a2, n) \
26 __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
29 * Copy a null terminated string from userspace.
32 #define __do_strncpy_from_user(dst, src, count, res) \
34 int __d0, __d1, __d2; \
36 __asm__ __volatile__( \
41 " testb %%al,%%al\n" \
47 ".section .fixup,\"ax\"\n" \
52 : "=d"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1), \
54 : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \
59 * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
60 * @dst: Destination address, in kernel space. This buffer must be at
61 * least @count bytes long.
62 * @src: Source address, in user space.
63 * @count: Maximum number of bytes to copy, including the trailing NUL.
65 * Copies a NUL-terminated string from userspace to kernel space.
66 * Caller must check the specified block with access_ok() before calling
69 * On success, returns the length of the string (not including the trailing
72 * If access to userspace fails, returns -EFAULT (some data may have been
75 * If @count is smaller than the length of the string, copies @count bytes
79 __strncpy_from_user(char *dst
, const char __user
*src
, long count
)
82 __do_strncpy_from_user(dst
, src
, count
, res
);
85 EXPORT_SYMBOL(__strncpy_from_user
);
88 * strncpy_from_user: - Copy a NUL terminated string from userspace.
89 * @dst: Destination address, in kernel space. This buffer must be at
90 * least @count bytes long.
91 * @src: Source address, in user space.
92 * @count: Maximum number of bytes to copy, including the trailing NUL.
94 * Copies a NUL-terminated string from userspace to kernel space.
96 * On success, returns the length of the string (not including the trailing
99 * If access to userspace fails, returns -EFAULT (some data may have been
102 * If @count is smaller than the length of the string, copies @count bytes
103 * and returns @count.
106 strncpy_from_user(char *dst
, const char __user
*src
, long count
)
109 if (access_ok(VERIFY_READ
, src
, 1))
110 __do_strncpy_from_user(dst
, src
, count
, res
);
113 EXPORT_SYMBOL(strncpy_from_user
);
119 #define __do_clear_user(addr,size) \
123 __asm__ __volatile__( \
128 ".section .fixup,\"ax\"\n" \
129 "3: lea 0(%2,%0,4),%0\n" \
132 _ASM_EXTABLE(0b,3b) \
133 _ASM_EXTABLE(1b,2b) \
134 : "=&c"(size), "=&D" (__d0) \
135 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \
139 * clear_user: - Zero a block of memory in user space.
140 * @to: Destination address, in user space.
141 * @n: Number of bytes to zero.
143 * Zero a block of memory in user space.
145 * Returns number of bytes that could not be cleared.
146 * On success, this will be zero.
149 clear_user(void __user
*to
, unsigned long n
)
151 if (access_ok(VERIFY_WRITE
, to
, n
))
152 __do_clear_user(to
, n
);
155 EXPORT_SYMBOL(clear_user
);
158 * __clear_user: - Zero a block of memory in user space, with less checking.
159 * @to: Destination address, in user space.
160 * @n: Number of bytes to zero.
162 * Zero a block of memory in user space. Caller must check
163 * the specified block with access_ok() before calling this function.
165 * Returns number of bytes that could not be cleared.
166 * On success, this will be zero.
169 __clear_user(void __user
*to
, unsigned long n
)
171 __do_clear_user(to
, n
);
174 EXPORT_SYMBOL(__clear_user
);
177 * strnlen_user: - Get the size of a string in user space.
178 * @s: The string to measure.
179 * @n: The maximum valid length
181 * Get the size of a NUL-terminated string in user space.
183 * Returns the size of the string INCLUDING the terminating NUL.
184 * On exception, returns 0.
185 * If the string is too long, returns a value greater than @n.
187 long strnlen_user(const char __user
*s
, long n
)
189 unsigned long mask
= -__addr_ok(s
);
190 unsigned long res
, tmp
;
194 __asm__
__volatile__(
203 ".section .fixup,\"ax\"\n"
204 "2: xorl %%eax,%%eax\n"
209 ".section __ex_table,\"a\"\n"
213 :"=r" (n
), "=D" (s
), "=a" (res
), "=c" (tmp
)
214 :"0" (n
), "1" (s
), "2" (0), "3" (mask
)
218 EXPORT_SYMBOL(strnlen_user
);
220 #ifdef CONFIG_X86_INTEL_USERCOPY
222 __copy_user_intel(void __user
*to
, const void *from
, unsigned long size
)
225 __asm__
__volatile__(
227 "1: movl 32(%4), %%eax\n"
230 "2: movl 64(%4), %%eax\n"
232 "3: movl 0(%4), %%eax\n"
233 "4: movl 4(%4), %%edx\n"
234 "5: movl %%eax, 0(%3)\n"
235 "6: movl %%edx, 4(%3)\n"
236 "7: movl 8(%4), %%eax\n"
237 "8: movl 12(%4),%%edx\n"
238 "9: movl %%eax, 8(%3)\n"
239 "10: movl %%edx, 12(%3)\n"
240 "11: movl 16(%4), %%eax\n"
241 "12: movl 20(%4), %%edx\n"
242 "13: movl %%eax, 16(%3)\n"
243 "14: movl %%edx, 20(%3)\n"
244 "15: movl 24(%4), %%eax\n"
245 "16: movl 28(%4), %%edx\n"
246 "17: movl %%eax, 24(%3)\n"
247 "18: movl %%edx, 28(%3)\n"
248 "19: movl 32(%4), %%eax\n"
249 "20: movl 36(%4), %%edx\n"
250 "21: movl %%eax, 32(%3)\n"
251 "22: movl %%edx, 36(%3)\n"
252 "23: movl 40(%4), %%eax\n"
253 "24: movl 44(%4), %%edx\n"
254 "25: movl %%eax, 40(%3)\n"
255 "26: movl %%edx, 44(%3)\n"
256 "27: movl 48(%4), %%eax\n"
257 "28: movl 52(%4), %%edx\n"
258 "29: movl %%eax, 48(%3)\n"
259 "30: movl %%edx, 52(%3)\n"
260 "31: movl 56(%4), %%eax\n"
261 "32: movl 60(%4), %%edx\n"
262 "33: movl %%eax, 56(%3)\n"
263 "34: movl %%edx, 60(%3)\n"
269 "35: movl %0, %%eax\n"
274 "36: movl %%eax, %0\n"
277 ".section .fixup,\"ax\"\n"
278 "101: lea 0(%%eax,%0,4),%0\n"
281 ".section __ex_table,\"a\"\n"
322 : "=&c"(size
), "=&D" (d0
), "=&S" (d1
)
323 : "1"(to
), "2"(from
), "0"(size
)
324 : "eax", "edx", "memory");
329 __copy_user_zeroing_intel(void *to
, const void __user
*from
, unsigned long size
)
332 __asm__
__volatile__(
334 "0: movl 32(%4), %%eax\n"
337 "1: movl 64(%4), %%eax\n"
339 "2: movl 0(%4), %%eax\n"
340 "21: movl 4(%4), %%edx\n"
341 " movl %%eax, 0(%3)\n"
342 " movl %%edx, 4(%3)\n"
343 "3: movl 8(%4), %%eax\n"
344 "31: movl 12(%4),%%edx\n"
345 " movl %%eax, 8(%3)\n"
346 " movl %%edx, 12(%3)\n"
347 "4: movl 16(%4), %%eax\n"
348 "41: movl 20(%4), %%edx\n"
349 " movl %%eax, 16(%3)\n"
350 " movl %%edx, 20(%3)\n"
351 "10: movl 24(%4), %%eax\n"
352 "51: movl 28(%4), %%edx\n"
353 " movl %%eax, 24(%3)\n"
354 " movl %%edx, 28(%3)\n"
355 "11: movl 32(%4), %%eax\n"
356 "61: movl 36(%4), %%edx\n"
357 " movl %%eax, 32(%3)\n"
358 " movl %%edx, 36(%3)\n"
359 "12: movl 40(%4), %%eax\n"
360 "71: movl 44(%4), %%edx\n"
361 " movl %%eax, 40(%3)\n"
362 " movl %%edx, 44(%3)\n"
363 "13: movl 48(%4), %%eax\n"
364 "81: movl 52(%4), %%edx\n"
365 " movl %%eax, 48(%3)\n"
366 " movl %%edx, 52(%3)\n"
367 "14: movl 56(%4), %%eax\n"
368 "91: movl 60(%4), %%edx\n"
369 " movl %%eax, 56(%3)\n"
370 " movl %%edx, 60(%3)\n"
376 "5: movl %0, %%eax\n"
384 ".section .fixup,\"ax\"\n"
385 "9: lea 0(%%eax,%0,4),%0\n"
388 " xorl %%eax,%%eax\n"
394 ".section __ex_table,\"a\"\n"
417 : "=&c"(size
), "=&D" (d0
), "=&S" (d1
)
418 : "1"(to
), "2"(from
), "0"(size
)
419 : "eax", "edx", "memory");
424 * Non Temporal Hint version of __copy_user_zeroing_intel. It is cache aware.
425 * hyoshiok@miraclelinux.com
428 static unsigned long __copy_user_zeroing_intel_nocache(void *to
,
429 const void __user
*from
, unsigned long size
)
433 __asm__
__volatile__(
435 "0: movl 32(%4), %%eax\n"
438 "1: movl 64(%4), %%eax\n"
440 "2: movl 0(%4), %%eax\n"
441 "21: movl 4(%4), %%edx\n"
442 " movnti %%eax, 0(%3)\n"
443 " movnti %%edx, 4(%3)\n"
444 "3: movl 8(%4), %%eax\n"
445 "31: movl 12(%4),%%edx\n"
446 " movnti %%eax, 8(%3)\n"
447 " movnti %%edx, 12(%3)\n"
448 "4: movl 16(%4), %%eax\n"
449 "41: movl 20(%4), %%edx\n"
450 " movnti %%eax, 16(%3)\n"
451 " movnti %%edx, 20(%3)\n"
452 "10: movl 24(%4), %%eax\n"
453 "51: movl 28(%4), %%edx\n"
454 " movnti %%eax, 24(%3)\n"
455 " movnti %%edx, 28(%3)\n"
456 "11: movl 32(%4), %%eax\n"
457 "61: movl 36(%4), %%edx\n"
458 " movnti %%eax, 32(%3)\n"
459 " movnti %%edx, 36(%3)\n"
460 "12: movl 40(%4), %%eax\n"
461 "71: movl 44(%4), %%edx\n"
462 " movnti %%eax, 40(%3)\n"
463 " movnti %%edx, 44(%3)\n"
464 "13: movl 48(%4), %%eax\n"
465 "81: movl 52(%4), %%edx\n"
466 " movnti %%eax, 48(%3)\n"
467 " movnti %%edx, 52(%3)\n"
468 "14: movl 56(%4), %%eax\n"
469 "91: movl 60(%4), %%edx\n"
470 " movnti %%eax, 56(%3)\n"
471 " movnti %%edx, 60(%3)\n"
478 "5: movl %0, %%eax\n"
486 ".section .fixup,\"ax\"\n"
487 "9: lea 0(%%eax,%0,4),%0\n"
490 " xorl %%eax,%%eax\n"
496 ".section __ex_table,\"a\"\n"
519 : "=&c"(size
), "=&D" (d0
), "=&S" (d1
)
520 : "1"(to
), "2"(from
), "0"(size
)
521 : "eax", "edx", "memory");
525 static unsigned long __copy_user_intel_nocache(void *to
,
526 const void __user
*from
, unsigned long size
)
530 __asm__
__volatile__(
532 "0: movl 32(%4), %%eax\n"
535 "1: movl 64(%4), %%eax\n"
537 "2: movl 0(%4), %%eax\n"
538 "21: movl 4(%4), %%edx\n"
539 " movnti %%eax, 0(%3)\n"
540 " movnti %%edx, 4(%3)\n"
541 "3: movl 8(%4), %%eax\n"
542 "31: movl 12(%4),%%edx\n"
543 " movnti %%eax, 8(%3)\n"
544 " movnti %%edx, 12(%3)\n"
545 "4: movl 16(%4), %%eax\n"
546 "41: movl 20(%4), %%edx\n"
547 " movnti %%eax, 16(%3)\n"
548 " movnti %%edx, 20(%3)\n"
549 "10: movl 24(%4), %%eax\n"
550 "51: movl 28(%4), %%edx\n"
551 " movnti %%eax, 24(%3)\n"
552 " movnti %%edx, 28(%3)\n"
553 "11: movl 32(%4), %%eax\n"
554 "61: movl 36(%4), %%edx\n"
555 " movnti %%eax, 32(%3)\n"
556 " movnti %%edx, 36(%3)\n"
557 "12: movl 40(%4), %%eax\n"
558 "71: movl 44(%4), %%edx\n"
559 " movnti %%eax, 40(%3)\n"
560 " movnti %%edx, 44(%3)\n"
561 "13: movl 48(%4), %%eax\n"
562 "81: movl 52(%4), %%edx\n"
563 " movnti %%eax, 48(%3)\n"
564 " movnti %%edx, 52(%3)\n"
565 "14: movl 56(%4), %%eax\n"
566 "91: movl 60(%4), %%edx\n"
567 " movnti %%eax, 56(%3)\n"
568 " movnti %%edx, 60(%3)\n"
575 "5: movl %0, %%eax\n"
583 ".section .fixup,\"ax\"\n"
584 "9: lea 0(%%eax,%0,4),%0\n"
587 ".section __ex_table,\"a\"\n"
610 : "=&c"(size
), "=&D" (d0
), "=&S" (d1
)
611 : "1"(to
), "2"(from
), "0"(size
)
612 : "eax", "edx", "memory");
619 * Leave these declared but undefined. They should not be any references to
622 unsigned long __copy_user_zeroing_intel(void *to
, const void __user
*from
,
624 unsigned long __copy_user_intel(void __user
*to
, const void *from
,
626 unsigned long __copy_user_zeroing_intel_nocache(void *to
,
627 const void __user
*from
, unsigned long size
);
628 #endif /* CONFIG_X86_INTEL_USERCOPY */
630 /* Generic arbitrary sized copy. */
631 #define __copy_user(to, from, size) \
633 int __d0, __d1, __d2; \
634 __asm__ __volatile__( \
650 ".section .fixup,\"ax\"\n" \
653 "3: lea 0(%3,%0,4),%0\n" \
656 ".section __ex_table,\"a\"\n" \
662 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
663 : "3"(size), "0"(size), "1"(to), "2"(from) \
667 #define __copy_user_zeroing(to, from, size) \
669 int __d0, __d1, __d2; \
670 __asm__ __volatile__( \
686 ".section .fixup,\"ax\"\n" \
689 "3: lea 0(%3,%0,4),%0\n" \
692 " xorl %%eax,%%eax\n" \
698 ".section __ex_table,\"a\"\n" \
704 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
705 : "3"(size), "0"(size), "1"(to), "2"(from) \
709 unsigned long __copy_to_user_ll(void __user
*to
, const void *from
,
712 #ifndef CONFIG_X86_WP_WORKS_OK
713 if (unlikely(boot_cpu_data
.wp_works_ok
== 0) &&
714 ((unsigned long)to
) < TASK_SIZE
) {
716 * When we are in an atomic section (see
717 * mm/filemap.c:file_read_actor), return the full
718 * length to take the slow path.
724 * CPU does not honor the WP bit when writing
725 * from supervisory mode, and due to preemption or SMP,
726 * the page tables can change at any time.
727 * Do it manually. Manfred <manfred@colorfullife.com>
730 unsigned long offset
= ((unsigned long)to
)%PAGE_SIZE
;
731 unsigned long len
= PAGE_SIZE
- offset
;
740 down_read(¤t
->mm
->mmap_sem
);
741 retval
= get_user_pages(current
, current
->mm
,
742 (unsigned long)to
, 1, 1, 0, &pg
, NULL
);
744 if (retval
== -ENOMEM
&& is_global_init(current
)) {
745 up_read(¤t
->mm
->mmap_sem
);
746 congestion_wait(WRITE
, HZ
/50);
751 up_read(¤t
->mm
->mmap_sem
);
755 maddr
= kmap_atomic(pg
, KM_USER0
);
756 memcpy(maddr
+ offset
, from
, len
);
757 kunmap_atomic(maddr
, KM_USER0
);
758 set_page_dirty_lock(pg
);
760 up_read(¤t
->mm
->mmap_sem
);
769 if (movsl_is_ok(to
, from
, n
))
770 __copy_user(to
, from
, n
);
772 n
= __copy_user_intel(to
, from
, n
);
775 EXPORT_SYMBOL(__copy_to_user_ll
);
777 unsigned long __copy_from_user_ll(void *to
, const void __user
*from
,
780 if (movsl_is_ok(to
, from
, n
))
781 __copy_user_zeroing(to
, from
, n
);
783 n
= __copy_user_zeroing_intel(to
, from
, n
);
786 EXPORT_SYMBOL(__copy_from_user_ll
);
788 unsigned long __copy_from_user_ll_nozero(void *to
, const void __user
*from
,
791 if (movsl_is_ok(to
, from
, n
))
792 __copy_user(to
, from
, n
);
794 n
= __copy_user_intel((void __user
*)to
,
795 (const void *)from
, n
);
798 EXPORT_SYMBOL(__copy_from_user_ll_nozero
);
800 unsigned long __copy_from_user_ll_nocache(void *to
, const void __user
*from
,
803 #ifdef CONFIG_X86_INTEL_USERCOPY
804 if (n
> 64 && cpu_has_xmm2
)
805 n
= __copy_user_zeroing_intel_nocache(to
, from
, n
);
807 __copy_user_zeroing(to
, from
, n
);
809 __copy_user_zeroing(to
, from
, n
);
813 EXPORT_SYMBOL(__copy_from_user_ll_nocache
);
815 unsigned long __copy_from_user_ll_nocache_nozero(void *to
, const void __user
*from
,
818 #ifdef CONFIG_X86_INTEL_USERCOPY
819 if (n
> 64 && cpu_has_xmm2
)
820 n
= __copy_user_intel_nocache(to
, from
, n
);
822 __copy_user(to
, from
, n
);
824 __copy_user(to
, from
, n
);
828 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero
);
831 * copy_to_user: - Copy a block of data into user space.
832 * @to: Destination address, in user space.
833 * @from: Source address, in kernel space.
834 * @n: Number of bytes to copy.
836 * Context: User context only. This function may sleep.
838 * Copy data from kernel space to user space.
840 * Returns number of bytes that could not be copied.
841 * On success, this will be zero.
844 copy_to_user(void __user
*to
, const void *from
, unsigned long n
)
846 if (access_ok(VERIFY_WRITE
, to
, n
))
847 n
= __copy_to_user(to
, from
, n
);
850 EXPORT_SYMBOL(copy_to_user
);
853 * copy_from_user: - Copy a block of data from user space.
854 * @to: Destination address, in kernel space.
855 * @from: Source address, in user space.
856 * @n: Number of bytes to copy.
858 * Context: User context only. This function may sleep.
860 * Copy data from user space to kernel space.
862 * Returns number of bytes that could not be copied.
863 * On success, this will be zero.
865 * If some data could not be copied, this function will pad the copied
866 * data to the requested size using zero bytes.
869 copy_from_user(void *to
, const void __user
*from
, unsigned long n
)
871 if (access_ok(VERIFY_READ
, from
, n
))
872 n
= __copy_from_user(to
, from
, n
);
877 EXPORT_SYMBOL(copy_from_user
);