Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * User address space access functions. | |
3 | * The non inlined parts of asm-i386/uaccess.h are here. | |
4 | * | |
5 | * Copyright 1997 Andi Kleen <ak@muc.de> | |
6 | * Copyright 1997 Linus Torvalds | |
7 | */ | |
1da177e4 LT |
8 | #include <linux/mm.h> |
9 | #include <linux/highmem.h> | |
10 | #include <linux/blkdev.h> | |
11 | #include <linux/module.h> | |
12 | #include <asm/uaccess.h> | |
13 | #include <asm/mmx.h> | |
14 | ||
15 | static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n) | |
16 | { | |
17 | #ifdef CONFIG_X86_INTEL_USERCOPY | |
18 | if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask)) | |
19 | return 0; | |
20 | #endif | |
21 | return 1; | |
22 | } | |
23 | #define movsl_is_ok(a1,a2,n) \ | |
24 | __movsl_is_ok((unsigned long)(a1),(unsigned long)(a2),(n)) | |
25 | ||
26 | /* | |
27 | * Copy a null terminated string from userspace. | |
28 | */ | |
29 | ||
30 | #define __do_strncpy_from_user(dst,src,count,res) \ | |
31 | do { \ | |
32 | int __d0, __d1, __d2; \ | |
33 | might_sleep(); \ | |
34 | __asm__ __volatile__( \ | |
35 | " testl %1,%1\n" \ | |
36 | " jz 2f\n" \ | |
37 | "0: lodsb\n" \ | |
38 | " stosb\n" \ | |
39 | " testb %%al,%%al\n" \ | |
40 | " jz 1f\n" \ | |
41 | " decl %1\n" \ | |
42 | " jnz 0b\n" \ | |
43 | "1: subl %1,%0\n" \ | |
44 | "2:\n" \ | |
45 | ".section .fixup,\"ax\"\n" \ | |
46 | "3: movl %5,%0\n" \ | |
47 | " jmp 2b\n" \ | |
48 | ".previous\n" \ | |
49 | ".section __ex_table,\"a\"\n" \ | |
50 | " .align 4\n" \ | |
51 | " .long 0b,3b\n" \ | |
52 | ".previous" \ | |
53 | : "=d"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1), \ | |
54 | "=&D" (__d2) \ | |
55 | : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \ | |
56 | : "memory"); \ | |
57 | } while (0) | |
58 | ||
59 | /** | |
60 | * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking. | |
61 | * @dst: Destination address, in kernel space. This buffer must be at | |
62 | * least @count bytes long. | |
63 | * @src: Source address, in user space. | |
64 | * @count: Maximum number of bytes to copy, including the trailing NUL. | |
65 | * | |
66 | * Copies a NUL-terminated string from userspace to kernel space. | |
67 | * Caller must check the specified block with access_ok() before calling | |
68 | * this function. | |
69 | * | |
70 | * On success, returns the length of the string (not including the trailing | |
71 | * NUL). | |
72 | * | |
73 | * If access to userspace fails, returns -EFAULT (some data may have been | |
74 | * copied). | |
75 | * | |
76 | * If @count is smaller than the length of the string, copies @count bytes | |
77 | * and returns @count. | |
78 | */ | |
79 | long | |
80 | __strncpy_from_user(char *dst, const char __user *src, long count) | |
81 | { | |
82 | long res; | |
83 | __do_strncpy_from_user(dst, src, count, res); | |
84 | return res; | |
85 | } | |
129f6946 | 86 | EXPORT_SYMBOL(__strncpy_from_user); |
1da177e4 LT |
87 | |
88 | /** | |
89 | * strncpy_from_user: - Copy a NUL terminated string from userspace. | |
90 | * @dst: Destination address, in kernel space. This buffer must be at | |
91 | * least @count bytes long. | |
92 | * @src: Source address, in user space. | |
93 | * @count: Maximum number of bytes to copy, including the trailing NUL. | |
94 | * | |
95 | * Copies a NUL-terminated string from userspace to kernel space. | |
96 | * | |
97 | * On success, returns the length of the string (not including the trailing | |
98 | * NUL). | |
99 | * | |
100 | * If access to userspace fails, returns -EFAULT (some data may have been | |
101 | * copied). | |
102 | * | |
103 | * If @count is smaller than the length of the string, copies @count bytes | |
104 | * and returns @count. | |
105 | */ | |
106 | long | |
107 | strncpy_from_user(char *dst, const char __user *src, long count) | |
108 | { | |
109 | long res = -EFAULT; | |
110 | if (access_ok(VERIFY_READ, src, 1)) | |
111 | __do_strncpy_from_user(dst, src, count, res); | |
112 | return res; | |
113 | } | |
129f6946 | 114 | EXPORT_SYMBOL(strncpy_from_user); |
1da177e4 LT |
115 | |
116 | /* | |
117 | * Zero Userspace | |
118 | */ | |
119 | ||
120 | #define __do_clear_user(addr,size) \ | |
121 | do { \ | |
122 | int __d0; \ | |
123 | might_sleep(); \ | |
124 | __asm__ __volatile__( \ | |
125 | "0: rep; stosl\n" \ | |
126 | " movl %2,%0\n" \ | |
127 | "1: rep; stosb\n" \ | |
128 | "2:\n" \ | |
129 | ".section .fixup,\"ax\"\n" \ | |
130 | "3: lea 0(%2,%0,4),%0\n" \ | |
131 | " jmp 2b\n" \ | |
132 | ".previous\n" \ | |
133 | ".section __ex_table,\"a\"\n" \ | |
134 | " .align 4\n" \ | |
135 | " .long 0b,3b\n" \ | |
136 | " .long 1b,2b\n" \ | |
137 | ".previous" \ | |
138 | : "=&c"(size), "=&D" (__d0) \ | |
139 | : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \ | |
140 | } while (0) | |
141 | ||
142 | /** | |
143 | * clear_user: - Zero a block of memory in user space. | |
144 | * @to: Destination address, in user space. | |
145 | * @n: Number of bytes to zero. | |
146 | * | |
147 | * Zero a block of memory in user space. | |
148 | * | |
149 | * Returns number of bytes that could not be cleared. | |
150 | * On success, this will be zero. | |
151 | */ | |
152 | unsigned long | |
153 | clear_user(void __user *to, unsigned long n) | |
154 | { | |
155 | might_sleep(); | |
156 | if (access_ok(VERIFY_WRITE, to, n)) | |
157 | __do_clear_user(to, n); | |
158 | return n; | |
159 | } | |
129f6946 | 160 | EXPORT_SYMBOL(clear_user); |
1da177e4 LT |
161 | |
162 | /** | |
163 | * __clear_user: - Zero a block of memory in user space, with less checking. | |
164 | * @to: Destination address, in user space. | |
165 | * @n: Number of bytes to zero. | |
166 | * | |
167 | * Zero a block of memory in user space. Caller must check | |
168 | * the specified block with access_ok() before calling this function. | |
169 | * | |
170 | * Returns number of bytes that could not be cleared. | |
171 | * On success, this will be zero. | |
172 | */ | |
173 | unsigned long | |
174 | __clear_user(void __user *to, unsigned long n) | |
175 | { | |
176 | __do_clear_user(to, n); | |
177 | return n; | |
178 | } | |
129f6946 | 179 | EXPORT_SYMBOL(__clear_user); |
1da177e4 LT |
180 | |
181 | /** | |
182 | * strlen_user: - Get the size of a string in user space. | |
183 | * @s: The string to measure. | |
184 | * @n: The maximum valid length | |
185 | * | |
186 | * Get the size of a NUL-terminated string in user space. | |
187 | * | |
188 | * Returns the size of the string INCLUDING the terminating NUL. | |
189 | * On exception, returns 0. | |
190 | * If the string is too long, returns a value greater than @n. | |
191 | */ | |
192 | long strnlen_user(const char __user *s, long n) | |
193 | { | |
194 | unsigned long mask = -__addr_ok(s); | |
195 | unsigned long res, tmp; | |
196 | ||
197 | might_sleep(); | |
198 | ||
199 | __asm__ __volatile__( | |
200 | " testl %0, %0\n" | |
201 | " jz 3f\n" | |
202 | " andl %0,%%ecx\n" | |
203 | "0: repne; scasb\n" | |
204 | " setne %%al\n" | |
205 | " subl %%ecx,%0\n" | |
206 | " addl %0,%%eax\n" | |
207 | "1:\n" | |
208 | ".section .fixup,\"ax\"\n" | |
209 | "2: xorl %%eax,%%eax\n" | |
210 | " jmp 1b\n" | |
211 | "3: movb $1,%%al\n" | |
212 | " jmp 1b\n" | |
213 | ".previous\n" | |
214 | ".section __ex_table,\"a\"\n" | |
215 | " .align 4\n" | |
216 | " .long 0b,2b\n" | |
217 | ".previous" | |
218 | :"=r" (n), "=D" (s), "=a" (res), "=c" (tmp) | |
219 | :"0" (n), "1" (s), "2" (0), "3" (mask) | |
220 | :"cc"); | |
221 | return res & mask; | |
222 | } | |
129f6946 | 223 | EXPORT_SYMBOL(strnlen_user); |
1da177e4 LT |
224 | |
225 | #ifdef CONFIG_X86_INTEL_USERCOPY | |
226 | static unsigned long | |
227 | __copy_user_intel(void __user *to, const void *from, unsigned long size) | |
228 | { | |
229 | int d0, d1; | |
230 | __asm__ __volatile__( | |
231 | " .align 2,0x90\n" | |
232 | "1: movl 32(%4), %%eax\n" | |
233 | " cmpl $67, %0\n" | |
234 | " jbe 3f\n" | |
235 | "2: movl 64(%4), %%eax\n" | |
236 | " .align 2,0x90\n" | |
237 | "3: movl 0(%4), %%eax\n" | |
238 | "4: movl 4(%4), %%edx\n" | |
239 | "5: movl %%eax, 0(%3)\n" | |
240 | "6: movl %%edx, 4(%3)\n" | |
241 | "7: movl 8(%4), %%eax\n" | |
242 | "8: movl 12(%4),%%edx\n" | |
243 | "9: movl %%eax, 8(%3)\n" | |
244 | "10: movl %%edx, 12(%3)\n" | |
245 | "11: movl 16(%4), %%eax\n" | |
246 | "12: movl 20(%4), %%edx\n" | |
247 | "13: movl %%eax, 16(%3)\n" | |
248 | "14: movl %%edx, 20(%3)\n" | |
249 | "15: movl 24(%4), %%eax\n" | |
250 | "16: movl 28(%4), %%edx\n" | |
251 | "17: movl %%eax, 24(%3)\n" | |
252 | "18: movl %%edx, 28(%3)\n" | |
253 | "19: movl 32(%4), %%eax\n" | |
254 | "20: movl 36(%4), %%edx\n" | |
255 | "21: movl %%eax, 32(%3)\n" | |
256 | "22: movl %%edx, 36(%3)\n" | |
257 | "23: movl 40(%4), %%eax\n" | |
258 | "24: movl 44(%4), %%edx\n" | |
259 | "25: movl %%eax, 40(%3)\n" | |
260 | "26: movl %%edx, 44(%3)\n" | |
261 | "27: movl 48(%4), %%eax\n" | |
262 | "28: movl 52(%4), %%edx\n" | |
263 | "29: movl %%eax, 48(%3)\n" | |
264 | "30: movl %%edx, 52(%3)\n" | |
265 | "31: movl 56(%4), %%eax\n" | |
266 | "32: movl 60(%4), %%edx\n" | |
267 | "33: movl %%eax, 56(%3)\n" | |
268 | "34: movl %%edx, 60(%3)\n" | |
269 | " addl $-64, %0\n" | |
270 | " addl $64, %4\n" | |
271 | " addl $64, %3\n" | |
272 | " cmpl $63, %0\n" | |
273 | " ja 1b\n" | |
274 | "35: movl %0, %%eax\n" | |
275 | " shrl $2, %0\n" | |
276 | " andl $3, %%eax\n" | |
277 | " cld\n" | |
278 | "99: rep; movsl\n" | |
279 | "36: movl %%eax, %0\n" | |
280 | "37: rep; movsb\n" | |
281 | "100:\n" | |
282 | ".section .fixup,\"ax\"\n" | |
283 | "101: lea 0(%%eax,%0,4),%0\n" | |
284 | " jmp 100b\n" | |
285 | ".previous\n" | |
286 | ".section __ex_table,\"a\"\n" | |
287 | " .align 4\n" | |
288 | " .long 1b,100b\n" | |
289 | " .long 2b,100b\n" | |
290 | " .long 3b,100b\n" | |
291 | " .long 4b,100b\n" | |
292 | " .long 5b,100b\n" | |
293 | " .long 6b,100b\n" | |
294 | " .long 7b,100b\n" | |
295 | " .long 8b,100b\n" | |
296 | " .long 9b,100b\n" | |
297 | " .long 10b,100b\n" | |
298 | " .long 11b,100b\n" | |
299 | " .long 12b,100b\n" | |
300 | " .long 13b,100b\n" | |
301 | " .long 14b,100b\n" | |
302 | " .long 15b,100b\n" | |
303 | " .long 16b,100b\n" | |
304 | " .long 17b,100b\n" | |
305 | " .long 18b,100b\n" | |
306 | " .long 19b,100b\n" | |
307 | " .long 20b,100b\n" | |
308 | " .long 21b,100b\n" | |
309 | " .long 22b,100b\n" | |
310 | " .long 23b,100b\n" | |
311 | " .long 24b,100b\n" | |
312 | " .long 25b,100b\n" | |
313 | " .long 26b,100b\n" | |
314 | " .long 27b,100b\n" | |
315 | " .long 28b,100b\n" | |
316 | " .long 29b,100b\n" | |
317 | " .long 30b,100b\n" | |
318 | " .long 31b,100b\n" | |
319 | " .long 32b,100b\n" | |
320 | " .long 33b,100b\n" | |
321 | " .long 34b,100b\n" | |
322 | " .long 35b,100b\n" | |
323 | " .long 36b,100b\n" | |
324 | " .long 37b,100b\n" | |
325 | " .long 99b,101b\n" | |
326 | ".previous" | |
327 | : "=&c"(size), "=&D" (d0), "=&S" (d1) | |
328 | : "1"(to), "2"(from), "0"(size) | |
329 | : "eax", "edx", "memory"); | |
330 | return size; | |
331 | } | |
332 | ||
333 | static unsigned long | |
334 | __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size) | |
335 | { | |
336 | int d0, d1; | |
337 | __asm__ __volatile__( | |
338 | " .align 2,0x90\n" | |
339 | "0: movl 32(%4), %%eax\n" | |
340 | " cmpl $67, %0\n" | |
341 | " jbe 2f\n" | |
342 | "1: movl 64(%4), %%eax\n" | |
343 | " .align 2,0x90\n" | |
344 | "2: movl 0(%4), %%eax\n" | |
345 | "21: movl 4(%4), %%edx\n" | |
346 | " movl %%eax, 0(%3)\n" | |
347 | " movl %%edx, 4(%3)\n" | |
348 | "3: movl 8(%4), %%eax\n" | |
349 | "31: movl 12(%4),%%edx\n" | |
350 | " movl %%eax, 8(%3)\n" | |
351 | " movl %%edx, 12(%3)\n" | |
352 | "4: movl 16(%4), %%eax\n" | |
353 | "41: movl 20(%4), %%edx\n" | |
354 | " movl %%eax, 16(%3)\n" | |
355 | " movl %%edx, 20(%3)\n" | |
356 | "10: movl 24(%4), %%eax\n" | |
357 | "51: movl 28(%4), %%edx\n" | |
358 | " movl %%eax, 24(%3)\n" | |
359 | " movl %%edx, 28(%3)\n" | |
360 | "11: movl 32(%4), %%eax\n" | |
361 | "61: movl 36(%4), %%edx\n" | |
362 | " movl %%eax, 32(%3)\n" | |
363 | " movl %%edx, 36(%3)\n" | |
364 | "12: movl 40(%4), %%eax\n" | |
365 | "71: movl 44(%4), %%edx\n" | |
366 | " movl %%eax, 40(%3)\n" | |
367 | " movl %%edx, 44(%3)\n" | |
368 | "13: movl 48(%4), %%eax\n" | |
369 | "81: movl 52(%4), %%edx\n" | |
370 | " movl %%eax, 48(%3)\n" | |
371 | " movl %%edx, 52(%3)\n" | |
372 | "14: movl 56(%4), %%eax\n" | |
373 | "91: movl 60(%4), %%edx\n" | |
374 | " movl %%eax, 56(%3)\n" | |
375 | " movl %%edx, 60(%3)\n" | |
376 | " addl $-64, %0\n" | |
377 | " addl $64, %4\n" | |
378 | " addl $64, %3\n" | |
379 | " cmpl $63, %0\n" | |
380 | " ja 0b\n" | |
381 | "5: movl %0, %%eax\n" | |
382 | " shrl $2, %0\n" | |
383 | " andl $3, %%eax\n" | |
384 | " cld\n" | |
385 | "6: rep; movsl\n" | |
386 | " movl %%eax,%0\n" | |
387 | "7: rep; movsb\n" | |
388 | "8:\n" | |
389 | ".section .fixup,\"ax\"\n" | |
390 | "9: lea 0(%%eax,%0,4),%0\n" | |
391 | "16: pushl %0\n" | |
392 | " pushl %%eax\n" | |
393 | " xorl %%eax,%%eax\n" | |
394 | " rep; stosb\n" | |
395 | " popl %%eax\n" | |
396 | " popl %0\n" | |
397 | " jmp 8b\n" | |
398 | ".previous\n" | |
399 | ".section __ex_table,\"a\"\n" | |
400 | " .align 4\n" | |
401 | " .long 0b,16b\n" | |
402 | " .long 1b,16b\n" | |
403 | " .long 2b,16b\n" | |
404 | " .long 21b,16b\n" | |
405 | " .long 3b,16b\n" | |
406 | " .long 31b,16b\n" | |
407 | " .long 4b,16b\n" | |
408 | " .long 41b,16b\n" | |
409 | " .long 10b,16b\n" | |
410 | " .long 51b,16b\n" | |
411 | " .long 11b,16b\n" | |
412 | " .long 61b,16b\n" | |
413 | " .long 12b,16b\n" | |
414 | " .long 71b,16b\n" | |
415 | " .long 13b,16b\n" | |
416 | " .long 81b,16b\n" | |
417 | " .long 14b,16b\n" | |
418 | " .long 91b,16b\n" | |
419 | " .long 6b,9b\n" | |
420 | " .long 7b,16b\n" | |
421 | ".previous" | |
422 | : "=&c"(size), "=&D" (d0), "=&S" (d1) | |
423 | : "1"(to), "2"(from), "0"(size) | |
424 | : "eax", "edx", "memory"); | |
425 | return size; | |
426 | } | |
c22ce143 HY |
427 | |
428 | /* | |
429 | * Non Temporal Hint version of __copy_user_zeroing_intel. It is cache aware. | |
430 | * hyoshiok@miraclelinux.com | |
431 | */ | |
432 | ||
433 | static unsigned long __copy_user_zeroing_intel_nocache(void *to, | |
434 | const void __user *from, unsigned long size) | |
435 | { | |
436 | int d0, d1; | |
437 | ||
438 | __asm__ __volatile__( | |
439 | " .align 2,0x90\n" | |
440 | "0: movl 32(%4), %%eax\n" | |
441 | " cmpl $67, %0\n" | |
442 | " jbe 2f\n" | |
443 | "1: movl 64(%4), %%eax\n" | |
444 | " .align 2,0x90\n" | |
445 | "2: movl 0(%4), %%eax\n" | |
446 | "21: movl 4(%4), %%edx\n" | |
447 | " movnti %%eax, 0(%3)\n" | |
448 | " movnti %%edx, 4(%3)\n" | |
449 | "3: movl 8(%4), %%eax\n" | |
450 | "31: movl 12(%4),%%edx\n" | |
451 | " movnti %%eax, 8(%3)\n" | |
452 | " movnti %%edx, 12(%3)\n" | |
453 | "4: movl 16(%4), %%eax\n" | |
454 | "41: movl 20(%4), %%edx\n" | |
455 | " movnti %%eax, 16(%3)\n" | |
456 | " movnti %%edx, 20(%3)\n" | |
457 | "10: movl 24(%4), %%eax\n" | |
458 | "51: movl 28(%4), %%edx\n" | |
459 | " movnti %%eax, 24(%3)\n" | |
460 | " movnti %%edx, 28(%3)\n" | |
461 | "11: movl 32(%4), %%eax\n" | |
462 | "61: movl 36(%4), %%edx\n" | |
463 | " movnti %%eax, 32(%3)\n" | |
464 | " movnti %%edx, 36(%3)\n" | |
465 | "12: movl 40(%4), %%eax\n" | |
466 | "71: movl 44(%4), %%edx\n" | |
467 | " movnti %%eax, 40(%3)\n" | |
468 | " movnti %%edx, 44(%3)\n" | |
469 | "13: movl 48(%4), %%eax\n" | |
470 | "81: movl 52(%4), %%edx\n" | |
471 | " movnti %%eax, 48(%3)\n" | |
472 | " movnti %%edx, 52(%3)\n" | |
473 | "14: movl 56(%4), %%eax\n" | |
474 | "91: movl 60(%4), %%edx\n" | |
475 | " movnti %%eax, 56(%3)\n" | |
476 | " movnti %%edx, 60(%3)\n" | |
477 | " addl $-64, %0\n" | |
478 | " addl $64, %4\n" | |
479 | " addl $64, %3\n" | |
480 | " cmpl $63, %0\n" | |
481 | " ja 0b\n" | |
482 | " sfence \n" | |
483 | "5: movl %0, %%eax\n" | |
484 | " shrl $2, %0\n" | |
485 | " andl $3, %%eax\n" | |
486 | " cld\n" | |
487 | "6: rep; movsl\n" | |
488 | " movl %%eax,%0\n" | |
489 | "7: rep; movsb\n" | |
490 | "8:\n" | |
491 | ".section .fixup,\"ax\"\n" | |
492 | "9: lea 0(%%eax,%0,4),%0\n" | |
493 | "16: pushl %0\n" | |
494 | " pushl %%eax\n" | |
495 | " xorl %%eax,%%eax\n" | |
496 | " rep; stosb\n" | |
497 | " popl %%eax\n" | |
498 | " popl %0\n" | |
499 | " jmp 8b\n" | |
500 | ".previous\n" | |
501 | ".section __ex_table,\"a\"\n" | |
502 | " .align 4\n" | |
503 | " .long 0b,16b\n" | |
504 | " .long 1b,16b\n" | |
505 | " .long 2b,16b\n" | |
506 | " .long 21b,16b\n" | |
507 | " .long 3b,16b\n" | |
508 | " .long 31b,16b\n" | |
509 | " .long 4b,16b\n" | |
510 | " .long 41b,16b\n" | |
511 | " .long 10b,16b\n" | |
512 | " .long 51b,16b\n" | |
513 | " .long 11b,16b\n" | |
514 | " .long 61b,16b\n" | |
515 | " .long 12b,16b\n" | |
516 | " .long 71b,16b\n" | |
517 | " .long 13b,16b\n" | |
518 | " .long 81b,16b\n" | |
519 | " .long 14b,16b\n" | |
520 | " .long 91b,16b\n" | |
521 | " .long 6b,9b\n" | |
522 | " .long 7b,16b\n" | |
523 | ".previous" | |
524 | : "=&c"(size), "=&D" (d0), "=&S" (d1) | |
525 | : "1"(to), "2"(from), "0"(size) | |
526 | : "eax", "edx", "memory"); | |
527 | return size; | |
528 | } | |
529 | ||
7c12d811 N |
530 | static unsigned long __copy_user_intel_nocache(void *to, |
531 | const void __user *from, unsigned long size) | |
532 | { | |
533 | int d0, d1; | |
534 | ||
535 | __asm__ __volatile__( | |
536 | " .align 2,0x90\n" | |
537 | "0: movl 32(%4), %%eax\n" | |
538 | " cmpl $67, %0\n" | |
539 | " jbe 2f\n" | |
540 | "1: movl 64(%4), %%eax\n" | |
541 | " .align 2,0x90\n" | |
542 | "2: movl 0(%4), %%eax\n" | |
543 | "21: movl 4(%4), %%edx\n" | |
544 | " movnti %%eax, 0(%3)\n" | |
545 | " movnti %%edx, 4(%3)\n" | |
546 | "3: movl 8(%4), %%eax\n" | |
547 | "31: movl 12(%4),%%edx\n" | |
548 | " movnti %%eax, 8(%3)\n" | |
549 | " movnti %%edx, 12(%3)\n" | |
550 | "4: movl 16(%4), %%eax\n" | |
551 | "41: movl 20(%4), %%edx\n" | |
552 | " movnti %%eax, 16(%3)\n" | |
553 | " movnti %%edx, 20(%3)\n" | |
554 | "10: movl 24(%4), %%eax\n" | |
555 | "51: movl 28(%4), %%edx\n" | |
556 | " movnti %%eax, 24(%3)\n" | |
557 | " movnti %%edx, 28(%3)\n" | |
558 | "11: movl 32(%4), %%eax\n" | |
559 | "61: movl 36(%4), %%edx\n" | |
560 | " movnti %%eax, 32(%3)\n" | |
561 | " movnti %%edx, 36(%3)\n" | |
562 | "12: movl 40(%4), %%eax\n" | |
563 | "71: movl 44(%4), %%edx\n" | |
564 | " movnti %%eax, 40(%3)\n" | |
565 | " movnti %%edx, 44(%3)\n" | |
566 | "13: movl 48(%4), %%eax\n" | |
567 | "81: movl 52(%4), %%edx\n" | |
568 | " movnti %%eax, 48(%3)\n" | |
569 | " movnti %%edx, 52(%3)\n" | |
570 | "14: movl 56(%4), %%eax\n" | |
571 | "91: movl 60(%4), %%edx\n" | |
572 | " movnti %%eax, 56(%3)\n" | |
573 | " movnti %%edx, 60(%3)\n" | |
574 | " addl $-64, %0\n" | |
575 | " addl $64, %4\n" | |
576 | " addl $64, %3\n" | |
577 | " cmpl $63, %0\n" | |
578 | " ja 0b\n" | |
579 | " sfence \n" | |
580 | "5: movl %0, %%eax\n" | |
581 | " shrl $2, %0\n" | |
582 | " andl $3, %%eax\n" | |
583 | " cld\n" | |
584 | "6: rep; movsl\n" | |
585 | " movl %%eax,%0\n" | |
586 | "7: rep; movsb\n" | |
587 | "8:\n" | |
588 | ".section .fixup,\"ax\"\n" | |
589 | "9: lea 0(%%eax,%0,4),%0\n" | |
590 | "16: jmp 8b\n" | |
591 | ".previous\n" | |
592 | ".section __ex_table,\"a\"\n" | |
593 | " .align 4\n" | |
594 | " .long 0b,16b\n" | |
595 | " .long 1b,16b\n" | |
596 | " .long 2b,16b\n" | |
597 | " .long 21b,16b\n" | |
598 | " .long 3b,16b\n" | |
599 | " .long 31b,16b\n" | |
600 | " .long 4b,16b\n" | |
601 | " .long 41b,16b\n" | |
602 | " .long 10b,16b\n" | |
603 | " .long 51b,16b\n" | |
604 | " .long 11b,16b\n" | |
605 | " .long 61b,16b\n" | |
606 | " .long 12b,16b\n" | |
607 | " .long 71b,16b\n" | |
608 | " .long 13b,16b\n" | |
609 | " .long 81b,16b\n" | |
610 | " .long 14b,16b\n" | |
611 | " .long 91b,16b\n" | |
612 | " .long 6b,9b\n" | |
613 | " .long 7b,16b\n" | |
614 | ".previous" | |
615 | : "=&c"(size), "=&D" (d0), "=&S" (d1) | |
616 | : "1"(to), "2"(from), "0"(size) | |
617 | : "eax", "edx", "memory"); | |
618 | return size; | |
619 | } | |
620 | ||
1da177e4 | 621 | #else |
c22ce143 | 622 | |
1da177e4 LT |
623 | /* |
624 | * Leave these declared but undefined. They should not be any references to | |
625 | * them | |
626 | */ | |
c22ce143 HY |
627 | unsigned long __copy_user_zeroing_intel(void *to, const void __user *from, |
628 | unsigned long size); | |
629 | unsigned long __copy_user_intel(void __user *to, const void *from, | |
630 | unsigned long size); | |
631 | unsigned long __copy_user_zeroing_intel_nocache(void *to, | |
632 | const void __user *from, unsigned long size); | |
1da177e4 LT |
633 | #endif /* CONFIG_X86_INTEL_USERCOPY */ |
634 | ||
635 | /* Generic arbitrary sized copy. */ | |
636 | #define __copy_user(to,from,size) \ | |
637 | do { \ | |
638 | int __d0, __d1, __d2; \ | |
639 | __asm__ __volatile__( \ | |
640 | " cmp $7,%0\n" \ | |
641 | " jbe 1f\n" \ | |
642 | " movl %1,%0\n" \ | |
643 | " negl %0\n" \ | |
644 | " andl $7,%0\n" \ | |
645 | " subl %0,%3\n" \ | |
646 | "4: rep; movsb\n" \ | |
647 | " movl %3,%0\n" \ | |
648 | " shrl $2,%0\n" \ | |
649 | " andl $3,%3\n" \ | |
650 | " .align 2,0x90\n" \ | |
651 | "0: rep; movsl\n" \ | |
652 | " movl %3,%0\n" \ | |
653 | "1: rep; movsb\n" \ | |
654 | "2:\n" \ | |
655 | ".section .fixup,\"ax\"\n" \ | |
656 | "5: addl %3,%0\n" \ | |
657 | " jmp 2b\n" \ | |
658 | "3: lea 0(%3,%0,4),%0\n" \ | |
659 | " jmp 2b\n" \ | |
660 | ".previous\n" \ | |
661 | ".section __ex_table,\"a\"\n" \ | |
662 | " .align 4\n" \ | |
663 | " .long 4b,5b\n" \ | |
664 | " .long 0b,3b\n" \ | |
665 | " .long 1b,2b\n" \ | |
666 | ".previous" \ | |
667 | : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ | |
668 | : "3"(size), "0"(size), "1"(to), "2"(from) \ | |
669 | : "memory"); \ | |
670 | } while (0) | |
671 | ||
672 | #define __copy_user_zeroing(to,from,size) \ | |
673 | do { \ | |
674 | int __d0, __d1, __d2; \ | |
675 | __asm__ __volatile__( \ | |
676 | " cmp $7,%0\n" \ | |
677 | " jbe 1f\n" \ | |
678 | " movl %1,%0\n" \ | |
679 | " negl %0\n" \ | |
680 | " andl $7,%0\n" \ | |
681 | " subl %0,%3\n" \ | |
682 | "4: rep; movsb\n" \ | |
683 | " movl %3,%0\n" \ | |
684 | " shrl $2,%0\n" \ | |
685 | " andl $3,%3\n" \ | |
686 | " .align 2,0x90\n" \ | |
687 | "0: rep; movsl\n" \ | |
688 | " movl %3,%0\n" \ | |
689 | "1: rep; movsb\n" \ | |
690 | "2:\n" \ | |
691 | ".section .fixup,\"ax\"\n" \ | |
692 | "5: addl %3,%0\n" \ | |
693 | " jmp 6f\n" \ | |
694 | "3: lea 0(%3,%0,4),%0\n" \ | |
695 | "6: pushl %0\n" \ | |
696 | " pushl %%eax\n" \ | |
697 | " xorl %%eax,%%eax\n" \ | |
698 | " rep; stosb\n" \ | |
699 | " popl %%eax\n" \ | |
700 | " popl %0\n" \ | |
701 | " jmp 2b\n" \ | |
702 | ".previous\n" \ | |
703 | ".section __ex_table,\"a\"\n" \ | |
704 | " .align 4\n" \ | |
705 | " .long 4b,5b\n" \ | |
706 | " .long 0b,3b\n" \ | |
707 | " .long 1b,6b\n" \ | |
708 | ".previous" \ | |
709 | : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ | |
710 | : "3"(size), "0"(size), "1"(to), "2"(from) \ | |
711 | : "memory"); \ | |
712 | } while (0) | |
713 | ||
c22ce143 HY |
714 | unsigned long __copy_to_user_ll(void __user *to, const void *from, |
715 | unsigned long n) | |
1da177e4 LT |
716 | { |
717 | BUG_ON((long) n < 0); | |
718 | #ifndef CONFIG_X86_WP_WORKS_OK | |
719 | if (unlikely(boot_cpu_data.wp_works_ok == 0) && | |
720 | ((unsigned long )to) < TASK_SIZE) { | |
721 | /* | |
722 | * CPU does not honor the WP bit when writing | |
723 | * from supervisory mode, and due to preemption or SMP, | |
724 | * the page tables can change at any time. | |
725 | * Do it manually. Manfred <manfred@colorfullife.com> | |
726 | */ | |
727 | while (n) { | |
728 | unsigned long offset = ((unsigned long)to)%PAGE_SIZE; | |
729 | unsigned long len = PAGE_SIZE - offset; | |
730 | int retval; | |
731 | struct page *pg; | |
732 | void *maddr; | |
733 | ||
734 | if (len > n) | |
735 | len = n; | |
736 | ||
737 | survive: | |
738 | down_read(¤t->mm->mmap_sem); | |
739 | retval = get_user_pages(current, current->mm, | |
740 | (unsigned long )to, 1, 1, 0, &pg, NULL); | |
741 | ||
f400e198 | 742 | if (retval == -ENOMEM && is_init(current)) { |
1da177e4 LT |
743 | up_read(¤t->mm->mmap_sem); |
744 | blk_congestion_wait(WRITE, HZ/50); | |
745 | goto survive; | |
746 | } | |
747 | ||
748 | if (retval != 1) { | |
749 | up_read(¤t->mm->mmap_sem); | |
750 | break; | |
751 | } | |
752 | ||
753 | maddr = kmap_atomic(pg, KM_USER0); | |
754 | memcpy(maddr + offset, from, len); | |
755 | kunmap_atomic(maddr, KM_USER0); | |
756 | set_page_dirty_lock(pg); | |
757 | put_page(pg); | |
758 | up_read(¤t->mm->mmap_sem); | |
759 | ||
760 | from += len; | |
761 | to += len; | |
762 | n -= len; | |
763 | } | |
764 | return n; | |
765 | } | |
766 | #endif | |
767 | if (movsl_is_ok(to, from, n)) | |
768 | __copy_user(to, from, n); | |
769 | else | |
770 | n = __copy_user_intel(to, from, n); | |
771 | return n; | |
772 | } | |
129f6946 | 773 | EXPORT_SYMBOL(__copy_to_user_ll); |
1da177e4 | 774 | |
c22ce143 HY |
775 | unsigned long __copy_from_user_ll(void *to, const void __user *from, |
776 | unsigned long n) | |
1da177e4 LT |
777 | { |
778 | BUG_ON((long)n < 0); | |
779 | if (movsl_is_ok(to, from, n)) | |
780 | __copy_user_zeroing(to, from, n); | |
781 | else | |
782 | n = __copy_user_zeroing_intel(to, from, n); | |
783 | return n; | |
784 | } | |
129f6946 | 785 | EXPORT_SYMBOL(__copy_from_user_ll); |
1da177e4 | 786 | |
7c12d811 N |
787 | unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from, |
788 | unsigned long n) | |
789 | { | |
790 | BUG_ON((long)n < 0); | |
791 | if (movsl_is_ok(to, from, n)) | |
792 | __copy_user(to, from, n); | |
793 | else | |
794 | n = __copy_user_intel((void __user *)to, | |
795 | (const void *)from, n); | |
796 | return n; | |
797 | } | |
798 | EXPORT_SYMBOL(__copy_from_user_ll_nozero); | |
799 | ||
c22ce143 HY |
800 | unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from, |
801 | unsigned long n) | |
802 | { | |
803 | BUG_ON((long)n < 0); | |
804 | #ifdef CONFIG_X86_INTEL_USERCOPY | |
805 | if ( n > 64 && cpu_has_xmm2) | |
806 | n = __copy_user_zeroing_intel_nocache(to, from, n); | |
807 | else | |
808 | __copy_user_zeroing(to, from, n); | |
809 | #else | |
810 | __copy_user_zeroing(to, from, n); | |
811 | #endif | |
812 | return n; | |
813 | } | |
814 | ||
7c12d811 N |
815 | unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, |
816 | unsigned long n) | |
817 | { | |
818 | BUG_ON((long)n < 0); | |
819 | #ifdef CONFIG_X86_INTEL_USERCOPY | |
820 | if ( n > 64 && cpu_has_xmm2) | |
821 | n = __copy_user_intel_nocache(to, from, n); | |
822 | else | |
823 | __copy_user(to, from, n); | |
824 | #else | |
825 | __copy_user(to, from, n); | |
826 | #endif | |
827 | return n; | |
828 | } | |
829 | ||
1da177e4 LT |
830 | /** |
831 | * copy_to_user: - Copy a block of data into user space. | |
832 | * @to: Destination address, in user space. | |
833 | * @from: Source address, in kernel space. | |
834 | * @n: Number of bytes to copy. | |
835 | * | |
836 | * Context: User context only. This function may sleep. | |
837 | * | |
838 | * Copy data from kernel space to user space. | |
839 | * | |
840 | * Returns number of bytes that could not be copied. | |
841 | * On success, this will be zero. | |
842 | */ | |
843 | unsigned long | |
844 | copy_to_user(void __user *to, const void *from, unsigned long n) | |
845 | { | |
1da177e4 LT |
846 | BUG_ON((long) n < 0); |
847 | if (access_ok(VERIFY_WRITE, to, n)) | |
848 | n = __copy_to_user(to, from, n); | |
849 | return n; | |
850 | } | |
851 | EXPORT_SYMBOL(copy_to_user); | |
852 | ||
853 | /** | |
854 | * copy_from_user: - Copy a block of data from user space. | |
855 | * @to: Destination address, in kernel space. | |
856 | * @from: Source address, in user space. | |
857 | * @n: Number of bytes to copy. | |
858 | * | |
859 | * Context: User context only. This function may sleep. | |
860 | * | |
861 | * Copy data from user space to kernel space. | |
862 | * | |
863 | * Returns number of bytes that could not be copied. | |
864 | * On success, this will be zero. | |
865 | * | |
866 | * If some data could not be copied, this function will pad the copied | |
867 | * data to the requested size using zero bytes. | |
868 | */ | |
869 | unsigned long | |
870 | copy_from_user(void *to, const void __user *from, unsigned long n) | |
871 | { | |
1da177e4 LT |
872 | BUG_ON((long) n < 0); |
873 | if (access_ok(VERIFY_READ, from, n)) | |
874 | n = __copy_from_user(to, from, n); | |
875 | else | |
876 | memset(to, 0, n); | |
877 | return n; | |
878 | } | |
879 | EXPORT_SYMBOL(copy_from_user); |