Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
1da177e4 LT |
2 | * Copyright (C) 1994 Linus Torvalds |
3 | * | |
4 | * Pentium III FXSR, SSE support | |
5 | * General FPU state handling cleanups | |
6 | * Gareth Hughes <gareth@valinux.com>, May 2000 | |
7 | */ | |
8 | ||
1da177e4 | 9 | #include <linux/sched.h> |
129f6946 | 10 | #include <linux/module.h> |
1da177e4 LT |
11 | #include <asm/processor.h> |
12 | #include <asm/i387.h> | |
13 | #include <asm/math_emu.h> | |
14 | #include <asm/sigcontext.h> | |
15 | #include <asm/user.h> | |
16 | #include <asm/ptrace.h> | |
17 | #include <asm/uaccess.h> | |
18 | ||
19 | #ifdef CONFIG_MATH_EMULATION | |
20 | #define HAVE_HWFP (boot_cpu_data.hard_math) | |
21 | #else | |
22 | #define HAVE_HWFP 1 | |
23 | #endif | |
24 | ||
7b0c2d92 | 25 | static unsigned long mxcsr_feature_mask __read_mostly = 0xffffffff; |
1da177e4 LT |
26 | |
27 | void mxcsr_feature_mask_init(void) | |
28 | { | |
29 | unsigned long mask = 0; | |
30 | clts(); | |
31 | if (cpu_has_fxsr) { | |
32 | memset(¤t->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct)); | |
33 | asm volatile("fxsave %0" : : "m" (current->thread.i387.fxsave)); | |
34 | mask = current->thread.i387.fxsave.mxcsr_mask; | |
35 | if (mask == 0) mask = 0x0000ffbf; | |
36 | } | |
37 | mxcsr_feature_mask &= mask; | |
38 | stts(); | |
39 | } | |
40 | ||
41 | /* | |
42 | * The _current_ task is using the FPU for the first time | |
43 | * so initialize it and set the mxcsr to its default | |
44 | * value at reset if we support XMM instructions and then | |
45 | * remeber the current task has used the FPU. | |
46 | */ | |
47 | void init_fpu(struct task_struct *tsk) | |
48 | { | |
49 | if (cpu_has_fxsr) { | |
50 | memset(&tsk->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct)); | |
51 | tsk->thread.i387.fxsave.cwd = 0x37f; | |
52 | if (cpu_has_xmm) | |
53 | tsk->thread.i387.fxsave.mxcsr = 0x1f80; | |
54 | } else { | |
55 | memset(&tsk->thread.i387.fsave, 0, sizeof(struct i387_fsave_struct)); | |
56 | tsk->thread.i387.fsave.cwd = 0xffff037fu; | |
57 | tsk->thread.i387.fsave.swd = 0xffff0000u; | |
58 | tsk->thread.i387.fsave.twd = 0xffffffffu; | |
59 | tsk->thread.i387.fsave.fos = 0xffff0000u; | |
60 | } | |
61 | /* only the device not available exception or ptrace can call init_fpu */ | |
62 | set_stopped_child_used_math(tsk); | |
63 | } | |
64 | ||
65 | /* | |
66 | * FPU lazy state save handling. | |
67 | */ | |
68 | ||
69 | void kernel_fpu_begin(void) | |
70 | { | |
71 | struct thread_info *thread = current_thread_info(); | |
72 | ||
73 | preempt_disable(); | |
74 | if (thread->status & TS_USEDFPU) { | |
75 | __save_init_fpu(thread->task); | |
76 | return; | |
77 | } | |
78 | clts(); | |
79 | } | |
129f6946 | 80 | EXPORT_SYMBOL_GPL(kernel_fpu_begin); |
1da177e4 | 81 | |
1da177e4 LT |
82 | /* |
83 | * FPU tag word conversions. | |
84 | */ | |
85 | ||
86 | static inline unsigned short twd_i387_to_fxsr( unsigned short twd ) | |
87 | { | |
88 | unsigned int tmp; /* to avoid 16 bit prefixes in the code */ | |
89 | ||
90 | /* Transform each pair of bits into 01 (valid) or 00 (empty) */ | |
91 | tmp = ~twd; | |
92 | tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */ | |
93 | /* and move the valid bits to the lower byte. */ | |
94 | tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */ | |
95 | tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */ | |
96 | tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */ | |
97 | return tmp; | |
98 | } | |
99 | ||
100 | static inline unsigned long twd_fxsr_to_i387( struct i387_fxsave_struct *fxsave ) | |
101 | { | |
102 | struct _fpxreg *st = NULL; | |
103 | unsigned long tos = (fxsave->swd >> 11) & 7; | |
104 | unsigned long twd = (unsigned long) fxsave->twd; | |
105 | unsigned long tag; | |
106 | unsigned long ret = 0xffff0000u; | |
107 | int i; | |
108 | ||
109 | #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16); | |
110 | ||
111 | for ( i = 0 ; i < 8 ; i++ ) { | |
112 | if ( twd & 0x1 ) { | |
113 | st = FPREG_ADDR( fxsave, (i - tos) & 7 ); | |
114 | ||
115 | switch ( st->exponent & 0x7fff ) { | |
116 | case 0x7fff: | |
117 | tag = 2; /* Special */ | |
118 | break; | |
119 | case 0x0000: | |
120 | if ( !st->significand[0] && | |
121 | !st->significand[1] && | |
122 | !st->significand[2] && | |
123 | !st->significand[3] ) { | |
124 | tag = 1; /* Zero */ | |
125 | } else { | |
126 | tag = 2; /* Special */ | |
127 | } | |
128 | break; | |
129 | default: | |
130 | if ( st->significand[3] & 0x8000 ) { | |
131 | tag = 0; /* Valid */ | |
132 | } else { | |
133 | tag = 2; /* Special */ | |
134 | } | |
135 | break; | |
136 | } | |
137 | } else { | |
138 | tag = 3; /* Empty */ | |
139 | } | |
140 | ret |= (tag << (2 * i)); | |
141 | twd = twd >> 1; | |
142 | } | |
143 | return ret; | |
144 | } | |
145 | ||
146 | /* | |
147 | * FPU state interaction. | |
148 | */ | |
149 | ||
150 | unsigned short get_fpu_cwd( struct task_struct *tsk ) | |
151 | { | |
152 | if ( cpu_has_fxsr ) { | |
153 | return tsk->thread.i387.fxsave.cwd; | |
154 | } else { | |
155 | return (unsigned short)tsk->thread.i387.fsave.cwd; | |
156 | } | |
157 | } | |
158 | ||
159 | unsigned short get_fpu_swd( struct task_struct *tsk ) | |
160 | { | |
161 | if ( cpu_has_fxsr ) { | |
162 | return tsk->thread.i387.fxsave.swd; | |
163 | } else { | |
164 | return (unsigned short)tsk->thread.i387.fsave.swd; | |
165 | } | |
166 | } | |
167 | ||
168 | #if 0 | |
169 | unsigned short get_fpu_twd( struct task_struct *tsk ) | |
170 | { | |
171 | if ( cpu_has_fxsr ) { | |
172 | return tsk->thread.i387.fxsave.twd; | |
173 | } else { | |
174 | return (unsigned short)tsk->thread.i387.fsave.twd; | |
175 | } | |
176 | } | |
177 | #endif /* 0 */ | |
178 | ||
179 | unsigned short get_fpu_mxcsr( struct task_struct *tsk ) | |
180 | { | |
181 | if ( cpu_has_xmm ) { | |
182 | return tsk->thread.i387.fxsave.mxcsr; | |
183 | } else { | |
184 | return 0x1f80; | |
185 | } | |
186 | } | |
187 | ||
188 | #if 0 | |
189 | ||
190 | void set_fpu_cwd( struct task_struct *tsk, unsigned short cwd ) | |
191 | { | |
192 | if ( cpu_has_fxsr ) { | |
193 | tsk->thread.i387.fxsave.cwd = cwd; | |
194 | } else { | |
195 | tsk->thread.i387.fsave.cwd = ((long)cwd | 0xffff0000u); | |
196 | } | |
197 | } | |
198 | ||
199 | void set_fpu_swd( struct task_struct *tsk, unsigned short swd ) | |
200 | { | |
201 | if ( cpu_has_fxsr ) { | |
202 | tsk->thread.i387.fxsave.swd = swd; | |
203 | } else { | |
204 | tsk->thread.i387.fsave.swd = ((long)swd | 0xffff0000u); | |
205 | } | |
206 | } | |
207 | ||
208 | void set_fpu_twd( struct task_struct *tsk, unsigned short twd ) | |
209 | { | |
210 | if ( cpu_has_fxsr ) { | |
211 | tsk->thread.i387.fxsave.twd = twd_i387_to_fxsr(twd); | |
212 | } else { | |
213 | tsk->thread.i387.fsave.twd = ((long)twd | 0xffff0000u); | |
214 | } | |
215 | } | |
216 | ||
217 | #endif /* 0 */ | |
218 | ||
219 | /* | |
220 | * FXSR floating point environment conversions. | |
221 | */ | |
222 | ||
223 | static int convert_fxsr_to_user( struct _fpstate __user *buf, | |
224 | struct i387_fxsave_struct *fxsave ) | |
225 | { | |
226 | unsigned long env[7]; | |
227 | struct _fpreg __user *to; | |
228 | struct _fpxreg *from; | |
229 | int i; | |
230 | ||
231 | env[0] = (unsigned long)fxsave->cwd | 0xffff0000ul; | |
232 | env[1] = (unsigned long)fxsave->swd | 0xffff0000ul; | |
233 | env[2] = twd_fxsr_to_i387(fxsave); | |
234 | env[3] = fxsave->fip; | |
235 | env[4] = fxsave->fcs | ((unsigned long)fxsave->fop << 16); | |
236 | env[5] = fxsave->foo; | |
237 | env[6] = fxsave->fos; | |
238 | ||
239 | if ( __copy_to_user( buf, env, 7 * sizeof(unsigned long) ) ) | |
240 | return 1; | |
241 | ||
242 | to = &buf->_st[0]; | |
243 | from = (struct _fpxreg *) &fxsave->st_space[0]; | |
244 | for ( i = 0 ; i < 8 ; i++, to++, from++ ) { | |
245 | unsigned long __user *t = (unsigned long __user *)to; | |
246 | unsigned long *f = (unsigned long *)from; | |
247 | ||
248 | if (__put_user(*f, t) || | |
249 | __put_user(*(f + 1), t + 1) || | |
250 | __put_user(from->exponent, &to->exponent)) | |
251 | return 1; | |
252 | } | |
253 | return 0; | |
254 | } | |
255 | ||
256 | static int convert_fxsr_from_user( struct i387_fxsave_struct *fxsave, | |
257 | struct _fpstate __user *buf ) | |
258 | { | |
259 | unsigned long env[7]; | |
260 | struct _fpxreg *to; | |
261 | struct _fpreg __user *from; | |
262 | int i; | |
263 | ||
264 | if ( __copy_from_user( env, buf, 7 * sizeof(long) ) ) | |
265 | return 1; | |
266 | ||
267 | fxsave->cwd = (unsigned short)(env[0] & 0xffff); | |
268 | fxsave->swd = (unsigned short)(env[1] & 0xffff); | |
269 | fxsave->twd = twd_i387_to_fxsr((unsigned short)(env[2] & 0xffff)); | |
270 | fxsave->fip = env[3]; | |
271 | fxsave->fop = (unsigned short)((env[4] & 0xffff0000ul) >> 16); | |
272 | fxsave->fcs = (env[4] & 0xffff); | |
273 | fxsave->foo = env[5]; | |
274 | fxsave->fos = env[6]; | |
275 | ||
276 | to = (struct _fpxreg *) &fxsave->st_space[0]; | |
277 | from = &buf->_st[0]; | |
278 | for ( i = 0 ; i < 8 ; i++, to++, from++ ) { | |
279 | unsigned long *t = (unsigned long *)to; | |
280 | unsigned long __user *f = (unsigned long __user *)from; | |
281 | ||
282 | if (__get_user(*t, f) || | |
283 | __get_user(*(t + 1), f + 1) || | |
284 | __get_user(to->exponent, &from->exponent)) | |
285 | return 1; | |
286 | } | |
287 | return 0; | |
288 | } | |
289 | ||
290 | /* | |
291 | * Signal frame handlers. | |
292 | */ | |
293 | ||
294 | static inline int save_i387_fsave( struct _fpstate __user *buf ) | |
295 | { | |
296 | struct task_struct *tsk = current; | |
297 | ||
298 | unlazy_fpu( tsk ); | |
299 | tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd; | |
300 | if ( __copy_to_user( buf, &tsk->thread.i387.fsave, | |
301 | sizeof(struct i387_fsave_struct) ) ) | |
302 | return -1; | |
303 | return 1; | |
304 | } | |
305 | ||
306 | static int save_i387_fxsave( struct _fpstate __user *buf ) | |
307 | { | |
308 | struct task_struct *tsk = current; | |
309 | int err = 0; | |
310 | ||
311 | unlazy_fpu( tsk ); | |
312 | ||
313 | if ( convert_fxsr_to_user( buf, &tsk->thread.i387.fxsave ) ) | |
314 | return -1; | |
315 | ||
316 | err |= __put_user( tsk->thread.i387.fxsave.swd, &buf->status ); | |
317 | err |= __put_user( X86_FXSR_MAGIC, &buf->magic ); | |
318 | if ( err ) | |
319 | return -1; | |
320 | ||
321 | if ( __copy_to_user( &buf->_fxsr_env[0], &tsk->thread.i387.fxsave, | |
322 | sizeof(struct i387_fxsave_struct) ) ) | |
323 | return -1; | |
324 | return 1; | |
325 | } | |
326 | ||
327 | int save_i387( struct _fpstate __user *buf ) | |
328 | { | |
329 | if ( !used_math() ) | |
330 | return 0; | |
331 | ||
332 | /* This will cause a "finit" to be triggered by the next | |
333 | * attempted FPU operation by the 'current' process. | |
334 | */ | |
335 | clear_used_math(); | |
336 | ||
337 | if ( HAVE_HWFP ) { | |
338 | if ( cpu_has_fxsr ) { | |
339 | return save_i387_fxsave( buf ); | |
340 | } else { | |
341 | return save_i387_fsave( buf ); | |
342 | } | |
343 | } else { | |
344 | return save_i387_soft( ¤t->thread.i387.soft, buf ); | |
345 | } | |
346 | } | |
347 | ||
348 | static inline int restore_i387_fsave( struct _fpstate __user *buf ) | |
349 | { | |
350 | struct task_struct *tsk = current; | |
351 | clear_fpu( tsk ); | |
352 | return __copy_from_user( &tsk->thread.i387.fsave, buf, | |
353 | sizeof(struct i387_fsave_struct) ); | |
354 | } | |
355 | ||
356 | static int restore_i387_fxsave( struct _fpstate __user *buf ) | |
357 | { | |
358 | int err; | |
359 | struct task_struct *tsk = current; | |
360 | clear_fpu( tsk ); | |
361 | err = __copy_from_user( &tsk->thread.i387.fxsave, &buf->_fxsr_env[0], | |
362 | sizeof(struct i387_fxsave_struct) ); | |
363 | /* mxcsr reserved bits must be masked to zero for security reasons */ | |
364 | tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; | |
365 | return err ? 1 : convert_fxsr_from_user( &tsk->thread.i387.fxsave, buf ); | |
366 | } | |
367 | ||
368 | int restore_i387( struct _fpstate __user *buf ) | |
369 | { | |
370 | int err; | |
371 | ||
372 | if ( HAVE_HWFP ) { | |
373 | if ( cpu_has_fxsr ) { | |
374 | err = restore_i387_fxsave( buf ); | |
375 | } else { | |
376 | err = restore_i387_fsave( buf ); | |
377 | } | |
378 | } else { | |
379 | err = restore_i387_soft( ¤t->thread.i387.soft, buf ); | |
380 | } | |
381 | set_used_math(); | |
382 | return err; | |
383 | } | |
384 | ||
385 | /* | |
386 | * ptrace request handlers. | |
387 | */ | |
388 | ||
389 | static inline int get_fpregs_fsave( struct user_i387_struct __user *buf, | |
390 | struct task_struct *tsk ) | |
391 | { | |
392 | return __copy_to_user( buf, &tsk->thread.i387.fsave, | |
393 | sizeof(struct user_i387_struct) ); | |
394 | } | |
395 | ||
396 | static inline int get_fpregs_fxsave( struct user_i387_struct __user *buf, | |
397 | struct task_struct *tsk ) | |
398 | { | |
399 | return convert_fxsr_to_user( (struct _fpstate __user *)buf, | |
400 | &tsk->thread.i387.fxsave ); | |
401 | } | |
402 | ||
403 | int get_fpregs( struct user_i387_struct __user *buf, struct task_struct *tsk ) | |
404 | { | |
405 | if ( HAVE_HWFP ) { | |
406 | if ( cpu_has_fxsr ) { | |
407 | return get_fpregs_fxsave( buf, tsk ); | |
408 | } else { | |
409 | return get_fpregs_fsave( buf, tsk ); | |
410 | } | |
411 | } else { | |
412 | return save_i387_soft( &tsk->thread.i387.soft, | |
413 | (struct _fpstate __user *)buf ); | |
414 | } | |
415 | } | |
416 | ||
417 | static inline int set_fpregs_fsave( struct task_struct *tsk, | |
418 | struct user_i387_struct __user *buf ) | |
419 | { | |
420 | return __copy_from_user( &tsk->thread.i387.fsave, buf, | |
421 | sizeof(struct user_i387_struct) ); | |
422 | } | |
423 | ||
424 | static inline int set_fpregs_fxsave( struct task_struct *tsk, | |
425 | struct user_i387_struct __user *buf ) | |
426 | { | |
427 | return convert_fxsr_from_user( &tsk->thread.i387.fxsave, | |
428 | (struct _fpstate __user *)buf ); | |
429 | } | |
430 | ||
431 | int set_fpregs( struct task_struct *tsk, struct user_i387_struct __user *buf ) | |
432 | { | |
433 | if ( HAVE_HWFP ) { | |
434 | if ( cpu_has_fxsr ) { | |
435 | return set_fpregs_fxsave( tsk, buf ); | |
436 | } else { | |
437 | return set_fpregs_fsave( tsk, buf ); | |
438 | } | |
439 | } else { | |
440 | return restore_i387_soft( &tsk->thread.i387.soft, | |
441 | (struct _fpstate __user *)buf ); | |
442 | } | |
443 | } | |
444 | ||
445 | int get_fpxregs( struct user_fxsr_struct __user *buf, struct task_struct *tsk ) | |
446 | { | |
447 | if ( cpu_has_fxsr ) { | |
448 | if (__copy_to_user( buf, &tsk->thread.i387.fxsave, | |
449 | sizeof(struct user_fxsr_struct) )) | |
450 | return -EFAULT; | |
451 | return 0; | |
452 | } else { | |
453 | return -EIO; | |
454 | } | |
455 | } | |
456 | ||
457 | int set_fpxregs( struct task_struct *tsk, struct user_fxsr_struct __user *buf ) | |
458 | { | |
459 | int ret = 0; | |
460 | ||
461 | if ( cpu_has_fxsr ) { | |
462 | if (__copy_from_user( &tsk->thread.i387.fxsave, buf, | |
463 | sizeof(struct user_fxsr_struct) )) | |
464 | ret = -EFAULT; | |
465 | /* mxcsr reserved bits must be masked to zero for security reasons */ | |
466 | tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; | |
467 | } else { | |
468 | ret = -EIO; | |
469 | } | |
470 | return ret; | |
471 | } | |
472 | ||
473 | /* | |
474 | * FPU state for core dumps. | |
475 | */ | |
476 | ||
477 | static inline void copy_fpu_fsave( struct task_struct *tsk, | |
478 | struct user_i387_struct *fpu ) | |
479 | { | |
480 | memcpy( fpu, &tsk->thread.i387.fsave, | |
481 | sizeof(struct user_i387_struct) ); | |
482 | } | |
483 | ||
484 | static inline void copy_fpu_fxsave( struct task_struct *tsk, | |
485 | struct user_i387_struct *fpu ) | |
486 | { | |
487 | unsigned short *to; | |
488 | unsigned short *from; | |
489 | int i; | |
490 | ||
491 | memcpy( fpu, &tsk->thread.i387.fxsave, 7 * sizeof(long) ); | |
492 | ||
493 | to = (unsigned short *)&fpu->st_space[0]; | |
494 | from = (unsigned short *)&tsk->thread.i387.fxsave.st_space[0]; | |
495 | for ( i = 0 ; i < 8 ; i++, to += 5, from += 8 ) { | |
496 | memcpy( to, from, 5 * sizeof(unsigned short) ); | |
497 | } | |
498 | } | |
499 | ||
500 | int dump_fpu( struct pt_regs *regs, struct user_i387_struct *fpu ) | |
501 | { | |
502 | int fpvalid; | |
503 | struct task_struct *tsk = current; | |
504 | ||
505 | fpvalid = !!used_math(); | |
506 | if ( fpvalid ) { | |
507 | unlazy_fpu( tsk ); | |
508 | if ( cpu_has_fxsr ) { | |
509 | copy_fpu_fxsave( tsk, fpu ); | |
510 | } else { | |
511 | copy_fpu_fsave( tsk, fpu ); | |
512 | } | |
513 | } | |
514 | ||
515 | return fpvalid; | |
516 | } | |
129f6946 | 517 | EXPORT_SYMBOL(dump_fpu); |
1da177e4 LT |
518 | |
519 | int dump_task_fpu(struct task_struct *tsk, struct user_i387_struct *fpu) | |
520 | { | |
521 | int fpvalid = !!tsk_used_math(tsk); | |
522 | ||
523 | if (fpvalid) { | |
524 | if (tsk == current) | |
525 | unlazy_fpu(tsk); | |
526 | if (cpu_has_fxsr) | |
527 | copy_fpu_fxsave(tsk, fpu); | |
528 | else | |
529 | copy_fpu_fsave(tsk, fpu); | |
530 | } | |
531 | return fpvalid; | |
532 | } | |
533 | ||
534 | int dump_task_extended_fpu(struct task_struct *tsk, struct user_fxsr_struct *fpu) | |
535 | { | |
536 | int fpvalid = tsk_used_math(tsk) && cpu_has_fxsr; | |
537 | ||
538 | if (fpvalid) { | |
539 | if (tsk == current) | |
540 | unlazy_fpu(tsk); | |
541 | memcpy(fpu, &tsk->thread.i387.fxsave, sizeof(*fpu)); | |
542 | } | |
543 | return fpvalid; | |
544 | } |