Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * include/asm-x86_64/i387.h | |
3 | * | |
4 | * Copyright (C) 1994 Linus Torvalds | |
5 | * | |
6 | * Pentium III FXSR, SSE support | |
7 | * General FPU state handling cleanups | |
8 | * Gareth Hughes <gareth@valinux.com>, May 2000 | |
9 | * x86-64 work by Andi Kleen 2002 | |
10 | */ | |
11 | ||
12 | #ifndef __ASM_X86_64_I387_H | |
13 | #define __ASM_X86_64_I387_H | |
14 | ||
15 | #include <linux/sched.h> | |
16 | #include <asm/processor.h> | |
17 | #include <asm/sigcontext.h> | |
18 | #include <asm/user.h> | |
19 | #include <asm/thread_info.h> | |
20 | #include <asm/uaccess.h> | |
21 | ||
22 | extern void fpu_init(void); | |
23 | extern unsigned int mxcsr_feature_mask; | |
24 | extern void mxcsr_feature_mask_init(void); | |
25 | extern void init_fpu(struct task_struct *child); | |
26 | extern int save_i387(struct _fpstate __user *buf); | |
e07e23e1 | 27 | extern asmlinkage void math_state_restore(void); |
1da177e4 LT |
28 | |
29 | /* | |
30 | * FPU lazy state save handling... | |
31 | */ | |
32 | ||
33 | #define unlazy_fpu(tsk) do { \ | |
e4f17c43 | 34 | if (task_thread_info(tsk)->status & TS_USEDFPU) \ |
e07e23e1 AV |
35 | save_init_fpu(tsk); \ |
36 | else \ | |
37 | tsk->fpu_counter = 0; \ | |
1da177e4 LT |
38 | } while (0) |
39 | ||
40 | /* Ignore delayed exceptions from user space */ | |
41 | static inline void tolerant_fwait(void) | |
42 | { | |
43 | asm volatile("1: fwait\n" | |
44 | "2:\n" | |
45 | " .section __ex_table,\"a\"\n" | |
46 | " .align 8\n" | |
47 | " .quad 1b,2b\n" | |
48 | " .previous\n"); | |
49 | } | |
50 | ||
51 | #define clear_fpu(tsk) do { \ | |
e4f17c43 | 52 | if (task_thread_info(tsk)->status & TS_USEDFPU) { \ |
1da177e4 | 53 | tolerant_fwait(); \ |
e4f17c43 | 54 | task_thread_info(tsk)->status &= ~TS_USEDFPU; \ |
1da177e4 LT |
55 | stts(); \ |
56 | } \ | |
57 | } while (0) | |
58 | ||
59 | /* | |
60 | * ptrace request handers... | |
61 | */ | |
62 | extern int get_fpregs(struct user_i387_struct __user *buf, | |
63 | struct task_struct *tsk); | |
64 | extern int set_fpregs(struct task_struct *tsk, | |
65 | struct user_i387_struct __user *buf); | |
66 | ||
67 | /* | |
68 | * i387 state interaction | |
69 | */ | |
70 | #define get_fpu_mxcsr(t) ((t)->thread.i387.fxsave.mxcsr) | |
71 | #define get_fpu_cwd(t) ((t)->thread.i387.fxsave.cwd) | |
72 | #define get_fpu_fxsr_twd(t) ((t)->thread.i387.fxsave.twd) | |
73 | #define get_fpu_swd(t) ((t)->thread.i387.fxsave.swd) | |
74 | #define set_fpu_cwd(t,val) ((t)->thread.i387.fxsave.cwd = (val)) | |
75 | #define set_fpu_swd(t,val) ((t)->thread.i387.fxsave.swd = (val)) | |
76 | #define set_fpu_fxsr_twd(t,val) ((t)->thread.i387.fxsave.twd = (val)) | |
77 | ||
18bd057b AK |
78 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ |
79 | ||
80 | /* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception | |
81 | is pending. Clear the x87 state here by setting it to fixed | |
82 | values. The kernel data segment can be sometimes 0 and sometimes | |
83 | new user value. Both should be ok. | |
84 | Use the PDA as safe address because it should be already in L1. */ | |
85 | static inline void clear_fpu_state(struct i387_fxsave_struct *fx) | |
86 | { | |
87 | if (unlikely(fx->swd & X87_FSW_ES)) | |
88 | asm volatile("fnclex"); | |
89 | alternative_input(ASM_NOP8 ASM_NOP2, | |
90 | " emms\n" /* clear stack tags */ | |
91 | " fildl %%gs:0", /* load to clear state */ | |
92 | X86_FEATURE_FXSAVE_LEAK); | |
93 | } | |
94 | ||
1da177e4 LT |
95 | static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) |
96 | { | |
97 | int err; | |
7180d4fb JB |
98 | |
99 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" | |
1da177e4 LT |
100 | "2:\n" |
101 | ".section .fixup,\"ax\"\n" | |
102 | "3: movl $-1,%[err]\n" | |
103 | " jmp 2b\n" | |
104 | ".previous\n" | |
105 | ".section __ex_table,\"a\"\n" | |
106 | " .align 8\n" | |
107 | " .quad 1b,3b\n" | |
108 | ".previous" | |
109 | : [err] "=r" (err) | |
7180d4fb JB |
110 | #if 0 /* See comment in __fxsave_clear() below. */ |
111 | : [fx] "r" (fx), "m" (*fx), "0" (0)); | |
112 | #else | |
113 | : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0)); | |
114 | #endif | |
1da177e4 LT |
115 | if (unlikely(err)) |
116 | init_fpu(current); | |
117 | return err; | |
118 | } | |
119 | ||
120 | static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) | |
121 | { | |
122 | int err; | |
7180d4fb JB |
123 | |
124 | asm volatile("1: rex64/fxsave (%[fx])\n\t" | |
1da177e4 LT |
125 | "2:\n" |
126 | ".section .fixup,\"ax\"\n" | |
127 | "3: movl $-1,%[err]\n" | |
128 | " jmp 2b\n" | |
129 | ".previous\n" | |
130 | ".section __ex_table,\"a\"\n" | |
131 | " .align 8\n" | |
132 | " .quad 1b,3b\n" | |
133 | ".previous" | |
7180d4fb JB |
134 | : [err] "=r" (err), "=m" (*fx) |
135 | #if 0 /* See comment in __fxsave_clear() below. */ | |
136 | : [fx] "r" (fx), "0" (0)); | |
137 | #else | |
138 | : [fx] "cdaSDb" (fx), "0" (0)); | |
139 | #endif | |
95912008 AK |
140 | if (unlikely(err) && __clear_user(fx, sizeof(struct i387_fxsave_struct))) |
141 | err = -EFAULT; | |
18bd057b | 142 | /* No need to clear here because the caller clears USED_MATH */ |
1da177e4 LT |
143 | return err; |
144 | } | |
145 | ||
7180d4fb JB |
146 | static inline void __fxsave_clear(struct task_struct *tsk) |
147 | { | |
148 | /* Using "rex64; fxsave %0" is broken because, if the memory operand | |
149 | uses any extended registers for addressing, a second REX prefix | |
150 | will be generated (to the assembler, rex64 followed by semicolon | |
151 | is a separate instruction), and hence the 64-bitness is lost. */ | |
152 | #if 0 | |
153 | /* Using "fxsaveq %0" would be the ideal choice, but is only supported | |
154 | starting with gas 2.16. */ | |
155 | __asm__ __volatile__("fxsaveq %0" | |
156 | : "=m" (tsk->thread.i387.fxsave)); | |
157 | #elif 0 | |
158 | /* Using, as a workaround, the properly prefixed form below isn't | |
159 | accepted by any binutils version so far released, complaining that | |
160 | the same type of prefix is used twice if an extended register is | |
161 | needed for addressing (fix submitted to mainline 2005-11-21). */ | |
162 | __asm__ __volatile__("rex64/fxsave %0" | |
163 | : "=m" (tsk->thread.i387.fxsave)); | |
164 | #else | |
165 | /* This, however, we can work around by forcing the compiler to select | |
166 | an addressing mode that doesn't require extended registers. */ | |
167 | __asm__ __volatile__("rex64/fxsave %P2(%1)" | |
168 | : "=m" (tsk->thread.i387.fxsave) | |
169 | : "cdaSDb" (tsk), | |
170 | "i" (offsetof(__typeof__(*tsk), | |
171 | thread.i387.fxsave))); | |
172 | #endif | |
18bd057b | 173 | clear_fpu_state(&tsk->thread.i387.fxsave); |
7180d4fb JB |
174 | } |
175 | ||
1da177e4 LT |
176 | static inline void kernel_fpu_begin(void) |
177 | { | |
178 | struct thread_info *me = current_thread_info(); | |
179 | preempt_disable(); | |
7180d4fb JB |
180 | if (me->status & TS_USEDFPU) { |
181 | __fxsave_clear(me->task); | |
1da177e4 LT |
182 | me->status &= ~TS_USEDFPU; |
183 | return; | |
184 | } | |
185 | clts(); | |
186 | } | |
187 | ||
188 | static inline void kernel_fpu_end(void) | |
189 | { | |
190 | stts(); | |
191 | preempt_enable(); | |
192 | } | |
193 | ||
e4f17c43 | 194 | static inline void save_init_fpu(struct task_struct *tsk) |
1da177e4 | 195 | { |
7180d4fb | 196 | __fxsave_clear(tsk); |
e4f17c43 | 197 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
1da177e4 LT |
198 | stts(); |
199 | } | |
200 | ||
201 | /* | |
202 | * This restores directly out of user space. Exceptions are handled. | |
203 | */ | |
204 | static inline int restore_i387(struct _fpstate __user *buf) | |
205 | { | |
206 | return restore_fpu_checking((__force struct i387_fxsave_struct *)buf); | |
207 | } | |
208 | ||
209 | #endif /* __ASM_X86_64_I387_H */ |