1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (C) 1996 Wolfgang Solfrank.
5 * Copyright (C) 1996 TooLs GmbH.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by TooLs GmbH.
19 * 4. The name of TooLs GmbH may not be used to endorse or promote products
20 * derived from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 *
33 * $NetBSD: fpu.c,v 1.5 2001/07/22 11:29:46 wiz Exp $
34 */
35
36 #include <sys/param.h>
37 #include <sys/proc.h>
38 #include <sys/systm.h>
39 #include <sys/limits.h>
40
41 #include <machine/fpu.h>
42 #include <machine/pcb.h>
43 #include <machine/psl.h>
44 #include <machine/altivec.h>
45
46 static void
save_fpu_int(struct thread * td)47 save_fpu_int(struct thread *td)
48 {
49 register_t msr;
50 struct pcb *pcb;
51
52 pcb = td->td_pcb;
53
54 /*
55 * Temporarily re-enable floating-point during the save
56 */
57 msr = mfmsr();
58 if (pcb->pcb_flags & PCB_VSX)
59 mtmsr(msr | PSL_FP | PSL_VSX);
60 else
61 mtmsr(msr | PSL_FP);
62
63 /*
64 * Save the floating-point registers and FPSCR to the PCB
65 */
66 if (pcb->pcb_flags & PCB_VSX) {
67 #if _BYTE_ORDER == _BIG_ENDIAN
68 #define SFP(n) __asm("stxvw4x " #n ", 0,%0" \
69 :: "b"(&pcb->pcb_fpu.fpr[n]));
70 #else
71 /*
72 * stxvw2x will swap words within the FP double word on LE systems,
73 * leading to corruption if VSX is used to store state and FP is
74 * subsequently used to restore state.
75 * Use stxvd2x instead.
76 */
77 #define SFP(n) __asm("stxvd2x " #n ", 0,%0" \
78 :: "b"(&pcb->pcb_fpu.fpr[n]));
79 #endif
80 SFP(0); SFP(1); SFP(2); SFP(3);
81 SFP(4); SFP(5); SFP(6); SFP(7);
82 SFP(8); SFP(9); SFP(10); SFP(11);
83 SFP(12); SFP(13); SFP(14); SFP(15);
84 SFP(16); SFP(17); SFP(18); SFP(19);
85 SFP(20); SFP(21); SFP(22); SFP(23);
86 SFP(24); SFP(25); SFP(26); SFP(27);
87 SFP(28); SFP(29); SFP(30); SFP(31);
88 #undef SFP
89 } else {
90 #define SFP(n) __asm("stfd " #n ", 0(%0)" \
91 :: "b"(&pcb->pcb_fpu.fpr[n].fpr));
92 SFP(0); SFP(1); SFP(2); SFP(3);
93 SFP(4); SFP(5); SFP(6); SFP(7);
94 SFP(8); SFP(9); SFP(10); SFP(11);
95 SFP(12); SFP(13); SFP(14); SFP(15);
96 SFP(16); SFP(17); SFP(18); SFP(19);
97 SFP(20); SFP(21); SFP(22); SFP(23);
98 SFP(24); SFP(25); SFP(26); SFP(27);
99 SFP(28); SFP(29); SFP(30); SFP(31);
100 #undef SFP
101 }
102 __asm __volatile ("mffs 0; stfd 0,0(%0)" :: "b"(&pcb->pcb_fpu.fpscr));
103
104 /*
105 * Disable floating-point again
106 */
107 isync();
108 mtmsr(msr);
109 }
110
111 void
enable_fpu(struct thread * td)112 enable_fpu(struct thread *td)
113 {
114 register_t msr;
115 struct pcb *pcb;
116 struct trapframe *tf;
117
118 pcb = td->td_pcb;
119 tf = trapframe(td);
120
121 /*
122 * Save the thread's FPU CPU number, and set the CPU's current
123 * FPU thread
124 */
125 td->td_pcb->pcb_fpcpu = PCPU_GET(cpuid);
126 PCPU_SET(fputhread, td);
127
128 /*
129 * Enable the FPU for when the thread returns from the exception.
130 * If this is the first time the FPU has been used by the thread,
131 * initialise the FPU registers and FPSCR to 0, and set the flag
132 * to indicate that the FPU is in use.
133 */
134 pcb->pcb_flags |= PCB_FPU;
135 if (pcb->pcb_flags & PCB_VSX)
136 tf->srr1 |= PSL_FP | PSL_VSX;
137 else
138 tf->srr1 |= PSL_FP;
139 if (!(pcb->pcb_flags & PCB_FPREGS)) {
140 memset(&pcb->pcb_fpu, 0, sizeof pcb->pcb_fpu);
141 pcb->pcb_flags |= PCB_FPREGS;
142 }
143
144 /*
145 * Temporarily enable floating-point so the registers
146 * can be restored.
147 */
148 msr = mfmsr();
149 if (pcb->pcb_flags & PCB_VSX)
150 mtmsr(msr | PSL_FP | PSL_VSX);
151 else
152 mtmsr(msr | PSL_FP);
153
154 /*
155 * Load the floating point registers and FPSCR from the PCB.
156 * (A value of 0xff for mtfsf specifies that all 8 4-bit fields
157 * of the saved FPSCR are to be loaded from the FPU reg).
158 */
159 __asm __volatile ("lfd 0,0(%0); mtfsf 0xff,0"
160 :: "b"(&pcb->pcb_fpu.fpscr));
161
162 if (pcb->pcb_flags & PCB_VSX) {
163 #if _BYTE_ORDER == _BIG_ENDIAN
164 #define LFP(n) __asm("lxvw4x " #n ", 0,%0" \
165 :: "b"(&pcb->pcb_fpu.fpr[n]));
166 #else
167 /*
168 * lxvw4x will swap words within the FP double word on LE systems,
169 * leading to corruption if FP is used to store state and VSX is
170 * subsequently used to restore state.
171 * Use lxvd2x instead.
172 */
173 #define LFP(n) __asm("lxvd2x " #n ", 0,%0" \
174 :: "b"(&pcb->pcb_fpu.fpr[n]));
175 #endif
176 LFP(0); LFP(1); LFP(2); LFP(3);
177 LFP(4); LFP(5); LFP(6); LFP(7);
178 LFP(8); LFP(9); LFP(10); LFP(11);
179 LFP(12); LFP(13); LFP(14); LFP(15);
180 LFP(16); LFP(17); LFP(18); LFP(19);
181 LFP(20); LFP(21); LFP(22); LFP(23);
182 LFP(24); LFP(25); LFP(26); LFP(27);
183 LFP(28); LFP(29); LFP(30); LFP(31);
184 #undef LFP
185 } else {
186 #define LFP(n) __asm("lfd " #n ", 0(%0)" \
187 :: "b"(&pcb->pcb_fpu.fpr[n].fpr));
188 LFP(0); LFP(1); LFP(2); LFP(3);
189 LFP(4); LFP(5); LFP(6); LFP(7);
190 LFP(8); LFP(9); LFP(10); LFP(11);
191 LFP(12); LFP(13); LFP(14); LFP(15);
192 LFP(16); LFP(17); LFP(18); LFP(19);
193 LFP(20); LFP(21); LFP(22); LFP(23);
194 LFP(24); LFP(25); LFP(26); LFP(27);
195 LFP(28); LFP(29); LFP(30); LFP(31);
196 #undef LFP
197 }
198
199 isync();
200 mtmsr(msr);
201 }
202
203 void
save_fpu(struct thread * td)204 save_fpu(struct thread *td)
205 {
206 struct pcb *pcb;
207
208 pcb = td->td_pcb;
209
210 save_fpu_int(td);
211
212 /*
213 * Clear the current fp thread and pcb's CPU id
214 * XXX should this be left clear to allow lazy save/restore ?
215 */
216 pcb->pcb_fpcpu = INT_MAX;
217 PCPU_SET(fputhread, NULL);
218 }
219
220 /*
221 * Save fpu state without dropping ownership. This will only save state if
222 * the current fpu thread is `td'.
223 */
224 void
save_fpu_nodrop(struct thread * td)225 save_fpu_nodrop(struct thread *td)
226 {
227
228 if (td == PCPU_GET(fputhread))
229 save_fpu_int(td);
230 }
231
232 /*
233 * Clear Floating-Point Status and Control Register
234 */
235 void
cleanup_fpscr(void)236 cleanup_fpscr(void)
237 {
238 register_t msr;
239
240 msr = mfmsr();
241 mtmsr(msr | PSL_FP);
242 mtfsf(0);
243
244 isync();
245 mtmsr(msr);
246 }
247
248 /*
249 * Get the current fp exception
250 */
251 u_int
get_fpu_exception(struct thread * td)252 get_fpu_exception(struct thread *td)
253 {
254 register_t msr;
255 u_int ucode;
256 register_t reg;
257
258 critical_enter();
259
260 msr = mfmsr();
261 mtmsr(msr | PSL_FP);
262
263 reg = mffs();
264
265 isync();
266 mtmsr(msr);
267
268 critical_exit();
269
270 if (reg & FPSCR_ZX)
271 ucode = FPE_FLTDIV;
272 else if (reg & FPSCR_OX)
273 ucode = FPE_FLTOVF;
274 else if (reg & FPSCR_UX)
275 ucode = FPE_FLTUND;
276 else if (reg & FPSCR_XX)
277 ucode = FPE_FLTRES;
278 else
279 ucode = FPE_FLTINV;
280
281 return ucode;
282 }
283
284 void
enable_fpu_kern(void)285 enable_fpu_kern(void)
286 {
287 register_t msr;
288
289 msr = mfmsr() | PSL_FP;
290
291 if (cpu_features & PPC_FEATURE_HAS_VSX)
292 msr |= PSL_VSX;
293
294 mtmsr(msr);
295 }
296
297 void
disable_fpu(struct thread * td)298 disable_fpu(struct thread *td)
299 {
300 register_t msr;
301 struct pcb *pcb;
302 struct trapframe *tf;
303
304 pcb = td->td_pcb;
305 tf = trapframe(td);
306
307 /* Disable FPU in kernel (if enabled) */
308 msr = mfmsr() & ~(PSL_FP | PSL_VSX);
309 isync();
310 mtmsr(msr);
311
312 /*
313 * Disable FPU in userspace. It will be re-enabled when
314 * an FP or VSX instruction is executed.
315 */
316 tf->srr1 &= ~(PSL_FP | PSL_VSX);
317 pcb->pcb_flags &= ~(PCB_FPU | PCB_VSX);
318 }
319
320 /*
321 * XXX: Implement fpu_kern_alloc_ctx/fpu_kern_free_ctx once fpu_kern_enter and
322 * fpu_kern_leave can handle !FPU_KERN_NOCTX.
323 */
324 struct fpu_kern_ctx {
325 #define FPU_KERN_CTX_DUMMY 0x01 /* avoided save for the kern thread */
326 #define FPU_KERN_CTX_INUSE 0x02
327 uint32_t flags;
328 };
329
330 void
fpu_kern_enter(struct thread * td,struct fpu_kern_ctx * ctx,u_int flags)331 fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags)
332 {
333 struct pcb *pcb;
334
335 pcb = td->td_pcb;
336
337 KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL,
338 ("ctx is required when !FPU_KERN_NOCTX"));
339 KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0,
340 ("using inuse ctx"));
341 KASSERT((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) == 0,
342 ("recursive fpu_kern_enter while in PCB_KERN_FPU_NOSAVE state"));
343
344 if ((flags & FPU_KERN_NOCTX) != 0) {
345 critical_enter();
346
347 if (pcb->pcb_flags & PCB_FPU) {
348 save_fpu(td);
349 pcb->pcb_flags |= PCB_FPREGS;
350 }
351 enable_fpu_kern();
352
353 if (pcb->pcb_flags & PCB_VEC) {
354 save_vec(td);
355 pcb->pcb_flags |= PCB_VECREGS;
356 }
357 enable_vec_kern();
358
359 pcb->pcb_flags |= PCB_KERN_FPU | PCB_KERN_FPU_NOSAVE;
360 return;
361 }
362
363 KASSERT(0, ("fpu_kern_enter with !FPU_KERN_NOCTX not implemented!"));
364 }
365
366 int
fpu_kern_leave(struct thread * td,struct fpu_kern_ctx * ctx)367 fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx)
368 {
369 struct pcb *pcb;
370
371 pcb = td->td_pcb;
372
373 if ((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) != 0) {
374 KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX"));
375 KASSERT(PCPU_GET(fpcurthread) == NULL,
376 ("non-NULL fpcurthread for PCB_FP_NOSAVE"));
377 CRITICAL_ASSERT(td);
378
379 /* Disable FPU, VMX, and VSX */
380 disable_fpu(td);
381 disable_vec(td);
382
383 pcb->pcb_flags &= ~PCB_KERN_FPU_NOSAVE;
384
385 critical_exit();
386 } else {
387 KASSERT(0, ("fpu_kern_leave with !FPU_KERN_NOCTX not implemented!"));
388 }
389
390 pcb->pcb_flags &= ~PCB_KERN_FPU;
391
392 return 0;
393 }
394
395 int
is_fpu_kern_thread(u_int flags __unused)396 is_fpu_kern_thread(u_int flags __unused)
397 {
398 struct pcb *curpcb;
399
400 if ((curthread->td_pflags & TDP_KTHREAD) == 0)
401 return (0);
402 curpcb = curthread->td_pcb;
403 return ((curpcb->pcb_flags & PCB_KERN_FPU) != 0);
404 }
405