1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (C) 1996 Wolfgang Solfrank.
5 * Copyright (C) 1996 TooLs GmbH.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by TooLs GmbH.
19 * 4. The name of TooLs GmbH may not be used to endorse or promote products
20 * derived from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 *
33 * $NetBSD: fpu.c,v 1.5 2001/07/22 11:29:46 wiz Exp $
34 */
35
36 #include <sys/cdefs.h>
37 #include <sys/param.h>
38 #include <sys/proc.h>
39 #include <sys/systm.h>
40 #include <sys/limits.h>
41
42 #include <machine/fpu.h>
43 #include <machine/pcb.h>
44 #include <machine/psl.h>
45 #include <machine/altivec.h>
46
47 static void
save_fpu_int(struct thread * td)48 save_fpu_int(struct thread *td)
49 {
50 register_t msr;
51 struct pcb *pcb;
52
53 pcb = td->td_pcb;
54
55 /*
56 * Temporarily re-enable floating-point during the save
57 */
58 msr = mfmsr();
59 if (pcb->pcb_flags & PCB_VSX)
60 mtmsr(msr | PSL_FP | PSL_VSX);
61 else
62 mtmsr(msr | PSL_FP);
63
64 /*
65 * Save the floating-point registers and FPSCR to the PCB
66 */
67 if (pcb->pcb_flags & PCB_VSX) {
68 #define SFP(n) __asm ("stxvw4x " #n ", 0,%0" \
69 :: "b"(&pcb->pcb_fpu.fpr[n]));
70 SFP(0); SFP(1); SFP(2); SFP(3);
71 SFP(4); SFP(5); SFP(6); SFP(7);
72 SFP(8); SFP(9); SFP(10); SFP(11);
73 SFP(12); SFP(13); SFP(14); SFP(15);
74 SFP(16); SFP(17); SFP(18); SFP(19);
75 SFP(20); SFP(21); SFP(22); SFP(23);
76 SFP(24); SFP(25); SFP(26); SFP(27);
77 SFP(28); SFP(29); SFP(30); SFP(31);
78 #undef SFP
79 } else {
80 #define SFP(n) __asm ("stfd " #n ", 0(%0)" \
81 :: "b"(&pcb->pcb_fpu.fpr[n].fpr));
82 SFP(0); SFP(1); SFP(2); SFP(3);
83 SFP(4); SFP(5); SFP(6); SFP(7);
84 SFP(8); SFP(9); SFP(10); SFP(11);
85 SFP(12); SFP(13); SFP(14); SFP(15);
86 SFP(16); SFP(17); SFP(18); SFP(19);
87 SFP(20); SFP(21); SFP(22); SFP(23);
88 SFP(24); SFP(25); SFP(26); SFP(27);
89 SFP(28); SFP(29); SFP(30); SFP(31);
90 #undef SFP
91 }
92 __asm __volatile ("mffs 0; stfd 0,0(%0)" :: "b"(&pcb->pcb_fpu.fpscr));
93
94 /*
95 * Disable floating-point again
96 */
97 isync();
98 mtmsr(msr);
99 }
100
101 void
enable_fpu(struct thread * td)102 enable_fpu(struct thread *td)
103 {
104 register_t msr;
105 struct pcb *pcb;
106 struct trapframe *tf;
107
108 pcb = td->td_pcb;
109 tf = trapframe(td);
110
111 /*
112 * Save the thread's FPU CPU number, and set the CPU's current
113 * FPU thread
114 */
115 td->td_pcb->pcb_fpcpu = PCPU_GET(cpuid);
116 PCPU_SET(fputhread, td);
117
118 /*
119 * Enable the FPU for when the thread returns from the exception.
120 * If this is the first time the FPU has been used by the thread,
121 * initialise the FPU registers and FPSCR to 0, and set the flag
122 * to indicate that the FPU is in use.
123 */
124 pcb->pcb_flags |= PCB_FPU;
125 if (pcb->pcb_flags & PCB_VSX)
126 tf->srr1 |= PSL_FP | PSL_VSX;
127 else
128 tf->srr1 |= PSL_FP;
129 if (!(pcb->pcb_flags & PCB_FPREGS)) {
130 memset(&pcb->pcb_fpu, 0, sizeof pcb->pcb_fpu);
131 pcb->pcb_flags |= PCB_FPREGS;
132 }
133
134 /*
135 * Temporarily enable floating-point so the registers
136 * can be restored.
137 */
138 msr = mfmsr();
139 if (pcb->pcb_flags & PCB_VSX)
140 mtmsr(msr | PSL_FP | PSL_VSX);
141 else
142 mtmsr(msr | PSL_FP);
143
144 /*
145 * Load the floating point registers and FPSCR from the PCB.
146 * (A value of 0xff for mtfsf specifies that all 8 4-bit fields
147 * of the saved FPSCR are to be loaded from the FPU reg).
148 */
149 __asm __volatile ("lfd 0,0(%0); mtfsf 0xff,0"
150 :: "b"(&pcb->pcb_fpu.fpscr));
151
152 if (pcb->pcb_flags & PCB_VSX) {
153 #define LFP(n) __asm ("lxvw4x " #n ", 0,%0" \
154 :: "b"(&pcb->pcb_fpu.fpr[n]));
155 LFP(0); LFP(1); LFP(2); LFP(3);
156 LFP(4); LFP(5); LFP(6); LFP(7);
157 LFP(8); LFP(9); LFP(10); LFP(11);
158 LFP(12); LFP(13); LFP(14); LFP(15);
159 LFP(16); LFP(17); LFP(18); LFP(19);
160 LFP(20); LFP(21); LFP(22); LFP(23);
161 LFP(24); LFP(25); LFP(26); LFP(27);
162 LFP(28); LFP(29); LFP(30); LFP(31);
163 #undef LFP
164 } else {
165 #define LFP(n) __asm ("lfd " #n ", 0(%0)" \
166 :: "b"(&pcb->pcb_fpu.fpr[n].fpr));
167 LFP(0); LFP(1); LFP(2); LFP(3);
168 LFP(4); LFP(5); LFP(6); LFP(7);
169 LFP(8); LFP(9); LFP(10); LFP(11);
170 LFP(12); LFP(13); LFP(14); LFP(15);
171 LFP(16); LFP(17); LFP(18); LFP(19);
172 LFP(20); LFP(21); LFP(22); LFP(23);
173 LFP(24); LFP(25); LFP(26); LFP(27);
174 LFP(28); LFP(29); LFP(30); LFP(31);
175 #undef LFP
176 }
177
178 isync();
179 mtmsr(msr);
180 }
181
182 void
save_fpu(struct thread * td)183 save_fpu(struct thread *td)
184 {
185 struct pcb *pcb;
186
187 pcb = td->td_pcb;
188
189 save_fpu_int(td);
190
191 /*
192 * Clear the current fp thread and pcb's CPU id
193 * XXX should this be left clear to allow lazy save/restore ?
194 */
195 pcb->pcb_fpcpu = INT_MAX;
196 PCPU_SET(fputhread, NULL);
197 }
198
199 /*
200 * Save fpu state without dropping ownership. This will only save state if
201 * the current fpu thread is `td'.
202 */
203 void
save_fpu_nodrop(struct thread * td)204 save_fpu_nodrop(struct thread *td)
205 {
206
207 if (td == PCPU_GET(fputhread))
208 save_fpu_int(td);
209 }
210
211 /*
212 * Clear Floating-Point Status and Control Register
213 */
214 void
cleanup_fpscr(void)215 cleanup_fpscr(void)
216 {
217 register_t msr;
218
219 msr = mfmsr();
220 mtmsr(msr | PSL_FP);
221 mtfsf(0);
222
223 isync();
224 mtmsr(msr);
225 }
226
227 /*
228 * Get the current fp exception
229 */
230 u_int
get_fpu_exception(struct thread * td)231 get_fpu_exception(struct thread *td)
232 {
233 register_t msr;
234 u_int ucode;
235 register_t reg;
236
237 critical_enter();
238
239 msr = mfmsr();
240 mtmsr(msr | PSL_FP);
241
242 reg = mffs();
243
244 isync();
245 mtmsr(msr);
246
247 critical_exit();
248
249 if (reg & FPSCR_ZX)
250 ucode = FPE_FLTDIV;
251 else if (reg & FPSCR_OX)
252 ucode = FPE_FLTOVF;
253 else if (reg & FPSCR_UX)
254 ucode = FPE_FLTUND;
255 else if (reg & FPSCR_XX)
256 ucode = FPE_FLTRES;
257 else
258 ucode = FPE_FLTINV;
259
260 return ucode;
261 }
262
263 void
enable_fpu_kern(void)264 enable_fpu_kern(void)
265 {
266 register_t msr;
267
268 msr = mfmsr() | PSL_FP;
269
270 if (cpu_features & PPC_FEATURE_HAS_VSX)
271 msr |= PSL_VSX;
272
273 mtmsr(msr);
274 }
275
276 void
disable_fpu(struct thread * td)277 disable_fpu(struct thread *td)
278 {
279 register_t msr;
280 struct pcb *pcb;
281 struct trapframe *tf;
282
283 pcb = td->td_pcb;
284 tf = trapframe(td);
285
286 /* Disable FPU in kernel (if enabled) */
287 msr = mfmsr() & ~(PSL_FP | PSL_VSX);
288 isync();
289 mtmsr(msr);
290
291 /*
292 * Disable FPU in userspace. It will be re-enabled when
293 * an FP or VSX instruction is executed.
294 */
295 tf->srr1 &= ~(PSL_FP | PSL_VSX);
296 pcb->pcb_flags &= ~(PCB_FPU | PCB_VSX);
297 }
298
299 #ifndef __SPE__
300 /*
301 * XXX: Implement fpu_kern_alloc_ctx/fpu_kern_free_ctx once fpu_kern_enter and
302 * fpu_kern_leave can handle !FPU_KERN_NOCTX.
303 */
304 struct fpu_kern_ctx {
305 #define FPU_KERN_CTX_DUMMY 0x01 /* avoided save for the kern thread */
306 #define FPU_KERN_CTX_INUSE 0x02
307 uint32_t flags;
308 };
309
310 void
fpu_kern_enter(struct thread * td,struct fpu_kern_ctx * ctx,u_int flags)311 fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags)
312 {
313 struct pcb *pcb;
314
315 pcb = td->td_pcb;
316
317 KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL,
318 ("ctx is required when !FPU_KERN_NOCTX"));
319 KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0,
320 ("using inuse ctx"));
321 KASSERT((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) == 0,
322 ("recursive fpu_kern_enter while in PCB_KERN_FPU_NOSAVE state"));
323
324 if ((flags & FPU_KERN_NOCTX) != 0) {
325 critical_enter();
326
327 if (pcb->pcb_flags & PCB_FPU) {
328 save_fpu(td);
329 pcb->pcb_flags |= PCB_FPREGS;
330 }
331 enable_fpu_kern();
332
333 if (pcb->pcb_flags & PCB_VEC) {
334 save_vec(td);
335 pcb->pcb_flags |= PCB_VECREGS;
336 }
337 enable_vec_kern();
338
339 pcb->pcb_flags |= PCB_KERN_FPU | PCB_KERN_FPU_NOSAVE;
340 return;
341 }
342
343 KASSERT(0, ("fpu_kern_enter with !FPU_KERN_NOCTX not implemented!"));
344 }
345
346 int
fpu_kern_leave(struct thread * td,struct fpu_kern_ctx * ctx)347 fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx)
348 {
349 struct pcb *pcb;
350
351 pcb = td->td_pcb;
352
353 if ((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) != 0) {
354 KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX"));
355 KASSERT(PCPU_GET(fpcurthread) == NULL,
356 ("non-NULL fpcurthread for PCB_FP_NOSAVE"));
357 CRITICAL_ASSERT(td);
358
359 /* Disable FPU, VMX, and VSX */
360 disable_fpu(td);
361 disable_vec(td);
362
363 pcb->pcb_flags &= ~PCB_KERN_FPU_NOSAVE;
364
365 critical_exit();
366 } else {
367 KASSERT(0, ("fpu_kern_leave with !FPU_KERN_NOCTX not implemented!"));
368 }
369
370 pcb->pcb_flags &= ~PCB_KERN_FPU;
371
372 return 0;
373 }
374
375 int
is_fpu_kern_thread(u_int flags __unused)376 is_fpu_kern_thread(u_int flags __unused)
377 {
378 struct pcb *curpcb;
379
380 if ((curthread->td_pflags & TDP_KTHREAD) == 0)
381 return (0);
382 curpcb = curthread->td_pcb;
383 return ((curpcb->pcb_flags & PCB_KERN_FPU) != 0);
384 }
385
386 #endif /* !__SPE__ */
387