1 /*-
2 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
3 * Copyright (C) 1995, 1996 TooLs GmbH.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. All advertising materials mentioning features or use of this software
15 * must display the following acknowledgement:
16 * This product includes software developed by TooLs GmbH.
17 * 4. The name of TooLs GmbH may not be used to endorse or promote products
18 * derived from this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
29 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * $NetBSD: trap.c,v 1.58 2002/03/04 04:07:35 dbj Exp $
32 */
33
34 #include <sys/cdefs.h>
35 #include <sys/param.h>
36 #include <sys/kdb.h>
37 #include <sys/proc.h>
38 #include <sys/ktr.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/ptrace.h>
42 #include <sys/reboot.h>
43 #include <sys/syscall.h>
44 #include <sys/sysent.h>
45 #include <sys/systm.h>
46 #include <sys/kernel.h>
47 #include <sys/uio.h>
48 #include <sys/signalvar.h>
49 #include <sys/vmmeter.h>
50
51 #include <security/audit/audit.h>
52
53 #include <vm/vm.h>
54 #include <vm/pmap.h>
55 #include <vm/vm_extern.h>
56 #include <vm/vm_param.h>
57 #include <vm/vm_kern.h>
58 #include <vm/vm_map.h>
59 #include <vm/vm_page.h>
60
61 #include <machine/_inttypes.h>
62 #include <machine/altivec.h>
63 #include <machine/cpu.h>
64 #include <machine/db_machdep.h>
65 #include <machine/fpu.h>
66 #include <machine/frame.h>
67 #include <machine/pcb.h>
68 #include <machine/psl.h>
69 #include <machine/slb.h>
70 #include <machine/spr.h>
71 #include <machine/sr.h>
72 #include <machine/trap.h>
73
74 /* Below matches setjmp.S */
75 #define FAULTBUF_LR 21
76 #define FAULTBUF_R1 1
77 #define FAULTBUF_R2 2
78 #define FAULTBUF_CR 22
79 #define FAULTBUF_R14 3
80
81 #define MOREARGS(sp) ((caddr_t)((uintptr_t)(sp) + \
82 sizeof(struct callframe) - 3*sizeof(register_t))) /* more args go here */
83
84 static void trap_fatal(struct trapframe *frame);
85 static void printtrap(u_int vector, struct trapframe *frame, int isfatal,
86 int user);
87 static bool trap_pfault(struct trapframe *frame, bool user, int *signo,
88 int *ucode);
89 static int fix_unaligned(struct thread *td, struct trapframe *frame);
90 static int handle_onfault(struct trapframe *frame);
91 static void syscall(struct trapframe *frame);
92
93 #if defined(__powerpc64__) && defined(AIM)
94 static void normalize_inputs(void);
95 #endif
96
97 extern vm_offset_t __startkernel;
98
99 extern int copy_fault(void);
100 extern int fusufault(void);
101
102 #ifdef KDB
103 int db_trap_glue(struct trapframe *); /* Called from trap_subr.S */
104 #endif
105
106 struct powerpc_exception {
107 u_int vector;
108 char *name;
109 };
110
111 #ifdef KDTRACE_HOOKS
112 #include <sys/dtrace_bsd.h>
113
114 int (*dtrace_invop_jump_addr)(struct trapframe *);
115 #endif
116
117 static struct powerpc_exception powerpc_exceptions[] = {
118 { EXC_CRIT, "critical input" },
119 { EXC_RST, "system reset" },
120 { EXC_MCHK, "machine check" },
121 { EXC_DSI, "data storage interrupt" },
122 { EXC_DSE, "data segment exception" },
123 { EXC_ISI, "instruction storage interrupt" },
124 { EXC_ISE, "instruction segment exception" },
125 { EXC_EXI, "external interrupt" },
126 { EXC_ALI, "alignment" },
127 { EXC_PGM, "program" },
128 { EXC_HEA, "hypervisor emulation assistance" },
129 { EXC_FPU, "floating-point unavailable" },
130 { EXC_APU, "auxiliary proc unavailable" },
131 { EXC_DECR, "decrementer" },
132 { EXC_FIT, "fixed-interval timer" },
133 { EXC_WDOG, "watchdog timer" },
134 { EXC_SC, "system call" },
135 { EXC_TRC, "trace" },
136 { EXC_FPA, "floating-point assist" },
137 { EXC_DEBUG, "debug" },
138 { EXC_PERF, "performance monitoring" },
139 { EXC_VEC, "altivec unavailable" },
140 { EXC_VSX, "vsx unavailable" },
141 { EXC_FAC, "facility unavailable" },
142 { EXC_HFAC, "hypervisor facility unavailable" },
143 { EXC_ITMISS, "instruction tlb miss" },
144 { EXC_DLMISS, "data load tlb miss" },
145 { EXC_DSMISS, "data store tlb miss" },
146 { EXC_BPT, "instruction breakpoint" },
147 { EXC_SMI, "system management" },
148 { EXC_VECAST_G4, "altivec assist" },
149 { EXC_THRM, "thermal management" },
150 { EXC_RUNMODETRC, "run mode/trace" },
151 { EXC_SOFT_PATCH, "soft patch exception" },
152 { EXC_HVI, "hypervisor virtualization" },
153 { EXC_LAST, NULL }
154 };
155
156 static int uprintf_signal;
157 SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN,
158 &uprintf_signal, 0,
159 "Print debugging information on trap signal to ctty");
160
161 #define ESR_BITMASK \
162 "\20" \
163 "\040b0\037b1\036b2\035b3\034PIL\033PRR\032PTR\031FP" \
164 "\030ST\027b9\026DLK\025ILK\024b12\023b13\022BO\021PIE" \
165 "\020b16\017b17\016b18\015b19\014b20\013b21\012b22\011b23" \
166 "\010SPE\007EPID\006b26\005b27\004b28\003b29\002b30\001b31"
167 #define MCSR_BITMASK \
168 "\20" \
169 "\040MCP\037ICERR\036DCERR\035TLBPERR\034L2MMU_MHIT\033b5\032b6\031b7" \
170 "\030b8\027b9\026b10\025NMI\024MAV\023MEA\022b14\021IF" \
171 "\020LD\017ST\016LDG\015b19\014b20\013b21\012b22\011b23" \
172 "\010b24\007b25\006b26\005b27\004b28\003b29\002TLBSYNC\001BSL2_ERR"
173 #define MSSSR_BITMASK \
174 "\20" \
175 "\040b0\037b1\036b2\035b3\034b4\033b5\032b6\031b7" \
176 "\030b8\027b9\026b10\025b11\024b12\023L2TAG\022L2DAT\021L3TAG" \
177 "\020L3DAT\017APE\016DPE\015TEA\014b20\013b21\012b22\011b23" \
178 "\010b24\007b25\006b26\005b27\004b28\003b29\002b30\001b31"
179
180 static const char *
trapname(u_int vector)181 trapname(u_int vector)
182 {
183 struct powerpc_exception *pe;
184
185 for (pe = powerpc_exceptions; pe->vector != EXC_LAST; pe++) {
186 if (pe->vector == vector)
187 return (pe->name);
188 }
189
190 return ("unknown");
191 }
192
193 static inline bool
frame_is_trap_inst(struct trapframe * frame)194 frame_is_trap_inst(struct trapframe *frame)
195 {
196 #ifdef AIM
197 return (frame->exc == EXC_PGM && frame->srr1 & EXC_PGM_TRAP);
198 #else
199 return ((frame->cpu.booke.esr & ESR_PTR) != 0);
200 #endif
201 }
202
203 void
trap(struct trapframe * frame)204 trap(struct trapframe *frame)
205 {
206 struct thread *td;
207 struct proc *p;
208 #ifdef KDTRACE_HOOKS
209 uint32_t inst;
210 #endif
211 int sig, type, user;
212 u_int ucode;
213 ksiginfo_t ksi;
214 register_t addr, fscr;
215
216 VM_CNT_INC(v_trap);
217
218 #ifdef KDB
219 if (kdb_active) {
220 kdb_reenter();
221 return;
222 }
223 #endif
224
225 td = curthread;
226 p = td->td_proc;
227
228 type = ucode = frame->exc;
229 sig = 0;
230 user = frame->srr1 & PSL_PR;
231 addr = 0;
232
233 CTR3(KTR_TRAP, "trap: %s type=%s (%s)", td->td_name,
234 trapname(type), user ? "user" : "kernel");
235
236 #ifdef KDTRACE_HOOKS
237 /*
238 * A trap can occur while DTrace executes a probe. Before
239 * executing the probe, DTrace blocks re-scheduling and sets
240 * a flag in its per-cpu flags to indicate that it doesn't
241 * want to fault. On returning from the probe, the no-fault
242 * flag is cleared and finally re-scheduling is enabled.
243 *
244 * If the DTrace kernel module has registered a trap handler,
245 * call it and if it returns non-zero, assume that it has
246 * handled the trap and modified the trap frame so that this
247 * function can return normally.
248 */
249 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type) != 0)
250 return;
251 #endif
252
253 if (user) {
254 td->td_pticks = 0;
255 td->td_frame = frame;
256 addr = frame->srr0;
257 if (td->td_cowgen != atomic_load_int(&p->p_cowgen))
258 thread_cow_update(td);
259
260 /* User Mode Traps */
261 switch (type) {
262 case EXC_RUNMODETRC:
263 case EXC_TRC:
264 frame->srr1 &= ~PSL_SE;
265 sig = SIGTRAP;
266 ucode = TRAP_TRACE;
267 break;
268
269 #if defined(__powerpc64__) && defined(AIM)
270 case EXC_DSE:
271 addr = frame->dar;
272 /* FALLTHROUGH */
273 case EXC_ISE:
274 /* DSE/ISE are automatically fatal with radix pmap. */
275 if (radix_mmu ||
276 handle_user_slb_spill(&p->p_vmspace->vm_pmap,
277 addr) != 0){
278 sig = SIGSEGV;
279 ucode = SEGV_MAPERR;
280 }
281 break;
282 #endif
283 case EXC_DSI:
284 addr = frame->dar;
285 /* FALLTHROUGH */
286 case EXC_ISI:
287 if (trap_pfault(frame, true, &sig, &ucode))
288 sig = 0;
289 break;
290
291 case EXC_SC:
292 syscall(frame);
293 break;
294
295 case EXC_FPU:
296 KASSERT((td->td_pcb->pcb_flags & PCB_FPU) != PCB_FPU,
297 ("FPU already enabled for thread"));
298 enable_fpu(td);
299 break;
300
301 case EXC_VEC:
302 KASSERT((td->td_pcb->pcb_flags & PCB_VEC) != PCB_VEC,
303 ("Altivec already enabled for thread"));
304 enable_vec(td);
305 break;
306
307 case EXC_VSX:
308 KASSERT((td->td_pcb->pcb_flags & PCB_VSX) != PCB_VSX,
309 ("VSX already enabled for thread"));
310 if (!(td->td_pcb->pcb_flags & PCB_VEC))
311 enable_vec(td);
312 if (td->td_pcb->pcb_flags & PCB_FPU)
313 save_fpu(td);
314 td->td_pcb->pcb_flags |= PCB_VSX;
315 enable_fpu(td);
316 break;
317
318 case EXC_FAC:
319 fscr = mfspr(SPR_FSCR);
320 switch (fscr & FSCR_IC_MASK) {
321 case FSCR_IC_HTM:
322 CTR0(KTR_TRAP,
323 "Hardware Transactional Memory subsystem disabled");
324 sig = SIGILL;
325 ucode = ILL_ILLOPC;
326 break;
327 case FSCR_IC_DSCR:
328 td->td_pcb->pcb_flags |= PCB_CFSCR | PCB_CDSCR;
329 fscr |= FSCR_DSCR;
330 mtspr(SPR_DSCR, 0);
331 break;
332 case FSCR_IC_EBB:
333 td->td_pcb->pcb_flags |= PCB_CFSCR;
334 fscr |= FSCR_EBB;
335 mtspr(SPR_EBBHR, 0);
336 mtspr(SPR_EBBRR, 0);
337 mtspr(SPR_BESCR, 0);
338 break;
339 case FSCR_IC_TAR:
340 td->td_pcb->pcb_flags |= PCB_CFSCR;
341 fscr |= FSCR_TAR;
342 mtspr(SPR_TAR, 0);
343 break;
344 case FSCR_IC_LM:
345 td->td_pcb->pcb_flags |= PCB_CFSCR;
346 fscr |= FSCR_LM;
347 mtspr(SPR_LMRR, 0);
348 mtspr(SPR_LMSER, 0);
349 break;
350 default:
351 sig = SIGILL;
352 ucode = ILL_ILLOPC;
353 }
354 mtspr(SPR_FSCR, fscr & ~FSCR_IC_MASK);
355 break;
356 case EXC_HEA:
357 case EXC_HFAC:
358 sig = SIGILL;
359 ucode = ILL_ILLOPC;
360 break;
361
362 case EXC_VECAST_E:
363 case EXC_VECAST_G4:
364 case EXC_VECAST_G5:
365 /*
366 * We get a VPU assist exception for IEEE mode
367 * vector operations on denormalized floats.
368 * Emulating this is a giant pain, so for now,
369 * just switch off IEEE mode and treat them as
370 * zero.
371 */
372
373 save_vec(td);
374 td->td_pcb->pcb_vec.vscr |= ALTIVEC_VSCR_NJ;
375 enable_vec(td);
376 break;
377
378 case EXC_ALI:
379 if (fix_unaligned(td, frame) != 0) {
380 sig = SIGBUS;
381 ucode = BUS_ADRALN;
382 addr = frame->dar;
383 }
384 else
385 frame->srr0 += 4;
386 break;
387
388 case EXC_DEBUG: /* Single stepping */
389 mtspr(SPR_DBSR, mfspr(SPR_DBSR));
390 frame->srr1 &= ~PSL_DE;
391 frame->cpu.booke.dbcr0 &= ~(DBCR0_IDM | DBCR0_IC);
392 sig = SIGTRAP;
393 ucode = TRAP_TRACE;
394 break;
395
396 case EXC_PGM:
397 /* Identify the trap reason */
398 if (frame_is_trap_inst(frame)) {
399 #ifdef KDTRACE_HOOKS
400 inst = fuword32((const void *)frame->srr0);
401 if (inst == 0x0FFFDDDD &&
402 dtrace_pid_probe_ptr != NULL) {
403 (*dtrace_pid_probe_ptr)(frame);
404 break;
405 }
406 #endif
407 sig = SIGTRAP;
408 ucode = TRAP_BRKPT;
409 break;
410 }
411
412 if ((frame->srr1 & EXC_PGM_FPENABLED) &&
413 (td->td_pcb->pcb_flags & PCB_FPU))
414 sig = SIGFPE;
415 else
416 sig = ppc_instr_emulate(frame, td);
417
418 if (sig == SIGILL) {
419 if (frame->srr1 & EXC_PGM_PRIV)
420 ucode = ILL_PRVOPC;
421 else if (frame->srr1 & EXC_PGM_ILLEGAL)
422 ucode = ILL_ILLOPC;
423 } else if (sig == SIGFPE) {
424 ucode = get_fpu_exception(td);
425 }
426
427 break;
428
429 case EXC_MCHK:
430 sig = cpu_machine_check(td, frame, &ucode);
431 printtrap(frame->exc, frame, 0, (frame->srr1 & PSL_PR));
432 break;
433
434 #if defined(__powerpc64__) && defined(AIM)
435 case EXC_SOFT_PATCH:
436 /*
437 * Point to the instruction that generated the exception to execute it again,
438 * and normalize the register values.
439 */
440 frame->srr0 -= 4;
441 normalize_inputs();
442 break;
443 #endif
444
445 default:
446 trap_fatal(frame);
447 }
448 } else {
449 /* Kernel Mode Traps */
450
451 KASSERT(cold || td->td_ucred != NULL,
452 ("kernel trap doesn't have ucred"));
453 switch (type) {
454 case EXC_PGM:
455 #ifdef KDTRACE_HOOKS
456 if (frame_is_trap_inst(frame)) {
457 if (*(uint32_t *)frame->srr0 == EXC_DTRACE) {
458 if (dtrace_invop_jump_addr != NULL) {
459 dtrace_invop_jump_addr(frame);
460 return;
461 }
462 }
463 }
464 #endif
465 #ifdef KDB
466 if (db_trap_glue(frame))
467 return;
468 #endif
469 break;
470 #if defined(__powerpc64__) && defined(AIM)
471 case EXC_DSE:
472 /* DSE on radix mmu is automatically fatal. */
473 if (radix_mmu)
474 break;
475 if (td->td_pcb->pcb_cpu.aim.usr_vsid != 0 &&
476 (frame->dar & SEGMENT_MASK) == USER_ADDR) {
477 __asm __volatile ("slbmte %0, %1" ::
478 "r"(td->td_pcb->pcb_cpu.aim.usr_vsid),
479 "r"(USER_SLB_SLBE));
480 return;
481 }
482 break;
483 #endif
484 case EXC_DSI:
485 if (trap_pfault(frame, false, NULL, NULL))
486 return;
487 break;
488 case EXC_MCHK:
489 if (handle_onfault(frame))
490 return;
491 break;
492 default:
493 break;
494 }
495 trap_fatal(frame);
496 }
497
498 if (sig != 0) {
499 ksiginfo_init_trap(&ksi);
500 ksi.ksi_signo = sig;
501 ksi.ksi_code = (int) ucode; /* XXX, not POSIX */
502 ksi.ksi_addr = (void *)addr;
503 ksi.ksi_trapno = type;
504 if (uprintf_signal) {
505 uprintf("pid %d comm %s: signal %d code %d type 0x%x "
506 "addr 0x%lx r1 0x%lx srr0 0x%lx srr1 0x%lx\n",
507 p->p_pid, p->p_comm, sig, ucode, type,
508 (u_long)addr, (u_long)frame->fixreg[1],
509 (u_long)frame->srr0, (u_long)frame->srr1);
510 }
511
512 trapsignal(td, &ksi);
513 }
514
515 userret(td, frame);
516 }
517
518 static void
trap_fatal(struct trapframe * frame)519 trap_fatal(struct trapframe *frame)
520 {
521 #ifdef KDB
522 bool handled;
523 #endif
524
525 printtrap(frame->exc, frame, 1, (frame->srr1 & PSL_PR));
526 #ifdef KDB
527 if (debugger_on_trap) {
528 kdb_why = KDB_WHY_TRAP;
529 handled = kdb_trap(frame->exc, 0, frame);
530 kdb_why = KDB_WHY_UNSET;
531 if (handled)
532 return;
533 }
534 #endif
535 panic("%s trap", trapname(frame->exc));
536 }
537
538 static void
cpu_printtrap(u_int vector,struct trapframe * frame,int isfatal,int user)539 cpu_printtrap(u_int vector, struct trapframe *frame, int isfatal, int user)
540 {
541 #ifdef AIM
542 uint16_t ver;
543
544 switch (vector) {
545 case EXC_MCHK:
546 ver = mfpvr() >> 16;
547 if (MPC745X_P(ver))
548 printf(" msssr0 = 0x%b\n",
549 (int)mfspr(SPR_MSSSR0), MSSSR_BITMASK);
550 case EXC_DSE:
551 case EXC_DSI:
552 case EXC_DTMISS:
553 printf(" dsisr = 0x%lx\n",
554 (u_long)frame->cpu.aim.dsisr);
555 break;
556 }
557 #elif defined(BOOKE)
558 vm_paddr_t pa;
559
560 switch (vector) {
561 case EXC_MCHK:
562 pa = mfspr(SPR_MCARU);
563 pa = (pa << 32) | (u_register_t)mfspr(SPR_MCAR);
564 printf(" mcsr = 0x%b\n",
565 (int)mfspr(SPR_MCSR), MCSR_BITMASK);
566 printf(" mcar = 0x%jx\n", (uintmax_t)pa);
567 }
568 printf(" esr = 0x%b\n",
569 (int)frame->cpu.booke.esr, ESR_BITMASK);
570 #endif
571 }
572
573 static void
printtrap(u_int vector,struct trapframe * frame,int isfatal,int user)574 printtrap(u_int vector, struct trapframe *frame, int isfatal, int user)
575 {
576
577 printf("\n");
578 printf("%s %s trap:\n", isfatal ? "fatal" : "handled",
579 user ? "user" : "kernel");
580 printf("\n");
581 printf(" exception = 0x%x (%s)\n", vector, trapname(vector));
582 switch (vector) {
583 case EXC_DSE:
584 case EXC_DSI:
585 case EXC_DTMISS:
586 case EXC_ALI:
587 case EXC_MCHK:
588 printf(" virtual address = 0x%" PRIxPTR "\n", frame->dar);
589 break;
590 case EXC_ISE:
591 case EXC_ISI:
592 case EXC_ITMISS:
593 printf(" virtual address = 0x%" PRIxPTR "\n", frame->srr0);
594 break;
595 }
596 cpu_printtrap(vector, frame, isfatal, user);
597 printf(" srr0 = 0x%" PRIxPTR " (0x%" PRIxPTR ")\n",
598 frame->srr0, frame->srr0 - (register_t)(__startkernel - KERNBASE));
599 printf(" srr1 = 0x%lx\n", (u_long)frame->srr1);
600 printf(" current msr = 0x%" PRIxPTR "\n", mfmsr());
601 printf(" lr = 0x%" PRIxPTR " (0x%" PRIxPTR ")\n",
602 frame->lr, frame->lr - (register_t)(__startkernel - KERNBASE));
603 printf(" frame = %p\n", frame);
604 printf(" curthread = %p\n", curthread);
605 if (curthread != NULL)
606 printf(" pid = %d, comm = %s\n",
607 curthread->td_proc->p_pid, curthread->td_name);
608 printf("\n");
609 }
610
611 /*
612 * Handles a fatal fault when we have onfault state to recover. Returns
613 * non-zero if there was onfault recovery state available.
614 */
615 static int
handle_onfault(struct trapframe * frame)616 handle_onfault(struct trapframe *frame)
617 {
618 struct thread *td;
619 jmp_buf *fb;
620
621 td = curthread;
622 #if defined(__powerpc64__) || defined(BOOKE)
623 uintptr_t dispatch = (uintptr_t)td->td_pcb->pcb_onfault;
624
625 if (dispatch == 0)
626 return (0);
627 /* Short-circuit radix and Book-E paths. */
628 switch (dispatch) {
629 case COPYFAULT:
630 frame->srr0 = (uintptr_t)copy_fault;
631 return (1);
632 case FUSUFAULT:
633 frame->srr0 = (uintptr_t)fusufault;
634 return (1);
635 default:
636 break;
637 }
638 #endif
639 fb = td->td_pcb->pcb_onfault;
640 if (fb != NULL) {
641 frame->srr0 = (*fb)->_jb[FAULTBUF_LR];
642 frame->fixreg[1] = (*fb)->_jb[FAULTBUF_R1];
643 frame->fixreg[2] = (*fb)->_jb[FAULTBUF_R2];
644 frame->fixreg[3] = 1;
645 frame->cr = (*fb)->_jb[FAULTBUF_CR];
646 bcopy(&(*fb)->_jb[FAULTBUF_R14], &frame->fixreg[14],
647 18 * sizeof(register_t));
648 td->td_pcb->pcb_onfault = NULL; /* Returns twice, not thrice */
649 return (1);
650 }
651 return (0);
652 }
653
654 int
cpu_fetch_syscall_args(struct thread * td)655 cpu_fetch_syscall_args(struct thread *td)
656 {
657 struct proc *p;
658 struct trapframe *frame;
659 struct syscall_args *sa;
660 caddr_t params;
661 size_t argsz;
662 int error, n, narg, i;
663
664 p = td->td_proc;
665 frame = td->td_frame;
666 sa = &td->td_sa;
667
668 sa->code = frame->fixreg[0];
669 sa->original_code = sa->code;
670 params = (caddr_t)(frame->fixreg + FIRSTARG);
671 n = NARGREG;
672
673 if (sa->code == SYS_syscall) {
674 /*
675 * code is first argument,
676 * followed by actual args.
677 */
678 sa->code = *(register_t *) params;
679 params += sizeof(register_t);
680 n -= 1;
681 } else if (sa->code == SYS___syscall) {
682 /*
683 * Like syscall, but code is a quad,
684 * so as to maintain quad alignment
685 * for the rest of the args.
686 */
687 if (SV_PROC_FLAG(p, SV_ILP32)) {
688 params += sizeof(register_t);
689 sa->code = *(register_t *) params;
690 params += sizeof(register_t);
691 n -= 2;
692 } else {
693 sa->code = *(register_t *) params;
694 params += sizeof(register_t);
695 n -= 1;
696 }
697 }
698
699 if (sa->code >= p->p_sysent->sv_size)
700 sa->callp = &nosys_sysent;
701 else
702 sa->callp = &p->p_sysent->sv_table[sa->code];
703
704 narg = sa->callp->sy_narg;
705
706 if (SV_PROC_FLAG(p, SV_ILP32)) {
707 argsz = sizeof(uint32_t);
708
709 for (i = 0; i < n; i++)
710 sa->args[i] = ((u_register_t *)(params))[i] &
711 0xffffffff;
712 } else {
713 argsz = sizeof(uint64_t);
714
715 for (i = 0; i < n; i++)
716 sa->args[i] = ((u_register_t *)(params))[i];
717 }
718
719 if (narg > n)
720 error = copyin(MOREARGS(frame->fixreg[1]), sa->args + n,
721 (narg - n) * argsz);
722 else
723 error = 0;
724
725 #ifdef __powerpc64__
726 if (SV_PROC_FLAG(p, SV_ILP32) && narg > n) {
727 /* Expand the size of arguments copied from the stack */
728
729 for (i = narg; i >= n; i--)
730 sa->args[i] = ((uint32_t *)(&sa->args[n]))[i-n];
731 }
732 #endif
733
734 if (error == 0) {
735 td->td_retval[0] = 0;
736 td->td_retval[1] = frame->fixreg[FIRSTARG + 1];
737 }
738 return (error);
739 }
740
741 #include "../../kern/subr_syscall.c"
742
743 void
syscall(struct trapframe * frame)744 syscall(struct trapframe *frame)
745 {
746 struct thread *td;
747
748 td = curthread;
749 td->td_frame = frame;
750
751 #if defined(__powerpc64__) && defined(AIM)
752 /*
753 * Speculatively restore last user SLB segment, which we know is
754 * invalid already, since we are likely to do copyin()/copyout().
755 */
756 if (td->td_pcb->pcb_cpu.aim.usr_vsid != 0)
757 __asm __volatile ("slbmte %0, %1; isync" ::
758 "r"(td->td_pcb->pcb_cpu.aim.usr_vsid), "r"(USER_SLB_SLBE));
759 #endif
760
761 syscallenter(td);
762 syscallret(td);
763 }
764
765 static bool
trap_pfault(struct trapframe * frame,bool user,int * signo,int * ucode)766 trap_pfault(struct trapframe *frame, bool user, int *signo, int *ucode)
767 {
768 vm_offset_t eva;
769 struct thread *td;
770 struct proc *p;
771 vm_map_t map;
772 vm_prot_t ftype;
773 int rv, is_user;
774
775 td = curthread;
776 p = td->td_proc;
777 if (frame->exc == EXC_ISI) {
778 eva = frame->srr0;
779 ftype = VM_PROT_EXECUTE;
780 if (frame->srr1 & SRR1_ISI_PFAULT)
781 ftype |= VM_PROT_READ;
782 } else {
783 eva = frame->dar;
784 #ifdef BOOKE
785 if (frame->cpu.booke.esr & ESR_ST)
786 #else
787 if (frame->cpu.aim.dsisr & DSISR_STORE)
788 #endif
789 ftype = VM_PROT_WRITE;
790 else
791 ftype = VM_PROT_READ;
792 }
793 #if defined(__powerpc64__) && defined(AIM)
794 if (radix_mmu && pmap_nofault(&p->p_vmspace->vm_pmap, eva, ftype) == 0)
795 return (true);
796 #endif
797
798 if (__predict_false((td->td_pflags & TDP_NOFAULTING) == 0)) {
799 /*
800 * If we get a page fault while in a critical section, then
801 * it is most likely a fatal kernel page fault. The kernel
802 * is already going to panic trying to get a sleep lock to
803 * do the VM lookup, so just consider it a fatal trap so the
804 * kernel can print out a useful trap message and even get
805 * to the debugger.
806 *
807 * If we get a page fault while holding a non-sleepable
808 * lock, then it is most likely a fatal kernel page fault.
809 * If WITNESS is enabled, then it's going to whine about
810 * bogus LORs with various VM locks, so just skip to the
811 * fatal trap handling directly.
812 */
813 if (td->td_critnest != 0 ||
814 WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
815 "Kernel page fault") != 0) {
816 trap_fatal(frame);
817 return (false);
818 }
819 }
820 if (user) {
821 KASSERT(p->p_vmspace != NULL, ("trap_pfault: vmspace NULL"));
822 map = &p->p_vmspace->vm_map;
823 } else {
824 rv = pmap_decode_kernel_ptr(eva, &is_user, &eva);
825 if (rv != 0)
826 return (false);
827
828 if (is_user)
829 map = &p->p_vmspace->vm_map;
830 else
831 map = kernel_map;
832 }
833
834 /* Fault in the page. */
835 rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode);
836 /*
837 * XXXDTRACE: add dtrace_doubletrap_func here?
838 */
839
840 if (rv == KERN_SUCCESS)
841 return (true);
842
843 if (!user && handle_onfault(frame))
844 return (true);
845
846 return (false);
847 }
848
849 /*
850 * For now, this only deals with the particular unaligned access case
851 * that gcc tends to generate. Eventually it should handle all of the
852 * possibilities that can happen on a 32-bit PowerPC in big-endian mode.
853 */
854
855 static int
fix_unaligned(struct thread * td,struct trapframe * frame)856 fix_unaligned(struct thread *td, struct trapframe *frame)
857 {
858 struct thread *fputhread;
859 #ifdef BOOKE
860 uint32_t inst;
861 #endif
862 int indicator, reg;
863 double *fpr;
864
865 #ifdef __SPE__
866 indicator = (frame->cpu.booke.esr & (ESR_ST|ESR_SPE));
867 if (indicator & ESR_SPE) {
868 if (copyin((void *)frame->srr0, &inst, sizeof(inst)) != 0)
869 return (-1);
870 reg = EXC_ALI_INST_RST(inst);
871 fpr = (double *)td->td_pcb->pcb_vec.vr[reg];
872 fputhread = PCPU_GET(vecthread);
873
874 /* Juggle the SPE to ensure that we've initialized
875 * the registers, and that their current state is in
876 * the PCB.
877 */
878 if (fputhread != td) {
879 if (fputhread)
880 save_vec(fputhread);
881 enable_vec(td);
882 }
883 save_vec(td);
884
885 if (!(indicator & ESR_ST)) {
886 if (copyin((void *)frame->dar, fpr,
887 sizeof(double)) != 0)
888 return (-1);
889 frame->fixreg[reg] = td->td_pcb->pcb_vec.vr[reg][1];
890 enable_vec(td);
891 } else {
892 td->td_pcb->pcb_vec.vr[reg][1] = frame->fixreg[reg];
893 if (copyout(fpr, (void *)frame->dar,
894 sizeof(double)) != 0)
895 return (-1);
896 }
897 return (0);
898 }
899 #else
900 #ifdef BOOKE
901 indicator = (frame->cpu.booke.esr & ESR_ST) ? EXC_ALI_STFD : EXC_ALI_LFD;
902 #else
903 indicator = EXC_ALI_OPCODE_INDICATOR(frame->cpu.aim.dsisr);
904 #endif
905
906 switch (indicator) {
907 case EXC_ALI_LFD:
908 case EXC_ALI_STFD:
909 #ifdef BOOKE
910 if (copyin((void *)frame->srr0, &inst, sizeof(inst)) != 0)
911 return (-1);
912 reg = EXC_ALI_INST_RST(inst);
913 #else
914 reg = EXC_ALI_RST(frame->cpu.aim.dsisr);
915 #endif
916 fpr = &td->td_pcb->pcb_fpu.fpr[reg].fpr;
917 fputhread = PCPU_GET(fputhread);
918
919 /* Juggle the FPU to ensure that we've initialized
920 * the FPRs, and that their current state is in
921 * the PCB.
922 */
923 if (fputhread != td) {
924 if (fputhread)
925 save_fpu(fputhread);
926 enable_fpu(td);
927 }
928 save_fpu(td);
929
930 if (indicator == EXC_ALI_LFD) {
931 if (copyin((void *)frame->dar, fpr,
932 sizeof(double)) != 0)
933 return (-1);
934 enable_fpu(td);
935 } else {
936 if (copyout(fpr, (void *)frame->dar,
937 sizeof(double)) != 0)
938 return (-1);
939 }
940 return (0);
941 break;
942 }
943 #endif
944
945 return (-1);
946 }
947
948 #if defined(__powerpc64__) && defined(AIM)
949 #define MSKNSHL(x, m, n) "(((" #x ") & " #m ") << " #n ")"
950 #define MSKNSHR(x, m, n) "(((" #x ") & " #m ") >> " #n ")"
951
952 /* xvcpsgndp instruction, built in opcode format.
953 * This can be changed to use mnemonic after a toolchain update.
954 */
955 #define XVCPSGNDP(xt, xa, xb) \
956 __asm __volatile(".long (" \
957 MSKNSHL(60, 0x3f, 26) " | " \
958 MSKNSHL(xt, 0x1f, 21) " | " \
959 MSKNSHL(xa, 0x1f, 16) " | " \
960 MSKNSHL(xb, 0x1f, 11) " | " \
961 MSKNSHL(240, 0xff, 3) " | " \
962 MSKNSHR(xa, 0x20, 3) " | " \
963 MSKNSHR(xa, 0x20, 4) " | " \
964 MSKNSHR(xa, 0x20, 5) ")")
965
966 /* Macros to normalize 1 or 10 VSX registers */
967 #define NORM(x) XVCPSGNDP(x, x, x)
968 #define NORM10(x) \
969 NORM(x ## 0); NORM(x ## 1); NORM(x ## 2); NORM(x ## 3); NORM(x ## 4); \
970 NORM(x ## 5); NORM(x ## 6); NORM(x ## 7); NORM(x ## 8); NORM(x ## 9)
971
972 static void
normalize_inputs(void)973 normalize_inputs(void)
974 {
975 register_t msr;
976
977 /* enable VSX */
978 msr = mfmsr();
979 mtmsr(msr | PSL_VSX);
980
981 NORM(0); NORM(1); NORM(2); NORM(3); NORM(4);
982 NORM(5); NORM(6); NORM(7); NORM(8); NORM(9);
983 NORM10(1); NORM10(2); NORM10(3); NORM10(4); NORM10(5);
984 NORM(60); NORM(61); NORM(62); NORM(63);
985
986 /* restore MSR */
987 mtmsr(msr);
988 }
989 #endif
990
991 #ifdef KDB
992 int
db_trap_glue(struct trapframe * frame)993 db_trap_glue(struct trapframe *frame)
994 {
995
996 if (!(frame->srr1 & PSL_PR)
997 && (frame->exc == EXC_TRC || frame->exc == EXC_RUNMODETRC
998 || frame_is_trap_inst(frame)
999 || frame->exc == EXC_BPT
1000 || frame->exc == EXC_DEBUG
1001 || frame->exc == EXC_DSI)) {
1002 int type = frame->exc;
1003
1004 /* Ignore DTrace traps. */
1005 if (*(uint32_t *)frame->srr0 == EXC_DTRACE)
1006 return (0);
1007 if (frame_is_trap_inst(frame)) {
1008 type = T_BREAKPOINT;
1009 }
1010 return (kdb_trap(type, 0, frame));
1011 }
1012
1013 return (0);
1014 }
1015 #endif
1016