1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28 #include "opt_ddb.h"
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/asan.h>
33 #include <sys/kernel.h>
34 #include <sys/ktr.h>
35 #include <sys/lock.h>
36 #include <sys/msan.h>
37 #include <sys/mutex.h>
38 #include <sys/proc.h>
39 #include <sys/ptrace.h>
40 #include <sys/syscall.h>
41 #include <sys/sysent.h>
42 #ifdef KDB
43 #include <sys/kdb.h>
44 #endif
45
46 #include <vm/vm.h>
47 #include <vm/pmap.h>
48 #include <vm/vm_kern.h>
49 #include <vm/vm_map.h>
50 #include <vm/vm_param.h>
51 #include <vm/vm_extern.h>
52
53 #include <machine/frame.h>
54 #include <machine/md_var.h>
55 #include <machine/pcb.h>
56 #include <machine/pcpu.h>
57 #include <machine/undefined.h>
58
59 #ifdef KDTRACE_HOOKS
60 #include <sys/dtrace_bsd.h>
61 #endif
62
63 #ifdef VFP
64 #include <machine/vfp.h>
65 #endif
66
67 #ifdef KDB
68 #include <machine/db_machdep.h>
69 #endif
70
71 #ifdef DDB
72 #include <ddb/ddb.h>
73 #include <ddb/db_sym.h>
74 #endif
75
76 /* Called from exception.S */
77 void do_el1h_sync(struct thread *, struct trapframe *);
78 void do_el0_sync(struct thread *, struct trapframe *);
79 void do_el0_error(struct trapframe *);
80 void do_serror(struct trapframe *);
81 void unhandled_exception(struct trapframe *);
82
83 static void print_gp_register(const char *name, uint64_t value);
84 static void print_registers(struct trapframe *frame);
85
86 int (*dtrace_invop_jump_addr)(struct trapframe *);
87
88 u_long cnt_efirt_faults;
89 int print_efirt_faults;
90
91 typedef void (abort_handler)(struct thread *, struct trapframe *, uint64_t,
92 uint64_t, int);
93
94 static abort_handler align_abort;
95 static abort_handler data_abort;
96 static abort_handler external_abort;
97
98 static abort_handler *abort_handlers[] = {
99 [ISS_DATA_DFSC_TF_L0] = data_abort,
100 [ISS_DATA_DFSC_TF_L1] = data_abort,
101 [ISS_DATA_DFSC_TF_L2] = data_abort,
102 [ISS_DATA_DFSC_TF_L3] = data_abort,
103 [ISS_DATA_DFSC_AFF_L1] = data_abort,
104 [ISS_DATA_DFSC_AFF_L2] = data_abort,
105 [ISS_DATA_DFSC_AFF_L3] = data_abort,
106 [ISS_DATA_DFSC_PF_L1] = data_abort,
107 [ISS_DATA_DFSC_PF_L2] = data_abort,
108 [ISS_DATA_DFSC_PF_L3] = data_abort,
109 [ISS_DATA_DFSC_ALIGN] = align_abort,
110 [ISS_DATA_DFSC_EXT] = external_abort,
111 [ISS_DATA_DFSC_EXT_L0] = external_abort,
112 [ISS_DATA_DFSC_EXT_L1] = external_abort,
113 [ISS_DATA_DFSC_EXT_L2] = external_abort,
114 [ISS_DATA_DFSC_EXT_L3] = external_abort,
115 [ISS_DATA_DFSC_ECC] = external_abort,
116 [ISS_DATA_DFSC_ECC_L0] = external_abort,
117 [ISS_DATA_DFSC_ECC_L1] = external_abort,
118 [ISS_DATA_DFSC_ECC_L2] = external_abort,
119 [ISS_DATA_DFSC_ECC_L3] = external_abort,
120 };
121
122 static __inline void
call_trapsignal(struct thread * td,int sig,int code,void * addr,int trapno)123 call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno)
124 {
125 ksiginfo_t ksi;
126
127 ksiginfo_init_trap(&ksi);
128 ksi.ksi_signo = sig;
129 ksi.ksi_code = code;
130 ksi.ksi_addr = addr;
131 ksi.ksi_trapno = trapno;
132 trapsignal(td, &ksi);
133 }
134
135 int
cpu_fetch_syscall_args(struct thread * td)136 cpu_fetch_syscall_args(struct thread *td)
137 {
138 struct proc *p;
139 syscallarg_t *ap, *dst_ap;
140 struct syscall_args *sa;
141
142 p = td->td_proc;
143 sa = &td->td_sa;
144 ap = td->td_frame->tf_x;
145 dst_ap = &sa->args[0];
146
147 sa->code = td->td_frame->tf_x[8];
148 sa->original_code = sa->code;
149
150 if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) {
151 sa->code = *ap++;
152 } else {
153 *dst_ap++ = *ap++;
154 }
155
156 if (__predict_false(sa->code >= p->p_sysent->sv_size))
157 sa->callp = &nosys_sysent;
158 else
159 sa->callp = &p->p_sysent->sv_table[sa->code];
160
161 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
162 ("Syscall %d takes too many arguments", sa->code));
163
164 memcpy(dst_ap, ap, (nitems(sa->args) - 1) * sizeof(*dst_ap));
165
166 td->td_retval[0] = 0;
167 td->td_retval[1] = 0;
168
169 return (0);
170 }
171
172 #include "../../kern/subr_syscall.c"
173
174 /*
175 * Test for fault generated by given access instruction in
176 * bus_peek_<foo> or bus_poke_<foo> bus function.
177 */
178 extern uint32_t generic_bs_peek_1f, generic_bs_peek_2f;
179 extern uint32_t generic_bs_peek_4f, generic_bs_peek_8f;
180 extern uint32_t generic_bs_poke_1f, generic_bs_poke_2f;
181 extern uint32_t generic_bs_poke_4f, generic_bs_poke_8f;
182
183 static bool
test_bs_fault(void * addr)184 test_bs_fault(void *addr)
185 {
186 return (addr == &generic_bs_peek_1f ||
187 addr == &generic_bs_peek_2f ||
188 addr == &generic_bs_peek_4f ||
189 addr == &generic_bs_peek_8f ||
190 addr == &generic_bs_poke_1f ||
191 addr == &generic_bs_poke_2f ||
192 addr == &generic_bs_poke_4f ||
193 addr == &generic_bs_poke_8f);
194 }
195
196 static bool
svc_handler(struct thread * td,struct trapframe * frame)197 svc_handler(struct thread *td, struct trapframe *frame)
198 {
199
200 if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) {
201 syscallenter(td);
202 syscallret(td);
203 /* Skip userret as syscallret already called it */
204 return (true);
205 } else {
206 call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr,
207 ESR_ELx_EXCEPTION(frame->tf_esr));
208 return (false);
209 }
210 }
211
212 static void
align_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)213 align_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
214 uint64_t far, int lower)
215 {
216 if (!lower) {
217 print_registers(frame);
218 print_gp_register("far", far);
219 printf(" esr: 0x%.16lx\n", esr);
220 panic("Misaligned access from kernel space!");
221 }
222
223 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
224 ESR_ELx_EXCEPTION(frame->tf_esr));
225 }
226
227
228 static void
external_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)229 external_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
230 uint64_t far, int lower)
231 {
232 if (lower) {
233 call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)far,
234 ESR_ELx_EXCEPTION(frame->tf_esr));
235 return;
236 }
237
238 /*
239 * Try to handle synchronous external aborts caused by
240 * bus_space_peek() and/or bus_space_poke() functions.
241 */
242 if (test_bs_fault((void *)frame->tf_elr)) {
243 frame->tf_elr = (uint64_t)generic_bs_fault;
244 return;
245 }
246
247 print_registers(frame);
248 print_gp_register("far", far);
249 printf(" esr: 0x%.16lx\n", esr);
250 panic("Unhandled external data abort");
251 }
252
253 /*
254 * It is unsafe to access the stack canary value stored in "td" until
255 * kernel map translation faults are handled, see the pmap_klookup() call below.
256 * Thus, stack-smashing detection with per-thread canaries must be disabled in
257 * this function.
258 */
259 static void NO_PERTHREAD_SSP
data_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)260 data_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
261 uint64_t far, int lower)
262 {
263 struct vm_map *map;
264 struct pcb *pcb;
265 vm_offset_t fault_va;
266 vm_prot_t ftype;
267 int error, sig, ucode;
268 #ifdef KDB
269 bool handled;
270 #endif
271
272 /*
273 * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive
274 * and Store-Exclusive instruction usage restrictions", state
275 * of the exclusive monitors after data abort exception is unknown.
276 */
277 clrex();
278
279 #ifdef KDB
280 if (kdb_active) {
281 kdb_reenter();
282 return;
283 }
284 #endif
285
286 fault_va = far;
287 if (lower) {
288 map = &td->td_proc->p_vmspace->vm_map;
289 if ((td->td_proc->p_md.md_tcr & TCR_TBI0) != 0)
290 fault_va = ADDR_MAKE_CANONICAL(far);
291 } else if (!ADDR_IS_CANONICAL(far)) {
292 /* We received a TBI/PAC/etc. fault from the kernel */
293 error = KERN_INVALID_ADDRESS;
294 pcb = td->td_pcb;
295 goto bad_far;
296 } else if (ADDR_IS_KERNEL(far)) {
297 /*
298 * Handle a special case: the data abort was caused by accessing
299 * a thread structure while its mapping was being promoted or
300 * demoted, as a consequence of the break-before-make rule. It
301 * is not safe to enable interrupts or dereference "td" before
302 * this case is handled.
303 *
304 * In principle, if pmap_klookup() fails, there is no need to
305 * call pmap_fault() below, but avoiding that call is not worth
306 * the effort.
307 */
308 if (ESR_ELx_EXCEPTION(esr) == EXCP_DATA_ABORT) {
309 switch (esr & ISS_DATA_DFSC_MASK) {
310 case ISS_DATA_DFSC_TF_L0:
311 case ISS_DATA_DFSC_TF_L1:
312 case ISS_DATA_DFSC_TF_L2:
313 case ISS_DATA_DFSC_TF_L3:
314 if (pmap_klookup(far, NULL))
315 return;
316 break;
317 }
318 }
319 if (td->td_md.md_spinlock_count == 0 &&
320 (frame->tf_spsr & PSR_DAIF_INTR) != PSR_DAIF_INTR) {
321 MPASS((frame->tf_spsr & PSR_DAIF_INTR) == 0);
322 intr_enable();
323 }
324 map = kernel_map;
325 } else {
326 if (td->td_md.md_spinlock_count == 0 &&
327 (frame->tf_spsr & PSR_DAIF_INTR) != PSR_DAIF_INTR) {
328 MPASS((frame->tf_spsr & PSR_DAIF_INTR) == 0);
329 intr_enable();
330 }
331 map = &td->td_proc->p_vmspace->vm_map;
332 if (map == NULL)
333 map = kernel_map;
334 }
335 pcb = td->td_pcb;
336
337 /*
338 * Try to handle translation, access flag, and permission faults.
339 * Translation faults may occur as a result of the required
340 * break-before-make sequence used when promoting or demoting
341 * superpages. Such faults must not occur while holding the pmap lock,
342 * or pmap_fault() will recurse on that lock.
343 */
344 if ((lower || map == kernel_map || pcb->pcb_onfault != 0) &&
345 pmap_fault(map->pmap, esr, fault_va) == KERN_SUCCESS)
346 return;
347
348 #ifdef INVARIANTS
349 if (td->td_md.md_spinlock_count != 0) {
350 print_registers(frame);
351 print_gp_register("far", far);
352 printf(" esr: 0x%.16lx\n", esr);
353 panic("data abort with spinlock held (spinlock count %d != 0)",
354 td->td_md.md_spinlock_count);
355 }
356 #endif
357 if ((td->td_pflags & TDP_NOFAULTING) == 0 &&
358 (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
359 WARN_GIANTOK, NULL, "Kernel page fault") != 0)) {
360 print_registers(frame);
361 print_gp_register("far", far);
362 printf(" esr: 0x%.16lx\n", esr);
363 panic("data abort in critical section or under mutex");
364 }
365
366 switch (ESR_ELx_EXCEPTION(esr)) {
367 case EXCP_INSN_ABORT:
368 case EXCP_INSN_ABORT_L:
369 ftype = VM_PROT_EXECUTE;
370 break;
371 default:
372 /*
373 * If the exception was because of a read or cache operation
374 * pass a read fault type into the vm code. Cache operations
375 * need read permission but will set the WnR flag when the
376 * memory is unmapped.
377 */
378 if ((esr & ISS_DATA_WnR) == 0 || (esr & ISS_DATA_CM) != 0)
379 ftype = VM_PROT_READ;
380 else
381 ftype = VM_PROT_WRITE;
382 break;
383 }
384
385 /* Fault in the page. */
386 error = vm_fault_trap(map, fault_va, ftype, VM_FAULT_NORMAL, &sig,
387 &ucode);
388 if (error != KERN_SUCCESS) {
389 if (lower) {
390 call_trapsignal(td, sig, ucode, (void *)far,
391 ESR_ELx_EXCEPTION(esr));
392 } else {
393 bad_far:
394 if (td->td_intr_nesting_level == 0 &&
395 pcb->pcb_onfault != 0) {
396 frame->tf_elr = pcb->pcb_onfault;
397 return;
398 }
399
400 printf("Fatal data abort:\n");
401 print_registers(frame);
402 print_gp_register("far", far);
403 printf(" esr: 0x%.16lx\n", esr);
404
405 #ifdef KDB
406 if (debugger_on_trap) {
407 kdb_why = KDB_WHY_TRAP;
408 handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0,
409 frame);
410 kdb_why = KDB_WHY_UNSET;
411 if (handled)
412 return;
413 }
414 #endif
415 panic("vm_fault failed: 0x%lx error %d",
416 frame->tf_elr, error);
417 }
418 }
419 }
420
421 static void
print_gp_register(const char * name,uint64_t value)422 print_gp_register(const char *name, uint64_t value)
423 {
424 #if defined(DDB)
425 c_db_sym_t sym;
426 const char *sym_name;
427 db_expr_t sym_value;
428 db_expr_t offset;
429 #endif
430
431 printf(" %s: 0x%.16lx", name, value);
432 #if defined(DDB)
433 /* If this looks like a kernel address try to find the symbol */
434 if (value >= VM_MIN_KERNEL_ADDRESS) {
435 sym = db_search_symbol(value, DB_STGY_ANY, &offset);
436 if (sym != C_DB_SYM_NULL) {
437 db_symbol_values(sym, &sym_name, &sym_value);
438 printf(" (%s + 0x%lx)", sym_name, offset);
439 }
440 }
441 #endif
442 printf("\n");
443 }
444
445 static void
print_registers(struct trapframe * frame)446 print_registers(struct trapframe *frame)
447 {
448 char name[4];
449 u_int reg;
450
451 for (reg = 0; reg < nitems(frame->tf_x); reg++) {
452 snprintf(name, sizeof(name), "%sx%d", (reg < 10) ? " " : "",
453 reg);
454 print_gp_register(name, frame->tf_x[reg]);
455 }
456 printf(" sp: 0x%.16lx\n", frame->tf_sp);
457 print_gp_register(" lr", frame->tf_lr);
458 print_gp_register("elr", frame->tf_elr);
459 printf("spsr: 0x%.16lx\n", frame->tf_spsr);
460 }
461
462 #ifdef VFP
463 static void
fpe_trap(struct thread * td,void * addr,uint32_t exception)464 fpe_trap(struct thread *td, void *addr, uint32_t exception)
465 {
466 int code;
467
468 code = FPE_FLTIDO;
469 if ((exception & ISS_FP_TFV) != 0) {
470 if ((exception & ISS_FP_IOF) != 0)
471 code = FPE_FLTINV;
472 else if ((exception & ISS_FP_DZF) != 0)
473 code = FPE_FLTDIV;
474 else if ((exception & ISS_FP_OFF) != 0)
475 code = FPE_FLTOVF;
476 else if ((exception & ISS_FP_UFF) != 0)
477 code = FPE_FLTUND;
478 else if ((exception & ISS_FP_IXF) != 0)
479 code = FPE_FLTRES;
480 }
481 call_trapsignal(td, SIGFPE, code, addr, exception);
482 }
483 #endif
484
485 static void
handle_moe(struct thread * td,struct trapframe * frame,uint64_t esr)486 handle_moe(struct thread *td, struct trapframe *frame, uint64_t esr)
487 {
488 uint64_t src;
489 uint64_t dest;
490 uint64_t size;
491 int src_reg;
492 int dest_reg;
493 int size_reg;
494 int format_option;
495
496 format_option = esr & ISS_MOE_FORMAT_OPTION_MASK;
497 dest_reg = (esr & ISS_MOE_DESTREG_MASK) >> ISS_MOE_DESTREG_SHIFT;
498 size_reg = (esr & ISS_MOE_SIZEREG_MASK) >> ISS_MOE_SIZEREG_SHIFT;
499 dest = frame->tf_x[dest_reg];
500 size = frame->tf_x[size_reg];
501
502 /*
503 * Put the registers back in the original format suitable for a
504 * prologue instruction, using the generic return routine from the
505 * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH.
506 */
507 if (esr & ISS_MOE_MEMINST) {
508 /* SET* instruction */
509 if (format_option == ISS_MOE_FORMAT_OPTION_A ||
510 format_option == ISS_MOE_FORMAT_OPTION_A2) {
511 /* Format is from Option A; forward set */
512 frame->tf_x[dest_reg] = dest + size;
513 frame->tf_x[size_reg] = -size;
514 }
515 } else {
516 /* CPY* instruction */
517 src_reg = (esr & ISS_MOE_SRCREG_MASK) >> ISS_MOE_SRCREG_SHIFT;
518 src = frame->tf_x[src_reg];
519
520 if (format_option == ISS_MOE_FORMAT_OPTION_B ||
521 format_option == ISS_MOE_FORMAT_OPTION_B2) {
522 /* Format is from Option B */
523 if (frame->tf_spsr & PSR_N) {
524 /* Backward copy */
525 frame->tf_x[dest_reg] = dest - size;
526 frame->tf_x[src_reg] = src + size;
527 }
528 } else {
529 /* Format is from Option A */
530 if (frame->tf_x[size_reg] & (1UL << 63)) {
531 /* Forward copy */
532 frame->tf_x[dest_reg] = dest + size;
533 frame->tf_x[src_reg] = src + size;
534 frame->tf_x[size_reg] = -size;
535 }
536 }
537 }
538
539 if (esr & ISS_MOE_FROM_EPILOGUE)
540 frame->tf_elr -= 8;
541 else
542 frame->tf_elr -= 4;
543 }
544
545 /*
546 * See the comment above data_abort().
547 */
548 void NO_PERTHREAD_SSP
do_el1h_sync(struct thread * td,struct trapframe * frame)549 do_el1h_sync(struct thread *td, struct trapframe *frame)
550 {
551 uint32_t exception;
552 uint64_t esr, far;
553 int dfsc;
554
555 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
556 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
557
558 far = frame->tf_far;
559 /* Read the esr register to get the exception details */
560 esr = frame->tf_esr;
561 exception = ESR_ELx_EXCEPTION(esr);
562
563 #ifdef KDTRACE_HOOKS
564 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
565 return;
566 #endif
567
568 CTR4(KTR_TRAP, "%s: exception=%lu, elr=0x%lx, esr=0x%lx",
569 __func__, exception, frame->tf_elr, esr);
570
571 /*
572 * Enable debug exceptions if we aren't already handling one. They will
573 * be masked again in the exception handler's epilogue.
574 */
575 switch (exception) {
576 case EXCP_BRK:
577 case EXCP_BRKPT_EL1:
578 case EXCP_WATCHPT_EL1:
579 case EXCP_SOFTSTP_EL1:
580 break;
581 default:
582 dbg_enable();
583 break;
584 }
585
586 switch (exception) {
587 case EXCP_FP_SIMD:
588 case EXCP_TRAP_FP:
589 #ifdef VFP
590 if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) {
591 vfp_restore_state();
592 } else
593 #endif
594 {
595 print_registers(frame);
596 printf(" esr: 0x%.16lx\n", esr);
597 panic("VFP exception in the kernel");
598 }
599 break;
600 case EXCP_INSN_ABORT:
601 case EXCP_DATA_ABORT:
602 dfsc = esr & ISS_DATA_DFSC_MASK;
603 if (dfsc < nitems(abort_handlers) &&
604 abort_handlers[dfsc] != NULL) {
605 abort_handlers[dfsc](td, frame, esr, far, 0);
606 } else {
607 print_registers(frame);
608 print_gp_register("far", far);
609 printf(" esr: 0x%.16lx\n", esr);
610 panic("Unhandled EL1 %s abort: 0x%x",
611 exception == EXCP_INSN_ABORT ? "instruction" :
612 "data", dfsc);
613 }
614 break;
615 case EXCP_BRK:
616 #ifdef KDTRACE_HOOKS
617 if ((esr & ESR_ELx_ISS_MASK) == 0x40d /* BRK_IMM16_VAL */ &&
618 dtrace_invop_jump_addr != NULL &&
619 dtrace_invop_jump_addr(frame) == 0)
620 break;
621 #endif
622 #ifdef KDB
623 kdb_trap(exception, 0, frame);
624 #else
625 panic("No debugger in kernel.");
626 #endif
627 break;
628 case EXCP_BRKPT_EL1:
629 case EXCP_WATCHPT_EL1:
630 case EXCP_SOFTSTP_EL1:
631 #ifdef KDB
632 kdb_trap(exception, 0, frame);
633 #else
634 panic("No debugger in kernel.");
635 #endif
636 break;
637 case EXCP_FPAC:
638 /* We can see this if the authentication on PAC fails */
639 print_registers(frame);
640 print_gp_register("far", far);
641 panic("FPAC kernel exception");
642 break;
643 case EXCP_UNKNOWN:
644 print_registers(frame);
645 print_gp_register("far", far);
646 panic("Undefined instruction: %08x",
647 *(uint32_t *)frame->tf_elr);
648 break;
649 case EXCP_BTI:
650 print_registers(frame);
651 print_gp_register("far", far);
652 panic("Branch Target exception");
653 break;
654 case EXCP_MOE:
655 handle_moe(td, frame, esr);
656 break;
657 default:
658 print_registers(frame);
659 print_gp_register("far", far);
660 panic("Unknown kernel exception 0x%x esr_el1 0x%lx", exception,
661 esr);
662 }
663 }
664
665 void
do_el0_sync(struct thread * td,struct trapframe * frame)666 do_el0_sync(struct thread *td, struct trapframe *frame)
667 {
668 pcpu_bp_harden bp_harden;
669 uint32_t exception;
670 uint64_t esr, far;
671 int dfsc;
672 bool skip_userret;
673
674 /* Check we have a sane environment when entering from userland */
675 KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
676 ("Invalid pcpu address from userland: %p (tpidr 0x%lx)",
677 get_pcpu(), READ_SPECIALREG(tpidr_el1)));
678
679 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
680 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
681
682 far = frame->tf_far;
683 esr = frame->tf_esr;
684 exception = ESR_ELx_EXCEPTION(esr);
685 if (exception == EXCP_INSN_ABORT_L && far > VM_MAXUSER_ADDRESS) {
686 /*
687 * Userspace may be trying to train the branch predictor to
688 * attack the kernel. If we are on a CPU affected by this
689 * call the handler to clear the branch predictor state.
690 */
691 bp_harden = PCPU_GET(bp_harden);
692 if (bp_harden != NULL)
693 bp_harden();
694 }
695 intr_enable();
696
697 CTR4(KTR_TRAP, "%s: exception=%lu, elr=0x%lx, esr=0x%lx",
698 __func__, exception, frame->tf_elr, esr);
699
700 skip_userret = false;
701 switch (exception) {
702 case EXCP_FP_SIMD:
703 #ifdef VFP
704 vfp_restore_state();
705 #else
706 panic("VFP exception in userland");
707 #endif
708 break;
709 case EXCP_TRAP_FP:
710 #ifdef VFP
711 fpe_trap(td, (void *)frame->tf_elr, esr);
712 #else
713 panic("VFP exception in userland");
714 #endif
715 break;
716 case EXCP_SVE:
717 /* Returns true if this thread can use SVE */
718 if (!sve_restore_state(td))
719 call_trapsignal(td, SIGILL, ILL_ILLTRP,
720 (void *)frame->tf_elr, exception);
721 break;
722 case EXCP_SVC32:
723 case EXCP_SVC64:
724 skip_userret = svc_handler(td, frame);
725 break;
726 case EXCP_INSN_ABORT_L:
727 case EXCP_DATA_ABORT_L:
728 case EXCP_DATA_ABORT:
729 dfsc = esr & ISS_DATA_DFSC_MASK;
730 if (dfsc < nitems(abort_handlers) &&
731 abort_handlers[dfsc] != NULL)
732 abort_handlers[dfsc](td, frame, esr, far, 1);
733 else {
734 print_registers(frame);
735 print_gp_register("far", far);
736 printf(" esr: 0x%.16lx\n", esr);
737 panic("Unhandled EL0 %s abort: 0x%x",
738 exception == EXCP_INSN_ABORT_L ? "instruction" :
739 "data", dfsc);
740 }
741 break;
742 case EXCP_UNKNOWN:
743 if (!undef_insn(frame))
744 call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far,
745 exception);
746 break;
747 case EXCP_FPAC:
748 call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr,
749 exception);
750 break;
751 case EXCP_SP_ALIGN:
752 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp,
753 exception);
754 break;
755 case EXCP_PC_ALIGN:
756 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
757 exception);
758 break;
759 case EXCP_BRKPT_EL0:
760 case EXCP_BRK:
761 #ifdef COMPAT_FREEBSD32
762 case EXCP_BRKPT_32:
763 #endif /* COMPAT_FREEBSD32 */
764 call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr,
765 exception);
766 break;
767 case EXCP_WATCHPT_EL0:
768 call_trapsignal(td, SIGTRAP, TRAP_TRACE, (void *)far,
769 exception);
770 break;
771 case EXCP_MSR:
772 /*
773 * The CPU can raise EXCP_MSR when userspace executes an mrs
774 * instruction to access a special register userspace doesn't
775 * have access to.
776 */
777 if (!undef_insn(frame))
778 call_trapsignal(td, SIGILL, ILL_PRVOPC,
779 (void *)frame->tf_elr, exception);
780 break;
781 case EXCP_SOFTSTP_EL0:
782 PROC_LOCK(td->td_proc);
783 if ((td->td_dbgflags & TDB_STEP) != 0) {
784 td->td_frame->tf_spsr &= ~PSR_SS;
785 td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
786 WRITE_SPECIALREG(mdscr_el1,
787 READ_SPECIALREG(mdscr_el1) & ~MDSCR_SS);
788 }
789 PROC_UNLOCK(td->td_proc);
790 call_trapsignal(td, SIGTRAP, TRAP_TRACE,
791 (void *)frame->tf_elr, exception);
792 break;
793 case EXCP_BTI:
794 call_trapsignal(td, SIGILL, ILL_ILLOPC, (void *)frame->tf_elr,
795 exception);
796 break;
797 case EXCP_MOE:
798 handle_moe(td, frame, esr);
799 break;
800 default:
801 call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr,
802 exception);
803 break;
804 }
805
806 if (!skip_userret)
807 userret(td, frame);
808 KASSERT(
809 (td->td_pcb->pcb_fpflags & ~(PCB_FP_USERMASK|PCB_FP_SVEVALID)) == 0,
810 ("Kernel VFP flags set while entering userspace"));
811 KASSERT(
812 td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate,
813 ("Kernel VFP state in use when entering userspace"));
814 }
815
816 /*
817 * TODO: We will need to handle these later when we support ARMv8.2 RAS.
818 */
819 void
do_serror(struct trapframe * frame)820 do_serror(struct trapframe *frame)
821 {
822 uint64_t esr, far;
823
824 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
825 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
826
827 far = frame->tf_far;
828 esr = frame->tf_esr;
829
830 print_registers(frame);
831 print_gp_register("far", far);
832 printf(" esr: 0x%.16lx\n", esr);
833 panic("Unhandled System Error");
834 }
835
836 void
unhandled_exception(struct trapframe * frame)837 unhandled_exception(struct trapframe *frame)
838 {
839 uint64_t esr, far;
840
841 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
842 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
843
844 far = frame->tf_far;
845 esr = frame->tf_esr;
846
847 print_registers(frame);
848 print_gp_register("far", far);
849 printf(" esr: 0x%.16lx\n", esr);
850 panic("Unhandled exception");
851 }
852