Lines Matching full:regs
38 static noinstr irqentry_state_t enter_from_kernel_mode(struct pt_regs *regs) in enter_from_kernel_mode() argument
42 state = irqentry_enter(regs); in enter_from_kernel_mode()
54 static void noinstr exit_to_kernel_mode(struct pt_regs *regs, in exit_to_kernel_mode() argument
58 irqentry_exit(regs, state); in exit_to_kernel_mode()
66 static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) in arm64_enter_from_user_mode() argument
68 enter_from_user_mode(regs); in arm64_enter_from_user_mode()
78 static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) in arm64_exit_to_user_mode() argument
81 exit_to_user_mode_prepare_legacy(regs); in arm64_exit_to_user_mode()
87 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) in asm_exit_to_user_mode() argument
89 arm64_exit_to_user_mode(regs); in asm_exit_to_user_mode()
97 static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs) in arm64_enter_el1_dbg() argument
116 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs, in arm64_exit_el1_dbg() argument
129 static void do_interrupt_handler(struct pt_regs *regs, in do_interrupt_handler() argument
132 struct pt_regs *old_regs = set_irq_regs(regs); in do_interrupt_handler()
135 call_on_irq_stack(regs, handler); in do_interrupt_handler()
137 handler(regs); in do_interrupt_handler()
145 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, in __panic_unhandled() argument
148 irqentry_nmi_enter(regs); in __panic_unhandled()
156 __show_regs(regs); in __panic_unhandled()
161 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
164 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
194 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) in cortex_a76_erratum_1463225_debug_handler() argument
206 regs->pstate |= PSR_D_BIT; in cortex_a76_erratum_1463225_debug_handler()
211 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) in cortex_a76_erratum_1463225_debug_handler() argument
276 static void debug_exception_enter(struct pt_regs *regs) in debug_exception_enter() argument
285 static void debug_exception_exit(struct pt_regs *regs) in debug_exception_exit() argument
296 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) in el1_abort() argument
301 state = enter_from_kernel_mode(regs); in el1_abort()
302 local_daif_inherit(regs); in el1_abort()
303 do_mem_abort(far, esr, regs); in el1_abort()
305 exit_to_kernel_mode(regs, state); in el1_abort()
308 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) in el1_pc() argument
313 state = enter_from_kernel_mode(regs); in el1_pc()
314 local_daif_inherit(regs); in el1_pc()
315 do_sp_pc_abort(far, esr, regs); in el1_pc()
317 exit_to_kernel_mode(regs, state); in el1_pc()
320 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) in el1_undef() argument
324 state = enter_from_kernel_mode(regs); in el1_undef()
325 local_daif_inherit(regs); in el1_undef()
326 do_el1_undef(regs, esr); in el1_undef()
328 exit_to_kernel_mode(regs, state); in el1_undef()
331 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) in el1_bti() argument
335 state = enter_from_kernel_mode(regs); in el1_bti()
336 local_daif_inherit(regs); in el1_bti()
337 do_el1_bti(regs, esr); in el1_bti()
339 exit_to_kernel_mode(regs, state); in el1_bti()
342 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) in el1_gcs() argument
346 state = enter_from_kernel_mode(regs); in el1_gcs()
347 local_daif_inherit(regs); in el1_gcs()
348 do_el1_gcs(regs, esr); in el1_gcs()
350 exit_to_kernel_mode(regs, state); in el1_gcs()
353 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) in el1_mops() argument
357 state = enter_from_kernel_mode(regs); in el1_mops()
358 local_daif_inherit(regs); in el1_mops()
359 do_el1_mops(regs, esr); in el1_mops()
361 exit_to_kernel_mode(regs, state); in el1_mops()
364 static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) in el1_breakpt() argument
368 state = arm64_enter_el1_dbg(regs); in el1_breakpt()
369 debug_exception_enter(regs); in el1_breakpt()
370 do_breakpoint(esr, regs); in el1_breakpt()
371 debug_exception_exit(regs); in el1_breakpt()
372 arm64_exit_el1_dbg(regs, state); in el1_breakpt()
375 static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) in el1_softstp() argument
379 state = arm64_enter_el1_dbg(regs); in el1_softstp()
380 if (!cortex_a76_erratum_1463225_debug_handler(regs)) { in el1_softstp()
381 debug_exception_enter(regs); in el1_softstp()
388 if (!try_step_suspended_breakpoints(regs)) in el1_softstp()
389 do_el1_softstep(esr, regs); in el1_softstp()
390 debug_exception_exit(regs); in el1_softstp()
392 arm64_exit_el1_dbg(regs, state); in el1_softstp()
395 static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) in el1_watchpt() argument
401 state = arm64_enter_el1_dbg(regs); in el1_watchpt()
402 debug_exception_enter(regs); in el1_watchpt()
403 do_watchpoint(far, esr, regs); in el1_watchpt()
404 debug_exception_exit(regs); in el1_watchpt()
405 arm64_exit_el1_dbg(regs, state); in el1_watchpt()
408 static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr) in el1_brk64() argument
412 state = arm64_enter_el1_dbg(regs); in el1_brk64()
413 debug_exception_enter(regs); in el1_brk64()
414 do_el1_brk64(esr, regs); in el1_brk64()
415 debug_exception_exit(regs); in el1_brk64()
416 arm64_exit_el1_dbg(regs, state); in el1_brk64()
419 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) in el1_fpac() argument
423 state = enter_from_kernel_mode(regs); in el1_fpac()
424 local_daif_inherit(regs); in el1_fpac()
425 do_el1_fpac(regs, esr); in el1_fpac()
427 exit_to_kernel_mode(regs, state); in el1_fpac()
430 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) in el1h_64_sync_handler() argument
437 el1_abort(regs, esr); in el1h_64_sync_handler()
444 el1_pc(regs, esr); in el1h_64_sync_handler()
448 el1_undef(regs, esr); in el1h_64_sync_handler()
451 el1_bti(regs, esr); in el1h_64_sync_handler()
454 el1_gcs(regs, esr); in el1h_64_sync_handler()
457 el1_mops(regs, esr); in el1h_64_sync_handler()
460 el1_breakpt(regs, esr); in el1h_64_sync_handler()
463 el1_softstp(regs, esr); in el1h_64_sync_handler()
466 el1_watchpt(regs, esr); in el1h_64_sync_handler()
469 el1_brk64(regs, esr); in el1h_64_sync_handler()
472 el1_fpac(regs, esr); in el1h_64_sync_handler()
475 __panic_unhandled(regs, "64-bit el1h sync", esr); in el1h_64_sync_handler()
479 static __always_inline void __el1_pnmi(struct pt_regs *regs, in __el1_pnmi() argument
484 state = irqentry_nmi_enter(regs); in __el1_pnmi()
485 do_interrupt_handler(regs, handler); in __el1_pnmi()
486 irqentry_nmi_exit(regs, state); in __el1_pnmi()
489 static __always_inline void __el1_irq(struct pt_regs *regs, in __el1_irq() argument
494 state = enter_from_kernel_mode(regs); in __el1_irq()
497 do_interrupt_handler(regs, handler); in __el1_irq()
500 exit_to_kernel_mode(regs, state); in __el1_irq()
502 static void noinstr el1_interrupt(struct pt_regs *regs, in el1_interrupt() argument
507 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs)) in el1_interrupt()
508 __el1_pnmi(regs, handler); in el1_interrupt()
510 __el1_irq(regs, handler); in el1_interrupt()
513 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) in el1h_64_irq_handler() argument
515 el1_interrupt(regs, handle_arch_irq); in el1h_64_irq_handler()
518 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) in el1h_64_fiq_handler() argument
520 el1_interrupt(regs, handle_arch_fiq); in el1h_64_fiq_handler()
523 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) in el1h_64_error_handler() argument
529 state = irqentry_nmi_enter(regs); in el1h_64_error_handler()
530 do_serror(regs, esr); in el1h_64_error_handler()
531 irqentry_nmi_exit(regs, state); in el1h_64_error_handler()
534 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) in el0_da() argument
538 arm64_enter_from_user_mode(regs); in el0_da()
540 do_mem_abort(far, esr, regs); in el0_da()
541 arm64_exit_to_user_mode(regs); in el0_da()
544 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) in el0_ia() argument
556 arm64_enter_from_user_mode(regs); in el0_ia()
558 do_mem_abort(far, esr, regs); in el0_ia()
559 arm64_exit_to_user_mode(regs); in el0_ia()
562 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) in el0_fpsimd_acc() argument
564 arm64_enter_from_user_mode(regs); in el0_fpsimd_acc()
566 do_fpsimd_acc(esr, regs); in el0_fpsimd_acc()
567 arm64_exit_to_user_mode(regs); in el0_fpsimd_acc()
570 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) in el0_sve_acc() argument
572 arm64_enter_from_user_mode(regs); in el0_sve_acc()
574 do_sve_acc(esr, regs); in el0_sve_acc()
575 arm64_exit_to_user_mode(regs); in el0_sve_acc()
578 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr) in el0_sme_acc() argument
580 arm64_enter_from_user_mode(regs); in el0_sme_acc()
582 do_sme_acc(esr, regs); in el0_sme_acc()
583 arm64_exit_to_user_mode(regs); in el0_sme_acc()
586 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) in el0_fpsimd_exc() argument
588 arm64_enter_from_user_mode(regs); in el0_fpsimd_exc()
590 do_fpsimd_exc(esr, regs); in el0_fpsimd_exc()
591 arm64_exit_to_user_mode(regs); in el0_fpsimd_exc()
594 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) in el0_sys() argument
596 arm64_enter_from_user_mode(regs); in el0_sys()
598 do_el0_sys(esr, regs); in el0_sys()
599 arm64_exit_to_user_mode(regs); in el0_sys()
602 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) in el0_pc() argument
606 if (!is_ttbr0_addr(instruction_pointer(regs))) in el0_pc()
609 arm64_enter_from_user_mode(regs); in el0_pc()
611 do_sp_pc_abort(far, esr, regs); in el0_pc()
612 arm64_exit_to_user_mode(regs); in el0_pc()
615 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) in el0_sp() argument
617 arm64_enter_from_user_mode(regs); in el0_sp()
619 do_sp_pc_abort(regs->sp, esr, regs); in el0_sp()
620 arm64_exit_to_user_mode(regs); in el0_sp()
623 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) in el0_undef() argument
625 arm64_enter_from_user_mode(regs); in el0_undef()
627 do_el0_undef(regs, esr); in el0_undef()
628 arm64_exit_to_user_mode(regs); in el0_undef()
631 static void noinstr el0_bti(struct pt_regs *regs) in el0_bti() argument
633 arm64_enter_from_user_mode(regs); in el0_bti()
635 do_el0_bti(regs); in el0_bti()
636 arm64_exit_to_user_mode(regs); in el0_bti()
639 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr) in el0_mops() argument
641 arm64_enter_from_user_mode(regs); in el0_mops()
643 do_el0_mops(regs, esr); in el0_mops()
644 arm64_exit_to_user_mode(regs); in el0_mops()
647 static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr) in el0_gcs() argument
649 arm64_enter_from_user_mode(regs); in el0_gcs()
651 do_el0_gcs(regs, esr); in el0_gcs()
652 arm64_exit_to_user_mode(regs); in el0_gcs()
655 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) in el0_inv() argument
657 arm64_enter_from_user_mode(regs); in el0_inv()
659 bad_el0_sync(regs, 0, esr); in el0_inv()
660 arm64_exit_to_user_mode(regs); in el0_inv()
663 static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr) in el0_breakpt() argument
665 if (!is_ttbr0_addr(regs->pc)) in el0_breakpt()
668 arm64_enter_from_user_mode(regs); in el0_breakpt()
669 debug_exception_enter(regs); in el0_breakpt()
670 do_breakpoint(esr, regs); in el0_breakpt()
671 debug_exception_exit(regs); in el0_breakpt()
673 arm64_exit_to_user_mode(regs); in el0_breakpt()
676 static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr) in el0_softstp() argument
680 if (!is_ttbr0_addr(regs->pc)) in el0_softstp()
683 arm64_enter_from_user_mode(regs); in el0_softstp()
690 step_done = try_step_suspended_breakpoints(regs); in el0_softstp()
693 do_el0_softstep(esr, regs); in el0_softstp()
694 arm64_exit_to_user_mode(regs); in el0_softstp()
697 static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr) in el0_watchpt() argument
702 arm64_enter_from_user_mode(regs); in el0_watchpt()
703 debug_exception_enter(regs); in el0_watchpt()
704 do_watchpoint(far, esr, regs); in el0_watchpt()
705 debug_exception_exit(regs); in el0_watchpt()
707 arm64_exit_to_user_mode(regs); in el0_watchpt()
710 static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) in el0_brk64() argument
712 arm64_enter_from_user_mode(regs); in el0_brk64()
714 do_el0_brk64(esr, regs); in el0_brk64()
715 arm64_exit_to_user_mode(regs); in el0_brk64()
718 static void noinstr el0_svc(struct pt_regs *regs) in el0_svc() argument
720 arm64_enter_from_user_mode(regs); in el0_svc()
724 do_el0_svc(regs); in el0_svc()
725 arm64_exit_to_user_mode(regs); in el0_svc()
729 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) in el0_fpac() argument
731 arm64_enter_from_user_mode(regs); in el0_fpac()
733 do_el0_fpac(regs, esr); in el0_fpac()
734 arm64_exit_to_user_mode(regs); in el0_fpac()
737 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) in el0t_64_sync_handler() argument
743 el0_svc(regs); in el0t_64_sync_handler()
746 el0_da(regs, esr); in el0t_64_sync_handler()
749 el0_ia(regs, esr); in el0t_64_sync_handler()
752 el0_fpsimd_acc(regs, esr); in el0t_64_sync_handler()
755 el0_sve_acc(regs, esr); in el0t_64_sync_handler()
758 el0_sme_acc(regs, esr); in el0t_64_sync_handler()
761 el0_fpsimd_exc(regs, esr); in el0t_64_sync_handler()
765 el0_sys(regs, esr); in el0t_64_sync_handler()
768 el0_sp(regs, esr); in el0t_64_sync_handler()
771 el0_pc(regs, esr); in el0t_64_sync_handler()
774 el0_undef(regs, esr); in el0t_64_sync_handler()
777 el0_bti(regs); in el0t_64_sync_handler()
780 el0_mops(regs, esr); in el0t_64_sync_handler()
783 el0_gcs(regs, esr); in el0t_64_sync_handler()
786 el0_breakpt(regs, esr); in el0t_64_sync_handler()
789 el0_softstp(regs, esr); in el0t_64_sync_handler()
792 el0_watchpt(regs, esr); in el0t_64_sync_handler()
795 el0_brk64(regs, esr); in el0t_64_sync_handler()
798 el0_fpac(regs, esr); in el0t_64_sync_handler()
801 el0_inv(regs, esr); in el0t_64_sync_handler()
805 static void noinstr el0_interrupt(struct pt_regs *regs, in el0_interrupt() argument
808 arm64_enter_from_user_mode(regs); in el0_interrupt()
812 if (regs->pc & BIT(55)) in el0_interrupt()
816 do_interrupt_handler(regs, handler); in el0_interrupt()
819 arm64_exit_to_user_mode(regs); in el0_interrupt()
822 static void noinstr __el0_irq_handler_common(struct pt_regs *regs) in __el0_irq_handler_common() argument
824 el0_interrupt(regs, handle_arch_irq); in __el0_irq_handler_common()
827 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) in el0t_64_irq_handler() argument
829 __el0_irq_handler_common(regs); in el0t_64_irq_handler()
832 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) in __el0_fiq_handler_common() argument
834 el0_interrupt(regs, handle_arch_fiq); in __el0_fiq_handler_common()
837 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) in el0t_64_fiq_handler() argument
839 __el0_fiq_handler_common(regs); in el0t_64_fiq_handler()
842 static void noinstr __el0_error_handler_common(struct pt_regs *regs) in __el0_error_handler_common() argument
847 arm64_enter_from_user_mode(regs); in __el0_error_handler_common()
849 state = irqentry_nmi_enter(regs); in __el0_error_handler_common()
850 do_serror(regs, esr); in __el0_error_handler_common()
851 irqentry_nmi_exit(regs, state); in __el0_error_handler_common()
853 arm64_exit_to_user_mode(regs); in __el0_error_handler_common()
856 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) in el0t_64_error_handler() argument
858 __el0_error_handler_common(regs); in el0t_64_error_handler()
862 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) in el0_cp15() argument
864 arm64_enter_from_user_mode(regs); in el0_cp15()
866 do_el0_cp15(esr, regs); in el0_cp15()
867 arm64_exit_to_user_mode(regs); in el0_cp15()
870 static void noinstr el0_svc_compat(struct pt_regs *regs) in el0_svc_compat() argument
872 arm64_enter_from_user_mode(regs); in el0_svc_compat()
875 do_el0_svc_compat(regs); in el0_svc_compat()
876 arm64_exit_to_user_mode(regs); in el0_svc_compat()
879 static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) in el0_bkpt32() argument
881 arm64_enter_from_user_mode(regs); in el0_bkpt32()
883 do_bkpt32(esr, regs); in el0_bkpt32()
884 arm64_exit_to_user_mode(regs); in el0_bkpt32()
887 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) in el0t_32_sync_handler() argument
893 el0_svc_compat(regs); in el0t_32_sync_handler()
896 el0_da(regs, esr); in el0t_32_sync_handler()
899 el0_ia(regs, esr); in el0t_32_sync_handler()
902 el0_fpsimd_acc(regs, esr); in el0t_32_sync_handler()
905 el0_fpsimd_exc(regs, esr); in el0t_32_sync_handler()
908 el0_pc(regs, esr); in el0t_32_sync_handler()
914 el0_undef(regs, esr); in el0t_32_sync_handler()
918 el0_cp15(regs, esr); in el0t_32_sync_handler()
921 el0_breakpt(regs, esr); in el0t_32_sync_handler()
924 el0_softstp(regs, esr); in el0t_32_sync_handler()
927 el0_watchpt(regs, esr); in el0t_32_sync_handler()
930 el0_bkpt32(regs, esr); in el0t_32_sync_handler()
933 el0_inv(regs, esr); in el0t_32_sync_handler()
937 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) in el0t_32_irq_handler() argument
939 __el0_irq_handler_common(regs); in el0t_32_irq_handler()
942 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) in el0t_32_fiq_handler() argument
944 __el0_fiq_handler_common(regs); in el0t_32_fiq_handler()
947 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) in el0t_32_error_handler() argument
949 __el0_error_handler_common(regs); in el0t_32_error_handler()
958 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) in handle_bad_stack() argument
963 irqentry_nmi_enter(regs); in handle_bad_stack()
964 panic_bad_stack(regs, esr, far); in handle_bad_stack()
969 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) in __sdei_handler() argument
996 state = irqentry_nmi_enter(regs); in __sdei_handler()
997 ret = do_sdei_event(regs, arg); in __sdei_handler()
998 irqentry_nmi_exit(regs, state); in __sdei_handler()