38 static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)  in __enter_from_kernel_mode()  argument
40 	regs->exit_rcu = false;  in __enter_from_kernel_mode()
47 		regs->exit_rcu = true;  in __enter_from_kernel_mode()
56 static void noinstr enter_from_kernel_mode(struct pt_regs *regs)  in enter_from_kernel_mode()  argument
58 	__enter_from_kernel_mode(regs);  in enter_from_kernel_mode()
71 static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)  in __exit_to_kernel_mode()  argument
75 	if (interrupts_enabled(regs)) {  in __exit_to_kernel_mode()
76 		if (regs->exit_rcu) {  in __exit_to_kernel_mode()
86 		if (regs->exit_rcu)  in __exit_to_kernel_mode()
91 static void noinstr exit_to_kernel_mode(struct pt_regs *regs)  in exit_to_kernel_mode()  argument
94 	__exit_to_kernel_mode(regs);  in exit_to_kernel_mode()
111 static __always_inline void enter_from_user_mode(struct pt_regs *regs)  in enter_from_user_mode()  argument
129 static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)  in exit_to_user_mode_prepare()  argument
137 		do_notify_resume(regs, flags);  in exit_to_user_mode_prepare()
142 static __always_inline void exit_to_user_mode(struct pt_regs *regs)  in exit_to_user_mode()  argument
144 	exit_to_user_mode_prepare(regs);  in exit_to_user_mode()
149 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)  in asm_exit_to_user_mode()  argument
151 	exit_to_user_mode(regs);  in asm_exit_to_user_mode()
159 static void noinstr arm64_enter_nmi(struct pt_regs *regs)  in arm64_enter_nmi()  argument
161 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_nmi()
177 static void noinstr arm64_exit_nmi(struct pt_regs *regs)  in arm64_exit_nmi()  argument
179 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_nmi()
199 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)  in arm64_enter_el1_dbg()  argument
201 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_el1_dbg()
214 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)  in arm64_exit_el1_dbg()  argument
216 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_el1_dbg()
270 static void do_interrupt_handler(struct pt_regs *regs,  in do_interrupt_handler()  argument
273 	struct pt_regs *old_regs = set_irq_regs(regs);  in do_interrupt_handler()
276 		call_on_irq_stack(regs, handler);  in do_interrupt_handler()
278 		handler(regs);  in do_interrupt_handler()
286 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,  in __panic_unhandled()  argument
289 	arm64_enter_nmi(regs);  in __panic_unhandled()
297 	__show_regs(regs);  in __panic_unhandled()
302 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs)	\
305 	__panic_unhandled(regs, desc, read_sysreg(esr_el1));				\
335 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
347 	regs->pstate |= PSR_D_BIT;  in cortex_a76_erratum_1463225_debug_handler()
352 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
392 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)  in el1_abort()  argument
396 	enter_from_kernel_mode(regs);  in el1_abort()
397 	local_daif_inherit(regs);  in el1_abort()
398 	do_mem_abort(far, esr, regs);  in el1_abort()
400 	exit_to_kernel_mode(regs);  in el1_abort()
403 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)  in el1_pc()  argument
407 	enter_from_kernel_mode(regs);  in el1_pc()
408 	local_daif_inherit(regs);  in el1_pc()
409 	do_sp_pc_abort(far, esr, regs);  in el1_pc()
411 	exit_to_kernel_mode(regs);  in el1_pc()
414 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)  in el1_undef()  argument
416 	enter_from_kernel_mode(regs);  in el1_undef()
417 	local_daif_inherit(regs);  in el1_undef()
418 	do_el1_undef(regs, esr);  in el1_undef()
420 	exit_to_kernel_mode(regs);  in el1_undef()
423 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)  in el1_bti()  argument
425 	enter_from_kernel_mode(regs);  in el1_bti()
426 	local_daif_inherit(regs);  in el1_bti()
427 	do_el1_bti(regs, esr);  in el1_bti()
429 	exit_to_kernel_mode(regs);  in el1_bti()
432 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)  in el1_dbg()  argument
436 	arm64_enter_el1_dbg(regs);  in el1_dbg()
437 	if (!cortex_a76_erratum_1463225_debug_handler(regs))  in el1_dbg()
438 		do_debug_exception(far, esr, regs);  in el1_dbg()
439 	arm64_exit_el1_dbg(regs);  in el1_dbg()
442 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)  in el1_fpac()  argument
444 	enter_from_kernel_mode(regs);  in el1_fpac()
445 	local_daif_inherit(regs);  in el1_fpac()
446 	do_el1_fpac(regs, esr);  in el1_fpac()
448 	exit_to_kernel_mode(regs);  in el1_fpac()
451 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)  in el1h_64_sync_handler()  argument
458 		el1_abort(regs, esr);  in el1h_64_sync_handler()
465 		el1_pc(regs, esr);  in el1h_64_sync_handler()
469 		el1_undef(regs, esr);  in el1h_64_sync_handler()
472 		el1_bti(regs, esr);  in el1h_64_sync_handler()
478 		el1_dbg(regs, esr);  in el1h_64_sync_handler()
481 		el1_fpac(regs, esr);  in el1h_64_sync_handler()
484 		__panic_unhandled(regs, "64-bit el1h sync", esr);  in el1h_64_sync_handler()
488 static __always_inline void __el1_pnmi(struct pt_regs *regs,  in __el1_pnmi()  argument
491 	arm64_enter_nmi(regs);  in __el1_pnmi()
492 	do_interrupt_handler(regs, handler);  in __el1_pnmi()
493 	arm64_exit_nmi(regs);  in __el1_pnmi()
496 static __always_inline void __el1_irq(struct pt_regs *regs,  in __el1_irq()  argument
499 	enter_from_kernel_mode(regs);  in __el1_irq()
502 	do_interrupt_handler(regs, handler);  in __el1_irq()
507 	exit_to_kernel_mode(regs);  in __el1_irq()
509 static void noinstr el1_interrupt(struct pt_regs *regs,  in el1_interrupt()  argument
514 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))  in el1_interrupt()
515 		__el1_pnmi(regs, handler);  in el1_interrupt()
517 		__el1_irq(regs, handler);  in el1_interrupt()
520 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)  in el1h_64_irq_handler()  argument
522 	el1_interrupt(regs, handle_arch_irq);  in el1h_64_irq_handler()
525 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)  in el1h_64_fiq_handler()  argument
527 	el1_interrupt(regs, handle_arch_fiq);  in el1h_64_fiq_handler()
530 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)  in el1h_64_error_handler()  argument
535 	arm64_enter_nmi(regs);  in el1h_64_error_handler()
536 	do_serror(regs, esr);  in el1h_64_error_handler()
537 	arm64_exit_nmi(regs);  in el1h_64_error_handler()
540 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)  in el0_da()  argument
544 	enter_from_user_mode(regs);  in el0_da()
546 	do_mem_abort(far, esr, regs);  in el0_da()
547 	exit_to_user_mode(regs);  in el0_da()
550 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)  in el0_ia()  argument
562 	enter_from_user_mode(regs);  in el0_ia()
564 	do_mem_abort(far, esr, regs);  in el0_ia()
565 	exit_to_user_mode(regs);  in el0_ia()
568 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_acc()  argument
570 	enter_from_user_mode(regs);  in el0_fpsimd_acc()
572 	do_fpsimd_acc(esr, regs);  in el0_fpsimd_acc()
573 	exit_to_user_mode(regs);  in el0_fpsimd_acc()
576 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)  in el0_sve_acc()  argument
578 	enter_from_user_mode(regs);  in el0_sve_acc()
580 	do_sve_acc(esr, regs);  in el0_sve_acc()
581 	exit_to_user_mode(regs);  in el0_sve_acc()
584 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)  in el0_sme_acc()  argument
586 	enter_from_user_mode(regs);  in el0_sme_acc()
588 	do_sme_acc(esr, regs);  in el0_sme_acc()
589 	exit_to_user_mode(regs);  in el0_sme_acc()
592 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_exc()  argument
594 	enter_from_user_mode(regs);  in el0_fpsimd_exc()
596 	do_fpsimd_exc(esr, regs);  in el0_fpsimd_exc()
597 	exit_to_user_mode(regs);  in el0_fpsimd_exc()
600 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)  in el0_sys()  argument
602 	enter_from_user_mode(regs);  in el0_sys()
604 	do_el0_sys(esr, regs);  in el0_sys()
605 	exit_to_user_mode(regs);  in el0_sys()
608 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)  in el0_pc()  argument
612 	if (!is_ttbr0_addr(instruction_pointer(regs)))  in el0_pc()
615 	enter_from_user_mode(regs);  in el0_pc()
617 	do_sp_pc_abort(far, esr, regs);  in el0_pc()
618 	exit_to_user_mode(regs);  in el0_pc()
621 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)  in el0_sp()  argument
623 	enter_from_user_mode(regs);  in el0_sp()
625 	do_sp_pc_abort(regs->sp, esr, regs);  in el0_sp()
626 	exit_to_user_mode(regs);  in el0_sp()
629 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)  in el0_undef()  argument
631 	enter_from_user_mode(regs);  in el0_undef()
633 	do_el0_undef(regs, esr);  in el0_undef()
634 	exit_to_user_mode(regs);  in el0_undef()
637 static void noinstr el0_bti(struct pt_regs *regs)  in el0_bti()  argument
639 	enter_from_user_mode(regs);  in el0_bti()
641 	do_el0_bti(regs);  in el0_bti()
642 	exit_to_user_mode(regs);  in el0_bti()
645 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)  in el0_mops()  argument
647 	enter_from_user_mode(regs);  in el0_mops()
649 	do_el0_mops(regs, esr);  in el0_mops()
650 	exit_to_user_mode(regs);  in el0_mops()
653 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)  in el0_inv()  argument
655 	enter_from_user_mode(regs);  in el0_inv()
657 	bad_el0_sync(regs, 0, esr);  in el0_inv()
658 	exit_to_user_mode(regs);  in el0_inv()
661 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)  in el0_dbg()  argument
666 	enter_from_user_mode(regs);  in el0_dbg()
667 	do_debug_exception(far, esr, regs);  in el0_dbg()
669 	exit_to_user_mode(regs);  in el0_dbg()
672 static void noinstr el0_svc(struct pt_regs *regs)  in el0_svc()  argument
674 	enter_from_user_mode(regs);  in el0_svc()
678 	do_el0_svc(regs);  in el0_svc()
679 	exit_to_user_mode(regs);  in el0_svc()
682 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)  in el0_fpac()  argument
684 	enter_from_user_mode(regs);  in el0_fpac()
686 	do_el0_fpac(regs, esr);  in el0_fpac()
687 	exit_to_user_mode(regs);  in el0_fpac()
690 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)  in el0t_64_sync_handler()  argument
696 		el0_svc(regs);  in el0t_64_sync_handler()
699 		el0_da(regs, esr);  in el0t_64_sync_handler()
702 		el0_ia(regs, esr);  in el0t_64_sync_handler()
705 		el0_fpsimd_acc(regs, esr);  in el0t_64_sync_handler()
708 		el0_sve_acc(regs, esr);  in el0t_64_sync_handler()
711 		el0_sme_acc(regs, esr);  in el0t_64_sync_handler()
714 		el0_fpsimd_exc(regs, esr);  in el0t_64_sync_handler()
718 		el0_sys(regs, esr);  in el0t_64_sync_handler()
721 		el0_sp(regs, esr);  in el0t_64_sync_handler()
724 		el0_pc(regs, esr);  in el0t_64_sync_handler()
727 		el0_undef(regs, esr);  in el0t_64_sync_handler()
730 		el0_bti(regs);  in el0t_64_sync_handler()
733 		el0_mops(regs, esr);  in el0t_64_sync_handler()
739 		el0_dbg(regs, esr);  in el0t_64_sync_handler()
742 		el0_fpac(regs, esr);  in el0t_64_sync_handler()
745 		el0_inv(regs, esr);  in el0t_64_sync_handler()
749 static void noinstr el0_interrupt(struct pt_regs *regs,  in el0_interrupt()  argument
752 	enter_from_user_mode(regs);  in el0_interrupt()
756 	if (regs->pc & BIT(55))  in el0_interrupt()
760 	do_interrupt_handler(regs, handler);  in el0_interrupt()
763 	exit_to_user_mode(regs);  in el0_interrupt()
766 static void noinstr __el0_irq_handler_common(struct pt_regs *regs)  in __el0_irq_handler_common()  argument
768 	el0_interrupt(regs, handle_arch_irq);  in __el0_irq_handler_common()
771 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)  in el0t_64_irq_handler()  argument
773 	__el0_irq_handler_common(regs);  in el0t_64_irq_handler()
776 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)  in __el0_fiq_handler_common()  argument
778 	el0_interrupt(regs, handle_arch_fiq);  in __el0_fiq_handler_common()
781 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)  in el0t_64_fiq_handler()  argument
783 	__el0_fiq_handler_common(regs);  in el0t_64_fiq_handler()
786 static void noinstr __el0_error_handler_common(struct pt_regs *regs)  in __el0_error_handler_common()  argument
790 	enter_from_user_mode(regs);  in __el0_error_handler_common()
792 	arm64_enter_nmi(regs);  in __el0_error_handler_common()
793 	do_serror(regs, esr);  in __el0_error_handler_common()
794 	arm64_exit_nmi(regs);  in __el0_error_handler_common()
796 	exit_to_user_mode(regs);  in __el0_error_handler_common()
799 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)  in el0t_64_error_handler()  argument
801 	__el0_error_handler_common(regs);  in el0t_64_error_handler()
805 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)  in el0_cp15()  argument
807 	enter_from_user_mode(regs);  in el0_cp15()
809 	do_el0_cp15(esr, regs);  in el0_cp15()
810 	exit_to_user_mode(regs);  in el0_cp15()
813 static void noinstr el0_svc_compat(struct pt_regs *regs)  in el0_svc_compat()  argument
815 	enter_from_user_mode(regs);  in el0_svc_compat()
818 	do_el0_svc_compat(regs);  in el0_svc_compat()
819 	exit_to_user_mode(regs);  in el0_svc_compat()
822 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)  in el0t_32_sync_handler()  argument
828 		el0_svc_compat(regs);  in el0t_32_sync_handler()
831 		el0_da(regs, esr);  in el0t_32_sync_handler()
834 		el0_ia(regs, esr);  in el0t_32_sync_handler()
837 		el0_fpsimd_acc(regs, esr);  in el0t_32_sync_handler()
840 		el0_fpsimd_exc(regs, esr);  in el0t_32_sync_handler()
843 		el0_pc(regs, esr);  in el0t_32_sync_handler()
849 		el0_undef(regs, esr);  in el0t_32_sync_handler()
853 		el0_cp15(regs, esr);  in el0t_32_sync_handler()
859 		el0_dbg(regs, esr);  in el0t_32_sync_handler()
862 		el0_inv(regs, esr);  in el0t_32_sync_handler()
866 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)  in el0t_32_irq_handler()  argument
868 	__el0_irq_handler_common(regs);  in el0t_32_irq_handler()
871 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)  in el0t_32_fiq_handler()  argument
873 	__el0_fiq_handler_common(regs);  in el0t_32_fiq_handler()
876 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)  in el0t_32_error_handler()  argument
878 	__el0_error_handler_common(regs);  in el0t_32_error_handler()
888 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)  in handle_bad_stack()  argument
893 	arm64_enter_nmi(regs);  in handle_bad_stack()
894 	panic_bad_stack(regs, esr, far);  in handle_bad_stack()
900 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)  in __sdei_handler()  argument
926 	arm64_enter_nmi(regs);  in __sdei_handler()
927 	ret = do_sdei_event(regs, arg);  in __sdei_handler()
928 	arm64_exit_nmi(regs);  in __sdei_handler()