arm64/mm/fault.c

38 #include <asm/esr.h>
48 	int	(*fn)(unsigned long far, unsigned long esr,
57 static inline const struct fault_info *esr_to_fault_info(unsigned long esr)
59 	return fault_info + (esr & ESR_ELx_FSC);
62 static void data_abort_decode(unsigned long esr)
64 	unsigned long iss2 = ESR_ELx_ISS2(esr);
68 	if (esr & ESR_ELx_ISV) {
70 			 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
72 			 (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,
73 			 (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT);
75 			 (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
76 			 (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
79 			 esr & ESR_ELx_ISS_MASK, iss2);
83 		 (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
84 		 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT,
95 static void mem_abort_decode(unsigned long esr)
99 	pr_alert("  ESR = 0x%016lx\n", esr);
101 		 ESR_ELx_EC(esr), esr_get_class_string(esr),
102 		 (esr & ESR_ELx_IL) ? 32 : 16);
104 		 (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
105 		 (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
107 		 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
108 		 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
109 	pr_alert("  FSC = 0x%02lx: %s\n", (esr & ESR_ELx_FSC),
110 		 esr_to_fault_info(esr)->name);
112 	if (esr_is_data_abort(esr))
113 		data_abort_decode(esr);
242 static bool is_el1_instruction_abort(unsigned long esr)
244 	return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
247 static bool is_el1_data_abort(unsigned long esr)
249 	return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_CUR;
252 static inline bool is_el1_permission_fault(unsigned long addr, unsigned long esr,
255 	if (!is_el1_data_abort(esr) && !is_el1_instruction_abort(esr))
258 	if (esr_fsc_is_permission_fault(esr))
262 		return esr_fsc_is_translation_fault(esr) &&
269 							unsigned long esr,
275 	if (!is_el1_data_abort(esr) || !esr_fsc_is_translation_fault(esr))
300 			     unsigned long esr, struct pt_regs *regs)
309 	mem_abort_decode(esr);
312 	die("Oops", regs, esr);
318 static void report_tag_fault(unsigned long addr, unsigned long esr,
325 	bool is_write = !!(esr & ESR_ELx_WNR);
330 static inline void report_tag_fault(unsigned long addr, unsigned long esr,
334 static void do_tag_recovery(unsigned long addr, unsigned long esr,
338 	report_tag_fault(addr, esr, regs);
350 static bool is_el1_mte_sync_tag_check_fault(unsigned long esr)
352 	unsigned long fsc = esr & ESR_ELx_FSC;
354 	if (!is_el1_data_abort(esr))
363 static void __do_kernel_fault(unsigned long addr, unsigned long esr,
372 	if (!is_el1_instruction_abort(esr) && fixup_exception(regs, esr))
375 	if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr, esr, regs),
379 	if (is_el1_mte_sync_tag_check_fault(esr)) {
380 		do_tag_recovery(addr, esr, regs);
385 	if (is_el1_permission_fault(addr, esr, regs)) {
386 		if (esr & ESR_ELx_WNR)
388 		else if (is_el1_instruction_abort(esr))
395 		if (esr_fsc_is_translation_fault(esr) &&
396 		    kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
405 	die_kernel_fault(msg, addr, esr, regs);
408 static void set_thread_esr(unsigned long address, unsigned long esr)
425 		switch (ESR_ELx_EC(esr)) {
436 			esr &= ESR_ELx_EC_MASK | ESR_ELx_IL |
438 			esr |= ESR_ELx_FSC_FAULT;
446 			esr &= ESR_ELx_EC_MASK | ESR_ELx_IL;
447 			esr |= ESR_ELx_FSC_FAULT;
456 			WARN(1, "ESR 0x%lx is not DABT or IABT from EL0\n", esr);
457 			esr = 0;
462 	current->thread.fault_code = esr;
465 static void do_bad_area(unsigned long far, unsigned long esr,
475 		const struct fault_info *inf = esr_to_fault_info(esr);
477 		set_thread_esr(addr, esr);
480 		__do_kernel_fault(addr, esr, regs);
513 static bool is_gcs_fault(unsigned long esr)
515 	if (!esr_is_data_abort(esr))
518 	return ESR_ELx_ISS2(esr) & ESR_ELx_GCS;
521 static bool is_el0_instruction_abort(unsigned long esr)
523 	return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
530 static bool is_write_abort(unsigned long esr)
532 	return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
535 static bool is_invalid_gcs_access(struct vm_area_struct *vma, u64 esr)
540 	if (unlikely(is_gcs_fault(esr))) {
546 		return esr_is_data_abort(esr) && is_write_abort(esr);
552 static int __kprobes do_page_fault(unsigned long far, unsigned long esr,
565 	if (kprobe_page_fault(regs, esr))
584 	if (is_el0_instruction_abort(esr)) {
588 	} else if (is_gcs_fault(esr)) {
596 	} else if (is_write_abort(esr)) {
610 	if (is_ttbr0_addr(addr) && is_el1_permission_fault(addr, esr, regs)) {
611 		if (is_el1_instruction_abort(esr))
613 					 addr, esr, regs);
615 		if (!insn_may_access_user(regs->pc, esr))
617 					 addr, esr, regs);
629 	if (is_invalid_gcs_access(vma, esr)) {
739 	inf = esr_to_fault_info(esr);
740 	set_thread_esr(addr, esr);
778 	__do_kernel_fault(addr, esr, regs);
783 					  unsigned long esr,
789 		return do_page_fault(far, esr, regs);
791 	do_bad_area(far, esr, regs);
795 static int do_alignment_fault(unsigned long far, unsigned long esr,
801 	do_bad_area(far, esr, regs);
805 static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs)
810 static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
815 	inf = esr_to_fault_info(esr);
825 	if (esr & ESR_ELx_FnV) {
836 	arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
841 static int do_tag_check_fault(unsigned long far, unsigned long esr,
853 	do_bad_area(far, esr, regs);
924 void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs)
926 	const struct fault_info *inf = esr_to_fault_info(esr);
929 	if (!inf->fn(far, esr, regs))
933 		die_kernel_fault(inf->name, addr, esr, regs);
940 	arm64_notify_die(inf->name, regs, inf->sig, inf->code, addr, esr);
944 void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs)
947 			 addr, esr);