mm/nohash/tlb_low_64e.S

1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  *  Low level TLB miss handlers for Book3E
5  *  Copyright (C) 2008-2009
15 #include <asm/asm-offsets.h>
17 #include <asm/exception-64e.h>
18 #include <asm/ppc-opcode.h>
21 #include <asm/feature-fixups.h>
36  * Note that, unlike non-bolted handlers, TLB_EXFRAME is not
95 	/* We pre-test some combination of permissions to avoid double
101 	 * So the shift is >> 19. This tests for supervisor writeability.
102 	 * If the page happens to be supervisor writeable and not user
116 	rldicl.	r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
117 	bne-	dtlb_miss_fault_bolted	/* Bail if fault addr is invalid */
119 	rlwinm	r10,r11,32-19,27,27
120 	rlwimi	r10,r11,32-16,19,19
131 	beq-	tlb_miss_fault_bolted /* KUAP fault */
136  * This is the guts of the TLB miss handler for bolted-linear.
146 	rldicl	r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
153 	rldicl	r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
159 	rldicl	r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
165 	rldicl	r15,r16,64-PAGE_SHIFT+3,64-PTE_INDEX_SIZE-3
173 	rldicr	r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
174 	bne-	tlb_miss_fault_bolted
180 	 *               - PID already updated by caller if necessary
181 	 *               - TSIZE need change if !base page size, not
188 	rlwimi	r11,r14,32-19,27,31	/* Insert WIMGE */
189 	rlwimi	r15,r14,32-8,22,25	/* Move in U bits */
192 	rlwimi	r15,r14,32-2,26,31	/* Move in BAP bits */
233 	rldicl.	r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
235 	bne-	itlb_miss_fault_bolted
253  * No HES or NV hint on TLB1, so we need to do software round-robin
255  *    with MAS-damage caused by tlbsx
303 	crclr	cr1*4+eq	/* set cr1.eq = 0 for non-recursive */
324 	 * Erratum A-008139 says that we can't use tlbwe to change
342 	rlwinm	r10,r10,0,0x80000fff  /* tgs,tlpid -> sgs,slpid */
347 	rlwinm	r15,r10,0,0x3fff0000  /* tid -> spid */
348 	rlwimi	r15,r10,20,0x00000003 /* ind,ts -> sind,sas */
382 	rldicl.	r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
383 	bne-	tlb_miss_fault_e6500
385 	rldicl	r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
388 	beq-	tlb_miss_fault_e6500 /* No PGDIR, bail */
391 	rldicl	r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
397 	rldicl	r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
409 	 * MAS 0   :	ESEL needs to be filled by software round-robin
411 	 *               - PID already updated by caller if necessary
412 	 *               - TSIZE for now is base ind page size always
413 	 *               - TID already cleared if necessary
414 	 * MAS 2   :	Default not 2M-aligned, need to be redone
421 	clrrdi	r15,r16,21		/* make EA 2M-aligned */
462 	 * MAS 0   :	ESEL needs to be filled by software round-robin
463 	 *		 - can be handled by indirect code
465 	 * MAS 2,3+7:	Needs to be redone similar to non-tablewalk handler
474 	li	r10,-0x400
477 	rldicr	r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
478 	rlwimi	r10,r14,32-19,27,31	/* Insert WIMGE */
480 	rlwimi	r15,r14,32-8,22,25	/* Move in U bits */
483 	rlwimi	r15,r14,32-2,26,31	/* Move in BAP bits */
535 	beq	tlb_load_linear		/* yes -> go to linear map load */
539 	 * we don't know whether we are trying to fault in a first level
544 	bne-	virt_page_table_tlb_miss
557 	/* We pre-test some combination of permissions to avoid double
563 	 * So the shift is >> 19. This tests for supervisor writeability.
564 	 * If the page happens to be supervisor writeable and not user
574 	rlwimi	r11,r14,32-19,27,27
575 	rlwimi	r11,r14,32-16,19,19
593 	/* If we take a recursive fault, the second level handler may need
598 	li	r14,-1	/* store to exception frame is done later */
602 	 * not re-enter. We could indeed optimize and also not save SRR0/1
610 	beq	tlb_load_linear		/* yes -> go to linear map load */
619 	std	r14,EX_TLB_ESR(r12)		/* write crazy -1 to frame */
635  * This is the guts of the first-level TLB miss handler for direct
650 	beq-	normal_tlb_miss_access_fault /* KUAP fault */
655 	 * PAGE_SHIFT-3, clearing the bottom 3 bits (get a PTE ptr) and
663 	rldicl	r14,r16,64-(PAGE_SHIFT-3),PAGE_SHIFT-3+4
673 	bne-	normal_tlb_miss_access_fault
679 	 *               - PID already updated by caller if necessary
680 	 *               - TSIZE need change if !base page size, not
688 	rlwimi	r10,r14,32-19,27,31	/* Insert WIMGE */
692 	rldicl	r10,r14,64-8,64-8
694 	beq-	1f
701 	rldicr	r11,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
703 	rlwimi	r15,r14,32-8,22,25	/* Move in U bits */
704 	rlwimi	r15,r14,32-2,26,31	/* Move in BAP bits */
720 	 * level 0 and just going back to userland. They are only needed
741  * This is the guts of the second-level TLB miss handler for direct
752  * Note that this should only ever be called as a second level handler
755  * EX_TLB_DEAR-EX_TLB_SIZE(r12)
757  * It can be re-entered by the linear mapping miss handler. However, to
758  * avoid too much complication, it will restart the whole fault at level
771 	 * pgdir in the PACA :-).
786 	beq-	virt_page_table_tlb_miss_fault /* KUAP fault */
795 	rldicl	r10,r16,64-(VPTE_INDEX_SIZE+3),VPTE_INDEX_SIZE+3+4
797 	bne-	virt_page_table_tlb_miss_fault
802 	beq-	virt_page_table_tlb_miss_fault
805 	rldicl	r11,r16,64-VPTE_PGD_SHIFT,64-PGD_INDEX_SIZE-3
812 	rldicl	r11,r16,64-VPTE_PUD_SHIFT,64-PUD_INDEX_SIZE-3
819 	rldicl	r11,r16,64-VPTE_PMD_SHIFT,64-PMD_INDEX_SIZE-3
826 	 * a 4K or 64K page from r16 -> r15.
832 	 *               - PID already updated by caller if necessary
833 	 *               - TSIZE for now is base page size always
840 	ori	r10,r11,1		/* Or-in SR */
858 	 * always called as a second level tlb miss for SW load or as a first
859 	 * level TLB miss for HW load, so we should be able to peek at the
868 	 * level as well. Since we are doing that, we don't need to clear or
873 	bne-	virt_page_table_tlb_miss_whacko_fault
880 	cmpdi	cr0,r16,-1
918 	beq	tlb_load_linear		/* yes -> go to linear map load */
944 	/* If we take a recursive fault, the second level handler may need
949 	li	r14,-1	/* store to exception frame is done later */
953 	 * not re-enter. We could indeed optimize and also not save SRR0/1
961 	beq	tlb_load_linear		/* yes -> go to linear map load */
983  * This is the guts of the second-level TLB miss handler for direct
994  * It can be re-entered by the linear mapping miss handler. However, to
1001 	beq-	htw_tlb_miss_fault /* KUAP fault */
1014 	rldicl.	r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
1015 	bne-	htw_tlb_miss_fault
1019 	beq-	htw_tlb_miss_fault
1022 	rldicl	r11,r16,64-(PGDIR_SHIFT-3),64-PGD_INDEX_SIZE-3
1029 	rldicl	r11,r16,64-(PUD_SHIFT-3),64-PUD_INDEX_SIZE-3
1036 	rldicl	r11,r16,64-(PMD_SHIFT-3),64-PMD_INDEX_SIZE-3
1052 	rlwimi	r15,r16,32-9,20,20
1057 	 *               - PID already updated by caller if necessary
1058 	 *               - TSIZE for now is base ind page size always
1072 	 * level 0 and just going back to userland. They are only needed
1080 	 * though because r14 would contain -1
1082 	cmpdi	cr0,r14,-1
1092  * This is the guts of "any" level TLB miss handler for kernel linear
1098  * r14 = ESR (data) or -1 (instruction)
1104  * In addition we know that we will not re-enter, so in theory, we could
1109  * handlers in which case we probably want to do a full restart at level
1164 	cmpdi	cr0,r14,-1