109cfefb7SHuacai Chen/* SPDX-License-Identifier: GPL-2.0 */ 209cfefb7SHuacai Chen/* 309cfefb7SHuacai Chen * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 409cfefb7SHuacai Chen */ 509cfefb7SHuacai Chen#include <asm/asm.h> 609cfefb7SHuacai Chen#include <asm/loongarch.h> 709cfefb7SHuacai Chen#include <asm/page.h> 809cfefb7SHuacai Chen#include <asm/pgtable.h> 909cfefb7SHuacai Chen#include <asm/regdef.h> 1009cfefb7SHuacai Chen#include <asm/stackframe.h> 1109cfefb7SHuacai Chen 12b681604eSHuacai Chen#define INVTLB_ADDR_GFALSE_AND_ASID 5 13b681604eSHuacai Chen 14a2a84e36SRui Wang#define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3) 15a2a84e36SRui Wang#define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3) 16a2a84e36SRui Wang#define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3) 17a2a84e36SRui Wang#define PTRS_PER_PTE_BITS (PAGE_SHIFT - 3) 18a2a84e36SRui Wang 1909cfefb7SHuacai Chen .macro tlb_do_page_fault, write 2000c2ca84STiezhu Yang SYM_CODE_START(tlb_do_page_fault_\write) 21cb8a2ef0STiezhu Yang UNWIND_HINT_UNDEFINED 2209cfefb7SHuacai Chen SAVE_ALL 2309cfefb7SHuacai Chen csrrd a2, LOONGARCH_CSR_BADV 2409cfefb7SHuacai Chen move a0, sp 2509cfefb7SHuacai Chen REG_S a2, sp, PT_BVADDR 2609cfefb7SHuacai Chen li.w a1, \write 27f733f119SXi Ruoyao bl do_page_fault 2809cfefb7SHuacai Chen RESTORE_ALL_AND_RET 2900c2ca84STiezhu Yang SYM_CODE_END(tlb_do_page_fault_\write) 3009cfefb7SHuacai Chen .endm 3109cfefb7SHuacai Chen 3209cfefb7SHuacai Chen tlb_do_page_fault 0 3309cfefb7SHuacai Chen tlb_do_page_fault 1 3409cfefb7SHuacai Chen 3500c2ca84STiezhu YangSYM_CODE_START(handle_tlb_protect) 36cb8a2ef0STiezhu Yang UNWIND_HINT_UNDEFINED 3709cfefb7SHuacai Chen BACKUP_T0T1 3809cfefb7SHuacai Chen SAVE_ALL 3909cfefb7SHuacai Chen move a0, sp 4009cfefb7SHuacai Chen move a1, zero 4109cfefb7SHuacai Chen csrrd a2, LOONGARCH_CSR_BADV 4209cfefb7SHuacai Chen REG_S a2, sp, PT_BVADDR 43396233c6SYouling Tang la_abs t0, do_page_fault 4409cfefb7SHuacai Chen jirl ra, t0, 0 4509cfefb7SHuacai Chen RESTORE_ALL_AND_RET 4600c2ca84STiezhu YangSYM_CODE_END(handle_tlb_protect) 4709cfefb7SHuacai Chen 4800c2ca84STiezhu YangSYM_CODE_START(handle_tlb_load) 49cb8a2ef0STiezhu Yang UNWIND_HINT_UNDEFINED 5009cfefb7SHuacai Chen csrwr t0, EXCEPTION_KS0 5109cfefb7SHuacai Chen csrwr t1, EXCEPTION_KS1 5209cfefb7SHuacai Chen csrwr ra, EXCEPTION_KS2 5309cfefb7SHuacai Chen 5409cfefb7SHuacai Chen /* 5509cfefb7SHuacai Chen * The vmalloc handling is not in the hotpath. 5609cfefb7SHuacai Chen */ 5709cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_BADV 58d1bc75d7SWANG Xuerui bltz t0, vmalloc_load 5909cfefb7SHuacai Chen csrrd t1, LOONGARCH_CSR_PGDL 6009cfefb7SHuacai Chen 6109cfefb7SHuacai Chenvmalloc_done_load: 6209cfefb7SHuacai Chen /* Get PGD offset in bytes */ 63a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 64a2a84e36SRui Wang alsl.d t1, ra, t1, 3 6509cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 3 6609cfefb7SHuacai Chen ld.d t1, t1, 0 67a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 68a2a84e36SRui Wang alsl.d t1, ra, t1, 3 6909cfefb7SHuacai Chen#endif 7009cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 2 7109cfefb7SHuacai Chen ld.d t1, t1, 0 72a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 73a2a84e36SRui Wang alsl.d t1, ra, t1, 3 7409cfefb7SHuacai Chen#endif 7509cfefb7SHuacai Chen ld.d ra, t1, 0 7609cfefb7SHuacai Chen 7709cfefb7SHuacai Chen /* 7809cfefb7SHuacai Chen * For huge tlb entries, pmde doesn't contain an address but 7909cfefb7SHuacai Chen * instead contains the tlb pte. Check the PAGE_HUGE bit and 8009cfefb7SHuacai Chen * see if we need to jump to huge tlb processing. 8109cfefb7SHuacai Chen */ 82a2a84e36SRui Wang rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 83a2a84e36SRui Wang bltz ra, tlb_huge_update_load 8409cfefb7SHuacai Chen 85a2a84e36SRui Wang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 86a2a84e36SRui Wang bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 87a2a84e36SRui Wang alsl.d t1, t0, ra, _PTE_T_LOG2 8809cfefb7SHuacai Chen 8946859ac8SHuacai Chen#ifdef CONFIG_SMP 9046859ac8SHuacai Chensmp_pgtable_change_load: 9146859ac8SHuacai Chen ll.d t0, t1, 0 9246859ac8SHuacai Chen#else 9309cfefb7SHuacai Chen ld.d t0, t1, 0 9446859ac8SHuacai Chen#endif 95a2a84e36SRui Wang andi ra, t0, _PAGE_PRESENT 96d47b2dc8SWANG Xuerui beqz ra, nopage_tlb_load 9709cfefb7SHuacai Chen 9809cfefb7SHuacai Chen ori t0, t0, _PAGE_VALID 9946859ac8SHuacai Chen#ifdef CONFIG_SMP 10046859ac8SHuacai Chen sc.d t0, t1, 0 101d47b2dc8SWANG Xuerui beqz t0, smp_pgtable_change_load 10246859ac8SHuacai Chen#else 10309cfefb7SHuacai Chen st.d t0, t1, 0 10446859ac8SHuacai Chen#endif 105a2a84e36SRui Wang tlbsrch 106a2a84e36SRui Wang bstrins.d t1, zero, 3, 3 10709cfefb7SHuacai Chen ld.d t0, t1, 0 10809cfefb7SHuacai Chen ld.d t1, t1, 8 10909cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO0 11009cfefb7SHuacai Chen csrwr t1, LOONGARCH_CSR_TLBELO1 11109cfefb7SHuacai Chen tlbwr 112a2a84e36SRui Wang 11309cfefb7SHuacai Chen csrrd t0, EXCEPTION_KS0 11409cfefb7SHuacai Chen csrrd t1, EXCEPTION_KS1 11509cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 11609cfefb7SHuacai Chen ertn 117a2a84e36SRui Wang 11809cfefb7SHuacai Chen#ifdef CONFIG_64BIT 11909cfefb7SHuacai Chenvmalloc_load: 120396233c6SYouling Tang la_abs t1, swapper_pg_dir 12109cfefb7SHuacai Chen b vmalloc_done_load 12209cfefb7SHuacai Chen#endif 12309cfefb7SHuacai Chen 124a2a84e36SRui Wang /* This is the entry point of a huge page. */ 12509cfefb7SHuacai Chentlb_huge_update_load: 12646859ac8SHuacai Chen#ifdef CONFIG_SMP 127a2a84e36SRui Wang ll.d ra, t1, 0 1285685d7fcSTiezhu Yang#else 1295685d7fcSTiezhu Yang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 13046859ac8SHuacai Chen#endif 131a2a84e36SRui Wang andi t0, ra, _PAGE_PRESENT 132a2a84e36SRui Wang beqz t0, nopage_tlb_load 13309cfefb7SHuacai Chen 13446859ac8SHuacai Chen#ifdef CONFIG_SMP 135a2a84e36SRui Wang ori t0, ra, _PAGE_VALID 13646859ac8SHuacai Chen sc.d t0, t1, 0 137d47b2dc8SWANG Xuerui beqz t0, tlb_huge_update_load 138a2a84e36SRui Wang ori t0, ra, _PAGE_VALID 13946859ac8SHuacai Chen#else 140a2a84e36SRui Wang ori t0, ra, _PAGE_VALID 14109cfefb7SHuacai Chen st.d t0, t1, 0 14246859ac8SHuacai Chen#endif 143b681604eSHuacai Chen csrrd ra, LOONGARCH_CSR_ASID 144b681604eSHuacai Chen csrrd t1, LOONGARCH_CSR_BADV 145b681604eSHuacai Chen andi ra, ra, CSR_ASID_ASID 146b681604eSHuacai Chen invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 14709cfefb7SHuacai Chen 14809cfefb7SHuacai Chen /* 14909cfefb7SHuacai Chen * A huge PTE describes an area the size of the 15009cfefb7SHuacai Chen * configured huge page size. This is twice the 15109cfefb7SHuacai Chen * of the large TLB entry size we intend to use. 15209cfefb7SHuacai Chen * A TLB entry half the size of the configured 15309cfefb7SHuacai Chen * huge page size is configured into entrylo0 15409cfefb7SHuacai Chen * and entrylo1 to cover the contiguous huge PTE 15509cfefb7SHuacai Chen * address space. 15609cfefb7SHuacai Chen */ 15709cfefb7SHuacai Chen /* Huge page: Move Global bit */ 15809cfefb7SHuacai Chen xori t0, t0, _PAGE_HUGE 15909cfefb7SHuacai Chen lu12i.w t1, _PAGE_HGLOBAL >> 12 16009cfefb7SHuacai Chen and t1, t0, t1 16109cfefb7SHuacai Chen srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 16209cfefb7SHuacai Chen or t0, t0, t1 16309cfefb7SHuacai Chen 164a2a84e36SRui Wang move ra, t0 165a2a84e36SRui Wang csrwr ra, LOONGARCH_CSR_TLBELO0 16609cfefb7SHuacai Chen 16709cfefb7SHuacai Chen /* Convert to entrylo1 */ 168d8e7f201SWANG Xuerui addi.d t1, zero, 1 16909cfefb7SHuacai Chen slli.d t1, t1, (HPAGE_SHIFT - 1) 17009cfefb7SHuacai Chen add.d t0, t0, t1 17109cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO1 17209cfefb7SHuacai Chen 17309cfefb7SHuacai Chen /* Set huge page tlb entry size */ 174d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 175d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 17609cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 17709cfefb7SHuacai Chen 17809cfefb7SHuacai Chen tlbfill 17909cfefb7SHuacai Chen 180d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 181d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 18209cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 18309cfefb7SHuacai Chen 184a2a84e36SRui Wang csrrd t0, EXCEPTION_KS0 185a2a84e36SRui Wang csrrd t1, EXCEPTION_KS1 186a2a84e36SRui Wang csrrd ra, EXCEPTION_KS2 187a2a84e36SRui Wang ertn 188a2a84e36SRui Wang 18909cfefb7SHuacai Chennopage_tlb_load: 190e031a5f3SHuacai Chen dbar 0x700 19109cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 192396233c6SYouling Tang la_abs t0, tlb_do_page_fault_0 19307b48069SWANG Xuerui jr t0 19400c2ca84STiezhu YangSYM_CODE_END(handle_tlb_load) 19509cfefb7SHuacai Chen 19600c2ca84STiezhu YangSYM_CODE_START(handle_tlb_load_ptw) 197cb8a2ef0STiezhu Yang UNWIND_HINT_UNDEFINED 19801158487SHuacai Chen csrwr t0, LOONGARCH_CSR_KS0 19901158487SHuacai Chen csrwr t1, LOONGARCH_CSR_KS1 20001158487SHuacai Chen la_abs t0, tlb_do_page_fault_0 20101158487SHuacai Chen jr t0 20200c2ca84STiezhu YangSYM_CODE_END(handle_tlb_load_ptw) 20301158487SHuacai Chen 20400c2ca84STiezhu YangSYM_CODE_START(handle_tlb_store) 205cb8a2ef0STiezhu Yang UNWIND_HINT_UNDEFINED 20609cfefb7SHuacai Chen csrwr t0, EXCEPTION_KS0 20709cfefb7SHuacai Chen csrwr t1, EXCEPTION_KS1 20809cfefb7SHuacai Chen csrwr ra, EXCEPTION_KS2 20909cfefb7SHuacai Chen 21009cfefb7SHuacai Chen /* 21109cfefb7SHuacai Chen * The vmalloc handling is not in the hotpath. 21209cfefb7SHuacai Chen */ 21309cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_BADV 214d1bc75d7SWANG Xuerui bltz t0, vmalloc_store 21509cfefb7SHuacai Chen csrrd t1, LOONGARCH_CSR_PGDL 21609cfefb7SHuacai Chen 21709cfefb7SHuacai Chenvmalloc_done_store: 21809cfefb7SHuacai Chen /* Get PGD offset in bytes */ 219a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 220a2a84e36SRui Wang alsl.d t1, ra, t1, 3 22109cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 3 22209cfefb7SHuacai Chen ld.d t1, t1, 0 223a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 224a2a84e36SRui Wang alsl.d t1, ra, t1, 3 22509cfefb7SHuacai Chen#endif 22609cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 2 22709cfefb7SHuacai Chen ld.d t1, t1, 0 228a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 229a2a84e36SRui Wang alsl.d t1, ra, t1, 3 23009cfefb7SHuacai Chen#endif 23109cfefb7SHuacai Chen ld.d ra, t1, 0 23209cfefb7SHuacai Chen 23309cfefb7SHuacai Chen /* 23409cfefb7SHuacai Chen * For huge tlb entries, pmde doesn't contain an address but 23509cfefb7SHuacai Chen * instead contains the tlb pte. Check the PAGE_HUGE bit and 23609cfefb7SHuacai Chen * see if we need to jump to huge tlb processing. 23709cfefb7SHuacai Chen */ 238a2a84e36SRui Wang rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 239a2a84e36SRui Wang bltz ra, tlb_huge_update_store 24009cfefb7SHuacai Chen 241a2a84e36SRui Wang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 242a2a84e36SRui Wang bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 243a2a84e36SRui Wang alsl.d t1, t0, ra, _PTE_T_LOG2 24409cfefb7SHuacai Chen 24546859ac8SHuacai Chen#ifdef CONFIG_SMP 24646859ac8SHuacai Chensmp_pgtable_change_store: 24746859ac8SHuacai Chen ll.d t0, t1, 0 24846859ac8SHuacai Chen#else 24909cfefb7SHuacai Chen ld.d t0, t1, 0 25046859ac8SHuacai Chen#endif 251a2a84e36SRui Wang andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE 252a2a84e36SRui Wang xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE 253d47b2dc8SWANG Xuerui bnez ra, nopage_tlb_store 25409cfefb7SHuacai Chen 25509cfefb7SHuacai Chen ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 25646859ac8SHuacai Chen#ifdef CONFIG_SMP 25746859ac8SHuacai Chen sc.d t0, t1, 0 258d47b2dc8SWANG Xuerui beqz t0, smp_pgtable_change_store 25946859ac8SHuacai Chen#else 26009cfefb7SHuacai Chen st.d t0, t1, 0 26146859ac8SHuacai Chen#endif 262a2a84e36SRui Wang tlbsrch 263a2a84e36SRui Wang bstrins.d t1, zero, 3, 3 26409cfefb7SHuacai Chen ld.d t0, t1, 0 26509cfefb7SHuacai Chen ld.d t1, t1, 8 26609cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO0 26709cfefb7SHuacai Chen csrwr t1, LOONGARCH_CSR_TLBELO1 26809cfefb7SHuacai Chen tlbwr 269a2a84e36SRui Wang 27009cfefb7SHuacai Chen csrrd t0, EXCEPTION_KS0 27109cfefb7SHuacai Chen csrrd t1, EXCEPTION_KS1 27209cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 27309cfefb7SHuacai Chen ertn 274a2a84e36SRui Wang 27509cfefb7SHuacai Chen#ifdef CONFIG_64BIT 27609cfefb7SHuacai Chenvmalloc_store: 277396233c6SYouling Tang la_abs t1, swapper_pg_dir 27809cfefb7SHuacai Chen b vmalloc_done_store 27909cfefb7SHuacai Chen#endif 28009cfefb7SHuacai Chen 281a2a84e36SRui Wang /* This is the entry point of a huge page. */ 28209cfefb7SHuacai Chentlb_huge_update_store: 28346859ac8SHuacai Chen#ifdef CONFIG_SMP 284a2a84e36SRui Wang ll.d ra, t1, 0 2855685d7fcSTiezhu Yang#else 2865685d7fcSTiezhu Yang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 28746859ac8SHuacai Chen#endif 288a2a84e36SRui Wang andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE 289a2a84e36SRui Wang xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE 290a2a84e36SRui Wang bnez t0, nopage_tlb_store 29109cfefb7SHuacai Chen 29246859ac8SHuacai Chen#ifdef CONFIG_SMP 293a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 29446859ac8SHuacai Chen sc.d t0, t1, 0 295d47b2dc8SWANG Xuerui beqz t0, tlb_huge_update_store 296a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 29746859ac8SHuacai Chen#else 298a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 29909cfefb7SHuacai Chen st.d t0, t1, 0 30046859ac8SHuacai Chen#endif 301b681604eSHuacai Chen csrrd ra, LOONGARCH_CSR_ASID 302b681604eSHuacai Chen csrrd t1, LOONGARCH_CSR_BADV 303b681604eSHuacai Chen andi ra, ra, CSR_ASID_ASID 304b681604eSHuacai Chen invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 30509cfefb7SHuacai Chen 30609cfefb7SHuacai Chen /* 30709cfefb7SHuacai Chen * A huge PTE describes an area the size of the 30809cfefb7SHuacai Chen * configured huge page size. This is twice the 30909cfefb7SHuacai Chen * of the large TLB entry size we intend to use. 31009cfefb7SHuacai Chen * A TLB entry half the size of the configured 31109cfefb7SHuacai Chen * huge page size is configured into entrylo0 31209cfefb7SHuacai Chen * and entrylo1 to cover the contiguous huge PTE 31309cfefb7SHuacai Chen * address space. 31409cfefb7SHuacai Chen */ 31509cfefb7SHuacai Chen /* Huge page: Move Global bit */ 31609cfefb7SHuacai Chen xori t0, t0, _PAGE_HUGE 31709cfefb7SHuacai Chen lu12i.w t1, _PAGE_HGLOBAL >> 12 31809cfefb7SHuacai Chen and t1, t0, t1 31909cfefb7SHuacai Chen srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 32009cfefb7SHuacai Chen or t0, t0, t1 32109cfefb7SHuacai Chen 322a2a84e36SRui Wang move ra, t0 323a2a84e36SRui Wang csrwr ra, LOONGARCH_CSR_TLBELO0 32409cfefb7SHuacai Chen 32509cfefb7SHuacai Chen /* Convert to entrylo1 */ 326d8e7f201SWANG Xuerui addi.d t1, zero, 1 32709cfefb7SHuacai Chen slli.d t1, t1, (HPAGE_SHIFT - 1) 32809cfefb7SHuacai Chen add.d t0, t0, t1 32909cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO1 33009cfefb7SHuacai Chen 33109cfefb7SHuacai Chen /* Set huge page tlb entry size */ 332d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 333d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 33409cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 33509cfefb7SHuacai Chen 33609cfefb7SHuacai Chen tlbfill 33709cfefb7SHuacai Chen 33809cfefb7SHuacai Chen /* Reset default page size */ 339d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 340d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 34109cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 34209cfefb7SHuacai Chen 343a2a84e36SRui Wang csrrd t0, EXCEPTION_KS0 344a2a84e36SRui Wang csrrd t1, EXCEPTION_KS1 345a2a84e36SRui Wang csrrd ra, EXCEPTION_KS2 346a2a84e36SRui Wang ertn 347a2a84e36SRui Wang 34809cfefb7SHuacai Chennopage_tlb_store: 349e031a5f3SHuacai Chen dbar 0x700 35009cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 351396233c6SYouling Tang la_abs t0, tlb_do_page_fault_1 35207b48069SWANG Xuerui jr t0 35300c2ca84STiezhu YangSYM_CODE_END(handle_tlb_store) 35409cfefb7SHuacai Chen 35500c2ca84STiezhu YangSYM_CODE_START(handle_tlb_store_ptw) 356cb8a2ef0STiezhu Yang UNWIND_HINT_UNDEFINED 35701158487SHuacai Chen csrwr t0, LOONGARCH_CSR_KS0 35801158487SHuacai Chen csrwr t1, LOONGARCH_CSR_KS1 35901158487SHuacai Chen la_abs t0, tlb_do_page_fault_1 36001158487SHuacai Chen jr t0 36100c2ca84STiezhu YangSYM_CODE_END(handle_tlb_store_ptw) 36201158487SHuacai Chen 36300c2ca84STiezhu YangSYM_CODE_START(handle_tlb_modify) 364cb8a2ef0STiezhu Yang UNWIND_HINT_UNDEFINED 36509cfefb7SHuacai Chen csrwr t0, EXCEPTION_KS0 36609cfefb7SHuacai Chen csrwr t1, EXCEPTION_KS1 36709cfefb7SHuacai Chen csrwr ra, EXCEPTION_KS2 36809cfefb7SHuacai Chen 36909cfefb7SHuacai Chen /* 37009cfefb7SHuacai Chen * The vmalloc handling is not in the hotpath. 37109cfefb7SHuacai Chen */ 37209cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_BADV 373d1bc75d7SWANG Xuerui bltz t0, vmalloc_modify 37409cfefb7SHuacai Chen csrrd t1, LOONGARCH_CSR_PGDL 37509cfefb7SHuacai Chen 37609cfefb7SHuacai Chenvmalloc_done_modify: 37709cfefb7SHuacai Chen /* Get PGD offset in bytes */ 378a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 379a2a84e36SRui Wang alsl.d t1, ra, t1, 3 38009cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 3 38109cfefb7SHuacai Chen ld.d t1, t1, 0 382a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 383a2a84e36SRui Wang alsl.d t1, ra, t1, 3 38409cfefb7SHuacai Chen#endif 38509cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 2 38609cfefb7SHuacai Chen ld.d t1, t1, 0 387a2a84e36SRui Wang bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 388a2a84e36SRui Wang alsl.d t1, ra, t1, 3 38909cfefb7SHuacai Chen#endif 39009cfefb7SHuacai Chen ld.d ra, t1, 0 39109cfefb7SHuacai Chen 39209cfefb7SHuacai Chen /* 39309cfefb7SHuacai Chen * For huge tlb entries, pmde doesn't contain an address but 39409cfefb7SHuacai Chen * instead contains the tlb pte. Check the PAGE_HUGE bit and 39509cfefb7SHuacai Chen * see if we need to jump to huge tlb processing. 39609cfefb7SHuacai Chen */ 397a2a84e36SRui Wang rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 398a2a84e36SRui Wang bltz ra, tlb_huge_update_modify 39909cfefb7SHuacai Chen 400a2a84e36SRui Wang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 401a2a84e36SRui Wang bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 402a2a84e36SRui Wang alsl.d t1, t0, ra, _PTE_T_LOG2 40309cfefb7SHuacai Chen 40446859ac8SHuacai Chen#ifdef CONFIG_SMP 40546859ac8SHuacai Chensmp_pgtable_change_modify: 40646859ac8SHuacai Chen ll.d t0, t1, 0 40746859ac8SHuacai Chen#else 40809cfefb7SHuacai Chen ld.d t0, t1, 0 40946859ac8SHuacai Chen#endif 410a2a84e36SRui Wang andi ra, t0, _PAGE_WRITE 411d47b2dc8SWANG Xuerui beqz ra, nopage_tlb_modify 41209cfefb7SHuacai Chen 41309cfefb7SHuacai Chen ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 41446859ac8SHuacai Chen#ifdef CONFIG_SMP 41546859ac8SHuacai Chen sc.d t0, t1, 0 416d47b2dc8SWANG Xuerui beqz t0, smp_pgtable_change_modify 41746859ac8SHuacai Chen#else 41809cfefb7SHuacai Chen st.d t0, t1, 0 41946859ac8SHuacai Chen#endif 420a2a84e36SRui Wang tlbsrch 421a2a84e36SRui Wang bstrins.d t1, zero, 3, 3 42209cfefb7SHuacai Chen ld.d t0, t1, 0 42309cfefb7SHuacai Chen ld.d t1, t1, 8 42409cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO0 42509cfefb7SHuacai Chen csrwr t1, LOONGARCH_CSR_TLBELO1 42609cfefb7SHuacai Chen tlbwr 427a2a84e36SRui Wang 42809cfefb7SHuacai Chen csrrd t0, EXCEPTION_KS0 42909cfefb7SHuacai Chen csrrd t1, EXCEPTION_KS1 43009cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 43109cfefb7SHuacai Chen ertn 432a2a84e36SRui Wang 43309cfefb7SHuacai Chen#ifdef CONFIG_64BIT 43409cfefb7SHuacai Chenvmalloc_modify: 435396233c6SYouling Tang la_abs t1, swapper_pg_dir 43609cfefb7SHuacai Chen b vmalloc_done_modify 43709cfefb7SHuacai Chen#endif 43809cfefb7SHuacai Chen 439a2a84e36SRui Wang /* This is the entry point of a huge page. */ 44009cfefb7SHuacai Chentlb_huge_update_modify: 44146859ac8SHuacai Chen#ifdef CONFIG_SMP 442a2a84e36SRui Wang ll.d ra, t1, 0 4435685d7fcSTiezhu Yang#else 4445685d7fcSTiezhu Yang rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 44546859ac8SHuacai Chen#endif 446a2a84e36SRui Wang andi t0, ra, _PAGE_WRITE 447a2a84e36SRui Wang beqz t0, nopage_tlb_modify 44809cfefb7SHuacai Chen 44946859ac8SHuacai Chen#ifdef CONFIG_SMP 450a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 45146859ac8SHuacai Chen sc.d t0, t1, 0 452d47b2dc8SWANG Xuerui beqz t0, tlb_huge_update_modify 453a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 45446859ac8SHuacai Chen#else 455a2a84e36SRui Wang ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 45609cfefb7SHuacai Chen st.d t0, t1, 0 45746859ac8SHuacai Chen#endif 458b681604eSHuacai Chen csrrd ra, LOONGARCH_CSR_ASID 459b681604eSHuacai Chen csrrd t1, LOONGARCH_CSR_BADV 460b681604eSHuacai Chen andi ra, ra, CSR_ASID_ASID 461b681604eSHuacai Chen invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 462b681604eSHuacai Chen 46309cfefb7SHuacai Chen /* 46409cfefb7SHuacai Chen * A huge PTE describes an area the size of the 46509cfefb7SHuacai Chen * configured huge page size. This is twice the 46609cfefb7SHuacai Chen * of the large TLB entry size we intend to use. 46709cfefb7SHuacai Chen * A TLB entry half the size of the configured 46809cfefb7SHuacai Chen * huge page size is configured into entrylo0 46909cfefb7SHuacai Chen * and entrylo1 to cover the contiguous huge PTE 47009cfefb7SHuacai Chen * address space. 47109cfefb7SHuacai Chen */ 47209cfefb7SHuacai Chen /* Huge page: Move Global bit */ 47309cfefb7SHuacai Chen xori t0, t0, _PAGE_HUGE 47409cfefb7SHuacai Chen lu12i.w t1, _PAGE_HGLOBAL >> 12 47509cfefb7SHuacai Chen and t1, t0, t1 47609cfefb7SHuacai Chen srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 47709cfefb7SHuacai Chen or t0, t0, t1 47809cfefb7SHuacai Chen 479a2a84e36SRui Wang move ra, t0 480a2a84e36SRui Wang csrwr ra, LOONGARCH_CSR_TLBELO0 48109cfefb7SHuacai Chen 48209cfefb7SHuacai Chen /* Convert to entrylo1 */ 483d8e7f201SWANG Xuerui addi.d t1, zero, 1 48409cfefb7SHuacai Chen slli.d t1, t1, (HPAGE_SHIFT - 1) 48509cfefb7SHuacai Chen add.d t0, t0, t1 48609cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBELO1 48709cfefb7SHuacai Chen 48809cfefb7SHuacai Chen /* Set huge page tlb entry size */ 489d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 490d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 49109cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 49209cfefb7SHuacai Chen 493b681604eSHuacai Chen tlbfill 49409cfefb7SHuacai Chen 49509cfefb7SHuacai Chen /* Reset default page size */ 496d8e7f201SWANG Xuerui addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 497d8e7f201SWANG Xuerui addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 49809cfefb7SHuacai Chen csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 49909cfefb7SHuacai Chen 500a2a84e36SRui Wang csrrd t0, EXCEPTION_KS0 501a2a84e36SRui Wang csrrd t1, EXCEPTION_KS1 502a2a84e36SRui Wang csrrd ra, EXCEPTION_KS2 503a2a84e36SRui Wang ertn 504a2a84e36SRui Wang 50509cfefb7SHuacai Chennopage_tlb_modify: 506e031a5f3SHuacai Chen dbar 0x700 50709cfefb7SHuacai Chen csrrd ra, EXCEPTION_KS2 508396233c6SYouling Tang la_abs t0, tlb_do_page_fault_1 50907b48069SWANG Xuerui jr t0 51000c2ca84STiezhu YangSYM_CODE_END(handle_tlb_modify) 51109cfefb7SHuacai Chen 51200c2ca84STiezhu YangSYM_CODE_START(handle_tlb_modify_ptw) 513cb8a2ef0STiezhu Yang UNWIND_HINT_UNDEFINED 51401158487SHuacai Chen csrwr t0, LOONGARCH_CSR_KS0 51501158487SHuacai Chen csrwr t1, LOONGARCH_CSR_KS1 51601158487SHuacai Chen la_abs t0, tlb_do_page_fault_1 51701158487SHuacai Chen jr t0 51800c2ca84STiezhu YangSYM_CODE_END(handle_tlb_modify_ptw) 51901158487SHuacai Chen 52000c2ca84STiezhu YangSYM_CODE_START(handle_tlb_refill) 521cb8a2ef0STiezhu Yang UNWIND_HINT_UNDEFINED 52209cfefb7SHuacai Chen csrwr t0, LOONGARCH_CSR_TLBRSAVE 52309cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_PGD 52409cfefb7SHuacai Chen lddir t0, t0, 3 52509cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 3 52609cfefb7SHuacai Chen lddir t0, t0, 2 52709cfefb7SHuacai Chen#endif 52809cfefb7SHuacai Chen#if CONFIG_PGTABLE_LEVELS > 2 52909cfefb7SHuacai Chen lddir t0, t0, 1 53009cfefb7SHuacai Chen#endif 53109cfefb7SHuacai Chen ldpte t0, 0 53209cfefb7SHuacai Chen ldpte t0, 1 53309cfefb7SHuacai Chen tlbfill 53409cfefb7SHuacai Chen csrrd t0, LOONGARCH_CSR_TLBRSAVE 53509cfefb7SHuacai Chen ertn 53600c2ca84STiezhu YangSYM_CODE_END(handle_tlb_refill) 537