Lines Matching +full:tlb +full:- +full:split
6 * Synthesize TLB refill handlers at runtime.
10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
34 #include <asm/cpu-type.h>
52 * TLB load/store/modify handlers.
131 * CVMSEG starts at address -32768 and extends for in scratchpad_offset()
135 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; in scratchpad_offset()
230 * TLB exception handlers.
262 unsigned int count = (end - start) / sizeof(u32); in dump_handler()
279 /* The only general purpose registers allowed in TLB handlers. */
307 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
338 return -1; in allocate_kscratch()
340 r--; /* make it zero based */ in allocate_kscratch()
406 * The R3000 TLB handler is simple.
435 panic("TLB refill handler space exceeded"); in build_r3000_tlb_refill_handler()
437 pr_debug("Wrote TLB refill handler (%u instructions).\n", in build_r3000_tlb_refill_handler()
438 (unsigned int)(p - tlb_handler)); in build_r3000_tlb_refill_handler()
447 * The R4000 TLB handler is much more complicated. We have two
470 * The software work-around is to not allow the instruction preceding the TLBP
471 * to stall - make it an NOP or some other instruction guaranteed not to stall.
595 panic("No TLB refill handler yet (CPU type: %d)", in build_tlb_write_entry()
616 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); in build_convert_pte_to_entrylo()
683 /* Set huge page tlb entry size */ in build_huge_tlb_write_entry()
717 * of the large TLB entry size we intend to use. in build_huge_update_entries()
718 * A TLB entry half the size of the configured in build_huge_update_entries()
802 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); in build_get_pmde64()
809 if (pgd_reg != -1) { in build_get_pmde64()
844 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); in build_get_pmde64()
846 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); in build_get_pmde64()
851 uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */ in build_get_pmde64()
852 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3); in build_get_pmde64()
858 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ in build_get_pmde64()
859 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); in build_get_pmde64()
947 if (pgd_reg != -1) { in build_get_pgde32()
976 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; in build_adjust_context()
977 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); in build_adjust_context()
1084 if (pgd_reg != -1) in build_fast_tlb_refill_handler()
1095 PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); in build_fast_tlb_refill_handler()
1098 if (pgd_reg == -1) { in build_fast_tlb_refill_handler()
1104 if (pgd_reg != -1) in build_fast_tlb_refill_handler()
1116 if (pgd_reg == -1) in build_fast_tlb_refill_handler()
1123 if (pgd_reg == -1) { in build_fast_tlb_refill_handler()
1139 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1145 * fall-through case = badvaddr *pgd_current in build_fast_tlb_refill_handler()
1151 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1156 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); in build_fast_tlb_refill_handler()
1167 uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3); in build_fast_tlb_refill_handler()
1168 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3); in build_fast_tlb_refill_handler()
1182 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); in build_fast_tlb_refill_handler()
1183 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); in build_fast_tlb_refill_handler()
1256 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
1259 * unused TLB refill exception.
1298 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_refill_handler()
1335 * free instruction slot for the wrap-around branch. In worst in build_r4000_tlb_refill_handler()
1346 if ((p - tlb_handler) > 64) in build_r4000_tlb_refill_handler()
1347 panic("TLB refill handler space exceeded"); in build_r4000_tlb_refill_handler()
1349 * Now fold the handler in the TLB refill handler space. in build_r4000_tlb_refill_handler()
1354 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1357 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) in build_r4000_tlb_refill_handler()
1358 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) in build_r4000_tlb_refill_handler()
1360 tlb_handler + MIPS64_REFILL_INSNS - 3))) in build_r4000_tlb_refill_handler()
1361 panic("TLB refill handler space exceeded"); in build_r4000_tlb_refill_handler()
1363 * Now fold the handler in the TLB refill handler space. in build_r4000_tlb_refill_handler()
1366 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { in build_r4000_tlb_refill_handler()
1369 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1376 u32 *split; in build_r4000_tlb_refill_handler() local
1383 split = labels[i].addr; in build_r4000_tlb_refill_handler()
1388 if (split > tlb_handler + MIPS64_REFILL_INSNS || in build_r4000_tlb_refill_handler()
1389 split < p - MIPS64_REFILL_INSNS) in build_r4000_tlb_refill_handler()
1394 * Split two instructions before the end. One in build_r4000_tlb_refill_handler()
1398 split = tlb_handler + MIPS64_REFILL_INSNS - 2; in build_r4000_tlb_refill_handler()
1405 if (uasm_insn_has_bdelay(relocs, split - 1)) in build_r4000_tlb_refill_handler()
1406 split--; in build_r4000_tlb_refill_handler()
1409 uasm_copy_handler(relocs, labels, tlb_handler, split, f); in build_r4000_tlb_refill_handler()
1410 f += split - tlb_handler; in build_r4000_tlb_refill_handler()
1416 if (uasm_insn_has_bdelay(relocs, split)) in build_r4000_tlb_refill_handler()
1420 split, split + 1, f); in build_r4000_tlb_refill_handler()
1421 uasm_move_labels(labels, f, f + 1, -1); in build_r4000_tlb_refill_handler()
1423 split++; in build_r4000_tlb_refill_handler()
1428 uasm_copy_handler(relocs, labels, split, p, final_handler); in build_r4000_tlb_refill_handler()
1429 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + in build_r4000_tlb_refill_handler()
1430 (p - split); in build_r4000_tlb_refill_handler()
1437 pr_debug("Wrote TLB refill handler (%u instructions).\n", in build_r4000_tlb_refill_handler()
1461 pgd_w = PGDIR_SHIFT - PMD_SHIFT + PGD_TABLE_ORDER; in setup_pw()
1464 pmd_w = PMD_SHIFT - PAGE_SHIFT; in setup_pw()
1466 pgd_w = PGDIR_SHIFT - PAGE_SHIFT + PGD_TABLE_ORDER; in setup_pw()
1470 pt_w = PAGE_SHIFT - 3; in setup_pw()
1504 uasm_i_dsrl_safe(&p, K1, K0, PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); in build_loongson3_tlb_refill_handler()
1561 memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p); in build_setup_pgd()
1566 if (pgd_reg == -1) { in build_setup_pgd()
1581 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); in build_setup_pgd()
1610 if (pgd_reg != -1) { in build_setup_pgd()
1624 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd)); in build_setup_pgd()
1717 * the page table where this PTE is located, PTE will be re-loaded
1739 /* You lose the SMP race :-(*/ in build_pte_present()
1752 /* You lose the SMP race :-(*/ in build_pte_present()
1789 /* You lose the SMP race :-(*/ in build_pte_writable()
1826 /* You lose the SMP race :-(*/ in build_pte_modifiable()
1835 * R3000 style TLB load/store/modify handlers.
1902 memset(p, 0, handle_tlbl_end - (char *)p); in build_r3000_tlb_load_handler()
1907 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); in build_r3000_tlb_load_handler()
1909 build_make_valid(&p, &r, K0, K1, -1); in build_r3000_tlb_load_handler()
1917 panic("TLB load handler fastpath space exceeded"); in build_r3000_tlb_load_handler()
1920 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", in build_r3000_tlb_load_handler()
1921 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r3000_tlb_load_handler()
1932 memset(p, 0, handle_tlbs_end - (char *)p); in build_r3000_tlb_store_handler()
1937 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); in build_r3000_tlb_store_handler()
1939 build_make_write(&p, &r, K0, K1, -1); in build_r3000_tlb_store_handler()
1947 panic("TLB store handler fastpath space exceeded"); in build_r3000_tlb_store_handler()
1950 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", in build_r3000_tlb_store_handler()
1951 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r3000_tlb_store_handler()
1962 memset(p, 0, handle_tlbm_end - (char *)p); in build_r3000_tlb_modify_handler()
1967 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); in build_r3000_tlb_modify_handler()
1969 build_make_write(&p, &r, K0, K1, -1); in build_r3000_tlb_modify_handler()
1977 panic("TLB modify handler fastpath space exceeded"); in build_r3000_tlb_modify_handler()
1980 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", in build_r3000_tlb_modify_handler()
1981 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r3000_tlb_modify_handler()
1990 * When a Hardware Table Walker is running it can replace TLB entries in cpu_has_tlbex_tlbp_race()
2008 * R4000 style TLB load/store/modify handlers.
2024 * For huge tlb entries, pmd doesn't contain an address but in build_r4000_tlbchange_handler_head()
2025 * instead contains the tlb pte. Check the PAGE_HUGE bit and in build_r4000_tlbchange_handler_head()
2026 * see if we need to jump to huge tlb processing. in build_r4000_tlbchange_handler_head()
2033 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT - PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2034 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2079 memset(p, 0, handle_tlbl_end - (char *)p); in build_r4000_tlb_load_handler()
2091 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_load_handler()
2117 * Warn if something may race with us & replace the TLB entry in build_r4000_tlb_load_handler()
2183 * Warn if something may race with us & replace the TLB entry in build_r4000_tlb_load_handler()
2245 panic("TLB load handler fastpath space exceeded"); in build_r4000_tlb_load_handler()
2248 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", in build_r4000_tlb_load_handler()
2249 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r4000_tlb_load_handler()
2261 memset(p, 0, handle_tlbs_end - (char *)p); in build_r4000_tlb_store_handler()
2301 panic("TLB store handler fastpath space exceeded"); in build_r4000_tlb_store_handler()
2304 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", in build_r4000_tlb_store_handler()
2305 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r4000_tlb_store_handler()
2317 memset(p, 0, handle_tlbm_end - (char *)p); in build_r4000_tlb_modify_handler()
2358 panic("TLB modify handler fastpath space exceeded"); in build_r4000_tlb_modify_handler()
2361 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", in build_r4000_tlb_modify_handler()
2362 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r4000_tlb_modify_handler()
2422 * We are using 2-level page tables, so we only need to in config_htw_params()
2431 /* re-initialize the GDI field */ in config_htw_params()
2434 /* re-initialize the PTI field including the even/odd bit */ in config_htw_params()
2469 pwsize |= ((PTE_T_LOG2 - PGD_T_LOG2) << MIPS_PWSIZE_PTEW_SHIFT) in config_htw_params()
2478 * Enable HTW (and only for XUSeg on 64-bit), and disable the rest of in config_htw_params()
2530 /* clear all non-PFN bits */ in check_pabits()
2531 entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1); in check_pabits()
2536 fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0); in check_pabits()
2539 fillbits -= min_t(unsigned, fillbits, 2); in check_pabits()
2550 * The refill handler is generated per-CPU, multi-node systems in build_tlb_refill_handler()
2563 …check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3… in build_tlb_refill_handler()
2578 panic("No R3000 TLB refill handler"); in build_tlb_refill_handler()