Lines Matching +full:d +full:- +full:tlb +full:- +full:size
1 /* SPDX-License-Identifier: GPL-2.0 */
5 * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co
8 * Copyright (C) 2000, 2002-2003 Intel Co
14 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> TLB handling for SMP
15 * 00/12/20 David Mosberger-Tang <davidm@hpl.hp.com> DTLB/ITLB handler now uses virtual PT.
17 * Copyright (C) 2005 Hewlett-Packard Co
33 * interruptions like TLB misses.
37 * // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
38 * entry offset ----/ / / / /
39 * entry number ---------/ / / /
40 * size of the entry -------------/ / /
41 * vector name -------------------------------------/ /
42 * interruptions triggering this vector ----------------------/
44 * The table is 32KB in size and must be aligned on 32KB boundary.
55 #include <asm/asm-offsets.h>
93 // 0x0000 Entry 0 (size 64 bundles) VHPT Translation (8,20,47)
97 * The VHPT vector is invoked when the TLB entry for the virtual page table
99 * (the "original") TLB miss, which may either be caused by an instruction
100 * fetch or a data access (or non-access).
102 * What we do here is normal TLB miss handing for the _original_ miss,
103 * followed by inserting the TLB entry for the virtual page table page
106 * mappings for the faulting address. The TLB entry for the original
111 * - the faulting virtual address uses unimplemented address bits
112 * - the faulting virtual address has no valid page table mapping
114 MOV_FROM_IFA(r16) // get address that caused the TLB miss
140 (p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
142 srlz.d
147 (p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
149 (p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
150 (p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
162 dep r28=r28,r17,3,(PAGE_SHIFT-3) // r28=pud_offset(pgd,addr)
168 dep r17=r18,r29,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
170 dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pgd,addr)
177 dep r21=r19,r20,3,(PAGE_SHIFT-3) // r21=pte_offset(pmd,addr)
183 MOV_FROM_IHA(r22) // get the VHPT address that caused the TLB miss
185 (p7) tbit.nz.unc p10,p11=r19,32 // is it an instruction TLB miss?
188 ITC_I_AND_D(p10, p11, r18, r24) // insert the instruction TLB entry and
189 // insert the data TLB entry
194 MOV_TO_ITIR(p8, r25, r24) // change to default page-size for VHPT
198 * Now compute and insert the TLB entry for the virtual page table. We never
208 * Tell the assemblers dependency-violation checker that the above "itc" instructions
214 * Re-check pagetable entry. If they changed, we may have received a ptc.g
245 mov pr=r31,-1 // restore predicate registers
251 // 0x0400 Entry 1 (size 64 bundles) ITLB (21)
256 * page table. If a nested TLB miss occurs, we switch into physical
257 * mode, walk the page table, and then re-execute the PTE read and
277 * Tell the assemblers dependency-violation checker that the above "itc" instructions
283 mov r20=PAGE_SHIFT<<2 // setup page size for purge
289 mov pr=r31,-1
295 // 0x0800 Entry 2 (size 64 bundles) DTLB (9,48)
300 * page table. If a nested TLB miss occurs, we switch into physical
301 * mode, walk the page table, and then re-execute the PTE read and
321 * Tell the assemblers dependency-violation checker that the above "itc" instructions
327 mov r20=PAGE_SHIFT<<2 // setup page size for purge
333 mov pr=r31,-1
339 // 0x0c00 Entry 3 (size 64 bundles) Alt ITLB (19)
342 MOV_FROM_IFA(r16) // get address that caused the TLB miss
345 movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
363 andcm r18=0x10,r18 // bit 4=~address-bit(61)
370 ITC_I(p0, r19, r18) // insert the TLB entry
371 mov pr=r31,-1
377 // 0x1000 Entry 4 (size 64 bundles) Alt DTLB (7,46)
380 MOV_FROM_IFA(r16) // get address that caused the TLB miss
383 movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
391 cmp.gt p8,p0=6,r22 // access to region 0-5
407 (p11) and r19=r19,r16 // clear non-ppn fields
411 tbit.nz p9,p0=r20,IA64_ISR_NA_BIT // is non-access bit on?
417 (p12) dep r17=-1,r17,4,1 // set ma=UC for region 6 addr
420 dep r21=-1,r21,IA64_PSR_ED_BIT,1
425 ITC_D(p7, r19, r18) // insert the TLB entry
426 mov pr=r31,-1
432 // 0x1400 Entry 5 (size 64 bundles) Data nested TLB (6,45)
436 * page table is accessed non-speculatively (e.g., in the Dirty-bit, Instruction
437 * Access-bit, or Data Access-bit faults). If the DTLB entry for the virtual page
438 * table is missing, a nested TLB miss fault is triggered and control is
462 extr.u r18=r18,2,6 // get the faulting page size
465 add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address
466 add r18=PGDIR_SHIFT-PAGE_SHIFT,r18
470 (p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
472 srlz.d
477 (p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
479 (p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
480 (p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
491 dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=p[u|m]d_offset(pgd,addr)
498 dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
505 dep r17=r19,r17,3,(PAGE_SHIFT-3) // r17=pte_offset(pmd,addr);
513 // 0x1800 Entry 6 (size 64 bundles) Instruction Key Miss (24)
521 // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
529 // 0x2000 Entry 8 (size 64 bundles) Dirty-bit (54)
534 * update both the page-table and the TLB entry. To efficiently access the PTE,
535 * we address it through the virtual page table. Most likely, the TLB entry for
536 * the relevant virtual page table page is still present in the TLB so we can
537 * normally do this without additional TLB misses. In case the necessary virtual
538 * page table TLB entry isn't present, we take a nested TLB miss hit where we look
564 * Tell the assemblers dependency-violation checker that the above "itc" instructions
586 mov pr=r31,-1 // restore pr
592 // 0x2400 Entry 9 (size 64 bundles) Instruction Access-bit (27)
630 * Tell the assemblers dependency-violation checker that the above "itc" instructions
652 mov pr=r31,-1
658 // 0x2800 Entry 10 (size 64 bundles) Data Access-bit (15,55)
684 * Tell the assemblers dependency-violation checker that the above "itc" instructions
705 mov pr=r31,-1
711 // 0x2c00 Entry 11 (size 64 bundles) Break instruction (33)
715 * of pt_regs. This implies that the callers of system-calls must adhere to the
723 * r8-r11: output value from the system call.
729 mov.m r16=IA64_KR(CURRENT) // M2 r16 <- current task (12 cyc)
754 (p7) br.cond.spnt non_syscall // B no ->
756 // From this point on, we are definitely on the syscall-path
757 // and we can use (non-banked) scratch registers.
760 mov r1=r16 // A move task-pointer to "addl"-addressable reg
762 add r9=TI_FLAGS+IA64_TASK_SIZE,r16 // A r9 = ¤t_thread_info()->flags
765 adds r15=-1024,r15 // A subtract 1024 from syscall number
766 mov r3=NR_syscalls - 1
768 ld1.bias r17=[r16] // M0|1 r17 = current->thread.on_ustack flag
769 ld4 r9=[r9] // M0|1 r9 = current_thread_info()->flags
772 shladd r30=r15,3,r30 // A r30 = sys_call_table + 8*(syscall-1024)
806 st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
816 (pKStk) br.cond.spnt .break_fixup // B we're already in kernel-mode -- fix up RBS
819 (pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A compute base of memory stack
829 ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // M get last stamp
830 ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // M time at leave
832 ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // M cumulated stime
836 st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // M update stamp
852 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r16) // M2 now it's safe to re-enable intr.-collection
857 (p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
860 (p14) br.call.sptk.many b6=b6 // B invoke syscall-handker (ignore return addr)
861 br.cond.spnt.many ia64_trace_syscall // B do syscall-tracing thingamagic
864 // On entry, we optimistically assumed that we're coming from user-space.
865 // For the rare cases where a system-call is done from within the kernel,
868 add r1=-IA64_PT_REGS_SIZE,sp // A allocate space for pt_regs structure
877 // 0x3000 Entry 12 (size 64 bundles) External Interrupt (4)
885 // 0x3400 Entry 13 (size 64 bundles) Reserved
891 // 0x3800 Entry 14 (size 64 bundles) Reserved
906 * - executing on bank 0 or bank 1 register set (doesn't matter)
907 * - r1: stack pointer
908 * - r2: current task pointer
909 * - r3: preserved
910 * - r11: original contents (saved ar.pfs to be saved)
911 * - r12: original contents (sp to be saved)
912 * - r13: original contents (tp to be saved)
913 * - r15: original contents (syscall # to be saved)
914 * - r18: saved bsp (after switching to kernel stack)
915 * - r19: saved b6
916 * - r20: saved r1 (gp)
917 * - r21: saved ar.fpsr
918 * - r22: kernel's register backing store base (krbs_base)
919 * - r23: saved ar.bspstore
920 * - r24: saved ar.rnat
921 * - r25: saved ar.unat
922 * - r26: saved ar.pfs
923 * - r27: saved ar.rsc
924 * - r28: saved cr.iip
925 * - r29: saved cr.ipsr
926 * - r30: ar.itc for accounting (don't touch)
927 * - r31: saved pr
928 * - b0: original contents (to be saved)
930 * - p10: TRUE if syscall is invoked with more than 8 out
932 * - r1: kernel's gp
933 * - r3: preserved (same as on entry)
934 * - r8: -EINVAL if p10 is true
935 * - r12: points to kernel stack
936 * - r13: points to current task
937 * - r14: preserved (same as on entry)
938 * - p13: preserved
939 * - p15: TRUE if interrupts need to be re-enabled
940 * - ar.fpsr: set to kernel settings
941 * - b6: preserved (same as on entry)
951 alloc r19=ar.pfs,8,0,0,0 // ensure in0-in7 are writable
952 st8 [r16]=r29,PT(AR_PFS)-PT(CR_IPSR) // save cr.ipsr
955 st8.spill [r17]=r11,PT(CR_IIP)-PT(R11) // save r11
960 st8 [r16]=r26,PT(CR_IFS)-PT(AR_PFS) // save ar.pfs
961 st8 [r17]=r28,PT(AR_UNAT)-PT(CR_IIP) // save cr.iip
965 st8 [r17]=r25,PT(AR_RSC)-PT(AR_UNAT) // save ar.unat
967 (p8) mov in0=-1
970 st8 [r16]=r19,PT(AR_RNAT)-PT(CR_IFS) // store ar.pfs.pfm in cr.ifs
974 st8 [r17]=r27,PT(AR_BSPSTORE)-PT(AR_RSC)// save ar.rsc
976 (p9) mov in1=-1
983 (pKStk) adds r16=PT(PR)-PT(AR_RNAT),r16 // skip over ar_rnat field
984 (pKStk) adds r17=PT(B0)-PT(AR_BSPSTORE),r17 // skip over ar_bspstore field
987 (p10) mov in2=-1
989 (p11) mov in3=-1
991 (pUStk) st8 [r16]=r24,PT(PR)-PT(AR_RNAT) // save ar.rnat
992 (pUStk) st8 [r17]=r23,PT(B0)-PT(AR_BSPSTORE) // save ar.bspstore
995 st8 [r16]=r31,PT(LOADRS)-PT(PR) // save predicates
996 st8 [r17]=r28,PT(R1)-PT(B0) // save b0
999 st8 [r16]=r18,PT(R12)-PT(LOADRS) // save ar.rsc value for "loadrs"
1000 st8.spill [r17]=r20,PT(R13)-PT(R1) // save original r1
1001 (p12) mov in4=-1
1004 .mem.offset 0,0; st8.spill [r16]=r12,PT(AR_FPSR)-PT(R12) // save r12
1005 .mem.offset 8,0; st8.spill [r17]=r13,PT(R15)-PT(R13) // save r13
1006 (p13) mov in5=-1
1008 st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr
1010 cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8
1014 adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch)
1024 (p13) mov in6=-1
1025 (p8) mov in7=-1
1031 (p10) mov r8=-EINVAL
1037 // 0x3c00 Entry 15 (size 64 bundles) Reserved
1043 // 0x4000 Entry 16 (size 64 bundles) Reserved
1062 ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // time at last check in kernel
1063 ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // time at left from kernel
1065 ld8 r23=[r16],TI_AC_STAMP-TI_AC_STIME // cumulated stime
1069 st8 [r16]=r20,TI_AC_STIME-TI_AC_STAMP // update stamp
1084 // 0x4400 Entry 17 (size 64 bundles) Reserved
1090 // 0x4800 Entry 18 (size 64 bundles) Reserved
1096 // 0x4c00 Entry 19 (size 64 bundles) Reserved
1101 // --- End of long entries, Beginning of short entries
1106 // 0x5000 Entry 20 (size 16 bundles) Page Not Present (10,22,49)
1112 * The Linux page fault handler doesn't expect non-present pages to be in
1113 * the TLB. Flush the existing entry now, so we meet that expectation.
1120 srlz.d
1126 // 0x5100 Entry 21 (size 16 bundles) Key Permission (13,25,52)
1133 srlz.d
1139 // 0x5200 Entry 22 (size 16 bundles) Instruction Access Rights (26)
1146 srlz.d
1152 // 0x5300 Entry 23 (size 16 bundles) Data Access Rights (14,53)
1159 srlz.d
1165 // 0x5400 Entry 24 (size 16 bundles) General Exception (5,32,34,36,38,39)
1180 // 0x5500 Entry 25 (size 16 bundles) Disabled FP-Register (35)
1185 srlz.d
1193 // 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50)
1205 dep r16=-1,r16,IA64_PSR_ED_BIT,1
1209 mov pr=r31,-1
1213 1: mov pr=r31,-1
1220 // 0x5700 Entry 27 (size 16 bundles) Speculation (40)
1236 shl r18=r18,43 // put sign bit in position (43=64-21)
1240 shr r18=r18,39 // sign extend (39=43-4)
1257 // 0x5800 Entry 28 (size 16 bundles) Reserved
1263 // 0x5900 Entry 29 (size 16 bundles) Debug (16,28,56)
1271 // 0x5a00 Entry 30 (size 16 bundles) Unaligned Reference (57)
1281 // 0x5b00 Entry 31 (size 16 bundles) Unsupported Data Reference (57)
1289 // 0x5c00 Entry 32 (size 16 bundles) Floating-Point Fault (64)
1297 // 0x5d00 Entry 33 (size 16 bundles) Floating Point Trap (66)
1305 // 0x5e00 Entry 34 (size 16 bundles) Lower Privilege Transfer Trap (66)
1313 // 0x5f00 Entry 35 (size 16 bundles) Taken Branch Trap (68)
1321 // 0x6000 Entry 36 (size 16 bundles) Single Step Trap (69)
1329 // 0x6100 Entry 37 (size 16 bundles) Reserved
1335 // 0x6200 Entry 38 (size 16 bundles) Reserved
1341 // 0x6300 Entry 39 (size 16 bundles) Reserved
1347 // 0x6400 Entry 40 (size 16 bundles) Reserved
1353 // 0x6500 Entry 41 (size 16 bundles) Reserved
1359 // 0x6600 Entry 42 (size 16 bundles) Reserved
1365 // 0x6700 Entry 43 (size 16 bundles) Reserved
1371 // 0x6800 Entry 44 (size 16 bundles) Reserved
1377 // 0x6900 Entry 45 (size 16 bundles) IA-32 Exeception (17,18,29,41,42,43,44,58,60,61,62,72,73,75,76…
1385 // 0x6a00 Entry 46 (size 16 bundles) IA-32 Intercept (30,31,59,70,71)
1393 // 0x6b00 Entry 47 (size 16 bundles) IA-32 Interrupt (74)
1401 // 0x6c00 Entry 48 (size 16 bundles) Reserved
1407 // 0x6d00 Entry 49 (size 16 bundles) Reserved
1413 // 0x6e00 Entry 50 (size 16 bundles) Reserved
1419 // 0x6f00 Entry 51 (size 16 bundles) Reserved
1425 // 0x7000 Entry 52 (size 16 bundles) Reserved
1431 // 0x7100 Entry 53 (size 16 bundles) Reserved
1437 // 0x7200 Entry 54 (size 16 bundles) Reserved
1443 // 0x7300 Entry 55 (size 16 bundles) Reserved
1449 // 0x7400 Entry 56 (size 16 bundles) Reserved
1455 // 0x7500 Entry 57 (size 16 bundles) Reserved
1461 // 0x7600 Entry 58 (size 16 bundles) Reserved
1467 // 0x7700 Entry 59 (size 16 bundles) Reserved
1473 // 0x7800 Entry 60 (size 16 bundles) Reserved
1479 // 0x7900 Entry 61 (size 16 bundles) Reserved
1485 // 0x7a00 Entry 62 (size 16 bundles) Reserved
1491 // 0x7b00 Entry 63 (size 16 bundles) Reserved
1497 // 0x7c00 Entry 64 (size 16 bundles) Reserved
1503 // 0x7d00 Entry 65 (size 16 bundles) Reserved
1509 // 0x7e00 Entry 66 (size 16 bundles) Reserved
1515 // 0x7f00 Entry 67 (size 16 bundles) Reserved
1519 //-----------------------------------------------------------------------------------
1582 srlz.d // make sure we see the effect of cr.ivr