Lines Matching +full:async +full:- +full:enum
2 * RISC-V CPU helpers for qemu.
4 * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
5 * Copyright (c) 2017-2018 SiFive, Inc.
22 #include "qemu/main-loop.h"
27 #include "exec/page-protection.h"
31 #include "tcg/tcg-op.h"
32 #include "accel/tcg/cpu-ops.h"
34 #include "semihosting/common-semi.h"
45 bool virt = env->virt_enabled; in riscv_env_mmu_index()
46 int mode = env->priv; in riscv_env_mmu_index()
48 /* All priv -> mmu_idx mapping are here */ in riscv_env_mmu_index()
50 uint64_t status = env->mstatus; in riscv_env_mmu_index()
53 mode = get_field(env->mstatus, MSTATUS_MPP); in riscv_env_mmu_index()
54 virt = get_field(env->mstatus, MSTATUS_MPV) && in riscv_env_mmu_index()
57 status = env->vsstatus; in riscv_env_mmu_index()
72 if (!env_archcpu(env)->cfg.ext_zicfilp) { in cpu_get_fcfien()
76 switch (env->priv) { in cpu_get_fcfien()
79 return env->senvcfg & SENVCFG_LPE; in cpu_get_fcfien()
81 return env->menvcfg & MENVCFG_LPE; in cpu_get_fcfien()
84 if (env->virt_enabled) { in cpu_get_fcfien()
85 return env->henvcfg & HENVCFG_LPE; in cpu_get_fcfien()
87 return env->menvcfg & MENVCFG_LPE; in cpu_get_fcfien()
89 return env->mseccfg & MSECCFG_MLPE; in cpu_get_fcfien()
99 if (!env_archcpu(env)->cfg.ext_zicfiss) { in cpu_get_bcfien()
103 switch (env->priv) { in cpu_get_bcfien()
110 return env->senvcfg & SENVCFG_SSE; in cpu_get_bcfien()
113 if (env->virt_enabled) { in cpu_get_bcfien()
114 return env->henvcfg & HENVCFG_SSE; in cpu_get_bcfien()
116 return env->menvcfg & MENVCFG_SSE; in cpu_get_bcfien()
117 case PRV_M: /* M-mode shadow stack is always off */ in cpu_get_bcfien()
131 return (env->henvcfg & HENVCFG_DTE) != 0; in riscv_env_smode_dbltrp_enabled()
133 return (env->menvcfg & MENVCFG_DTE) != 0; in riscv_env_smode_dbltrp_enabled()
143 if (get_field(env->mstatus, MSTATUS_MPRV) && in riscv_pm_get_pmm()
144 get_field(env->mstatus, MSTATUS_MXR)) { in riscv_pm_get_pmm()
151 if (riscv_cpu_cfg(env)->ext_smmpm) { in riscv_pm_get_pmm()
152 return get_field(env->mseccfg, MSECCFG_PMM); in riscv_pm_get_pmm()
156 if (riscv_cpu_cfg(env)->ext_smnpm) { in riscv_pm_get_pmm()
157 if (get_field(env->mstatus, MSTATUS_MPV)) { in riscv_pm_get_pmm()
158 return get_field(env->henvcfg, HENVCFG_PMM); in riscv_pm_get_pmm()
160 return get_field(env->menvcfg, MENVCFG_PMM); in riscv_pm_get_pmm()
166 if (riscv_cpu_cfg(env)->ext_ssnpm) { in riscv_pm_get_pmm()
167 return get_field(env->senvcfg, SENVCFG_PMM); in riscv_pm_get_pmm()
170 if (riscv_cpu_cfg(env)->ext_smnpm) { in riscv_pm_get_pmm()
171 return get_field(env->menvcfg, MENVCFG_PMM); in riscv_pm_get_pmm()
190 return get_field(env->hstatus, HSTATUS_HUPMM); in riscv_pm_get_virt_pmm()
192 if (get_field(env->hstatus, HSTATUS_SPVP)) { in riscv_pm_get_virt_pmm()
193 return get_field(env->henvcfg, HENVCFG_PMM); in riscv_pm_get_virt_pmm()
195 return get_field(env->senvcfg, SENVCFG_PMM); in riscv_pm_get_virt_pmm()
210 satp_mode = get_field(env->satp, SATP32_MODE); in riscv_cpu_virt_mem_enabled()
212 satp_mode = get_field(env->satp, SATP64_MODE); in riscv_cpu_virt_mem_enabled()
238 * The HS-mode is allowed to configure priority only for the
239 * following VS-mode local interrupts:
267 return -EINVAL; in riscv_cpu_hviprio_index2irq()
283 * RISC-V Advanced Interrupt Architecture specification.
285 * ----------------------------------------------------------------
288 * ----------------------------------------------------------------
299 * ----------------------------------------------------------------
394 if (!((extirq == IRQ_M_EXT) ? riscv_cpu_cfg(env)->ext_smaia : in riscv_cpu_pending_to_irq()
395 riscv_cpu_cfg(env)->ext_ssaia)) { in riscv_cpu_pending_to_irq()
422 * Doesn't report interrupts inserted using mvip from M-mode firmware or
423 * using hvip bits 13:63 from HS-mode. Those are returned in
428 uint32_t gein = get_field(env->hstatus, HSTATUS_VGEIN); in riscv_cpu_all_pending()
429 uint64_t vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0; in riscv_cpu_all_pending()
430 uint64_t vstip = (env->vstime_irq) ? MIP_VSTIP : 0; in riscv_cpu_all_pending()
432 return (env->mip | vsgein | vstip) & env->mie; in riscv_cpu_all_pending()
437 uint64_t irqs = riscv_cpu_all_pending(env) & ~env->mideleg & in riscv_cpu_mirq_pending()
441 irqs, env->miprio); in riscv_cpu_mirq_pending()
446 uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg & in riscv_cpu_sirq_pending()
448 uint64_t irqs_f = env->mvip & env->mvien & ~env->mideleg & env->sie; in riscv_cpu_sirq_pending()
451 irqs | irqs_f, env->siprio); in riscv_cpu_sirq_pending()
456 uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg & env->hideleg; in riscv_cpu_vsirq_pending()
457 uint64_t irqs_f_vs = env->hvip & env->hvien & ~env->hideleg & env->vsie; in riscv_cpu_vsirq_pending()
460 /* Bring VS-level bits to correct position */ in riscv_cpu_vsirq_pending()
466 (irqs | irqs_f_vs), env->hviprio); in riscv_cpu_vsirq_pending()
476 if (riscv_cpu_cfg(env)->ext_smrnmi) { in riscv_cpu_local_irq_pending()
478 if (!get_field(env->mnstatus, MNSTATUS_NMIE)) { in riscv_cpu_local_irq_pending()
482 if (env->rnmip) { in riscv_cpu_local_irq_pending()
483 return ctz64(env->rnmip); /* since non-zero */ in riscv_cpu_local_irq_pending()
488 if (env->virt_enabled) { in riscv_cpu_local_irq_pending()
491 vsie = (env->priv < PRV_S) || in riscv_cpu_local_irq_pending()
492 (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE)); in riscv_cpu_local_irq_pending()
494 mie = (env->priv < PRV_M) || in riscv_cpu_local_irq_pending()
495 (env->priv == PRV_M && get_field(env->mstatus, MSTATUS_MIE)); in riscv_cpu_local_irq_pending()
496 hsie = (env->priv < PRV_S) || in riscv_cpu_local_irq_pending()
497 (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE)); in riscv_cpu_local_irq_pending()
504 /* Check M-mode interrupts */ in riscv_cpu_local_irq_pending()
505 irqs = pending & ~env->mideleg & -mie; in riscv_cpu_local_irq_pending()
508 irqs, env->miprio); in riscv_cpu_local_irq_pending()
511 /* Check for virtual S-mode interrupts. */ in riscv_cpu_local_irq_pending()
512 irqs_f = env->mvip & (env->mvien & ~env->mideleg) & env->sie; in riscv_cpu_local_irq_pending()
514 /* Check HS-mode interrupts */ in riscv_cpu_local_irq_pending()
515 irqs = ((pending & env->mideleg & ~env->hideleg) | irqs_f) & -hsie; in riscv_cpu_local_irq_pending()
518 irqs, env->siprio); in riscv_cpu_local_irq_pending()
521 /* Check for virtual VS-mode interrupts. */ in riscv_cpu_local_irq_pending()
522 irqs_f_vs = env->hvip & env->hvien & ~env->hideleg & env->vsie; in riscv_cpu_local_irq_pending()
524 /* Check VS-mode interrupts */ in riscv_cpu_local_irq_pending()
525 irq_delegated = pending & env->mideleg & env->hideleg; in riscv_cpu_local_irq_pending()
527 /* Bring VS-level bits to correct position */ in riscv_cpu_local_irq_pending()
532 irqs = (irq_delegated | irqs_f_vs) & -vsie; in riscv_cpu_local_irq_pending()
535 irqs, env->hviprio); in riscv_cpu_local_irq_pending()
553 CPURISCVState *env = &cpu->env; in riscv_cpu_exec_interrupt()
556 cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno; in riscv_cpu_exec_interrupt()
567 if (env->mstatus & MSTATUS_FS) { in riscv_cpu_fp_enabled()
568 if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_FS)) { in riscv_cpu_fp_enabled()
580 if (env->mstatus & MSTATUS_VS) { in riscv_cpu_vector_enabled()
581 if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_VS)) { in riscv_cpu_vector_enabled()
599 bool current_virt = env->virt_enabled; in riscv_cpu_swap_hypervisor_regs()
605 if (env_archcpu(env)->cfg.ext_zicfilp && in riscv_cpu_swap_hypervisor_regs()
606 get_field(env->henvcfg, HENVCFG_LPE)) { in riscv_cpu_swap_hypervisor_regs()
618 env->vsstatus = env->mstatus & mstatus_mask; in riscv_cpu_swap_hypervisor_regs()
619 env->mstatus &= ~mstatus_mask; in riscv_cpu_swap_hypervisor_regs()
620 env->mstatus |= env->mstatus_hs; in riscv_cpu_swap_hypervisor_regs()
622 env->vstvec = env->stvec; in riscv_cpu_swap_hypervisor_regs()
623 env->stvec = env->stvec_hs; in riscv_cpu_swap_hypervisor_regs()
625 env->vsscratch = env->sscratch; in riscv_cpu_swap_hypervisor_regs()
626 env->sscratch = env->sscratch_hs; in riscv_cpu_swap_hypervisor_regs()
628 env->vsepc = env->sepc; in riscv_cpu_swap_hypervisor_regs()
629 env->sepc = env->sepc_hs; in riscv_cpu_swap_hypervisor_regs()
631 env->vscause = env->scause; in riscv_cpu_swap_hypervisor_regs()
632 env->scause = env->scause_hs; in riscv_cpu_swap_hypervisor_regs()
634 env->vstval = env->stval; in riscv_cpu_swap_hypervisor_regs()
635 env->stval = env->stval_hs; in riscv_cpu_swap_hypervisor_regs()
637 env->vsatp = env->satp; in riscv_cpu_swap_hypervisor_regs()
638 env->satp = env->satp_hs; in riscv_cpu_swap_hypervisor_regs()
641 env->mstatus_hs = env->mstatus & mstatus_mask; in riscv_cpu_swap_hypervisor_regs()
642 env->mstatus &= ~mstatus_mask; in riscv_cpu_swap_hypervisor_regs()
643 env->mstatus |= env->vsstatus; in riscv_cpu_swap_hypervisor_regs()
645 env->stvec_hs = env->stvec; in riscv_cpu_swap_hypervisor_regs()
646 env->stvec = env->vstvec; in riscv_cpu_swap_hypervisor_regs()
648 env->sscratch_hs = env->sscratch; in riscv_cpu_swap_hypervisor_regs()
649 env->sscratch = env->vsscratch; in riscv_cpu_swap_hypervisor_regs()
651 env->sepc_hs = env->sepc; in riscv_cpu_swap_hypervisor_regs()
652 env->sepc = env->vsepc; in riscv_cpu_swap_hypervisor_regs()
654 env->scause_hs = env->scause; in riscv_cpu_swap_hypervisor_regs()
655 env->scause = env->vscause; in riscv_cpu_swap_hypervisor_regs()
657 env->stval_hs = env->stval; in riscv_cpu_swap_hypervisor_regs()
658 env->stval = env->vstval; in riscv_cpu_swap_hypervisor_regs()
660 env->satp_hs = env->satp; in riscv_cpu_swap_hypervisor_regs()
661 env->satp = env->vsatp; in riscv_cpu_swap_hypervisor_regs()
671 return env->geilen; in riscv_cpu_get_geilen()
680 if (geilen > (TARGET_LONG_BITS - 1)) { in riscv_cpu_set_geilen()
684 env->geilen = geilen; in riscv_cpu_set_geilen()
689 CPURISCVState *env = &cpu->env; in riscv_cpu_set_rnmi()
699 env->rnmip |= 1 << irq; in riscv_cpu_set_rnmi()
702 env->rnmip &= ~(1 << irq); in riscv_cpu_set_rnmi()
713 CPURISCVState *env = &cpu->env; in riscv_cpu_claim_interrupts()
714 if (env->miclaim & interrupts) { in riscv_cpu_claim_interrupts()
715 return -1; in riscv_cpu_claim_interrupts()
717 env->miclaim |= interrupts; in riscv_cpu_claim_interrupts()
729 if (env->virt_enabled) { in riscv_cpu_interrupt()
730 gein = get_field(env->hstatus, HSTATUS_VGEIN); in riscv_cpu_interrupt()
731 vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0; in riscv_cpu_interrupt()
732 irqf = env->hvien & env->hvip & env->vsie; in riscv_cpu_interrupt()
734 irqf = env->mvien & env->mvip & env->sie; in riscv_cpu_interrupt()
737 vstip = env->vstime_irq ? MIP_VSTIP : 0; in riscv_cpu_interrupt()
739 if (env->mip | vsgein | vstip | irqf) { in riscv_cpu_interrupt()
748 uint64_t old = env->mip; in riscv_cpu_update_mip()
751 mask = ((mask == MIP_VSTIP) && env->vstime_irq) ? 0 : mask; in riscv_cpu_update_mip()
755 env->mip = (env->mip & ~mask) | (value & mask); in riscv_cpu_update_mip()
765 env->rdtime_fn = fn; in riscv_cpu_set_rdtime_fn()
766 env->rdtime_fn_arg = arg; in riscv_cpu_set_rdtime_fn()
778 env->aia_ireg_rmw_fn[priv] = rmw_fn; in riscv_cpu_set_aia_ireg_rmw_fn()
779 env->aia_ireg_rmw_fn_arg[priv] = rmw_fn_arg; in riscv_cpu_set_aia_ireg_rmw_fn()
786 uint64_t ctl = virt ? env->vsctrctl : env->mctrctl; in riscv_ctr_freeze()
791 env->sctrstatus |= SCTRSTATUS_FROZEN; in riscv_ctr_freeze()
797 memset(env->ctr_src, 0x0, sizeof(env->ctr_src)); in riscv_ctr_clear()
798 memset(env->ctr_dst, 0x0, sizeof(env->ctr_dst)); in riscv_ctr_clear()
799 memset(env->ctr_data, 0x0, sizeof(env->ctr_data)); in riscv_ctr_clear()
827 return env->mctrctl; in riscv_ctr_get_control()
831 return env->vsctrctl; in riscv_ctr_get_control()
833 return env->mctrctl; in riscv_ctr_get_control()
847 target_long tgt_prv = env->priv; in riscv_ctr_check_xte()
857 } else if (env->virt_enabled && tgt_prv == PRV_S) { in riscv_ctr_check_xte()
863 res &= !!(env->vsctrctl & XCTRCTL_STE); in riscv_ctr_check_xte()
871 res &= !!(env->mctrctl & XCTRCTL_STE); in riscv_ctr_check_xte()
872 /* fall-through */ in riscv_ctr_check_xte()
877 res &= !!(env->mctrctl & MCTRCTL_MTE); in riscv_ctr_check_xte()
878 /* fall-through */ in riscv_ctr_check_xte()
889 * 1- Traps, and trap returns, between enabled modes are recorded as normal.
890 * 2- Traps from an inhibited mode to an enabled mode, and trap returns from an
895 * 3- Trap returns from an inhibited mode to an enabled mode are not recorded.
900 * can opt-in to allowing CTR to record traps into that mode even when the mode
901 * is inhibited. The MTE, STE, and VSTE bits allow M-mode, S-mode, and VS-mode,
902 * respectively, to opt-in. When an External Trap occurs, and xTE=1, such that
926 * will return entry 0 i-e buffer[8] and 0x201 will return entry 1 i-e
929 * entry = isel - CTR_ENTRIES_FIRST;
930 * idx = (sctrstatus.WRPTR - entry - 1) & (depth - 1);
933 enum CTRType type, target_ulong src_priv, bool src_virt) in riscv_ctr_add_entry()
935 bool tgt_virt = env->virt_enabled; in riscv_ctr_add_entry()
937 uint64_t tgt_mask = riscv_ctr_priv_to_mask(env->priv, tgt_virt); in riscv_ctr_add_entry()
939 uint64_t tgt_ctrl = riscv_ctr_get_control(env, env->priv, tgt_virt); in riscv_ctr_add_entry()
948 env->sctrstatus & SCTRSTATUS_FROZEN) { in riscv_ctr_add_entry()
954 * returns and Co-routine swap types. in riscv_ctr_add_entry()
969 /* Check if target priv-mode has allowed external trap recording. */ in riscv_ctr_add_entry()
1005 head = get_field(env->sctrstatus, SCTRSTATUS_WRPTR_MASK); in riscv_ctr_add_entry()
1007 depth = 16 << get_field(env->sctrdepth, SCTRDEPTH_MASK); in riscv_ctr_add_entry()
1009 head = (head - 1) & (depth - 1); in riscv_ctr_add_entry()
1011 env->ctr_src[head] &= ~CTRSOURCE_VALID; in riscv_ctr_add_entry()
1012 env->sctrstatus = in riscv_ctr_add_entry()
1013 set_field(env->sctrstatus, SCTRSTATUS_WRPTR_MASK, head); in riscv_ctr_add_entry()
1017 /* In case of Co-routine SWAP we overwrite latest entry. */ in riscv_ctr_add_entry()
1019 head = (head - 1) & (depth - 1); in riscv_ctr_add_entry()
1022 env->ctr_src[head] = src | CTRSOURCE_VALID; in riscv_ctr_add_entry()
1023 env->ctr_dst[head] = dst & ~CTRTARGET_MISP; in riscv_ctr_add_entry()
1024 env->ctr_data[head] = set_field(0, CTRDATA_TYPE_MASK, type); in riscv_ctr_add_entry()
1026 head = (head + 1) & (depth - 1); in riscv_ctr_add_entry()
1028 env->sctrstatus = set_field(env->sctrstatus, SCTRSTATUS_WRPTR_MASK, head); in riscv_ctr_add_entry()
1035 if (newpriv != env->priv || env->virt_enabled != virt_en) { in riscv_cpu_set_mode()
1044 env->priv = newpriv; in riscv_cpu_set_mode()
1045 env->xl = cpu_recompute_xl(env); in riscv_cpu_set_mode()
1048 * Clear the load reservation - otherwise a reservation placed in one in riscv_cpu_set_mode()
1055 env->load_res = -1; in riscv_cpu_set_mode()
1059 if (env->virt_enabled != virt_en) { in riscv_cpu_set_mode()
1063 env->virt_enabled = virt_en; in riscv_cpu_set_mode()
1081 * get_physical_address_pmp - check PMP permission for this physical address
1099 if (!riscv_cpu_cfg(env)->pmp) { in get_physical_address_pmp()
1121 if (!(env_archcpu(env)->cfg.ext_svukte || in do_svukte_check()
1123 VM_1_10_SV39 != get_field(env->satp, SATP64_MODE))) { in do_svukte_check()
1128 * Check hstatus.HUKTE if the effective mode is switched to VU-mode by in do_svukte_check()
1129 * executing HLV/HLVX/HSV in U-mode. in do_svukte_check()
1132 if (env->priv == PRV_U && !env->virt_enabled && virt) { in do_svukte_check()
1133 if (!get_field(env->hstatus, HSTATUS_HUKTE)) { in do_svukte_check()
1136 } else if (!get_field(env->senvcfg, SENVCFG_UKTE)) { in do_svukte_check()
1141 * Svukte extension is qualified only in U or VU-mode. in do_svukte_check()
1143 * Effective mode can be switched to U or VU-mode by: in do_svukte_check()
1144 * - M-mode + mstatus.MPRV=1 + mstatus.MPP=U-mode. in do_svukte_check()
1145 * - Execute HLV/HLVX/HSV from HS-mode + hstatus.SPVP=0. in do_svukte_check()
1146 * - U-mode. in do_svukte_check()
1147 * - VU-mode. in do_svukte_check()
1148 * - Execute HLV/HLVX/HSV from U-mode + hstatus.HU=1. in do_svukte_check()
1161 uint64_t high_bit = addr & (1UL << (sxlen - 1)); in check_svukte_addr()
1166 * get_physical_address - get the physical address for this virtual address
1195 * NOTE: the env->pc value visible here will not be in get_physical_address()
1222 if (!env->virt_enabled && two_stage) { in get_physical_address()
1226 if (mode == PRV_M || !riscv_cpu_cfg(env)->mmu) { in get_physical_address()
1240 base = (hwaddr)get_field(env->vsatp, SATP32_PPN) << PGSHIFT; in get_physical_address()
1241 vm = get_field(env->vsatp, SATP32_MODE); in get_physical_address()
1243 base = (hwaddr)get_field(env->vsatp, SATP64_PPN) << PGSHIFT; in get_physical_address()
1244 vm = get_field(env->vsatp, SATP64_MODE); in get_physical_address()
1248 base = (hwaddr)get_field(env->satp, SATP32_PPN) << PGSHIFT; in get_physical_address()
1249 vm = get_field(env->satp, SATP32_MODE); in get_physical_address()
1251 base = (hwaddr)get_field(env->satp, SATP64_PPN) << PGSHIFT; in get_physical_address()
1252 vm = get_field(env->satp, SATP64_MODE); in get_physical_address()
1258 base = (hwaddr)get_field(env->hgatp, SATP32_PPN) << PGSHIFT; in get_physical_address()
1259 vm = get_field(env->hgatp, SATP32_MODE); in get_physical_address()
1261 base = (hwaddr)get_field(env->hgatp, SATP64_PPN) << PGSHIFT; in get_physical_address()
1262 vm = get_field(env->hgatp, SATP64_MODE); in get_physical_address()
1292 if (sxlen > (va_bits - 1)) { in get_physical_address()
1293 mask = (1L << (sxlen - (va_bits - 1))) - 1; in get_physical_address()
1297 masked_msbs = (addr >> (va_bits - 1)) & mask; in get_physical_address()
1308 bool pbmte = env->menvcfg & MENVCFG_PBMTE; in get_physical_address()
1309 bool svade = riscv_cpu_cfg(env)->ext_svade; in get_physical_address()
1310 bool svadu = riscv_cpu_cfg(env)->ext_svadu; in get_physical_address()
1311 bool adue = svadu ? env->menvcfg & MENVCFG_ADUE : !svade; in get_physical_address()
1313 if (first_stage && two_stage && env->virt_enabled) { in get_physical_address()
1314 pbmte = pbmte && (env->henvcfg & HENVCFG_PBMTE); in get_physical_address()
1315 adue = adue && (env->henvcfg & HENVCFG_ADUE); in get_physical_address()
1318 int ptshift = (levels - 1) * ptidxbits; in get_physical_address()
1324 for (i = 0; i < levels; i++, ptshift -= ptidxbits) { in get_physical_address()
1328 ((1 << (ptidxbits + widened)) - 1); in get_physical_address()
1331 ((1 << ptidxbits) - 1); in get_physical_address()
1367 pte = address_space_ldl(cs->as, pte_addr, attrs, &res); in get_physical_address()
1369 pte = address_space_ldq(cs->as, pte_addr, attrs, &res); in get_physical_address()
1395 if (!riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) { in get_physical_address()
1417 /* D, A, and U bits are reserved in non-leaf/inner PTEs */ in get_physical_address()
1418 qemu_log_mask(LOG_GUEST_ERROR, "%s: D, A, or U bits set in non-leaf PTE: " in get_physical_address()
1431 if (ppn & ((1ULL << ptshift) - 1)) { in get_physical_address()
1490 if (first_stage || !env->virt_enabled) { in get_physical_address()
1491 mxr = get_field(env->mstatus, MSTATUS_MXR); in get_physical_address()
1495 if (first_stage && two_stage && !env->virt_enabled) { in get_physical_address()
1496 mxr |= get_field(env->vsstatus, MSTATUS_MXR); in get_physical_address()
1500 * Setting MXR at HS-level overrides both VS-stage and G-stage in get_physical_address()
1501 * execute-only permissions in get_physical_address()
1503 if (env->virt_enabled) { in get_physical_address()
1504 mxr |= get_field(env->mstatus_hs, MSTATUS_MXR); in get_physical_address()
1554 * - if accessed or dirty bits need updating, and the PTE is in get_physical_address()
1556 * - if the PTE is in IO space or ROM, then it can't be updated in get_physical_address()
1558 * - if the PTE changed by the time we went to update it, then in get_physical_address()
1559 * it is no longer valid and we must re-walk the page table. in get_physical_address()
1563 mr = address_space_translate(cs->as, pte_addr, &addr1, &l, in get_physical_address()
1566 target_ulong *pte_pa = qemu_map_ram_ptr(mr->ram_block, addr1); in get_physical_address()
1591 if (riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) { in get_physical_address()
1593 if ((i != (levels - 1)) || (napot_bits != 4)) { in get_physical_address()
1598 napot_mask = (1 << napot_bits) - 1; in get_physical_address()
1600 (vpn & (((target_ulong)1 << ptshift) - 1)) in get_physical_address()
1626 cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT; in raise_mmu_exception()
1627 } else if (env->virt_enabled && !first_stage) { in raise_mmu_exception()
1628 cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT; in raise_mmu_exception()
1630 cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT; in raise_mmu_exception()
1635 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT; in raise_mmu_exception()
1637 cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT; in raise_mmu_exception()
1639 cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT; in raise_mmu_exception()
1644 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT; in raise_mmu_exception()
1646 cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT; in raise_mmu_exception()
1648 cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT; in raise_mmu_exception()
1654 env->badaddr = address; in raise_mmu_exception()
1655 env->two_stage_lookup = two_stage; in raise_mmu_exception()
1656 env->two_stage_indirect_lookup = two_stage_indirect; in raise_mmu_exception()
1662 CPURISCVState *env = &cpu->env; in riscv_cpu_get_phys_page_debug()
1665 int mmu_idx = riscv_env_mmu_index(&cpu->env, false); in riscv_cpu_get_phys_page_debug()
1668 true, env->virt_enabled, true, false)) { in riscv_cpu_get_phys_page_debug()
1669 return -1; in riscv_cpu_get_phys_page_debug()
1672 if (env->virt_enabled) { in riscv_cpu_get_phys_page_debug()
1675 return -1; in riscv_cpu_get_phys_page_debug()
1689 CPURISCVState *env = &cpu->env; in riscv_cpu_do_transaction_failed()
1692 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT; in riscv_cpu_do_transaction_failed()
1694 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT; in riscv_cpu_do_transaction_failed()
1696 cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT; in riscv_cpu_do_transaction_failed()
1699 env->badaddr = addr; in riscv_cpu_do_transaction_failed()
1700 env->two_stage_lookup = mmuidx_2stage(mmu_idx); in riscv_cpu_do_transaction_failed()
1701 env->two_stage_indirect_lookup = false; in riscv_cpu_do_transaction_failed()
1710 CPURISCVState *env = &cpu->env; in riscv_cpu_do_unaligned_access()
1713 cs->exception_index = RISCV_EXCP_INST_ADDR_MIS; in riscv_cpu_do_unaligned_access()
1716 cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS; in riscv_cpu_do_unaligned_access()
1719 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT; in riscv_cpu_do_unaligned_access()
1723 cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS; in riscv_cpu_do_unaligned_access()
1726 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT; in riscv_cpu_do_unaligned_access()
1732 env->badaddr = addr; in riscv_cpu_do_unaligned_access()
1733 env->two_stage_lookup = mmuidx_2stage(mmu_idx); in riscv_cpu_do_unaligned_access()
1734 env->two_stage_indirect_lookup = false; in riscv_cpu_do_unaligned_access()
1741 enum riscv_pmu_event_idx pmu_event_type; in pmu_tlb_fill_incr_ctr()
1765 CPURISCVState *env = &cpu->env; in riscv_cpu_tlb_fill()
1778 env->guest_phys_fault_addr = 0; in riscv_cpu_tlb_fill()
1787 &env->guest_phys_fault_addr, access_type, in riscv_cpu_tlb_fill()
1791 * A G-stage exception may be triggered during two state lookup. in riscv_cpu_tlb_fill()
1792 * And the env->guest_phys_fault_addr has already been set in in riscv_cpu_tlb_fill()
1801 "%s 1st-stage address=%" VADDR_PRIx " ret %d physical " in riscv_cpu_tlb_fill()
1814 "%s 2nd-stage address=%" VADDR_PRIx in riscv_cpu_tlb_fill()
1839 env->guest_phys_fault_addr = (im_address | in riscv_cpu_tlb_fill()
1841 (TARGET_PAGE_SIZE - 1))) >> 2; in riscv_cpu_tlb_fill()
1875 tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1), in riscv_cpu_tlb_fill()
1891 * be a no-op and we'll resume the mmu_exception path. in riscv_cpu_tlb_fill()
2105 xinsn = SET_RS1(xinsn, (taddr - (env->gpr[access_rs1] + access_imm)) & in riscv_transformed_insn()
2106 (access_size - 1)); in riscv_transformed_insn()
2131 env->mnstatus = set_field(env->mnstatus, MNSTATUS_NMIE, false); in riscv_do_nmi()
2132 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPV, virt); in riscv_do_nmi()
2133 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPP, env->priv); in riscv_do_nmi()
2134 env->mncause = cause; in riscv_do_nmi()
2135 env->mnepc = env->pc; in riscv_do_nmi()
2136 env->pc = env->rnmi_irqvec; in riscv_do_nmi()
2139 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPELP, env->elp); in riscv_do_nmi()
2155 CPURISCVState *env = &cpu->env; in riscv_cpu_do_interrupt()
2156 bool virt = env->virt_enabled; in riscv_cpu_do_interrupt()
2158 bool always_storeamo = (env->excp_uw2 & RISCV_UW2_ALWAYS_STORE_AMO); in riscv_cpu_do_interrupt()
2164 * cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide in riscv_cpu_do_interrupt()
2167 bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG); in riscv_cpu_do_interrupt() local
2168 target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK; in riscv_cpu_do_interrupt()
2169 uint64_t deleg = async ? env->mideleg : env->medeleg; in riscv_cpu_do_interrupt()
2170 bool s_injected = env->mvip & (1ULL << cause) & env->mvien && in riscv_cpu_do_interrupt()
2171 !(env->mip & (1ULL << cause)); in riscv_cpu_do_interrupt()
2172 bool vs_injected = env->hvip & (1ULL << cause) & env->hvien && in riscv_cpu_do_interrupt()
2173 !(env->mip & (1ULL << cause)); in riscv_cpu_do_interrupt()
2175 uint64_t hdeleg = async ? env->hideleg : env->hedeleg; in riscv_cpu_do_interrupt()
2176 const bool prev_virt = env->virt_enabled; in riscv_cpu_do_interrupt()
2177 const target_ulong prev_priv = env->priv; in riscv_cpu_do_interrupt()
2187 if (cpu->cfg.ext_smrnmi && env->rnmip && async) { in riscv_cpu_do_interrupt()
2188 riscv_do_nmi(env, cause | ((target_ulong)1U << (mxlen - 1)), in riscv_cpu_do_interrupt()
2189 env->virt_enabled); in riscv_cpu_do_interrupt()
2193 if (!async) { in riscv_cpu_do_interrupt()
2199 env->pc += 4; in riscv_cpu_do_interrupt()
2213 write_gva = env->two_stage_lookup; in riscv_cpu_do_interrupt()
2214 tval = env->badaddr; in riscv_cpu_do_interrupt()
2215 if (env->two_stage_indirect_lookup) { in riscv_cpu_do_interrupt()
2217 * special pseudoinstruction for G-stage fault taken while in riscv_cpu_do_interrupt()
2218 * doing VS-stage page table walk. in riscv_cpu_do_interrupt()
2224 * non-zero only for misaligned access. in riscv_cpu_do_interrupt()
2226 tinst = riscv_transformed_insn(env, env->bins, tval); in riscv_cpu_do_interrupt()
2233 write_gva = env->two_stage_lookup; in riscv_cpu_do_interrupt()
2234 tval = env->badaddr; in riscv_cpu_do_interrupt()
2235 if (env->two_stage_indirect_lookup) { in riscv_cpu_do_interrupt()
2237 * special pseudoinstruction for G-stage fault taken while in riscv_cpu_do_interrupt()
2238 * doing VS-stage page table walk. in riscv_cpu_do_interrupt()
2245 tval = env->bins; in riscv_cpu_do_interrupt()
2248 tval = env->badaddr; in riscv_cpu_do_interrupt()
2249 if (cs->watchpoint_hit) { in riscv_cpu_do_interrupt()
2250 tval = cs->watchpoint_hit->hitaddr; in riscv_cpu_do_interrupt()
2251 cs->watchpoint_hit = NULL; in riscv_cpu_do_interrupt()
2255 tval = env->sw_check_code; in riscv_cpu_do_interrupt()
2262 assert(env->priv <= 3); in riscv_cpu_do_interrupt()
2264 if (env->priv == PRV_M) { in riscv_cpu_do_interrupt()
2266 } else if (env->priv == PRV_S && env->virt_enabled) { in riscv_cpu_do_interrupt()
2268 } else if (env->priv == PRV_S && !env->virt_enabled) { in riscv_cpu_do_interrupt()
2270 } else if (env->priv == PRV_U) { in riscv_cpu_do_interrupt()
2276 trace_riscv_trap(env->mhartid, async, cause, env->pc, tval, in riscv_cpu_do_interrupt()
2277 riscv_cpu_get_trap_name(cause, async)); in riscv_cpu_do_interrupt()
2280 "%s: hart:"TARGET_FMT_ld", async:%d, cause:"TARGET_FMT_lx", " in riscv_cpu_do_interrupt()
2282 __func__, env->mhartid, async, cause, env->pc, tval, in riscv_cpu_do_interrupt()
2283 riscv_cpu_get_trap_name(cause, async)); in riscv_cpu_do_interrupt()
2285 mode = env->priv <= PRV_S && cause < 64 && in riscv_cpu_do_interrupt()
2288 vsmode_exc = env->virt_enabled && cause < 64 && in riscv_cpu_do_interrupt()
2292 * Check double trap condition only if already in S-mode and targeting in riscv_cpu_do_interrupt()
2293 * S-mode in riscv_cpu_do_interrupt()
2295 if (cpu->cfg.ext_ssdbltrp && env->priv == PRV_S && mode == PRV_S) { in riscv_cpu_do_interrupt()
2296 bool dte = (env->menvcfg & MENVCFG_DTE) != 0; in riscv_cpu_do_interrupt()
2297 bool sdt = (env->mstatus & MSTATUS_SDT) != 0; in riscv_cpu_do_interrupt()
2301 /* VS -> VS, use henvcfg instead of menvcfg*/ in riscv_cpu_do_interrupt()
2302 dte = (env->henvcfg & HENVCFG_DTE) != 0; in riscv_cpu_do_interrupt()
2303 } else if (env->virt_enabled) { in riscv_cpu_do_interrupt()
2304 /* VS -> HS, use mstatus_hs */ in riscv_cpu_do_interrupt()
2305 sdt = (env->mstatus_hs & MSTATUS_SDT) != 0; in riscv_cpu_do_interrupt()
2315 /* handle the trap in S-mode */ in riscv_cpu_do_interrupt()
2318 env->mstatus = set_field(env->mstatus, MSTATUS_SPELP, env->elp); in riscv_cpu_do_interrupt()
2328 if (async && (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT || in riscv_cpu_do_interrupt()
2330 cause = cause - 1; in riscv_cpu_do_interrupt()
2333 } else if (env->virt_enabled) { in riscv_cpu_do_interrupt()
2336 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP, in riscv_cpu_do_interrupt()
2337 env->priv); in riscv_cpu_do_interrupt()
2338 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, true); in riscv_cpu_do_interrupt()
2340 htval = env->guest_phys_fault_addr; in riscv_cpu_do_interrupt()
2345 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, false); in riscv_cpu_do_interrupt()
2346 htval = env->guest_phys_fault_addr; in riscv_cpu_do_interrupt()
2348 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, write_gva); in riscv_cpu_do_interrupt()
2351 s = env->mstatus; in riscv_cpu_do_interrupt()
2353 s = set_field(s, MSTATUS_SPP, env->priv); in riscv_cpu_do_interrupt()
2358 env->mstatus = s; in riscv_cpu_do_interrupt()
2360 env->scause = cause | ((target_ulong)async << (sxlen - 1)); in riscv_cpu_do_interrupt()
2361 env->sepc = env->pc; in riscv_cpu_do_interrupt()
2362 env->stval = tval; in riscv_cpu_do_interrupt()
2363 env->htval = htval; in riscv_cpu_do_interrupt()
2364 env->htinst = tinst; in riscv_cpu_do_interrupt()
2365 env->pc = (env->stvec >> 2 << 2) + in riscv_cpu_do_interrupt()
2366 ((async && (env->stvec & 3) == 1) ? cause * 4 : 0); in riscv_cpu_do_interrupt()
2369 src = env->sepc; in riscv_cpu_do_interrupt()
2372 * If the hart encounters an exception while executing in M-mode in riscv_cpu_do_interrupt()
2375 nnmi_excep = cpu->cfg.ext_smrnmi && in riscv_cpu_do_interrupt()
2376 !get_field(env->mnstatus, MNSTATUS_NMIE) && in riscv_cpu_do_interrupt()
2377 !async; in riscv_cpu_do_interrupt()
2379 /* handle the trap in M-mode */ in riscv_cpu_do_interrupt()
2383 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPELP, in riscv_cpu_do_interrupt()
2384 env->elp); in riscv_cpu_do_interrupt()
2386 env->mstatus = set_field(env->mstatus, MSTATUS_MPELP, env->elp); in riscv_cpu_do_interrupt()
2391 if (env->virt_enabled) { in riscv_cpu_do_interrupt()
2394 env->mstatus = set_field(env->mstatus, MSTATUS_MPV, in riscv_cpu_do_interrupt()
2395 env->virt_enabled); in riscv_cpu_do_interrupt()
2396 if (env->virt_enabled && tval) { in riscv_cpu_do_interrupt()
2397 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1); in riscv_cpu_do_interrupt()
2400 mtval2 = env->guest_phys_fault_addr; in riscv_cpu_do_interrupt()
2406 * If the hart encounters an exception while executing in M-mode, in riscv_cpu_do_interrupt()
2410 nnmi_excep = cpu->cfg.ext_smrnmi && in riscv_cpu_do_interrupt()
2411 !get_field(env->mnstatus, MNSTATUS_NMIE) && in riscv_cpu_do_interrupt()
2412 !async; in riscv_cpu_do_interrupt()
2414 s = env->mstatus; in riscv_cpu_do_interrupt()
2416 s = set_field(s, MSTATUS_MPP, env->priv); in riscv_cpu_do_interrupt()
2418 if (cpu->cfg.ext_smdbltrp) { in riscv_cpu_do_interrupt()
2419 if (env->mstatus & MSTATUS_MDT) { in riscv_cpu_do_interrupt()
2420 assert(env->priv == PRV_M); in riscv_cpu_do_interrupt()
2421 if (!cpu->cfg.ext_smrnmi || nnmi_excep) { in riscv_cpu_do_interrupt()
2422 cpu_abort(CPU(cpu), "M-mode double trap\n"); in riscv_cpu_do_interrupt()
2431 env->mstatus = s; in riscv_cpu_do_interrupt()
2432 env->mcause = cause | ((target_ulong)async << (mxlen - 1)); in riscv_cpu_do_interrupt()
2434 env->mtval2 = env->mcause; in riscv_cpu_do_interrupt()
2435 env->mcause = RISCV_EXCP_DOUBLE_TRAP; in riscv_cpu_do_interrupt()
2437 env->mtval2 = mtval2; in riscv_cpu_do_interrupt()
2439 env->mepc = env->pc; in riscv_cpu_do_interrupt()
2440 env->mtval = tval; in riscv_cpu_do_interrupt()
2441 env->mtinst = tinst; in riscv_cpu_do_interrupt()
2448 env->pc = env->rnmi_excpvec; in riscv_cpu_do_interrupt()
2450 env->pc = (env->mtvec >> 2 << 2) + in riscv_cpu_do_interrupt()
2451 ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0); in riscv_cpu_do_interrupt()
2454 src = env->mepc; in riscv_cpu_do_interrupt()
2457 if (riscv_cpu_cfg(env)->ext_smctr || riscv_cpu_cfg(env)->ext_ssctr) { in riscv_cpu_do_interrupt()
2458 if (async && cause == IRQ_PMU_OVF) { in riscv_cpu_do_interrupt()
2460 } else if (!async && cause == RISCV_EXCP_BREAKPOINT) { in riscv_cpu_do_interrupt()
2464 riscv_ctr_add_entry(env, src, env->pc, in riscv_cpu_do_interrupt()
2465 async ? CTRDATA_TYPE_INTERRUPT : CTRDATA_TYPE_EXCEPTION, in riscv_cpu_do_interrupt()
2474 env->elp = false; in riscv_cpu_do_interrupt()
2480 * RISC-V ISA Specification. in riscv_cpu_do_interrupt()
2483 env->two_stage_lookup = false; in riscv_cpu_do_interrupt()
2484 env->two_stage_indirect_lookup = false; in riscv_cpu_do_interrupt()