32 G_NORETURN void riscv_raise_exception(CPURISCVState *env,  in riscv_raise_exception()  argument
36     CPUState *cs = env_cpu(env);  in riscv_raise_exception()
40                           env->pc);  in riscv_raise_exception()
46 void helper_raise_exception(CPURISCVState *env, uint32_t exception)  in helper_raise_exception()  argument
48     riscv_raise_exception(env, exception, 0);  in helper_raise_exception()
51 target_ulong helper_csrr(CPURISCVState *env, int csr)  in helper_csrr()  argument
59         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());  in helper_csrr()
63     RISCVException ret = riscv_csrr(env, csr, &val);  in helper_csrr()
66         riscv_raise_exception(env, ret, GETPC());  in helper_csrr()
71 void helper_csrw(CPURISCVState *env, int csr, target_ulong src)  in helper_csrw()  argument
73     target_ulong mask = env->xl == MXL_RV32 ? UINT32_MAX : (target_ulong)-1;  in helper_csrw()
74     RISCVException ret = riscv_csrrw(env, csr, NULL, src, mask, GETPC());  in helper_csrw()
77         riscv_raise_exception(env, ret, GETPC());  in helper_csrw()
81 target_ulong helper_csrrw(CPURISCVState *env, int csr,  in helper_csrrw()  argument
85     RISCVException ret = riscv_csrrw(env, csr, &val, src, write_mask, GETPC());  in helper_csrrw()
88         riscv_raise_exception(env, ret, GETPC());  in helper_csrrw()
93 target_ulong helper_csrr_i128(CPURISCVState *env, int csr)  in helper_csrr_i128()  argument
96     RISCVException ret = riscv_csrr_i128(env, csr, &rv);  in helper_csrr_i128()
99         riscv_raise_exception(env, ret, GETPC());  in helper_csrr_i128()
102     env->retxh = int128_gethi(rv);  in helper_csrr_i128()
106 void helper_csrw_i128(CPURISCVState *env, int csr,  in helper_csrw_i128()  argument
109     RISCVException ret = riscv_csrrw_i128(env, csr, NULL,  in helper_csrw_i128()
114         riscv_raise_exception(env, ret, GETPC());  in helper_csrw_i128()
118 target_ulong helper_csrrw_i128(CPURISCVState *env, int csr,  in helper_csrrw_i128()  argument
123     RISCVException ret = riscv_csrrw_i128(env, csr, &rv,  in helper_csrrw_i128()
129         riscv_raise_exception(env, ret, GETPC());  in helper_csrrw_i128()
132     env->retxh = int128_gethi(rv);  in helper_csrrw_i128()
144 static void check_zicbo_envcfg(CPURISCVState *env, target_ulong envbits,  in check_zicbo_envcfg()  argument
148     if ((env->priv < PRV_M) && !get_field(env->menvcfg, envbits)) {  in check_zicbo_envcfg()
149         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, ra);  in check_zicbo_envcfg()
152     if (env->virt_enabled &&  in check_zicbo_envcfg()
153         (((env->priv <= PRV_S) && !get_field(env->henvcfg, envbits)) ||  in check_zicbo_envcfg()
154          ((env->priv < PRV_S) && !get_field(env->senvcfg, envbits)))) {  in check_zicbo_envcfg()
155         riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, ra);  in check_zicbo_envcfg()
158     if ((env->priv < PRV_S) && !get_field(env->senvcfg, envbits)) {  in check_zicbo_envcfg()
159         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, ra);  in check_zicbo_envcfg()
164 void helper_cbo_zero(CPURISCVState *env, target_ulong address)  in helper_cbo_zero()  argument
166     RISCVCPU *cpu = env_archcpu(env);  in helper_cbo_zero()
168     int mmu_idx = riscv_env_mmu_index(env, false);  in helper_cbo_zero()
172     check_zicbo_envcfg(env, MENVCFG_CBZE, ra);  in helper_cbo_zero()
181     mem = probe_write(env, address, cbozlen, mmu_idx, ra);  in helper_cbo_zero()
198             cpu_stb_mmuidx_ra(env, address + i, 0, mmu_idx, ra);  in helper_cbo_zero()
211 static void check_zicbom_access(CPURISCVState *env,  in check_zicbom_access()  argument
215     RISCVCPU *cpu = env_archcpu(env);  in check_zicbom_access()
216     int mmu_idx = riscv_env_mmu_index(env, false);  in check_zicbom_access()
236     ret = probe_access_flags(env, address, cbomlen, MMU_DATA_LOAD,  in check_zicbom_access()
249     probe_write(env, address, cbomlen, mmu_idx, ra);  in check_zicbom_access()
252 void helper_cbo_clean_flush(CPURISCVState *env, target_ulong address)  in helper_cbo_clean_flush()  argument
255     check_zicbo_envcfg(env, MENVCFG_CBCFE, ra);  in helper_cbo_clean_flush()
256     check_zicbom_access(env, address, ra);  in helper_cbo_clean_flush()
261 void helper_cbo_inval(CPURISCVState *env, target_ulong address)  in helper_cbo_inval()  argument
264     check_zicbo_envcfg(env, MENVCFG_CBIE, ra);  in helper_cbo_inval()
265     check_zicbom_access(env, address, ra);  in helper_cbo_inval()
272 target_ulong helper_sret(CPURISCVState *env)  in helper_sret()  argument
275     target_ulong prev_priv, prev_virt = env->virt_enabled;  in helper_sret()
276     const target_ulong src_priv = env->priv;  in helper_sret()
277     const bool src_virt = env->virt_enabled;  in helper_sret()
279     if (!(env->priv >= PRV_S)) {  in helper_sret()
280         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());  in helper_sret()
283     target_ulong retpc = env->sepc & get_xepc_mask(env);  in helper_sret()
284     if (!riscv_cpu_allow_16bit_insn(&env_archcpu(env)->cfg,  in helper_sret()
285                                     env->priv_ver,  in helper_sret()
286                                     env->misa_ext) && (retpc & 0x3)) {  in helper_sret()
287         riscv_raise_exception(env, RISCV_EXCP_INST_ADDR_MIS, GETPC());  in helper_sret()
290     if (get_field(env->mstatus, MSTATUS_TSR) && !(env->priv >= PRV_M)) {  in helper_sret()
291         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());  in helper_sret()
294     if (env->virt_enabled && get_field(env->hstatus, HSTATUS_VTSR)) {  in helper_sret()
295         riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());  in helper_sret()
298     mstatus = env->mstatus;  in helper_sret()
305     if (riscv_cpu_cfg(env)->ext_ssdbltrp) {  in helper_sret()
306         if (riscv_has_ext(env, RVH)) {  in helper_sret()
307             target_ulong prev_vu = get_field(env->hstatus, HSTATUS_SPV) &&  in helper_sret()
310             if (!env->virt_enabled && prev_vu) {  in helper_sret()
311                 env->vsstatus = set_field(env->vsstatus, MSTATUS_SDT, 0);  in helper_sret()
316     if (riscv_cpu_cfg(env)->ext_smdbltrp && env->priv >= PRV_M) {  in helper_sret()
319     if (env->priv_ver >= PRIV_VERSION_1_12_0) {  in helper_sret()
322     env->mstatus = mstatus;  in helper_sret()
324     if (riscv_has_ext(env, RVH) && !env->virt_enabled) {  in helper_sret()
326         target_ulong hstatus = env->hstatus;  in helper_sret()
331         env->hstatus = hstatus;  in helper_sret()
334             riscv_cpu_swap_hypervisor_regs(env);  in helper_sret()
338     riscv_cpu_set_mode(env, prev_priv, prev_virt);  in helper_sret()
344     if (cpu_get_fcfien(env)) {  in helper_sret()
345         env->elp = get_field(env->mstatus, MSTATUS_SPELP);  in helper_sret()
347     env->mstatus = set_field(env->mstatus, MSTATUS_SPELP, 0);  in helper_sret()
349     if (riscv_cpu_cfg(env)->ext_smctr || riscv_cpu_cfg(env)->ext_ssctr) {  in helper_sret()
350         riscv_ctr_add_entry(env, env->pc, retpc, CTRDATA_TYPE_EXCEP_INT_RET,  in helper_sret()
357 static void check_ret_from_m_mode(CPURISCVState *env, target_ulong retpc,  in check_ret_from_m_mode()  argument
360     if (!(env->priv >= PRV_M)) {  in check_ret_from_m_mode()
361         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());  in check_ret_from_m_mode()
364     if (!riscv_cpu_allow_16bit_insn(&env_archcpu(env)->cfg,  in check_ret_from_m_mode()
365                                     env->priv_ver,  in check_ret_from_m_mode()
366                                     env->misa_ext) && (retpc & 0x3)) {  in check_ret_from_m_mode()
367         riscv_raise_exception(env, RISCV_EXCP_INST_ADDR_MIS, GETPC());  in check_ret_from_m_mode()
370     if (riscv_cpu_cfg(env)->pmp &&  in check_ret_from_m_mode()
371         !pmp_get_num_rules(env) && (prev_priv != PRV_M)) {  in check_ret_from_m_mode()
372         riscv_raise_exception(env, RISCV_EXCP_INST_ACCESS_FAULT, GETPC());  in check_ret_from_m_mode()
375 static target_ulong ssdbltrp_mxret(CPURISCVState *env, target_ulong mstatus,  in ssdbltrp_mxret()  argument
385             env->vsstatus = set_field(env->vsstatus, MSTATUS_SDT, 0);  in ssdbltrp_mxret()
392 target_ulong helper_mret(CPURISCVState *env)  in helper_mret()  argument
394     target_ulong retpc = env->mepc & get_xepc_mask(env);  in helper_mret()
395     uint64_t mstatus = env->mstatus;  in helper_mret()
398     check_ret_from_m_mode(env, retpc, prev_priv);  in helper_mret()
400     target_ulong prev_virt = get_field(env->mstatus, MSTATUS_MPV) &&  in helper_mret()
406                         riscv_has_ext(env, RVU) ? PRV_U : PRV_M);  in helper_mret()
408     if (riscv_cpu_cfg(env)->ext_ssdbltrp) {  in helper_mret()
409         mstatus = ssdbltrp_mxret(env, mstatus, prev_priv, prev_virt);  in helper_mret()
411     if (riscv_cpu_cfg(env)->ext_smdbltrp) {  in helper_mret()
414     if ((env->priv_ver >= PRIV_VERSION_1_12_0) && (prev_priv != PRV_M)) {  in helper_mret()
417     env->mstatus = mstatus;  in helper_mret()
419     if (riscv_has_ext(env, RVH) && prev_virt) {  in helper_mret()
420         riscv_cpu_swap_hypervisor_regs(env);  in helper_mret()
423     riscv_cpu_set_mode(env, prev_priv, prev_virt);  in helper_mret()
428     if (cpu_get_fcfien(env)) {  in helper_mret()
429         env->elp = get_field(env->mstatus, MSTATUS_MPELP);  in helper_mret()
431     env->mstatus = set_field(env->mstatus, MSTATUS_MPELP, 0);  in helper_mret()
433     if (riscv_cpu_cfg(env)->ext_smctr || riscv_cpu_cfg(env)->ext_ssctr) {  in helper_mret()
434         riscv_ctr_add_entry(env, env->pc, retpc, CTRDATA_TYPE_EXCEP_INT_RET,  in helper_mret()
441 target_ulong helper_mnret(CPURISCVState *env)  in helper_mnret()  argument
443     target_ulong retpc = env->mnepc;  in helper_mnret()
444     target_ulong prev_priv = get_field(env->mnstatus, MNSTATUS_MNPP);  in helper_mnret()
447     check_ret_from_m_mode(env, retpc, prev_priv);  in helper_mnret()
449     prev_virt = get_field(env->mnstatus, MNSTATUS_MNPV) &&  in helper_mnret()
451     env->mnstatus = set_field(env->mnstatus, MNSTATUS_NMIE, true);  in helper_mnret()
458         env->mstatus = set_field(env->mstatus, MSTATUS_MPRV, false);  in helper_mnret()
460     if (riscv_cpu_cfg(env)->ext_ssdbltrp) {  in helper_mnret()
461         env->mstatus = ssdbltrp_mxret(env, env->mstatus, prev_priv, prev_virt);  in helper_mnret()
464     if (riscv_cpu_cfg(env)->ext_smdbltrp) {  in helper_mnret()
466             env->mstatus = set_field(env->mstatus, MSTATUS_MDT, 0);  in helper_mnret()
470     if (riscv_has_ext(env, RVH) && prev_virt) {  in helper_mnret()
471         riscv_cpu_swap_hypervisor_regs(env);  in helper_mnret()
474     riscv_cpu_set_mode(env, prev_priv, prev_virt);  in helper_mnret()
480     if (cpu_get_fcfien(env)) {  in helper_mnret()
481         env->elp = get_field(env->mnstatus, MNSTATUS_MNPELP);  in helper_mnret()
483     env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPELP, 0);  in helper_mnret()
488 void helper_ctr_add_entry(CPURISCVState *env, target_ulong src,  in helper_ctr_add_entry()  argument
491     riscv_ctr_add_entry(env, src, dest, (enum CTRType)type,  in helper_ctr_add_entry()
492                         env->priv, env->virt_enabled);  in helper_ctr_add_entry()
495 void helper_ctr_clear(CPURISCVState *env)  in helper_ctr_clear()  argument
504     RISCVException ret = smstateen_acc_ok(env, 0, SMSTATEEN0_CTR);  in helper_ctr_clear()
506         riscv_raise_exception(env, ret, GETPC());  in helper_ctr_clear()
509     if (env->priv == PRV_U) {  in helper_ctr_clear()
516         uint32_t excep = env->virt_enabled ? RISCV_EXCP_VIRT_INSTRUCTION_FAULT :  in helper_ctr_clear()
518         riscv_raise_exception(env, excep, GETPC());  in helper_ctr_clear()
521     riscv_ctr_clear(env);  in helper_ctr_clear()
524 void helper_wfi(CPURISCVState *env)  in helper_wfi()  argument
526     CPUState *cs = env_cpu(env);  in helper_wfi()
527     bool rvs = riscv_has_ext(env, RVS);  in helper_wfi()
528     bool prv_u = env->priv == PRV_U;  in helper_wfi()
529     bool prv_s = env->priv == PRV_S;  in helper_wfi()
531     if (((prv_s || (!rvs && prv_u)) && get_field(env->mstatus, MSTATUS_TW)) ||  in helper_wfi()
532         (rvs && prv_u && !env->virt_enabled)) {  in helper_wfi()
533         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());  in helper_wfi()
534     } else if (env->virt_enabled &&  in helper_wfi()
535                (prv_u || (prv_s && get_field(env->hstatus, HSTATUS_VTW)))) {  in helper_wfi()
536         riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());  in helper_wfi()
544 void helper_wrs_nto(CPURISCVState *env)  in helper_wrs_nto()  argument
546     if (env->virt_enabled && (env->priv == PRV_S || env->priv == PRV_U) &&  in helper_wrs_nto()
547         get_field(env->hstatus, HSTATUS_VTW) &&  in helper_wrs_nto()
548         !get_field(env->mstatus, MSTATUS_TW)) {  in helper_wrs_nto()
549         riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());  in helper_wrs_nto()
550     } else if (env->priv != PRV_M && get_field(env->mstatus, MSTATUS_TW)) {  in helper_wrs_nto()
551         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());  in helper_wrs_nto()
555 void helper_tlb_flush(CPURISCVState *env)  in helper_tlb_flush()  argument
557     CPUState *cs = env_cpu(env);  in helper_tlb_flush()
558     if (!env->virt_enabled &&  in helper_tlb_flush()
559         (env->priv == PRV_U ||  in helper_tlb_flush()
560          (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_TVM)))) {  in helper_tlb_flush()
561         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());  in helper_tlb_flush()
562     } else if (env->virt_enabled &&  in helper_tlb_flush()
563                (env->priv == PRV_U || get_field(env->hstatus, HSTATUS_VTVM))) {  in helper_tlb_flush()
564         riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());  in helper_tlb_flush()
570 void helper_tlb_flush_all(CPURISCVState *env)  in helper_tlb_flush_all()  argument
572     CPUState *cs = env_cpu(env);  in helper_tlb_flush_all()
576 void helper_hyp_tlb_flush(CPURISCVState *env)  in helper_hyp_tlb_flush()  argument
578     CPUState *cs = env_cpu(env);  in helper_hyp_tlb_flush()
580     if (env->virt_enabled) {  in helper_hyp_tlb_flush()
581         riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());  in helper_hyp_tlb_flush()
584     if (env->priv == PRV_M ||  in helper_hyp_tlb_flush()
585         (env->priv == PRV_S && !env->virt_enabled)) {  in helper_hyp_tlb_flush()
590     riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());  in helper_hyp_tlb_flush()
593 void helper_hyp_gvma_tlb_flush(CPURISCVState *env)  in helper_hyp_gvma_tlb_flush()  argument
595     if (env->priv == PRV_S && !env->virt_enabled &&  in helper_hyp_gvma_tlb_flush()
596         get_field(env->mstatus, MSTATUS_TVM)) {  in helper_hyp_gvma_tlb_flush()
597         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());  in helper_hyp_gvma_tlb_flush()
600     helper_hyp_tlb_flush(env);  in helper_hyp_gvma_tlb_flush()
603 static int check_access_hlsv(CPURISCVState *env, bool x, uintptr_t ra)  in check_access_hlsv()  argument
605     if (env->priv == PRV_M) {  in check_access_hlsv()
607     } else if (env->virt_enabled) {  in check_access_hlsv()
608         riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, ra);  in check_access_hlsv()
609     } else if (env->priv == PRV_U && !get_field(env->hstatus, HSTATUS_HU)) {  in check_access_hlsv()
610         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, ra);  in check_access_hlsv()
613     int mode = get_field(env->hstatus, HSTATUS_SPVP);  in check_access_hlsv()
614     if (!x && mode == PRV_S && get_field(env->vsstatus, MSTATUS_SUM)) {  in check_access_hlsv()
620 target_ulong helper_hyp_hlv_bu(CPURISCVState *env, target_ulong addr)  in helper_hyp_hlv_bu()  argument
623     int mmu_idx = check_access_hlsv(env, false, ra);  in helper_hyp_hlv_bu()
626     return cpu_ldb_mmu(env, adjust_addr_virt(env, addr), oi, ra);  in helper_hyp_hlv_bu()
629 target_ulong helper_hyp_hlv_hu(CPURISCVState *env, target_ulong addr)  in helper_hyp_hlv_hu()  argument
632     int mmu_idx = check_access_hlsv(env, false, ra);  in helper_hyp_hlv_hu()
635     return cpu_ldw_mmu(env, adjust_addr_virt(env, addr), oi, ra);  in helper_hyp_hlv_hu()
638 target_ulong helper_hyp_hlv_wu(CPURISCVState *env, target_ulong addr)  in helper_hyp_hlv_wu()  argument
641     int mmu_idx = check_access_hlsv(env, false, ra);  in helper_hyp_hlv_wu()
644     return cpu_ldl_mmu(env, adjust_addr_virt(env, addr), oi, ra);  in helper_hyp_hlv_wu()
647 target_ulong helper_hyp_hlv_d(CPURISCVState *env, target_ulong addr)  in helper_hyp_hlv_d()  argument
650     int mmu_idx = check_access_hlsv(env, false, ra);  in helper_hyp_hlv_d()
653     return cpu_ldq_mmu(env, adjust_addr_virt(env, addr), oi, ra);  in helper_hyp_hlv_d()
656 void helper_hyp_hsv_b(CPURISCVState *env, target_ulong addr, target_ulong val)  in helper_hyp_hsv_b()  argument
659     int mmu_idx = check_access_hlsv(env, false, ra);  in helper_hyp_hsv_b()
662     cpu_stb_mmu(env, adjust_addr_virt(env, addr), val, oi, ra);  in helper_hyp_hsv_b()
665 void helper_hyp_hsv_h(CPURISCVState *env, target_ulong addr, target_ulong val)  in helper_hyp_hsv_h()  argument
668     int mmu_idx = check_access_hlsv(env, false, ra);  in helper_hyp_hsv_h()
671     cpu_stw_mmu(env, adjust_addr_virt(env, addr), val, oi, ra);  in helper_hyp_hsv_h()
674 void helper_hyp_hsv_w(CPURISCVState *env, target_ulong addr, target_ulong val)  in helper_hyp_hsv_w()  argument
677     int mmu_idx = check_access_hlsv(env, false, ra);  in helper_hyp_hsv_w()
680     cpu_stl_mmu(env, adjust_addr_virt(env, addr), val, oi, ra);  in helper_hyp_hsv_w()
683 void helper_hyp_hsv_d(CPURISCVState *env, target_ulong addr, target_ulong val)  in helper_hyp_hsv_d()  argument
686     int mmu_idx = check_access_hlsv(env, false, ra);  in helper_hyp_hsv_d()
689     cpu_stq_mmu(env, adjust_addr_virt(env, addr), val, oi, ra);  in helper_hyp_hsv_d()
699 target_ulong helper_hyp_hlvx_hu(CPURISCVState *env, target_ulong addr)  in helper_hyp_hlvx_hu()  argument
702     int mmu_idx = check_access_hlsv(env, true, ra);  in helper_hyp_hlvx_hu()
705     return cpu_ldw_code_mmu(env, addr, oi, GETPC());  in helper_hyp_hlvx_hu()
708 target_ulong helper_hyp_hlvx_wu(CPURISCVState *env, target_ulong addr)  in helper_hyp_hlvx_wu()  argument
711     int mmu_idx = check_access_hlsv(env, true, ra);  in helper_hyp_hlvx_wu()
714     return cpu_ldl_code_mmu(env, addr, oi, ra);  in helper_hyp_hlvx_wu()