1 /*
2 * RISC-V CPU helpers for qemu.
3 *
4 * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
5 * Copyright (c) 2017-2018 SiFive, Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2 or later, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "qemu/main-loop.h"
23 #include "cpu.h"
24 #include "internals.h"
25 #include "pmu.h"
26 #include "exec/cputlb.h"
27 #include "exec/page-protection.h"
28 #include "exec/target_page.h"
29 #include "system/memory.h"
30 #include "instmap.h"
31 #include "tcg/tcg-op.h"
32 #include "accel/tcg/cpu-ops.h"
33 #include "trace.h"
34 #include "semihosting/common-semi.h"
35 #include "exec/icount.h"
36 #include "cpu_bits.h"
37 #include "debug.h"
38 #include "pmp.h"
39
riscv_env_mmu_index(CPURISCVState * env,bool ifetch)40 int riscv_env_mmu_index(CPURISCVState *env, bool ifetch)
41 {
42 #ifdef CONFIG_USER_ONLY
43 return 0;
44 #else
45 bool virt = env->virt_enabled;
46 int mode = env->priv;
47
48 /* All priv -> mmu_idx mapping are here */
49 if (!ifetch) {
50 uint64_t status = env->mstatus;
51
52 if (mode == PRV_M && get_field(status, MSTATUS_MPRV)) {
53 mode = get_field(env->mstatus, MSTATUS_MPP);
54 virt = get_field(env->mstatus, MSTATUS_MPV) &&
55 (mode != PRV_M);
56 if (virt) {
57 status = env->vsstatus;
58 }
59 }
60 if (mode == PRV_S && get_field(status, MSTATUS_SUM)) {
61 mode = MMUIdx_S_SUM;
62 }
63 }
64
65 return mode | (virt ? MMU_2STAGE_BIT : 0);
66 #endif
67 }
68
cpu_get_fcfien(CPURISCVState * env)69 bool cpu_get_fcfien(CPURISCVState *env)
70 {
71 /* no cfi extension, return false */
72 if (!env_archcpu(env)->cfg.ext_zicfilp) {
73 return false;
74 }
75
76 switch (env->priv) {
77 case PRV_U:
78 if (riscv_has_ext(env, RVS)) {
79 return env->senvcfg & SENVCFG_LPE;
80 }
81 return env->menvcfg & MENVCFG_LPE;
82 #ifndef CONFIG_USER_ONLY
83 case PRV_S:
84 if (env->virt_enabled) {
85 return env->henvcfg & HENVCFG_LPE;
86 }
87 return env->menvcfg & MENVCFG_LPE;
88 case PRV_M:
89 return env->mseccfg & MSECCFG_MLPE;
90 #endif
91 default:
92 g_assert_not_reached();
93 }
94 }
95
cpu_get_bcfien(CPURISCVState * env)96 bool cpu_get_bcfien(CPURISCVState *env)
97 {
98 /* no cfi extension, return false */
99 if (!env_archcpu(env)->cfg.ext_zicfiss) {
100 return false;
101 }
102
103 switch (env->priv) {
104 case PRV_U:
105 /*
106 * If S is not implemented then shadow stack for U can't be turned on
107 * It is checked in `riscv_cpu_validate_set_extensions`, so no need to
108 * check here or assert here
109 */
110 return env->senvcfg & SENVCFG_SSE;
111 #ifndef CONFIG_USER_ONLY
112 case PRV_S:
113 if (env->virt_enabled) {
114 return env->henvcfg & HENVCFG_SSE;
115 }
116 return env->menvcfg & MENVCFG_SSE;
117 case PRV_M: /* M-mode shadow stack is always off */
118 return false;
119 #endif
120 default:
121 g_assert_not_reached();
122 }
123 }
124
riscv_env_smode_dbltrp_enabled(CPURISCVState * env,bool virt)125 bool riscv_env_smode_dbltrp_enabled(CPURISCVState *env, bool virt)
126 {
127 #ifdef CONFIG_USER_ONLY
128 return false;
129 #else
130 if (virt) {
131 return (env->henvcfg & HENVCFG_DTE) != 0;
132 } else {
133 return (env->menvcfg & MENVCFG_DTE) != 0;
134 }
135 #endif
136 }
137
riscv_pm_get_pmm(CPURISCVState * env)138 RISCVPmPmm riscv_pm_get_pmm(CPURISCVState *env)
139 {
140 #ifndef CONFIG_USER_ONLY
141 int priv_mode = cpu_address_mode(env);
142
143 if (get_field(env->mstatus, MSTATUS_MPRV) &&
144 get_field(env->mstatus, MSTATUS_MXR)) {
145 return PMM_FIELD_DISABLED;
146 }
147
148 /* Get current PMM field */
149 switch (priv_mode) {
150 case PRV_M:
151 if (riscv_cpu_cfg(env)->ext_smmpm) {
152 return get_field(env->mseccfg, MSECCFG_PMM);
153 }
154 break;
155 case PRV_S:
156 if (riscv_cpu_cfg(env)->ext_smnpm) {
157 if (get_field(env->mstatus, MSTATUS_MPV)) {
158 return get_field(env->henvcfg, HENVCFG_PMM);
159 } else {
160 return get_field(env->menvcfg, MENVCFG_PMM);
161 }
162 }
163 break;
164 case PRV_U:
165 if (riscv_has_ext(env, RVS)) {
166 if (riscv_cpu_cfg(env)->ext_ssnpm) {
167 return get_field(env->senvcfg, SENVCFG_PMM);
168 }
169 } else {
170 if (riscv_cpu_cfg(env)->ext_smnpm) {
171 return get_field(env->menvcfg, MENVCFG_PMM);
172 }
173 }
174 break;
175 default:
176 g_assert_not_reached();
177 }
178 return PMM_FIELD_DISABLED;
179 #else
180 return PMM_FIELD_DISABLED;
181 #endif
182 }
183
riscv_pm_get_virt_pmm(CPURISCVState * env)184 RISCVPmPmm riscv_pm_get_virt_pmm(CPURISCVState *env)
185 {
186 #ifndef CONFIG_USER_ONLY
187 int priv_mode = cpu_address_mode(env);
188
189 if (priv_mode == PRV_U) {
190 return get_field(env->hstatus, HSTATUS_HUPMM);
191 } else {
192 if (get_field(env->hstatus, HSTATUS_SPVP)) {
193 return get_field(env->henvcfg, HENVCFG_PMM);
194 } else {
195 return get_field(env->senvcfg, SENVCFG_PMM);
196 }
197 }
198 #else
199 return PMM_FIELD_DISABLED;
200 #endif
201 }
202
riscv_cpu_virt_mem_enabled(CPURISCVState * env)203 bool riscv_cpu_virt_mem_enabled(CPURISCVState *env)
204 {
205 #ifndef CONFIG_USER_ONLY
206 int satp_mode = 0;
207 int priv_mode = cpu_address_mode(env);
208
209 if (riscv_cpu_mxl(env) == MXL_RV32) {
210 satp_mode = get_field(env->satp, SATP32_MODE);
211 } else {
212 satp_mode = get_field(env->satp, SATP64_MODE);
213 }
214
215 return ((satp_mode != VM_1_10_MBARE) && (priv_mode != PRV_M));
216 #else
217 return false;
218 #endif
219 }
220
riscv_pm_get_pmlen(RISCVPmPmm pmm)221 uint32_t riscv_pm_get_pmlen(RISCVPmPmm pmm)
222 {
223 switch (pmm) {
224 case PMM_FIELD_DISABLED:
225 return 0;
226 case PMM_FIELD_PMLEN7:
227 return 7;
228 case PMM_FIELD_PMLEN16:
229 return 16;
230 default:
231 g_assert_not_reached();
232 }
233 }
234
235 #ifndef CONFIG_USER_ONLY
236
237 /*
238 * The HS-mode is allowed to configure priority only for the
239 * following VS-mode local interrupts:
240 *
241 * 0 (Reserved interrupt, reads as zero)
242 * 1 Supervisor software interrupt
243 * 4 (Reserved interrupt, reads as zero)
244 * 5 Supervisor timer interrupt
245 * 8 (Reserved interrupt, reads as zero)
246 * 13 (Reserved interrupt)
247 * 14 "
248 * 15 "
249 * 16 "
250 * 17 "
251 * 18 "
252 * 19 "
253 * 20 "
254 * 21 "
255 * 22 "
256 * 23 "
257 */
258
259 static const int hviprio_index2irq[] = {
260 0, 1, 4, 5, 8, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 };
261 static const int hviprio_index2rdzero[] = {
262 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
263
riscv_cpu_hviprio_index2irq(int index,int * out_irq,int * out_rdzero)264 int riscv_cpu_hviprio_index2irq(int index, int *out_irq, int *out_rdzero)
265 {
266 if (index < 0 || ARRAY_SIZE(hviprio_index2irq) <= index) {
267 return -EINVAL;
268 }
269
270 if (out_irq) {
271 *out_irq = hviprio_index2irq[index];
272 }
273
274 if (out_rdzero) {
275 *out_rdzero = hviprio_index2rdzero[index];
276 }
277
278 return 0;
279 }
280
281 /*
282 * Default priorities of local interrupts are defined in the
283 * RISC-V Advanced Interrupt Architecture specification.
284 *
285 * ----------------------------------------------------------------
286 * Default |
287 * Priority | Major Interrupt Numbers
288 * ----------------------------------------------------------------
289 * Highest | 47, 23, 46, 45, 22, 44,
290 * | 43, 21, 42, 41, 20, 40
291 * |
292 * | 11 (0b), 3 (03), 7 (07)
293 * | 9 (09), 1 (01), 5 (05)
294 * | 12 (0c)
295 * | 10 (0a), 2 (02), 6 (06)
296 * |
297 * | 39, 19, 38, 37, 18, 36,
298 * Lowest | 35, 17, 34, 33, 16, 32
299 * ----------------------------------------------------------------
300 */
301 static const uint8_t default_iprio[64] = {
302 /* Custom interrupts 48 to 63 */
303 [63] = IPRIO_MMAXIPRIO,
304 [62] = IPRIO_MMAXIPRIO,
305 [61] = IPRIO_MMAXIPRIO,
306 [60] = IPRIO_MMAXIPRIO,
307 [59] = IPRIO_MMAXIPRIO,
308 [58] = IPRIO_MMAXIPRIO,
309 [57] = IPRIO_MMAXIPRIO,
310 [56] = IPRIO_MMAXIPRIO,
311 [55] = IPRIO_MMAXIPRIO,
312 [54] = IPRIO_MMAXIPRIO,
313 [53] = IPRIO_MMAXIPRIO,
314 [52] = IPRIO_MMAXIPRIO,
315 [51] = IPRIO_MMAXIPRIO,
316 [50] = IPRIO_MMAXIPRIO,
317 [49] = IPRIO_MMAXIPRIO,
318 [48] = IPRIO_MMAXIPRIO,
319
320 /* Custom interrupts 24 to 31 */
321 [31] = IPRIO_MMAXIPRIO,
322 [30] = IPRIO_MMAXIPRIO,
323 [29] = IPRIO_MMAXIPRIO,
324 [28] = IPRIO_MMAXIPRIO,
325 [27] = IPRIO_MMAXIPRIO,
326 [26] = IPRIO_MMAXIPRIO,
327 [25] = IPRIO_MMAXIPRIO,
328 [24] = IPRIO_MMAXIPRIO,
329
330 [47] = IPRIO_DEFAULT_UPPER,
331 [23] = IPRIO_DEFAULT_UPPER + 1,
332 [46] = IPRIO_DEFAULT_UPPER + 2,
333 [45] = IPRIO_DEFAULT_UPPER + 3,
334 [22] = IPRIO_DEFAULT_UPPER + 4,
335 [44] = IPRIO_DEFAULT_UPPER + 5,
336
337 [43] = IPRIO_DEFAULT_UPPER + 6,
338 [21] = IPRIO_DEFAULT_UPPER + 7,
339 [42] = IPRIO_DEFAULT_UPPER + 8,
340 [41] = IPRIO_DEFAULT_UPPER + 9,
341 [20] = IPRIO_DEFAULT_UPPER + 10,
342 [40] = IPRIO_DEFAULT_UPPER + 11,
343
344 [11] = IPRIO_DEFAULT_M,
345 [3] = IPRIO_DEFAULT_M + 1,
346 [7] = IPRIO_DEFAULT_M + 2,
347
348 [9] = IPRIO_DEFAULT_S,
349 [1] = IPRIO_DEFAULT_S + 1,
350 [5] = IPRIO_DEFAULT_S + 2,
351
352 [12] = IPRIO_DEFAULT_SGEXT,
353
354 [10] = IPRIO_DEFAULT_VS,
355 [2] = IPRIO_DEFAULT_VS + 1,
356 [6] = IPRIO_DEFAULT_VS + 2,
357
358 [39] = IPRIO_DEFAULT_LOWER,
359 [19] = IPRIO_DEFAULT_LOWER + 1,
360 [38] = IPRIO_DEFAULT_LOWER + 2,
361 [37] = IPRIO_DEFAULT_LOWER + 3,
362 [18] = IPRIO_DEFAULT_LOWER + 4,
363 [36] = IPRIO_DEFAULT_LOWER + 5,
364
365 [35] = IPRIO_DEFAULT_LOWER + 6,
366 [17] = IPRIO_DEFAULT_LOWER + 7,
367 [34] = IPRIO_DEFAULT_LOWER + 8,
368 [33] = IPRIO_DEFAULT_LOWER + 9,
369 [16] = IPRIO_DEFAULT_LOWER + 10,
370 [32] = IPRIO_DEFAULT_LOWER + 11,
371 };
372
riscv_cpu_default_priority(int irq)373 uint8_t riscv_cpu_default_priority(int irq)
374 {
375 if (irq < 0 || irq > 63) {
376 return IPRIO_MMAXIPRIO;
377 }
378
379 return default_iprio[irq] ? default_iprio[irq] : IPRIO_MMAXIPRIO;
380 };
381
riscv_cpu_pending_to_irq(CPURISCVState * env,int extirq,unsigned int extirq_def_prio,uint64_t pending,uint8_t * iprio)382 static int riscv_cpu_pending_to_irq(CPURISCVState *env,
383 int extirq, unsigned int extirq_def_prio,
384 uint64_t pending, uint8_t *iprio)
385 {
386 int irq, best_irq = RISCV_EXCP_NONE;
387 unsigned int prio, best_prio = UINT_MAX;
388
389 if (!pending) {
390 return RISCV_EXCP_NONE;
391 }
392
393 irq = ctz64(pending);
394 if (!((extirq == IRQ_M_EXT) ? riscv_cpu_cfg(env)->ext_smaia :
395 riscv_cpu_cfg(env)->ext_ssaia)) {
396 return irq;
397 }
398
399 pending = pending >> irq;
400 while (pending) {
401 prio = iprio[irq];
402 if (!prio) {
403 if (irq == extirq) {
404 prio = extirq_def_prio;
405 } else {
406 prio = (riscv_cpu_default_priority(irq) < extirq_def_prio) ?
407 1 : IPRIO_MMAXIPRIO;
408 }
409 }
410 if ((pending & 0x1) && (prio <= best_prio)) {
411 best_irq = irq;
412 best_prio = prio;
413 }
414 irq++;
415 pending = pending >> 1;
416 }
417
418 return best_irq;
419 }
420
421 /*
422 * Doesn't report interrupts inserted using mvip from M-mode firmware or
423 * using hvip bits 13:63 from HS-mode. Those are returned in
424 * riscv_cpu_sirq_pending() and riscv_cpu_vsirq_pending().
425 */
riscv_cpu_all_pending(CPURISCVState * env)426 uint64_t riscv_cpu_all_pending(CPURISCVState *env)
427 {
428 uint32_t gein = get_field(env->hstatus, HSTATUS_VGEIN);
429 uint64_t vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0;
430 uint64_t vstip = (env->vstime_irq) ? MIP_VSTIP : 0;
431
432 return (env->mip | vsgein | vstip) & env->mie;
433 }
434
riscv_cpu_mirq_pending(CPURISCVState * env)435 int riscv_cpu_mirq_pending(CPURISCVState *env)
436 {
437 uint64_t irqs = riscv_cpu_all_pending(env) & ~env->mideleg &
438 ~(MIP_SGEIP | MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
439
440 return riscv_cpu_pending_to_irq(env, IRQ_M_EXT, IPRIO_DEFAULT_M,
441 irqs, env->miprio);
442 }
443
riscv_cpu_sirq_pending(CPURISCVState * env)444 int riscv_cpu_sirq_pending(CPURISCVState *env)
445 {
446 uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg &
447 ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
448 uint64_t irqs_f = env->mvip & env->mvien & ~env->mideleg & env->sie;
449
450 return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
451 irqs | irqs_f, env->siprio);
452 }
453
riscv_cpu_vsirq_pending(CPURISCVState * env)454 int riscv_cpu_vsirq_pending(CPURISCVState *env)
455 {
456 uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg & env->hideleg;
457 uint64_t irqs_f_vs = env->hvip & env->hvien & ~env->hideleg & env->vsie;
458 uint64_t vsbits;
459
460 /* Bring VS-level bits to correct position */
461 vsbits = irqs & VS_MODE_INTERRUPTS;
462 irqs &= ~VS_MODE_INTERRUPTS;
463 irqs |= vsbits >> 1;
464
465 return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
466 (irqs | irqs_f_vs), env->hviprio);
467 }
468
riscv_cpu_local_irq_pending(CPURISCVState * env)469 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
470 {
471 uint64_t irqs, pending, mie, hsie, vsie, irqs_f, irqs_f_vs;
472 uint64_t vsbits, irq_delegated;
473 int virq;
474
475 /* Priority: RNMI > Other interrupt. */
476 if (riscv_cpu_cfg(env)->ext_smrnmi) {
477 /* If mnstatus.NMIE == 0, all interrupts are disabled. */
478 if (!get_field(env->mnstatus, MNSTATUS_NMIE)) {
479 return RISCV_EXCP_NONE;
480 }
481
482 if (env->rnmip) {
483 return ctz64(env->rnmip); /* since non-zero */
484 }
485 }
486
487 /* Determine interrupt enable state of all privilege modes */
488 if (env->virt_enabled) {
489 mie = 1;
490 hsie = 1;
491 vsie = (env->priv < PRV_S) ||
492 (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
493 } else {
494 mie = (env->priv < PRV_M) ||
495 (env->priv == PRV_M && get_field(env->mstatus, MSTATUS_MIE));
496 hsie = (env->priv < PRV_S) ||
497 (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
498 vsie = 0;
499 }
500
501 /* Determine all pending interrupts */
502 pending = riscv_cpu_all_pending(env);
503
504 /* Check M-mode interrupts */
505 irqs = pending & ~env->mideleg & -mie;
506 if (irqs) {
507 return riscv_cpu_pending_to_irq(env, IRQ_M_EXT, IPRIO_DEFAULT_M,
508 irqs, env->miprio);
509 }
510
511 /* Check for virtual S-mode interrupts. */
512 irqs_f = env->mvip & (env->mvien & ~env->mideleg) & env->sie;
513
514 /* Check HS-mode interrupts */
515 irqs = ((pending & env->mideleg & ~env->hideleg) | irqs_f) & -hsie;
516 if (irqs) {
517 return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
518 irqs, env->siprio);
519 }
520
521 /* Check for virtual VS-mode interrupts. */
522 irqs_f_vs = env->hvip & env->hvien & ~env->hideleg & env->vsie;
523
524 /* Check VS-mode interrupts */
525 irq_delegated = pending & env->mideleg & env->hideleg;
526
527 /* Bring VS-level bits to correct position */
528 vsbits = irq_delegated & VS_MODE_INTERRUPTS;
529 irq_delegated &= ~VS_MODE_INTERRUPTS;
530 irq_delegated |= vsbits >> 1;
531
532 irqs = (irq_delegated | irqs_f_vs) & -vsie;
533 if (irqs) {
534 virq = riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
535 irqs, env->hviprio);
536 if (virq <= 0 || (virq > 12 && virq <= 63)) {
537 return virq;
538 } else {
539 return virq + 1;
540 }
541 }
542
543 /* Indicate no pending interrupt */
544 return RISCV_EXCP_NONE;
545 }
546
riscv_cpu_exec_interrupt(CPUState * cs,int interrupt_request)547 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
548 {
549 uint32_t mask = CPU_INTERRUPT_HARD | CPU_INTERRUPT_RNMI;
550
551 if (interrupt_request & mask) {
552 RISCVCPU *cpu = RISCV_CPU(cs);
553 CPURISCVState *env = &cpu->env;
554 int interruptno = riscv_cpu_local_irq_pending(env);
555 if (interruptno >= 0) {
556 cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
557 riscv_cpu_do_interrupt(cs);
558 return true;
559 }
560 }
561 return false;
562 }
563
564 /* Return true is floating point support is currently enabled */
riscv_cpu_fp_enabled(CPURISCVState * env)565 bool riscv_cpu_fp_enabled(CPURISCVState *env)
566 {
567 if (env->mstatus & MSTATUS_FS) {
568 if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_FS)) {
569 return false;
570 }
571 return true;
572 }
573
574 return false;
575 }
576
577 /* Return true is vector support is currently enabled */
riscv_cpu_vector_enabled(CPURISCVState * env)578 bool riscv_cpu_vector_enabled(CPURISCVState *env)
579 {
580 if (env->mstatus & MSTATUS_VS) {
581 if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_VS)) {
582 return false;
583 }
584 return true;
585 }
586
587 return false;
588 }
589
riscv_cpu_swap_hypervisor_regs(CPURISCVState * env)590 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
591 {
592 uint64_t mstatus_mask = MSTATUS_MXR | MSTATUS_SUM |
593 MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE |
594 MSTATUS64_UXL | MSTATUS_VS;
595
596 if (riscv_has_ext(env, RVF)) {
597 mstatus_mask |= MSTATUS_FS;
598 }
599 bool current_virt = env->virt_enabled;
600
601 /*
602 * If zicfilp extension available and henvcfg.LPE = 1,
603 * then apply SPELP mask on mstatus
604 */
605 if (env_archcpu(env)->cfg.ext_zicfilp &&
606 get_field(env->henvcfg, HENVCFG_LPE)) {
607 mstatus_mask |= SSTATUS_SPELP;
608 }
609
610 g_assert(riscv_has_ext(env, RVH));
611
612 if (riscv_env_smode_dbltrp_enabled(env, current_virt)) {
613 mstatus_mask |= MSTATUS_SDT;
614 }
615
616 if (current_virt) {
617 /* Current V=1 and we are about to change to V=0 */
618 env->vsstatus = env->mstatus & mstatus_mask;
619 env->mstatus &= ~mstatus_mask;
620 env->mstatus |= env->mstatus_hs;
621
622 env->vstvec = env->stvec;
623 env->stvec = env->stvec_hs;
624
625 env->vsscratch = env->sscratch;
626 env->sscratch = env->sscratch_hs;
627
628 env->vsepc = env->sepc;
629 env->sepc = env->sepc_hs;
630
631 env->vscause = env->scause;
632 env->scause = env->scause_hs;
633
634 env->vstval = env->stval;
635 env->stval = env->stval_hs;
636
637 env->vsatp = env->satp;
638 env->satp = env->satp_hs;
639 } else {
640 /* Current V=0 and we are about to change to V=1 */
641 env->mstatus_hs = env->mstatus & mstatus_mask;
642 env->mstatus &= ~mstatus_mask;
643 env->mstatus |= env->vsstatus;
644
645 env->stvec_hs = env->stvec;
646 env->stvec = env->vstvec;
647
648 env->sscratch_hs = env->sscratch;
649 env->sscratch = env->vsscratch;
650
651 env->sepc_hs = env->sepc;
652 env->sepc = env->vsepc;
653
654 env->scause_hs = env->scause;
655 env->scause = env->vscause;
656
657 env->stval_hs = env->stval;
658 env->stval = env->vstval;
659
660 env->satp_hs = env->satp;
661 env->satp = env->vsatp;
662 }
663 }
664
riscv_cpu_get_geilen(CPURISCVState * env)665 target_ulong riscv_cpu_get_geilen(CPURISCVState *env)
666 {
667 if (!riscv_has_ext(env, RVH)) {
668 return 0;
669 }
670
671 return env->geilen;
672 }
673
riscv_cpu_set_geilen(CPURISCVState * env,target_ulong geilen)674 void riscv_cpu_set_geilen(CPURISCVState *env, target_ulong geilen)
675 {
676 if (!riscv_has_ext(env, RVH)) {
677 return;
678 }
679
680 if (geilen > (TARGET_LONG_BITS - 1)) {
681 return;
682 }
683
684 env->geilen = geilen;
685 }
686
riscv_cpu_set_rnmi(RISCVCPU * cpu,uint32_t irq,bool level)687 void riscv_cpu_set_rnmi(RISCVCPU *cpu, uint32_t irq, bool level)
688 {
689 CPURISCVState *env = &cpu->env;
690 CPUState *cs = CPU(cpu);
691 bool release_lock = false;
692
693 if (!bql_locked()) {
694 release_lock = true;
695 bql_lock();
696 }
697
698 if (level) {
699 env->rnmip |= 1 << irq;
700 cpu_interrupt(cs, CPU_INTERRUPT_RNMI);
701 } else {
702 env->rnmip &= ~(1 << irq);
703 cpu_reset_interrupt(cs, CPU_INTERRUPT_RNMI);
704 }
705
706 if (release_lock) {
707 bql_unlock();
708 }
709 }
710
riscv_cpu_claim_interrupts(RISCVCPU * cpu,uint64_t interrupts)711 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint64_t interrupts)
712 {
713 CPURISCVState *env = &cpu->env;
714 if (env->miclaim & interrupts) {
715 return -1;
716 } else {
717 env->miclaim |= interrupts;
718 return 0;
719 }
720 }
721
riscv_cpu_interrupt(CPURISCVState * env)722 void riscv_cpu_interrupt(CPURISCVState *env)
723 {
724 uint64_t gein, vsgein = 0, vstip = 0, irqf = 0;
725 CPUState *cs = env_cpu(env);
726
727 BQL_LOCK_GUARD();
728
729 if (env->virt_enabled) {
730 gein = get_field(env->hstatus, HSTATUS_VGEIN);
731 vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0;
732 irqf = env->hvien & env->hvip & env->vsie;
733 } else {
734 irqf = env->mvien & env->mvip & env->sie;
735 }
736
737 vstip = env->vstime_irq ? MIP_VSTIP : 0;
738
739 if (env->mip | vsgein | vstip | irqf) {
740 cpu_interrupt(cs, CPU_INTERRUPT_HARD);
741 } else {
742 cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
743 }
744 }
745
riscv_cpu_update_mip(CPURISCVState * env,uint64_t mask,uint64_t value)746 uint64_t riscv_cpu_update_mip(CPURISCVState *env, uint64_t mask, uint64_t value)
747 {
748 uint64_t old = env->mip;
749
750 /* No need to update mip for VSTIP */
751 mask = ((mask == MIP_VSTIP) && env->vstime_irq) ? 0 : mask;
752
753 BQL_LOCK_GUARD();
754
755 env->mip = (env->mip & ~mask) | (value & mask);
756
757 riscv_cpu_interrupt(env);
758
759 return old;
760 }
761
riscv_cpu_set_rdtime_fn(CPURISCVState * env,uint64_t (* fn)(void *),void * arg)762 void riscv_cpu_set_rdtime_fn(CPURISCVState *env, uint64_t (*fn)(void *),
763 void *arg)
764 {
765 env->rdtime_fn = fn;
766 env->rdtime_fn_arg = arg;
767 }
768
riscv_cpu_set_aia_ireg_rmw_fn(CPURISCVState * env,uint32_t priv,int (* rmw_fn)(void * arg,target_ulong reg,target_ulong * val,target_ulong new_val,target_ulong write_mask),void * rmw_fn_arg)769 void riscv_cpu_set_aia_ireg_rmw_fn(CPURISCVState *env, uint32_t priv,
770 int (*rmw_fn)(void *arg,
771 target_ulong reg,
772 target_ulong *val,
773 target_ulong new_val,
774 target_ulong write_mask),
775 void *rmw_fn_arg)
776 {
777 if (priv <= PRV_M) {
778 env->aia_ireg_rmw_fn[priv] = rmw_fn;
779 env->aia_ireg_rmw_fn_arg[priv] = rmw_fn_arg;
780 }
781 }
782
riscv_ctr_freeze(CPURISCVState * env,uint64_t freeze_mask,bool virt)783 static void riscv_ctr_freeze(CPURISCVState *env, uint64_t freeze_mask,
784 bool virt)
785 {
786 uint64_t ctl = virt ? env->vsctrctl : env->mctrctl;
787
788 assert((freeze_mask & (~(XCTRCTL_BPFRZ | XCTRCTL_LCOFIFRZ))) == 0);
789
790 if (ctl & freeze_mask) {
791 env->sctrstatus |= SCTRSTATUS_FROZEN;
792 }
793 }
794
riscv_ctr_clear(CPURISCVState * env)795 void riscv_ctr_clear(CPURISCVState *env)
796 {
797 memset(env->ctr_src, 0x0, sizeof(env->ctr_src));
798 memset(env->ctr_dst, 0x0, sizeof(env->ctr_dst));
799 memset(env->ctr_data, 0x0, sizeof(env->ctr_data));
800 }
801
riscv_ctr_priv_to_mask(target_ulong priv,bool virt)802 static uint64_t riscv_ctr_priv_to_mask(target_ulong priv, bool virt)
803 {
804 switch (priv) {
805 case PRV_M:
806 return MCTRCTL_M;
807 case PRV_S:
808 if (virt) {
809 return XCTRCTL_S;
810 }
811 return XCTRCTL_S;
812 case PRV_U:
813 if (virt) {
814 return XCTRCTL_U;
815 }
816 return XCTRCTL_U;
817 }
818
819 g_assert_not_reached();
820 }
821
riscv_ctr_get_control(CPURISCVState * env,target_long priv,bool virt)822 static uint64_t riscv_ctr_get_control(CPURISCVState *env, target_long priv,
823 bool virt)
824 {
825 switch (priv) {
826 case PRV_M:
827 return env->mctrctl;
828 case PRV_S:
829 case PRV_U:
830 if (virt) {
831 return env->vsctrctl;
832 }
833 return env->mctrctl;
834 }
835
836 g_assert_not_reached();
837 }
838
839 /*
840 * This function assumes that src privilege and target privilege are not same
841 * and src privilege is less than target privilege. This includes the virtual
842 * state as well.
843 */
riscv_ctr_check_xte(CPURISCVState * env,target_long src_prv,bool src_virt)844 static bool riscv_ctr_check_xte(CPURISCVState *env, target_long src_prv,
845 bool src_virt)
846 {
847 target_long tgt_prv = env->priv;
848 bool res = true;
849
850 /*
851 * VS and U mode are same in terms of xTE bits required to record an
852 * external trap. See 6.1.2. External Traps, table 8 External Trap Enable
853 * Requirements. This changes VS to U to simplify the logic a bit.
854 */
855 if (src_virt && src_prv == PRV_S) {
856 src_prv = PRV_U;
857 } else if (env->virt_enabled && tgt_prv == PRV_S) {
858 tgt_prv = PRV_U;
859 }
860
861 /* VU mode is an outlier here. */
862 if (src_virt && src_prv == PRV_U) {
863 res &= !!(env->vsctrctl & XCTRCTL_STE);
864 }
865
866 switch (src_prv) {
867 case PRV_U:
868 if (tgt_prv == PRV_U) {
869 break;
870 }
871 res &= !!(env->mctrctl & XCTRCTL_STE);
872 /* fall-through */
873 case PRV_S:
874 if (tgt_prv == PRV_S) {
875 break;
876 }
877 res &= !!(env->mctrctl & MCTRCTL_MTE);
878 /* fall-through */
879 case PRV_M:
880 break;
881 }
882
883 return res;
884 }
885
886 /*
887 * Special cases for traps and trap returns:
888 *
889 * 1- Traps, and trap returns, between enabled modes are recorded as normal.
890 * 2- Traps from an inhibited mode to an enabled mode, and trap returns from an
891 * enabled mode back to an inhibited mode, are partially recorded. In such
892 * cases, the PC from the inhibited mode (source PC for traps, and target PC
893 * for trap returns) is 0.
894 *
895 * 3- Trap returns from an inhibited mode to an enabled mode are not recorded.
896 * Traps from an enabled mode to an inhibited mode, known as external traps,
897 * receive special handling.
898 * By default external traps are not recorded, but a handshake mechanism exists
899 * to allow partial recording. Software running in the target mode of the trap
900 * can opt-in to allowing CTR to record traps into that mode even when the mode
901 * is inhibited. The MTE, STE, and VSTE bits allow M-mode, S-mode, and VS-mode,
902 * respectively, to opt-in. When an External Trap occurs, and xTE=1, such that
903 * x is the target privilege mode of the trap, will CTR record the trap. In such
904 * cases, the target PC is 0.
905 */
906 /*
907 * CTR arrays are implemented as circular buffers and new entry is stored at
908 * sctrstatus.WRPTR, but they are presented to software as moving circular
909 * buffers. Which means, software get's the illusion that whenever a new entry
910 * is added the whole buffer is moved by one place and the new entry is added at
911 * the start keeping new entry at idx 0 and older ones follow.
912 *
913 * Depth = 16.
914 *
915 * buffer [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [A] [B] [C] [D] [E] [F]
916 * WRPTR W
917 * entry 7 6 5 4 3 2 1 0 F E D C B A 9 8
918 *
919 * When a new entry is added:
920 * buffer [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [A] [B] [C] [D] [E] [F]
921 * WRPTR W
922 * entry 8 7 6 5 4 3 2 1 0 F E D C B A 9
923 *
924 * entry here denotes the logical entry number that software can access
925 * using ctrsource, ctrtarget and ctrdata registers. So xiselect 0x200
926 * will return entry 0 i-e buffer[8] and 0x201 will return entry 1 i-e
927 * buffer[7]. Here is how we convert entry to buffer idx.
928 *
929 * entry = isel - CTR_ENTRIES_FIRST;
930 * idx = (sctrstatus.WRPTR - entry - 1) & (depth - 1);
931 */
riscv_ctr_add_entry(CPURISCVState * env,target_long src,target_long dst,enum CTRType type,target_ulong src_priv,bool src_virt)932 void riscv_ctr_add_entry(CPURISCVState *env, target_long src, target_long dst,
933 enum CTRType type, target_ulong src_priv, bool src_virt)
934 {
935 bool tgt_virt = env->virt_enabled;
936 uint64_t src_mask = riscv_ctr_priv_to_mask(src_priv, src_virt);
937 uint64_t tgt_mask = riscv_ctr_priv_to_mask(env->priv, tgt_virt);
938 uint64_t src_ctrl = riscv_ctr_get_control(env, src_priv, src_virt);
939 uint64_t tgt_ctrl = riscv_ctr_get_control(env, env->priv, tgt_virt);
940 uint64_t depth, head;
941 bool ext_trap = false;
942
943 /*
944 * Return immediately if both target and src recording is disabled or if
945 * CTR is in frozen state.
946 */
947 if ((!(src_ctrl & src_mask) && !(tgt_ctrl & tgt_mask)) ||
948 env->sctrstatus & SCTRSTATUS_FROZEN) {
949 return;
950 }
951
952 /*
953 * With RAS Emul enabled, only allow Indirect, direct calls, Function
954 * returns and Co-routine swap types.
955 */
956 if (tgt_ctrl & XCTRCTL_RASEMU &&
957 type != CTRDATA_TYPE_INDIRECT_CALL &&
958 type != CTRDATA_TYPE_DIRECT_CALL &&
959 type != CTRDATA_TYPE_RETURN &&
960 type != CTRDATA_TYPE_CO_ROUTINE_SWAP) {
961 return;
962 }
963
964 if (type == CTRDATA_TYPE_EXCEPTION || type == CTRDATA_TYPE_INTERRUPT) {
965 /* Case 2 for traps. */
966 if (!(src_ctrl & src_mask)) {
967 src = 0;
968 } else if (!(tgt_ctrl & tgt_mask)) {
969 /* Check if target priv-mode has allowed external trap recording. */
970 if (!riscv_ctr_check_xte(env, src_priv, src_virt)) {
971 return;
972 }
973
974 ext_trap = true;
975 dst = 0;
976 }
977 } else if (type == CTRDATA_TYPE_EXCEP_INT_RET) {
978 /*
979 * Case 3 for trap returns. Trap returns from inhibited mode are not
980 * recorded.
981 */
982 if (!(src_ctrl & src_mask)) {
983 return;
984 }
985
986 /* Case 2 for trap returns. */
987 if (!(tgt_ctrl & tgt_mask)) {
988 dst = 0;
989 }
990 }
991
992 /* Ignore filters in case of RASEMU mode or External trap. */
993 if (!(tgt_ctrl & XCTRCTL_RASEMU) && !ext_trap) {
994 /*
995 * Check if the specific type is inhibited. Not taken branch filter is
996 * an enable bit and needs to be checked separatly.
997 */
998 bool check = tgt_ctrl & BIT_ULL(type + XCTRCTL_INH_START);
999 if ((type == CTRDATA_TYPE_NONTAKEN_BRANCH && !check) ||
1000 (type != CTRDATA_TYPE_NONTAKEN_BRANCH && check)) {
1001 return;
1002 }
1003 }
1004
1005 head = get_field(env->sctrstatus, SCTRSTATUS_WRPTR_MASK);
1006
1007 depth = 16 << get_field(env->sctrdepth, SCTRDEPTH_MASK);
1008 if (tgt_ctrl & XCTRCTL_RASEMU && type == CTRDATA_TYPE_RETURN) {
1009 head = (head - 1) & (depth - 1);
1010
1011 env->ctr_src[head] &= ~CTRSOURCE_VALID;
1012 env->sctrstatus =
1013 set_field(env->sctrstatus, SCTRSTATUS_WRPTR_MASK, head);
1014 return;
1015 }
1016
1017 /* In case of Co-routine SWAP we overwrite latest entry. */
1018 if (tgt_ctrl & XCTRCTL_RASEMU && type == CTRDATA_TYPE_CO_ROUTINE_SWAP) {
1019 head = (head - 1) & (depth - 1);
1020 }
1021
1022 env->ctr_src[head] = src | CTRSOURCE_VALID;
1023 env->ctr_dst[head] = dst & ~CTRTARGET_MISP;
1024 env->ctr_data[head] = set_field(0, CTRDATA_TYPE_MASK, type);
1025
1026 head = (head + 1) & (depth - 1);
1027
1028 env->sctrstatus = set_field(env->sctrstatus, SCTRSTATUS_WRPTR_MASK, head);
1029 }
1030
riscv_cpu_set_mode(CPURISCVState * env,target_ulong newpriv,bool virt_en)1031 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv, bool virt_en)
1032 {
1033 g_assert(newpriv <= PRV_M && newpriv != PRV_RESERVED);
1034
1035 if (newpriv != env->priv || env->virt_enabled != virt_en) {
1036 if (icount_enabled()) {
1037 riscv_itrigger_update_priv(env);
1038 }
1039
1040 riscv_pmu_update_fixed_ctrs(env, newpriv, virt_en);
1041 }
1042
1043 /* tlb_flush is unnecessary as mode is contained in mmu_idx */
1044 env->priv = newpriv;
1045 env->xl = cpu_recompute_xl(env);
1046
1047 /*
1048 * Clear the load reservation - otherwise a reservation placed in one
1049 * context/process can be used by another, resulting in an SC succeeding
1050 * incorrectly. Version 2.2 of the ISA specification explicitly requires
1051 * this behaviour, while later revisions say that the kernel "should" use
1052 * an SC instruction to force the yielding of a load reservation on a
1053 * preemptive context switch. As a result, do both.
1054 */
1055 env->load_res = -1;
1056
1057 if (riscv_has_ext(env, RVH)) {
1058 /* Flush the TLB on all virt mode changes. */
1059 if (env->virt_enabled != virt_en) {
1060 tlb_flush(env_cpu(env));
1061 }
1062
1063 env->virt_enabled = virt_en;
1064 if (virt_en) {
1065 /*
1066 * The guest external interrupts from an interrupt controller are
1067 * delivered only when the Guest/VM is running (i.e. V=1). This
1068 * means any guest external interrupt which is triggered while the
1069 * Guest/VM is not running (i.e. V=0) will be missed on QEMU
1070 * resulting in guest with sluggish response to serial console
1071 * input and other I/O events.
1072 *
1073 * To solve this, we check and inject interrupt after setting V=1.
1074 */
1075 riscv_cpu_update_mip(env, 0, 0);
1076 }
1077 }
1078 }
1079
1080 /*
1081 * get_physical_address_pmp - check PMP permission for this physical address
1082 *
1083 * Match the PMP region and check permission for this physical address and it's
1084 * TLB page. Returns 0 if the permission checking was successful
1085 *
1086 * @env: CPURISCVState
1087 * @prot: The returned protection attributes
1088 * @addr: The physical address to be checked permission
1089 * @access_type: The type of MMU access
1090 * @mode: Indicates current privilege level.
1091 */
get_physical_address_pmp(CPURISCVState * env,int * prot,hwaddr addr,int size,MMUAccessType access_type,int mode)1092 static int get_physical_address_pmp(CPURISCVState *env, int *prot, hwaddr addr,
1093 int size, MMUAccessType access_type,
1094 int mode)
1095 {
1096 pmp_priv_t pmp_priv;
1097 bool pmp_has_privs;
1098
1099 if (!riscv_cpu_cfg(env)->pmp) {
1100 *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
1101 return TRANSLATE_SUCCESS;
1102 }
1103
1104 pmp_has_privs = pmp_hart_has_privs(env, addr, size, 1 << access_type,
1105 &pmp_priv, mode);
1106 if (!pmp_has_privs) {
1107 *prot = 0;
1108 return TRANSLATE_PMP_FAIL;
1109 }
1110
1111 *prot = pmp_priv_to_page_prot(pmp_priv);
1112
1113 return TRANSLATE_SUCCESS;
1114 }
1115
1116 /* Returns 'true' if a svukte address check is needed */
do_svukte_check(CPURISCVState * env,bool first_stage,int mode,bool virt)1117 static bool do_svukte_check(CPURISCVState *env, bool first_stage,
1118 int mode, bool virt)
1119 {
1120 /* Svukte extension depends on Sv39. */
1121 if (!(env_archcpu(env)->cfg.ext_svukte ||
1122 !first_stage ||
1123 VM_1_10_SV39 != get_field(env->satp, SATP64_MODE))) {
1124 return false;
1125 }
1126
1127 /*
1128 * Check hstatus.HUKTE if the effective mode is switched to VU-mode by
1129 * executing HLV/HLVX/HSV in U-mode.
1130 * For other cases, check senvcfg.UKTE.
1131 */
1132 if (env->priv == PRV_U && !env->virt_enabled && virt) {
1133 if (!get_field(env->hstatus, HSTATUS_HUKTE)) {
1134 return false;
1135 }
1136 } else if (!get_field(env->senvcfg, SENVCFG_UKTE)) {
1137 return false;
1138 }
1139
1140 /*
1141 * Svukte extension is qualified only in U or VU-mode.
1142 *
1143 * Effective mode can be switched to U or VU-mode by:
1144 * - M-mode + mstatus.MPRV=1 + mstatus.MPP=U-mode.
1145 * - Execute HLV/HLVX/HSV from HS-mode + hstatus.SPVP=0.
1146 * - U-mode.
1147 * - VU-mode.
1148 * - Execute HLV/HLVX/HSV from U-mode + hstatus.HU=1.
1149 */
1150 if (mode != PRV_U) {
1151 return false;
1152 }
1153
1154 return true;
1155 }
1156
check_svukte_addr(CPURISCVState * env,vaddr addr)1157 static bool check_svukte_addr(CPURISCVState *env, vaddr addr)
1158 {
1159 /* svukte extension excludes RV32 */
1160 uint32_t sxlen = 32 * riscv_cpu_sxl(env);
1161 uint64_t high_bit = addr & (1UL << (sxlen - 1));
1162 return !high_bit;
1163 }
1164
1165 /*
1166 * get_physical_address - get the physical address for this virtual address
1167 *
1168 * Do a page table walk to obtain the physical address corresponding to a
1169 * virtual address. Returns 0 if the translation was successful
1170 *
1171 * Adapted from Spike's mmu_t::translate and mmu_t::walk
1172 *
1173 * @env: CPURISCVState
1174 * @physical: This will be set to the calculated physical address
1175 * @prot: The returned protection attributes
1176 * @addr: The virtual address or guest physical address to be translated
1177 * @fault_pte_addr: If not NULL, this will be set to fault pte address
1178 * when a error occurs on pte address translation.
1179 * This will already be shifted to match htval.
1180 * @access_type: The type of MMU access
1181 * @mmu_idx: Indicates current privilege level
1182 * @first_stage: Are we in first stage translation?
1183 * Second stage is used for hypervisor guest translation
1184 * @two_stage: Are we going to perform two stage translation
1185 * @is_debug: Is this access from a debugger or the monitor?
1186 */
get_physical_address(CPURISCVState * env,hwaddr * physical,int * ret_prot,vaddr addr,target_ulong * fault_pte_addr,int access_type,int mmu_idx,bool first_stage,bool two_stage,bool is_debug,bool is_probe)1187 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
1188 int *ret_prot, vaddr addr,
1189 target_ulong *fault_pte_addr,
1190 int access_type, int mmu_idx,
1191 bool first_stage, bool two_stage,
1192 bool is_debug, bool is_probe)
1193 {
1194 /*
1195 * NOTE: the env->pc value visible here will not be
1196 * correct, but the value visible to the exception handler
1197 * (riscv_cpu_do_interrupt) is correct
1198 */
1199 MemTxResult res;
1200 MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
1201 int mode = mmuidx_priv(mmu_idx);
1202 bool virt = mmuidx_2stage(mmu_idx);
1203 bool use_background = false;
1204 hwaddr ppn;
1205 int napot_bits = 0;
1206 target_ulong napot_mask;
1207 bool is_sstack_idx = ((mmu_idx & MMU_IDX_SS_WRITE) == MMU_IDX_SS_WRITE);
1208 bool sstack_page = false;
1209
1210 if (do_svukte_check(env, first_stage, mode, virt) &&
1211 !check_svukte_addr(env, addr)) {
1212 return TRANSLATE_FAIL;
1213 }
1214
1215 /*
1216 * Check if we should use the background registers for the two
1217 * stage translation. We don't need to check if we actually need
1218 * two stage translation as that happened before this function
1219 * was called. Background registers will be used if the guest has
1220 * forced a two stage translation to be on (in HS or M mode).
1221 */
1222 if (!env->virt_enabled && two_stage) {
1223 use_background = true;
1224 }
1225
1226 if (mode == PRV_M || !riscv_cpu_cfg(env)->mmu) {
1227 *physical = addr;
1228 *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
1229 return TRANSLATE_SUCCESS;
1230 }
1231
1232 *ret_prot = 0;
1233
1234 hwaddr base;
1235 int levels, ptidxbits, ptesize, vm, widened;
1236
1237 if (first_stage == true) {
1238 if (use_background) {
1239 if (riscv_cpu_mxl(env) == MXL_RV32) {
1240 base = (hwaddr)get_field(env->vsatp, SATP32_PPN) << PGSHIFT;
1241 vm = get_field(env->vsatp, SATP32_MODE);
1242 } else {
1243 base = (hwaddr)get_field(env->vsatp, SATP64_PPN) << PGSHIFT;
1244 vm = get_field(env->vsatp, SATP64_MODE);
1245 }
1246 } else {
1247 if (riscv_cpu_mxl(env) == MXL_RV32) {
1248 base = (hwaddr)get_field(env->satp, SATP32_PPN) << PGSHIFT;
1249 vm = get_field(env->satp, SATP32_MODE);
1250 } else {
1251 base = (hwaddr)get_field(env->satp, SATP64_PPN) << PGSHIFT;
1252 vm = get_field(env->satp, SATP64_MODE);
1253 }
1254 }
1255 widened = 0;
1256 } else {
1257 if (riscv_cpu_mxl(env) == MXL_RV32) {
1258 base = (hwaddr)get_field(env->hgatp, SATP32_PPN) << PGSHIFT;
1259 vm = get_field(env->hgatp, SATP32_MODE);
1260 } else {
1261 base = (hwaddr)get_field(env->hgatp, SATP64_PPN) << PGSHIFT;
1262 vm = get_field(env->hgatp, SATP64_MODE);
1263 }
1264 widened = 2;
1265 }
1266
1267 switch (vm) {
1268 case VM_1_10_SV32:
1269 levels = 2; ptidxbits = 10; ptesize = 4; break;
1270 case VM_1_10_SV39:
1271 levels = 3; ptidxbits = 9; ptesize = 8; break;
1272 case VM_1_10_SV48:
1273 levels = 4; ptidxbits = 9; ptesize = 8; break;
1274 case VM_1_10_SV57:
1275 levels = 5; ptidxbits = 9; ptesize = 8; break;
1276 case VM_1_10_MBARE:
1277 *physical = addr;
1278 *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
1279 return TRANSLATE_SUCCESS;
1280 default:
1281 g_assert_not_reached();
1282 }
1283
1284 CPUState *cs = env_cpu(env);
1285 int va_bits = PGSHIFT + levels * ptidxbits + widened;
1286 int sxlen = 16 << riscv_cpu_sxl(env);
1287 int sxlen_bytes = sxlen / 8;
1288
1289 if (first_stage == true) {
1290 target_ulong mask, masked_msbs;
1291
1292 if (sxlen > (va_bits - 1)) {
1293 mask = (1L << (sxlen - (va_bits - 1))) - 1;
1294 } else {
1295 mask = 0;
1296 }
1297 masked_msbs = (addr >> (va_bits - 1)) & mask;
1298
1299 if (masked_msbs != 0 && masked_msbs != mask) {
1300 return TRANSLATE_FAIL;
1301 }
1302 } else {
1303 if (vm != VM_1_10_SV32 && addr >> va_bits != 0) {
1304 return TRANSLATE_FAIL;
1305 }
1306 }
1307
1308 bool pbmte = env->menvcfg & MENVCFG_PBMTE;
1309 bool svade = riscv_cpu_cfg(env)->ext_svade;
1310 bool svadu = riscv_cpu_cfg(env)->ext_svadu;
1311 bool adue = svadu ? env->menvcfg & MENVCFG_ADUE : !svade;
1312
1313 if (first_stage && two_stage && env->virt_enabled) {
1314 pbmte = pbmte && (env->henvcfg & HENVCFG_PBMTE);
1315 adue = adue && (env->henvcfg & HENVCFG_ADUE);
1316 }
1317
1318 int ptshift = (levels - 1) * ptidxbits;
1319 target_ulong pte;
1320 hwaddr pte_addr;
1321 int i;
1322
1323 restart:
1324 for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
1325 target_ulong idx;
1326 if (i == 0) {
1327 idx = (addr >> (PGSHIFT + ptshift)) &
1328 ((1 << (ptidxbits + widened)) - 1);
1329 } else {
1330 idx = (addr >> (PGSHIFT + ptshift)) &
1331 ((1 << ptidxbits) - 1);
1332 }
1333
1334 /* check that physical address of PTE is legal */
1335
1336 if (two_stage && first_stage) {
1337 int vbase_prot;
1338 hwaddr vbase;
1339
1340 /* Do the second stage translation on the base PTE address. */
1341 int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
1342 base, NULL, MMU_DATA_LOAD,
1343 MMUIdx_U, false, true,
1344 is_debug, false);
1345
1346 if (vbase_ret != TRANSLATE_SUCCESS) {
1347 if (fault_pte_addr) {
1348 *fault_pte_addr = (base + idx * ptesize) >> 2;
1349 }
1350 return TRANSLATE_G_STAGE_FAIL;
1351 }
1352
1353 pte_addr = vbase + idx * ptesize;
1354 } else {
1355 pte_addr = base + idx * ptesize;
1356 }
1357
1358 int pmp_prot;
1359 int pmp_ret = get_physical_address_pmp(env, &pmp_prot, pte_addr,
1360 sxlen_bytes,
1361 MMU_DATA_LOAD, PRV_S);
1362 if (pmp_ret != TRANSLATE_SUCCESS) {
1363 return TRANSLATE_PMP_FAIL;
1364 }
1365
1366 if (riscv_cpu_mxl(env) == MXL_RV32) {
1367 pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
1368 } else {
1369 pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
1370 }
1371
1372 if (res != MEMTX_OK) {
1373 return TRANSLATE_FAIL;
1374 }
1375
1376 if (riscv_cpu_sxl(env) == MXL_RV32) {
1377 ppn = pte >> PTE_PPN_SHIFT;
1378 } else {
1379 if (pte & PTE_RESERVED) {
1380 qemu_log_mask(LOG_GUEST_ERROR, "%s: reserved bits set in PTE: "
1381 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1382 __func__, pte_addr, pte);
1383 return TRANSLATE_FAIL;
1384 }
1385
1386 if (!pbmte && (pte & PTE_PBMT)) {
1387 /* Reserved without Svpbmt. */
1388 qemu_log_mask(LOG_GUEST_ERROR, "%s: PBMT bits set in PTE, "
1389 "and Svpbmt extension is disabled: "
1390 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1391 __func__, pte_addr, pte);
1392 return TRANSLATE_FAIL;
1393 }
1394
1395 if (!riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
1396 /* Reserved without Svnapot extension */
1397 qemu_log_mask(LOG_GUEST_ERROR, "%s: N bit set in PTE, "
1398 "and Svnapot extension is disabled: "
1399 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1400 __func__, pte_addr, pte);
1401 return TRANSLATE_FAIL;
1402 }
1403
1404 ppn = (pte & (target_ulong)PTE_PPN_MASK) >> PTE_PPN_SHIFT;
1405 }
1406
1407 if (!(pte & PTE_V)) {
1408 /* Invalid PTE */
1409 return TRANSLATE_FAIL;
1410 }
1411
1412 if (pte & (PTE_R | PTE_W | PTE_X)) {
1413 goto leaf;
1414 }
1415
1416 if (pte & (PTE_D | PTE_A | PTE_U | PTE_ATTR)) {
1417 /* D, A, and U bits are reserved in non-leaf/inner PTEs */
1418 qemu_log_mask(LOG_GUEST_ERROR, "%s: D, A, or U bits set in non-leaf PTE: "
1419 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1420 __func__, pte_addr, pte);
1421 return TRANSLATE_FAIL;
1422 }
1423 /* Inner PTE, continue walking */
1424 base = ppn << PGSHIFT;
1425 }
1426
1427 /* No leaf pte at any translation level. */
1428 return TRANSLATE_FAIL;
1429
1430 leaf:
1431 if (ppn & ((1ULL << ptshift) - 1)) {
1432 /* Misaligned PPN */
1433 qemu_log_mask(LOG_GUEST_ERROR, "%s: PPN bits in PTE is misaligned: "
1434 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1435 __func__, pte_addr, pte);
1436 return TRANSLATE_FAIL;
1437 }
1438 if (!pbmte && (pte & PTE_PBMT)) {
1439 /* Reserved without Svpbmt. */
1440 qemu_log_mask(LOG_GUEST_ERROR, "%s: PBMT bits set in PTE, "
1441 "and Svpbmt extension is disabled: "
1442 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1443 __func__, pte_addr, pte);
1444 return TRANSLATE_FAIL;
1445 }
1446
1447 target_ulong rwx = pte & (PTE_R | PTE_W | PTE_X);
1448 /* Check for reserved combinations of RWX flags. */
1449 switch (rwx) {
1450 case PTE_W | PTE_X:
1451 return TRANSLATE_FAIL;
1452 case PTE_W:
1453 /* if bcfi enabled, PTE_W is not reserved and shadow stack page */
1454 if (cpu_get_bcfien(env) && first_stage) {
1455 sstack_page = true;
1456 /*
1457 * if ss index, read and write allowed. else if not a probe
1458 * then only read allowed
1459 */
1460 rwx = is_sstack_idx ? (PTE_R | PTE_W) : (is_probe ? 0 : PTE_R);
1461 break;
1462 }
1463 return TRANSLATE_FAIL;
1464 case PTE_R:
1465 /*
1466 * no matter what's the `access_type`, shadow stack access to readonly
1467 * memory are always store page faults. During unwind, loads will be
1468 * promoted as store fault.
1469 */
1470 if (is_sstack_idx) {
1471 return TRANSLATE_FAIL;
1472 }
1473 break;
1474 }
1475
1476 int prot = 0;
1477 if (rwx & PTE_R) {
1478 prot |= PAGE_READ;
1479 }
1480 if (rwx & PTE_W) {
1481 prot |= PAGE_WRITE;
1482 }
1483 if (rwx & PTE_X) {
1484 bool mxr = false;
1485
1486 /*
1487 * Use mstatus for first stage or for the second stage without
1488 * virt_enabled (MPRV+MPV)
1489 */
1490 if (first_stage || !env->virt_enabled) {
1491 mxr = get_field(env->mstatus, MSTATUS_MXR);
1492 }
1493
1494 /* MPRV+MPV case, check VSSTATUS */
1495 if (first_stage && two_stage && !env->virt_enabled) {
1496 mxr |= get_field(env->vsstatus, MSTATUS_MXR);
1497 }
1498
1499 /*
1500 * Setting MXR at HS-level overrides both VS-stage and G-stage
1501 * execute-only permissions
1502 */
1503 if (env->virt_enabled) {
1504 mxr |= get_field(env->mstatus_hs, MSTATUS_MXR);
1505 }
1506
1507 if (mxr) {
1508 prot |= PAGE_READ;
1509 }
1510 prot |= PAGE_EXEC;
1511 }
1512
1513 if (pte & PTE_U) {
1514 if (mode != PRV_U) {
1515 if (!mmuidx_sum(mmu_idx)) {
1516 return TRANSLATE_FAIL;
1517 }
1518 /* SUM allows only read+write, not execute. */
1519 prot &= PAGE_READ | PAGE_WRITE;
1520 }
1521 } else if (mode != PRV_S) {
1522 /* Supervisor PTE flags when not S mode */
1523 return TRANSLATE_FAIL;
1524 }
1525
1526 if (!((prot >> access_type) & 1)) {
1527 /*
1528 * Access check failed, access check failures for shadow stack are
1529 * access faults.
1530 */
1531 return sstack_page ? TRANSLATE_PMP_FAIL : TRANSLATE_FAIL;
1532 }
1533
1534 target_ulong updated_pte = pte;
1535
1536 /*
1537 * If ADUE is enabled, set accessed and dirty bits.
1538 * Otherwise raise an exception if necessary.
1539 */
1540 if (adue) {
1541 updated_pte |= PTE_A | (access_type == MMU_DATA_STORE ? PTE_D : 0);
1542 } else if (!(pte & PTE_A) ||
1543 (access_type == MMU_DATA_STORE && !(pte & PTE_D))) {
1544 return TRANSLATE_FAIL;
1545 }
1546
1547 /* Page table updates need to be atomic with MTTCG enabled */
1548 if (updated_pte != pte && !is_debug) {
1549 if (!adue) {
1550 return TRANSLATE_FAIL;
1551 }
1552
1553 /*
1554 * - if accessed or dirty bits need updating, and the PTE is
1555 * in RAM, then we do so atomically with a compare and swap.
1556 * - if the PTE is in IO space or ROM, then it can't be updated
1557 * and we return TRANSLATE_FAIL.
1558 * - if the PTE changed by the time we went to update it, then
1559 * it is no longer valid and we must re-walk the page table.
1560 */
1561 MemoryRegion *mr;
1562 hwaddr l = sxlen_bytes, addr1;
1563 mr = address_space_translate(cs->as, pte_addr, &addr1, &l,
1564 false, MEMTXATTRS_UNSPECIFIED);
1565 if (memory_region_is_ram(mr)) {
1566 target_ulong *pte_pa = qemu_map_ram_ptr(mr->ram_block, addr1);
1567 target_ulong old_pte;
1568 if (riscv_cpu_sxl(env) == MXL_RV32) {
1569 old_pte = qatomic_cmpxchg((uint32_t *)pte_pa, cpu_to_le32(pte), cpu_to_le32(updated_pte));
1570 old_pte = le32_to_cpu(old_pte);
1571 } else {
1572 old_pte = qatomic_cmpxchg(pte_pa, cpu_to_le64(pte), cpu_to_le64(updated_pte));
1573 old_pte = le64_to_cpu(old_pte);
1574 }
1575 if (old_pte != pte) {
1576 goto restart;
1577 }
1578 pte = updated_pte;
1579 } else {
1580 /*
1581 * Misconfigured PTE in ROM (AD bits are not preset) or
1582 * PTE is in IO space and can't be updated atomically.
1583 */
1584 return TRANSLATE_FAIL;
1585 }
1586 }
1587
1588 /* For superpage mappings, make a fake leaf PTE for the TLB's benefit. */
1589 target_ulong vpn = addr >> PGSHIFT;
1590
1591 if (riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
1592 napot_bits = ctzl(ppn) + 1;
1593 if ((i != (levels - 1)) || (napot_bits != 4)) {
1594 return TRANSLATE_FAIL;
1595 }
1596 }
1597
1598 napot_mask = (1 << napot_bits) - 1;
1599 *physical = (((ppn & ~napot_mask) | (vpn & napot_mask) |
1600 (vpn & (((target_ulong)1 << ptshift) - 1))
1601 ) << PGSHIFT) | (addr & ~TARGET_PAGE_MASK);
1602
1603 /*
1604 * Remove write permission unless this is a store, or the page is
1605 * already dirty, so that we TLB miss on later writes to update
1606 * the dirty bit.
1607 */
1608 if (access_type != MMU_DATA_STORE && !(pte & PTE_D)) {
1609 prot &= ~PAGE_WRITE;
1610 }
1611 *ret_prot = prot;
1612
1613 return TRANSLATE_SUCCESS;
1614 }
1615
raise_mmu_exception(CPURISCVState * env,target_ulong address,MMUAccessType access_type,bool pmp_violation,bool first_stage,bool two_stage,bool two_stage_indirect)1616 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
1617 MMUAccessType access_type, bool pmp_violation,
1618 bool first_stage, bool two_stage,
1619 bool two_stage_indirect)
1620 {
1621 CPUState *cs = env_cpu(env);
1622
1623 switch (access_type) {
1624 case MMU_INST_FETCH:
1625 if (pmp_violation) {
1626 cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
1627 } else if (env->virt_enabled && !first_stage) {
1628 cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
1629 } else {
1630 cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
1631 }
1632 break;
1633 case MMU_DATA_LOAD:
1634 if (pmp_violation) {
1635 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1636 } else if (two_stage && !first_stage) {
1637 cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
1638 } else {
1639 cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
1640 }
1641 break;
1642 case MMU_DATA_STORE:
1643 if (pmp_violation) {
1644 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1645 } else if (two_stage && !first_stage) {
1646 cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
1647 } else {
1648 cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
1649 }
1650 break;
1651 default:
1652 g_assert_not_reached();
1653 }
1654 env->badaddr = address;
1655 env->two_stage_lookup = two_stage;
1656 env->two_stage_indirect_lookup = two_stage_indirect;
1657 }
1658
riscv_cpu_get_phys_page_debug(CPUState * cs,vaddr addr)1659 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
1660 {
1661 RISCVCPU *cpu = RISCV_CPU(cs);
1662 CPURISCVState *env = &cpu->env;
1663 hwaddr phys_addr;
1664 int prot;
1665 int mmu_idx = riscv_env_mmu_index(&cpu->env, false);
1666
1667 if (get_physical_address(env, &phys_addr, &prot, addr, NULL, 0, mmu_idx,
1668 true, env->virt_enabled, true, false)) {
1669 return -1;
1670 }
1671
1672 if (env->virt_enabled) {
1673 if (get_physical_address(env, &phys_addr, &prot, phys_addr, NULL,
1674 0, MMUIdx_U, false, true, true, false)) {
1675 return -1;
1676 }
1677 }
1678
1679 return phys_addr & TARGET_PAGE_MASK;
1680 }
1681
riscv_cpu_do_transaction_failed(CPUState * cs,hwaddr physaddr,vaddr addr,unsigned size,MMUAccessType access_type,int mmu_idx,MemTxAttrs attrs,MemTxResult response,uintptr_t retaddr)1682 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
1683 vaddr addr, unsigned size,
1684 MMUAccessType access_type,
1685 int mmu_idx, MemTxAttrs attrs,
1686 MemTxResult response, uintptr_t retaddr)
1687 {
1688 RISCVCPU *cpu = RISCV_CPU(cs);
1689 CPURISCVState *env = &cpu->env;
1690
1691 if (access_type == MMU_DATA_STORE) {
1692 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1693 } else if (access_type == MMU_DATA_LOAD) {
1694 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1695 } else {
1696 cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
1697 }
1698
1699 env->badaddr = addr;
1700 env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1701 env->two_stage_indirect_lookup = false;
1702 cpu_loop_exit_restore(cs, retaddr);
1703 }
1704
riscv_cpu_do_unaligned_access(CPUState * cs,vaddr addr,MMUAccessType access_type,int mmu_idx,uintptr_t retaddr)1705 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
1706 MMUAccessType access_type, int mmu_idx,
1707 uintptr_t retaddr)
1708 {
1709 RISCVCPU *cpu = RISCV_CPU(cs);
1710 CPURISCVState *env = &cpu->env;
1711 switch (access_type) {
1712 case MMU_INST_FETCH:
1713 cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
1714 break;
1715 case MMU_DATA_LOAD:
1716 cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
1717 /* shadow stack mis aligned accesses are access faults */
1718 if (mmu_idx & MMU_IDX_SS_WRITE) {
1719 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1720 }
1721 break;
1722 case MMU_DATA_STORE:
1723 cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
1724 /* shadow stack mis aligned accesses are access faults */
1725 if (mmu_idx & MMU_IDX_SS_WRITE) {
1726 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1727 }
1728 break;
1729 default:
1730 g_assert_not_reached();
1731 }
1732 env->badaddr = addr;
1733 env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1734 env->two_stage_indirect_lookup = false;
1735 cpu_loop_exit_restore(cs, retaddr);
1736 }
1737
1738
pmu_tlb_fill_incr_ctr(RISCVCPU * cpu,MMUAccessType access_type)1739 static void pmu_tlb_fill_incr_ctr(RISCVCPU *cpu, MMUAccessType access_type)
1740 {
1741 enum riscv_pmu_event_idx pmu_event_type;
1742
1743 switch (access_type) {
1744 case MMU_INST_FETCH:
1745 pmu_event_type = RISCV_PMU_EVENT_CACHE_ITLB_PREFETCH_MISS;
1746 break;
1747 case MMU_DATA_LOAD:
1748 pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_READ_MISS;
1749 break;
1750 case MMU_DATA_STORE:
1751 pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_WRITE_MISS;
1752 break;
1753 default:
1754 return;
1755 }
1756
1757 riscv_pmu_incr_ctr(cpu, pmu_event_type);
1758 }
1759
riscv_cpu_tlb_fill(CPUState * cs,vaddr address,int size,MMUAccessType access_type,int mmu_idx,bool probe,uintptr_t retaddr)1760 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
1761 MMUAccessType access_type, int mmu_idx,
1762 bool probe, uintptr_t retaddr)
1763 {
1764 RISCVCPU *cpu = RISCV_CPU(cs);
1765 CPURISCVState *env = &cpu->env;
1766 vaddr im_address;
1767 hwaddr pa = 0;
1768 int prot, prot2, prot_pmp;
1769 bool pmp_violation = false;
1770 bool first_stage_error = true;
1771 bool two_stage_lookup = mmuidx_2stage(mmu_idx);
1772 bool two_stage_indirect_error = false;
1773 int ret = TRANSLATE_FAIL;
1774 int mode = mmuidx_priv(mmu_idx);
1775 /* default TLB page size */
1776 hwaddr tlb_size = TARGET_PAGE_SIZE;
1777
1778 env->guest_phys_fault_addr = 0;
1779
1780 qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
1781 __func__, address, access_type, mmu_idx);
1782
1783 pmu_tlb_fill_incr_ctr(cpu, access_type);
1784 if (two_stage_lookup) {
1785 /* Two stage lookup */
1786 ret = get_physical_address(env, &pa, &prot, address,
1787 &env->guest_phys_fault_addr, access_type,
1788 mmu_idx, true, true, false, probe);
1789
1790 /*
1791 * A G-stage exception may be triggered during two state lookup.
1792 * And the env->guest_phys_fault_addr has already been set in
1793 * get_physical_address().
1794 */
1795 if (ret == TRANSLATE_G_STAGE_FAIL) {
1796 first_stage_error = false;
1797 two_stage_indirect_error = true;
1798 }
1799
1800 qemu_log_mask(CPU_LOG_MMU,
1801 "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
1802 HWADDR_FMT_plx " prot %d\n",
1803 __func__, address, ret, pa, prot);
1804
1805 if (ret == TRANSLATE_SUCCESS) {
1806 /* Second stage lookup */
1807 im_address = pa;
1808
1809 ret = get_physical_address(env, &pa, &prot2, im_address, NULL,
1810 access_type, MMUIdx_U, false, true,
1811 false, probe);
1812
1813 qemu_log_mask(CPU_LOG_MMU,
1814 "%s 2nd-stage address=%" VADDR_PRIx
1815 " ret %d physical "
1816 HWADDR_FMT_plx " prot %d\n",
1817 __func__, im_address, ret, pa, prot2);
1818
1819 prot &= prot2;
1820
1821 if (ret == TRANSLATE_SUCCESS) {
1822 ret = get_physical_address_pmp(env, &prot_pmp, pa,
1823 size, access_type, mode);
1824 tlb_size = pmp_get_tlb_size(env, pa);
1825
1826 qemu_log_mask(CPU_LOG_MMU,
1827 "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1828 " %d tlb_size %" HWADDR_PRIu "\n",
1829 __func__, pa, ret, prot_pmp, tlb_size);
1830
1831 prot &= prot_pmp;
1832 } else {
1833 /*
1834 * Guest physical address translation failed, this is a HS
1835 * level exception
1836 */
1837 first_stage_error = false;
1838 if (ret != TRANSLATE_PMP_FAIL) {
1839 env->guest_phys_fault_addr = (im_address |
1840 (address &
1841 (TARGET_PAGE_SIZE - 1))) >> 2;
1842 }
1843 }
1844 }
1845 } else {
1846 /* Single stage lookup */
1847 ret = get_physical_address(env, &pa, &prot, address, NULL,
1848 access_type, mmu_idx, true, false, false,
1849 probe);
1850
1851 qemu_log_mask(CPU_LOG_MMU,
1852 "%s address=%" VADDR_PRIx " ret %d physical "
1853 HWADDR_FMT_plx " prot %d\n",
1854 __func__, address, ret, pa, prot);
1855
1856 if (ret == TRANSLATE_SUCCESS) {
1857 ret = get_physical_address_pmp(env, &prot_pmp, pa,
1858 size, access_type, mode);
1859 tlb_size = pmp_get_tlb_size(env, pa);
1860
1861 qemu_log_mask(CPU_LOG_MMU,
1862 "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1863 " %d tlb_size %" HWADDR_PRIu "\n",
1864 __func__, pa, ret, prot_pmp, tlb_size);
1865
1866 prot &= prot_pmp;
1867 }
1868 }
1869
1870 if (ret == TRANSLATE_PMP_FAIL) {
1871 pmp_violation = true;
1872 }
1873
1874 if (ret == TRANSLATE_SUCCESS) {
1875 tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
1876 prot, mmu_idx, tlb_size);
1877 return true;
1878 } else if (probe) {
1879 return false;
1880 } else {
1881 int wp_access = 0;
1882
1883 if (access_type == MMU_DATA_LOAD) {
1884 wp_access |= BP_MEM_READ;
1885 } else if (access_type == MMU_DATA_STORE) {
1886 wp_access |= BP_MEM_WRITE;
1887 }
1888
1889 /*
1890 * If a watchpoint isn't found for 'addr' this will
1891 * be a no-op and we'll resume the mmu_exception path.
1892 * Otherwise we'll throw a debug exception and execution
1893 * will continue elsewhere.
1894 */
1895 cpu_check_watchpoint(cs, address, size, MEMTXATTRS_UNSPECIFIED,
1896 wp_access, retaddr);
1897
1898 raise_mmu_exception(env, address, access_type, pmp_violation,
1899 first_stage_error, two_stage_lookup,
1900 two_stage_indirect_error);
1901 cpu_loop_exit_restore(cs, retaddr);
1902 }
1903
1904 return true;
1905 }
1906
riscv_transformed_insn(CPURISCVState * env,target_ulong insn,target_ulong taddr)1907 static target_ulong riscv_transformed_insn(CPURISCVState *env,
1908 target_ulong insn,
1909 target_ulong taddr)
1910 {
1911 target_ulong xinsn = 0;
1912 target_ulong access_rs1 = 0, access_imm = 0, access_size = 0;
1913
1914 /*
1915 * Only Quadrant 0 and Quadrant 2 of RVC instruction space need to
1916 * be uncompressed. The Quadrant 1 of RVC instruction space need
1917 * not be transformed because these instructions won't generate
1918 * any load/store trap.
1919 */
1920
1921 if ((insn & 0x3) != 0x3) {
1922 /* Transform 16bit instruction into 32bit instruction */
1923 switch (GET_C_OP(insn)) {
1924 case OPC_RISC_C_OP_QUAD0: /* Quadrant 0 */
1925 switch (GET_C_FUNC(insn)) {
1926 case OPC_RISC_C_FUNC_FLD_LQ:
1927 if (riscv_cpu_xlen(env) != 128) { /* C.FLD (RV32/64) */
1928 xinsn = OPC_RISC_FLD;
1929 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1930 access_rs1 = GET_C_RS1S(insn);
1931 access_imm = GET_C_LD_IMM(insn);
1932 access_size = 8;
1933 }
1934 break;
1935 case OPC_RISC_C_FUNC_LW: /* C.LW */
1936 xinsn = OPC_RISC_LW;
1937 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1938 access_rs1 = GET_C_RS1S(insn);
1939 access_imm = GET_C_LW_IMM(insn);
1940 access_size = 4;
1941 break;
1942 case OPC_RISC_C_FUNC_FLW_LD:
1943 if (riscv_cpu_xlen(env) == 32) { /* C.FLW (RV32) */
1944 xinsn = OPC_RISC_FLW;
1945 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1946 access_rs1 = GET_C_RS1S(insn);
1947 access_imm = GET_C_LW_IMM(insn);
1948 access_size = 4;
1949 } else { /* C.LD (RV64/RV128) */
1950 xinsn = OPC_RISC_LD;
1951 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1952 access_rs1 = GET_C_RS1S(insn);
1953 access_imm = GET_C_LD_IMM(insn);
1954 access_size = 8;
1955 }
1956 break;
1957 case OPC_RISC_C_FUNC_FSD_SQ:
1958 if (riscv_cpu_xlen(env) != 128) { /* C.FSD (RV32/64) */
1959 xinsn = OPC_RISC_FSD;
1960 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1961 access_rs1 = GET_C_RS1S(insn);
1962 access_imm = GET_C_SD_IMM(insn);
1963 access_size = 8;
1964 }
1965 break;
1966 case OPC_RISC_C_FUNC_SW: /* C.SW */
1967 xinsn = OPC_RISC_SW;
1968 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1969 access_rs1 = GET_C_RS1S(insn);
1970 access_imm = GET_C_SW_IMM(insn);
1971 access_size = 4;
1972 break;
1973 case OPC_RISC_C_FUNC_FSW_SD:
1974 if (riscv_cpu_xlen(env) == 32) { /* C.FSW (RV32) */
1975 xinsn = OPC_RISC_FSW;
1976 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1977 access_rs1 = GET_C_RS1S(insn);
1978 access_imm = GET_C_SW_IMM(insn);
1979 access_size = 4;
1980 } else { /* C.SD (RV64/RV128) */
1981 xinsn = OPC_RISC_SD;
1982 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1983 access_rs1 = GET_C_RS1S(insn);
1984 access_imm = GET_C_SD_IMM(insn);
1985 access_size = 8;
1986 }
1987 break;
1988 default:
1989 break;
1990 }
1991 break;
1992 case OPC_RISC_C_OP_QUAD2: /* Quadrant 2 */
1993 switch (GET_C_FUNC(insn)) {
1994 case OPC_RISC_C_FUNC_FLDSP_LQSP:
1995 if (riscv_cpu_xlen(env) != 128) { /* C.FLDSP (RV32/64) */
1996 xinsn = OPC_RISC_FLD;
1997 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1998 access_rs1 = 2;
1999 access_imm = GET_C_LDSP_IMM(insn);
2000 access_size = 8;
2001 }
2002 break;
2003 case OPC_RISC_C_FUNC_LWSP: /* C.LWSP */
2004 xinsn = OPC_RISC_LW;
2005 xinsn = SET_RD(xinsn, GET_C_RD(insn));
2006 access_rs1 = 2;
2007 access_imm = GET_C_LWSP_IMM(insn);
2008 access_size = 4;
2009 break;
2010 case OPC_RISC_C_FUNC_FLWSP_LDSP:
2011 if (riscv_cpu_xlen(env) == 32) { /* C.FLWSP (RV32) */
2012 xinsn = OPC_RISC_FLW;
2013 xinsn = SET_RD(xinsn, GET_C_RD(insn));
2014 access_rs1 = 2;
2015 access_imm = GET_C_LWSP_IMM(insn);
2016 access_size = 4;
2017 } else { /* C.LDSP (RV64/RV128) */
2018 xinsn = OPC_RISC_LD;
2019 xinsn = SET_RD(xinsn, GET_C_RD(insn));
2020 access_rs1 = 2;
2021 access_imm = GET_C_LDSP_IMM(insn);
2022 access_size = 8;
2023 }
2024 break;
2025 case OPC_RISC_C_FUNC_FSDSP_SQSP:
2026 if (riscv_cpu_xlen(env) != 128) { /* C.FSDSP (RV32/64) */
2027 xinsn = OPC_RISC_FSD;
2028 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
2029 access_rs1 = 2;
2030 access_imm = GET_C_SDSP_IMM(insn);
2031 access_size = 8;
2032 }
2033 break;
2034 case OPC_RISC_C_FUNC_SWSP: /* C.SWSP */
2035 xinsn = OPC_RISC_SW;
2036 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
2037 access_rs1 = 2;
2038 access_imm = GET_C_SWSP_IMM(insn);
2039 access_size = 4;
2040 break;
2041 case 7:
2042 if (riscv_cpu_xlen(env) == 32) { /* C.FSWSP (RV32) */
2043 xinsn = OPC_RISC_FSW;
2044 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
2045 access_rs1 = 2;
2046 access_imm = GET_C_SWSP_IMM(insn);
2047 access_size = 4;
2048 } else { /* C.SDSP (RV64/RV128) */
2049 xinsn = OPC_RISC_SD;
2050 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
2051 access_rs1 = 2;
2052 access_imm = GET_C_SDSP_IMM(insn);
2053 access_size = 8;
2054 }
2055 break;
2056 default:
2057 break;
2058 }
2059 break;
2060 default:
2061 break;
2062 }
2063
2064 /*
2065 * Clear Bit1 of transformed instruction to indicate that
2066 * original insruction was a 16bit instruction
2067 */
2068 xinsn &= ~((target_ulong)0x2);
2069 } else {
2070 /* Transform 32bit (or wider) instructions */
2071 switch (MASK_OP_MAJOR(insn)) {
2072 case OPC_RISC_ATOMIC:
2073 xinsn = insn;
2074 access_rs1 = GET_RS1(insn);
2075 access_size = 1 << GET_FUNCT3(insn);
2076 break;
2077 case OPC_RISC_LOAD:
2078 case OPC_RISC_FP_LOAD:
2079 xinsn = SET_I_IMM(insn, 0);
2080 access_rs1 = GET_RS1(insn);
2081 access_imm = GET_IMM(insn);
2082 access_size = 1 << GET_FUNCT3(insn);
2083 break;
2084 case OPC_RISC_STORE:
2085 case OPC_RISC_FP_STORE:
2086 xinsn = SET_S_IMM(insn, 0);
2087 access_rs1 = GET_RS1(insn);
2088 access_imm = GET_STORE_IMM(insn);
2089 access_size = 1 << GET_FUNCT3(insn);
2090 break;
2091 case OPC_RISC_SYSTEM:
2092 if (MASK_OP_SYSTEM(insn) == OPC_RISC_HLVHSV) {
2093 xinsn = insn;
2094 access_rs1 = GET_RS1(insn);
2095 access_size = 1 << ((GET_FUNCT7(insn) >> 1) & 0x3);
2096 access_size = 1 << access_size;
2097 }
2098 break;
2099 default:
2100 break;
2101 }
2102 }
2103
2104 if (access_size) {
2105 xinsn = SET_RS1(xinsn, (taddr - (env->gpr[access_rs1] + access_imm)) &
2106 (access_size - 1));
2107 }
2108
2109 return xinsn;
2110 }
2111
promote_load_fault(target_ulong orig_cause)2112 static target_ulong promote_load_fault(target_ulong orig_cause)
2113 {
2114 switch (orig_cause) {
2115 case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
2116 return RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
2117
2118 case RISCV_EXCP_LOAD_ACCESS_FAULT:
2119 return RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
2120
2121 case RISCV_EXCP_LOAD_PAGE_FAULT:
2122 return RISCV_EXCP_STORE_PAGE_FAULT;
2123 }
2124
2125 /* if no promotion, return original cause */
2126 return orig_cause;
2127 }
2128
riscv_do_nmi(CPURISCVState * env,target_ulong cause,bool virt)2129 static void riscv_do_nmi(CPURISCVState *env, target_ulong cause, bool virt)
2130 {
2131 env->mnstatus = set_field(env->mnstatus, MNSTATUS_NMIE, false);
2132 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPV, virt);
2133 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPP, env->priv);
2134 env->mncause = cause;
2135 env->mnepc = env->pc;
2136 env->pc = env->rnmi_irqvec;
2137
2138 if (cpu_get_fcfien(env)) {
2139 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPELP, env->elp);
2140 }
2141
2142 /* Trapping to M mode, virt is disabled */
2143 riscv_cpu_set_mode(env, PRV_M, false);
2144 }
2145
2146 /*
2147 * Handle Traps
2148 *
2149 * Adapted from Spike's processor_t::take_trap.
2150 *
2151 */
riscv_cpu_do_interrupt(CPUState * cs)2152 void riscv_cpu_do_interrupt(CPUState *cs)
2153 {
2154 RISCVCPU *cpu = RISCV_CPU(cs);
2155 CPURISCVState *env = &cpu->env;
2156 bool virt = env->virt_enabled;
2157 bool write_gva = false;
2158 bool always_storeamo = (env->excp_uw2 & RISCV_UW2_ALWAYS_STORE_AMO);
2159 bool vsmode_exc;
2160 uint64_t s;
2161 int mode;
2162
2163 /*
2164 * cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
2165 * so we mask off the MSB and separate into trap type and cause.
2166 */
2167 bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
2168 target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
2169 uint64_t deleg = async ? env->mideleg : env->medeleg;
2170 bool s_injected = env->mvip & (1ULL << cause) & env->mvien &&
2171 !(env->mip & (1ULL << cause));
2172 bool vs_injected = env->hvip & (1ULL << cause) & env->hvien &&
2173 !(env->mip & (1ULL << cause));
2174 bool smode_double_trap = false;
2175 uint64_t hdeleg = async ? env->hideleg : env->hedeleg;
2176 const bool prev_virt = env->virt_enabled;
2177 const target_ulong prev_priv = env->priv;
2178 target_ulong tval = 0;
2179 target_ulong tinst = 0;
2180 target_ulong htval = 0;
2181 target_ulong mtval2 = 0;
2182 target_ulong src;
2183 int sxlen = 0;
2184 int mxlen = 16 << riscv_cpu_mxl(env);
2185 bool nnmi_excep = false;
2186
2187 if (cpu->cfg.ext_smrnmi && env->rnmip && async) {
2188 riscv_do_nmi(env, cause | ((target_ulong)1U << (mxlen - 1)),
2189 env->virt_enabled);
2190 return;
2191 }
2192
2193 if (!async) {
2194 /* set tval to badaddr for traps with address information */
2195 switch (cause) {
2196 #ifdef CONFIG_TCG
2197 case RISCV_EXCP_SEMIHOST:
2198 do_common_semihosting(cs);
2199 env->pc += 4;
2200 return;
2201 #endif
2202 case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
2203 case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
2204 case RISCV_EXCP_LOAD_ADDR_MIS:
2205 case RISCV_EXCP_STORE_AMO_ADDR_MIS:
2206 case RISCV_EXCP_LOAD_ACCESS_FAULT:
2207 case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
2208 case RISCV_EXCP_LOAD_PAGE_FAULT:
2209 case RISCV_EXCP_STORE_PAGE_FAULT:
2210 if (always_storeamo) {
2211 cause = promote_load_fault(cause);
2212 }
2213 write_gva = env->two_stage_lookup;
2214 tval = env->badaddr;
2215 if (env->two_stage_indirect_lookup) {
2216 /*
2217 * special pseudoinstruction for G-stage fault taken while
2218 * doing VS-stage page table walk.
2219 */
2220 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
2221 } else {
2222 /*
2223 * The "Addr. Offset" field in transformed instruction is
2224 * non-zero only for misaligned access.
2225 */
2226 tinst = riscv_transformed_insn(env, env->bins, tval);
2227 }
2228 break;
2229 case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
2230 case RISCV_EXCP_INST_ADDR_MIS:
2231 case RISCV_EXCP_INST_ACCESS_FAULT:
2232 case RISCV_EXCP_INST_PAGE_FAULT:
2233 write_gva = env->two_stage_lookup;
2234 tval = env->badaddr;
2235 if (env->two_stage_indirect_lookup) {
2236 /*
2237 * special pseudoinstruction for G-stage fault taken while
2238 * doing VS-stage page table walk.
2239 */
2240 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
2241 }
2242 break;
2243 case RISCV_EXCP_ILLEGAL_INST:
2244 case RISCV_EXCP_VIRT_INSTRUCTION_FAULT:
2245 tval = env->bins;
2246 break;
2247 case RISCV_EXCP_BREAKPOINT:
2248 tval = env->badaddr;
2249 if (cs->watchpoint_hit) {
2250 tval = cs->watchpoint_hit->hitaddr;
2251 cs->watchpoint_hit = NULL;
2252 }
2253 break;
2254 case RISCV_EXCP_SW_CHECK:
2255 tval = env->sw_check_code;
2256 break;
2257 default:
2258 break;
2259 }
2260 /* ecall is dispatched as one cause so translate based on mode */
2261 if (cause == RISCV_EXCP_U_ECALL) {
2262 assert(env->priv <= 3);
2263
2264 if (env->priv == PRV_M) {
2265 cause = RISCV_EXCP_M_ECALL;
2266 } else if (env->priv == PRV_S && env->virt_enabled) {
2267 cause = RISCV_EXCP_VS_ECALL;
2268 } else if (env->priv == PRV_S && !env->virt_enabled) {
2269 cause = RISCV_EXCP_S_ECALL;
2270 } else if (env->priv == PRV_U) {
2271 cause = RISCV_EXCP_U_ECALL;
2272 }
2273 }
2274 }
2275
2276 trace_riscv_trap(env->mhartid, async, cause, env->pc, tval,
2277 riscv_cpu_get_trap_name(cause, async));
2278
2279 qemu_log_mask(CPU_LOG_INT,
2280 "%s: hart:"TARGET_FMT_ld", async:%d, cause:"TARGET_FMT_lx", "
2281 "epc:0x"TARGET_FMT_lx", tval:0x"TARGET_FMT_lx", desc=%s\n",
2282 __func__, env->mhartid, async, cause, env->pc, tval,
2283 riscv_cpu_get_trap_name(cause, async));
2284
2285 mode = env->priv <= PRV_S && cause < 64 &&
2286 (((deleg >> cause) & 1) || s_injected || vs_injected) ? PRV_S : PRV_M;
2287
2288 vsmode_exc = env->virt_enabled && cause < 64 &&
2289 (((hdeleg >> cause) & 1) || vs_injected);
2290
2291 /*
2292 * Check double trap condition only if already in S-mode and targeting
2293 * S-mode
2294 */
2295 if (cpu->cfg.ext_ssdbltrp && env->priv == PRV_S && mode == PRV_S) {
2296 bool dte = (env->menvcfg & MENVCFG_DTE) != 0;
2297 bool sdt = (env->mstatus & MSTATUS_SDT) != 0;
2298 /* In VS or HS */
2299 if (riscv_has_ext(env, RVH)) {
2300 if (vsmode_exc) {
2301 /* VS -> VS, use henvcfg instead of menvcfg*/
2302 dte = (env->henvcfg & HENVCFG_DTE) != 0;
2303 } else if (env->virt_enabled) {
2304 /* VS -> HS, use mstatus_hs */
2305 sdt = (env->mstatus_hs & MSTATUS_SDT) != 0;
2306 }
2307 }
2308 smode_double_trap = dte && sdt;
2309 if (smode_double_trap) {
2310 mode = PRV_M;
2311 }
2312 }
2313
2314 if (mode == PRV_S) {
2315 /* handle the trap in S-mode */
2316 /* save elp status */
2317 if (cpu_get_fcfien(env)) {
2318 env->mstatus = set_field(env->mstatus, MSTATUS_SPELP, env->elp);
2319 }
2320
2321 if (riscv_has_ext(env, RVH)) {
2322 if (vsmode_exc) {
2323 /* Trap to VS mode */
2324 /*
2325 * See if we need to adjust cause. Yes if its VS mode interrupt
2326 * no if hypervisor has delegated one of hs mode's interrupt
2327 */
2328 if (async && (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
2329 cause == IRQ_VS_EXT)) {
2330 cause = cause - 1;
2331 }
2332 write_gva = false;
2333 } else if (env->virt_enabled) {
2334 /* Trap into HS mode, from virt */
2335 riscv_cpu_swap_hypervisor_regs(env);
2336 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
2337 env->priv);
2338 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, true);
2339
2340 htval = env->guest_phys_fault_addr;
2341
2342 virt = false;
2343 } else {
2344 /* Trap into HS mode */
2345 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, false);
2346 htval = env->guest_phys_fault_addr;
2347 }
2348 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, write_gva);
2349 }
2350
2351 s = env->mstatus;
2352 s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
2353 s = set_field(s, MSTATUS_SPP, env->priv);
2354 s = set_field(s, MSTATUS_SIE, 0);
2355 if (riscv_env_smode_dbltrp_enabled(env, virt)) {
2356 s = set_field(s, MSTATUS_SDT, 1);
2357 }
2358 env->mstatus = s;
2359 sxlen = 16 << riscv_cpu_sxl(env);
2360 env->scause = cause | ((target_ulong)async << (sxlen - 1));
2361 env->sepc = env->pc;
2362 env->stval = tval;
2363 env->htval = htval;
2364 env->htinst = tinst;
2365 env->pc = (env->stvec >> 2 << 2) +
2366 ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
2367 riscv_cpu_set_mode(env, PRV_S, virt);
2368
2369 src = env->sepc;
2370 } else {
2371 /*
2372 * If the hart encounters an exception while executing in M-mode
2373 * with the mnstatus.NMIE bit clear, the exception is an RNMI exception.
2374 */
2375 nnmi_excep = cpu->cfg.ext_smrnmi &&
2376 !get_field(env->mnstatus, MNSTATUS_NMIE) &&
2377 !async;
2378
2379 /* handle the trap in M-mode */
2380 /* save elp status */
2381 if (cpu_get_fcfien(env)) {
2382 if (nnmi_excep) {
2383 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPELP,
2384 env->elp);
2385 } else {
2386 env->mstatus = set_field(env->mstatus, MSTATUS_MPELP, env->elp);
2387 }
2388 }
2389
2390 if (riscv_has_ext(env, RVH)) {
2391 if (env->virt_enabled) {
2392 riscv_cpu_swap_hypervisor_regs(env);
2393 }
2394 env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
2395 env->virt_enabled);
2396 if (env->virt_enabled && tval) {
2397 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
2398 }
2399
2400 mtval2 = env->guest_phys_fault_addr;
2401
2402 /* Trapping to M mode, virt is disabled */
2403 virt = false;
2404 }
2405 /*
2406 * If the hart encounters an exception while executing in M-mode,
2407 * with the mnstatus.NMIE bit clear, the program counter is set to
2408 * the RNMI exception trap handler address.
2409 */
2410 nnmi_excep = cpu->cfg.ext_smrnmi &&
2411 !get_field(env->mnstatus, MNSTATUS_NMIE) &&
2412 !async;
2413
2414 s = env->mstatus;
2415 s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
2416 s = set_field(s, MSTATUS_MPP, env->priv);
2417 s = set_field(s, MSTATUS_MIE, 0);
2418 if (cpu->cfg.ext_smdbltrp) {
2419 if (env->mstatus & MSTATUS_MDT) {
2420 assert(env->priv == PRV_M);
2421 if (!cpu->cfg.ext_smrnmi || nnmi_excep) {
2422 cpu_abort(CPU(cpu), "M-mode double trap\n");
2423 } else {
2424 riscv_do_nmi(env, cause, false);
2425 return;
2426 }
2427 }
2428
2429 s = set_field(s, MSTATUS_MDT, 1);
2430 }
2431 env->mstatus = s;
2432 env->mcause = cause | ((target_ulong)async << (mxlen - 1));
2433 if (smode_double_trap) {
2434 env->mtval2 = env->mcause;
2435 env->mcause = RISCV_EXCP_DOUBLE_TRAP;
2436 } else {
2437 env->mtval2 = mtval2;
2438 }
2439 env->mepc = env->pc;
2440 env->mtval = tval;
2441 env->mtinst = tinst;
2442
2443 /*
2444 * For RNMI exception, program counter is set to the RNMI exception
2445 * trap handler address.
2446 */
2447 if (nnmi_excep) {
2448 env->pc = env->rnmi_excpvec;
2449 } else {
2450 env->pc = (env->mtvec >> 2 << 2) +
2451 ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
2452 }
2453 riscv_cpu_set_mode(env, PRV_M, virt);
2454 src = env->mepc;
2455 }
2456
2457 if (riscv_cpu_cfg(env)->ext_smctr || riscv_cpu_cfg(env)->ext_ssctr) {
2458 if (async && cause == IRQ_PMU_OVF) {
2459 riscv_ctr_freeze(env, XCTRCTL_LCOFIFRZ, virt);
2460 } else if (!async && cause == RISCV_EXCP_BREAKPOINT) {
2461 riscv_ctr_freeze(env, XCTRCTL_BPFRZ, virt);
2462 }
2463
2464 riscv_ctr_add_entry(env, src, env->pc,
2465 async ? CTRDATA_TYPE_INTERRUPT : CTRDATA_TYPE_EXCEPTION,
2466 prev_priv, prev_virt);
2467 }
2468
2469 /*
2470 * Interrupt/exception/trap delivery is asynchronous event and as per
2471 * zicfilp spec CPU should clear up the ELP state. No harm in clearing
2472 * unconditionally.
2473 */
2474 env->elp = false;
2475
2476 /*
2477 * NOTE: it is not necessary to yield load reservations here. It is only
2478 * necessary for an SC from "another hart" to cause a load reservation
2479 * to be yielded. Refer to the memory consistency model section of the
2480 * RISC-V ISA Specification.
2481 */
2482
2483 env->two_stage_lookup = false;
2484 env->two_stage_indirect_lookup = false;
2485 }
2486
2487 #endif /* !CONFIG_USER_ONLY */
2488