1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
4 */
5
6 #include <linux/err.h>
7 #include <linux/errno.h>
8 #include <linux/kvm_host.h>
9 #include <linux/module.h>
10 #include <linux/preempt.h>
11 #include <linux/vmalloc.h>
12 #include <trace/events/kvm.h>
13 #include <asm/fpu.h>
14 #include <asm/inst.h>
15 #include <asm/loongarch.h>
16 #include <asm/mmzone.h>
17 #include <asm/numa.h>
18 #include <asm/time.h>
19 #include <asm/tlb.h>
20 #include <asm/kvm_csr.h>
21 #include <asm/kvm_vcpu.h>
22 #include "trace.h"
23
kvm_emu_cpucfg(struct kvm_vcpu * vcpu,larch_inst inst)24 static int kvm_emu_cpucfg(struct kvm_vcpu *vcpu, larch_inst inst)
25 {
26 int rd, rj;
27 unsigned int index, ret;
28
29 if (inst.reg2_format.opcode != cpucfg_op)
30 return EMULATE_FAIL;
31
32 rd = inst.reg2_format.rd;
33 rj = inst.reg2_format.rj;
34 ++vcpu->stat.cpucfg_exits;
35 index = vcpu->arch.gprs[rj];
36
37 /*
38 * By LoongArch Reference Manual 2.2.10.5
39 * Return value is 0 for undefined CPUCFG index
40 *
41 * Disable preemption since hw gcsr is accessed
42 */
43 preempt_disable();
44 switch (index) {
45 case 0 ... (KVM_MAX_CPUCFG_REGS - 1):
46 vcpu->arch.gprs[rd] = vcpu->arch.cpucfg[index];
47 break;
48 case CPUCFG_KVM_SIG:
49 /* CPUCFG emulation between 0x40000000 -- 0x400000ff */
50 vcpu->arch.gprs[rd] = *(unsigned int *)KVM_SIGNATURE;
51 break;
52 case CPUCFG_KVM_FEATURE:
53 ret = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK;
54 vcpu->arch.gprs[rd] = ret;
55 break;
56 default:
57 vcpu->arch.gprs[rd] = 0;
58 break;
59 }
60 preempt_enable();
61
62 return EMULATE_DONE;
63 }
64
kvm_emu_read_csr(struct kvm_vcpu * vcpu,int csrid)65 static unsigned long kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid)
66 {
67 unsigned long val = 0;
68 struct loongarch_csrs *csr = vcpu->arch.csr;
69
70 /*
71 * From LoongArch Reference Manual Volume 1 Chapter 4.2.1
72 * For undefined CSR id, return value is 0
73 */
74 if (get_gcsr_flag(csrid) & SW_GCSR)
75 val = kvm_read_sw_gcsr(csr, csrid);
76 else
77 pr_warn_once("Unsupported csrrd 0x%x with pc %lx\n", csrid, vcpu->arch.pc);
78
79 return val;
80 }
81
kvm_emu_write_csr(struct kvm_vcpu * vcpu,int csrid,unsigned long val)82 static unsigned long kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val)
83 {
84 unsigned long old = 0;
85 struct loongarch_csrs *csr = vcpu->arch.csr;
86
87 if (get_gcsr_flag(csrid) & SW_GCSR) {
88 old = kvm_read_sw_gcsr(csr, csrid);
89 kvm_write_sw_gcsr(csr, csrid, val);
90 } else
91 pr_warn_once("Unsupported csrwr 0x%x with pc %lx\n", csrid, vcpu->arch.pc);
92
93 return old;
94 }
95
kvm_emu_xchg_csr(struct kvm_vcpu * vcpu,int csrid,unsigned long csr_mask,unsigned long val)96 static unsigned long kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid,
97 unsigned long csr_mask, unsigned long val)
98 {
99 unsigned long old = 0;
100 struct loongarch_csrs *csr = vcpu->arch.csr;
101
102 if (get_gcsr_flag(csrid) & SW_GCSR) {
103 old = kvm_read_sw_gcsr(csr, csrid);
104 val = (old & ~csr_mask) | (val & csr_mask);
105 kvm_write_sw_gcsr(csr, csrid, val);
106 old = old & csr_mask;
107 } else
108 pr_warn_once("Unsupported csrxchg 0x%x with pc %lx\n", csrid, vcpu->arch.pc);
109
110 return old;
111 }
112
kvm_handle_csr(struct kvm_vcpu * vcpu,larch_inst inst)113 static int kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst)
114 {
115 unsigned int rd, rj, csrid;
116 unsigned long csr_mask, val = 0;
117
118 /*
119 * CSR value mask imm
120 * rj = 0 means csrrd
121 * rj = 1 means csrwr
122 * rj != 0,1 means csrxchg
123 */
124 rd = inst.reg2csr_format.rd;
125 rj = inst.reg2csr_format.rj;
126 csrid = inst.reg2csr_format.csr;
127
128 if (csrid >= LOONGARCH_CSR_PERFCTRL0 && csrid <= vcpu->arch.max_pmu_csrid) {
129 if (kvm_guest_has_pmu(&vcpu->arch)) {
130 vcpu->arch.pc -= 4;
131 kvm_make_request(KVM_REQ_PMU, vcpu);
132 return EMULATE_DONE;
133 }
134 }
135
136 /* Process CSR ops */
137 switch (rj) {
138 case 0: /* process csrrd */
139 val = kvm_emu_read_csr(vcpu, csrid);
140 vcpu->arch.gprs[rd] = val;
141 break;
142 case 1: /* process csrwr */
143 val = vcpu->arch.gprs[rd];
144 val = kvm_emu_write_csr(vcpu, csrid, val);
145 vcpu->arch.gprs[rd] = val;
146 break;
147 default: /* process csrxchg */
148 val = vcpu->arch.gprs[rd];
149 csr_mask = vcpu->arch.gprs[rj];
150 val = kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val);
151 vcpu->arch.gprs[rd] = val;
152 }
153
154 return EMULATE_DONE;
155 }
156
kvm_emu_iocsr(larch_inst inst,struct kvm_run * run,struct kvm_vcpu * vcpu)157 int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu)
158 {
159 int idx, ret;
160 unsigned long *val;
161 u32 addr, rd, rj, opcode;
162
163 /*
164 * Each IOCSR with different opcode
165 */
166 rd = inst.reg2_format.rd;
167 rj = inst.reg2_format.rj;
168 opcode = inst.reg2_format.opcode;
169 addr = vcpu->arch.gprs[rj];
170 run->iocsr_io.phys_addr = addr;
171 run->iocsr_io.is_write = 0;
172 val = &vcpu->arch.gprs[rd];
173
174 /* LoongArch is Little endian */
175 switch (opcode) {
176 case iocsrrdb_op:
177 run->iocsr_io.len = 1;
178 break;
179 case iocsrrdh_op:
180 run->iocsr_io.len = 2;
181 break;
182 case iocsrrdw_op:
183 run->iocsr_io.len = 4;
184 break;
185 case iocsrrdd_op:
186 run->iocsr_io.len = 8;
187 break;
188 case iocsrwrb_op:
189 run->iocsr_io.len = 1;
190 run->iocsr_io.is_write = 1;
191 break;
192 case iocsrwrh_op:
193 run->iocsr_io.len = 2;
194 run->iocsr_io.is_write = 1;
195 break;
196 case iocsrwrw_op:
197 run->iocsr_io.len = 4;
198 run->iocsr_io.is_write = 1;
199 break;
200 case iocsrwrd_op:
201 run->iocsr_io.len = 8;
202 run->iocsr_io.is_write = 1;
203 break;
204 default:
205 return EMULATE_FAIL;
206 }
207
208 if (run->iocsr_io.is_write) {
209 idx = srcu_read_lock(&vcpu->kvm->srcu);
210 ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, run->iocsr_io.len, val);
211 srcu_read_unlock(&vcpu->kvm->srcu, idx);
212 if (ret == 0)
213 ret = EMULATE_DONE;
214 else {
215 ret = EMULATE_DO_IOCSR;
216 /* Save data and let user space to write it */
217 memcpy(run->iocsr_io.data, val, run->iocsr_io.len);
218 }
219 trace_kvm_iocsr(KVM_TRACE_IOCSR_WRITE, run->iocsr_io.len, addr, val);
220 } else {
221 vcpu->arch.io_gpr = rd; /* Set register id for iocsr read completion */
222 idx = srcu_read_lock(&vcpu->kvm->srcu);
223 ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr,
224 run->iocsr_io.len, run->iocsr_io.data);
225 srcu_read_unlock(&vcpu->kvm->srcu, idx);
226 if (ret == 0) {
227 kvm_complete_iocsr_read(vcpu, run);
228 ret = EMULATE_DONE;
229 } else
230 ret = EMULATE_DO_IOCSR;
231 trace_kvm_iocsr(KVM_TRACE_IOCSR_READ, run->iocsr_io.len, addr, NULL);
232 }
233
234 return ret;
235 }
236
kvm_complete_iocsr_read(struct kvm_vcpu * vcpu,struct kvm_run * run)237 int kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
238 {
239 enum emulation_result er = EMULATE_DONE;
240 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
241
242 switch (run->iocsr_io.len) {
243 case 1:
244 *gpr = *(s8 *)run->iocsr_io.data;
245 break;
246 case 2:
247 *gpr = *(s16 *)run->iocsr_io.data;
248 break;
249 case 4:
250 *gpr = *(s32 *)run->iocsr_io.data;
251 break;
252 case 8:
253 *gpr = *(s64 *)run->iocsr_io.data;
254 break;
255 default:
256 kvm_err("Bad IOCSR length: %d, addr is 0x%lx\n",
257 run->iocsr_io.len, vcpu->arch.badv);
258 er = EMULATE_FAIL;
259 break;
260 }
261
262 return er;
263 }
264
kvm_emu_idle(struct kvm_vcpu * vcpu)265 int kvm_emu_idle(struct kvm_vcpu *vcpu)
266 {
267 ++vcpu->stat.idle_exits;
268 trace_kvm_exit_idle(vcpu, KVM_TRACE_EXIT_IDLE);
269
270 if (!kvm_arch_vcpu_runnable(vcpu))
271 kvm_vcpu_halt(vcpu);
272
273 return EMULATE_DONE;
274 }
275
kvm_trap_handle_gspr(struct kvm_vcpu * vcpu)276 static int kvm_trap_handle_gspr(struct kvm_vcpu *vcpu)
277 {
278 unsigned long curr_pc;
279 larch_inst inst;
280 enum emulation_result er = EMULATE_DONE;
281 struct kvm_run *run = vcpu->run;
282
283 /* Fetch the instruction */
284 inst.word = vcpu->arch.badi;
285 curr_pc = vcpu->arch.pc;
286 update_pc(&vcpu->arch);
287
288 trace_kvm_exit_gspr(vcpu, inst.word);
289 er = EMULATE_FAIL;
290 switch (((inst.word >> 24) & 0xff)) {
291 case 0x0: /* CPUCFG GSPR */
292 trace_kvm_exit_cpucfg(vcpu, KVM_TRACE_EXIT_CPUCFG);
293 er = kvm_emu_cpucfg(vcpu, inst);
294 break;
295 case 0x4: /* CSR{RD,WR,XCHG} GSPR */
296 trace_kvm_exit_csr(vcpu, KVM_TRACE_EXIT_CSR);
297 er = kvm_handle_csr(vcpu, inst);
298 break;
299 case 0x6: /* Cache, Idle and IOCSR GSPR */
300 switch (((inst.word >> 22) & 0x3ff)) {
301 case 0x18: /* Cache GSPR */
302 er = EMULATE_DONE;
303 trace_kvm_exit_cache(vcpu, KVM_TRACE_EXIT_CACHE);
304 break;
305 case 0x19: /* Idle/IOCSR GSPR */
306 switch (((inst.word >> 15) & 0x1ffff)) {
307 case 0xc90: /* IOCSR GSPR */
308 er = kvm_emu_iocsr(inst, run, vcpu);
309 break;
310 case 0xc91: /* Idle GSPR */
311 er = kvm_emu_idle(vcpu);
312 break;
313 default:
314 er = EMULATE_FAIL;
315 break;
316 }
317 break;
318 default:
319 er = EMULATE_FAIL;
320 break;
321 }
322 break;
323 default:
324 er = EMULATE_FAIL;
325 break;
326 }
327
328 /* Rollback PC only if emulation was unsuccessful */
329 if (er == EMULATE_FAIL) {
330 kvm_err("[%#lx]%s: unsupported gspr instruction 0x%08x\n",
331 curr_pc, __func__, inst.word);
332
333 kvm_arch_vcpu_dump_regs(vcpu);
334 vcpu->arch.pc = curr_pc;
335 }
336
337 return er;
338 }
339
340 /*
341 * Trigger GSPR:
342 * 1) Execute CPUCFG instruction;
343 * 2) Execute CACOP/IDLE instructions;
344 * 3) Access to unimplemented CSRs/IOCSRs.
345 */
kvm_handle_gspr(struct kvm_vcpu * vcpu,int ecode)346 static int kvm_handle_gspr(struct kvm_vcpu *vcpu, int ecode)
347 {
348 int ret = RESUME_GUEST;
349 enum emulation_result er = EMULATE_DONE;
350
351 er = kvm_trap_handle_gspr(vcpu);
352
353 if (er == EMULATE_DONE) {
354 ret = RESUME_GUEST;
355 } else if (er == EMULATE_DO_MMIO) {
356 vcpu->run->exit_reason = KVM_EXIT_MMIO;
357 ret = RESUME_HOST;
358 } else if (er == EMULATE_DO_IOCSR) {
359 vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR;
360 ret = RESUME_HOST;
361 } else {
362 kvm_queue_exception(vcpu, EXCCODE_INE, 0);
363 ret = RESUME_GUEST;
364 }
365
366 return ret;
367 }
368
kvm_emu_mmio_read(struct kvm_vcpu * vcpu,larch_inst inst)369 int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst)
370 {
371 int idx, ret;
372 unsigned int op8, opcode, rd;
373 struct kvm_run *run = vcpu->run;
374
375 run->mmio.phys_addr = vcpu->arch.badv;
376 vcpu->mmio_needed = 2; /* signed */
377 op8 = (inst.word >> 24) & 0xff;
378 ret = EMULATE_DO_MMIO;
379
380 switch (op8) {
381 case 0x24 ... 0x27: /* ldptr.w/d process */
382 rd = inst.reg2i14_format.rd;
383 opcode = inst.reg2i14_format.opcode;
384
385 switch (opcode) {
386 case ldptrw_op:
387 run->mmio.len = 4;
388 break;
389 case ldptrd_op:
390 run->mmio.len = 8;
391 break;
392 default:
393 break;
394 }
395 break;
396 case 0x28 ... 0x2e: /* ld.b/h/w/d, ld.bu/hu/wu process */
397 rd = inst.reg2i12_format.rd;
398 opcode = inst.reg2i12_format.opcode;
399
400 switch (opcode) {
401 case ldb_op:
402 run->mmio.len = 1;
403 break;
404 case ldbu_op:
405 vcpu->mmio_needed = 1; /* unsigned */
406 run->mmio.len = 1;
407 break;
408 case ldh_op:
409 run->mmio.len = 2;
410 break;
411 case ldhu_op:
412 vcpu->mmio_needed = 1; /* unsigned */
413 run->mmio.len = 2;
414 break;
415 case ldw_op:
416 run->mmio.len = 4;
417 break;
418 case ldwu_op:
419 vcpu->mmio_needed = 1; /* unsigned */
420 run->mmio.len = 4;
421 break;
422 case ldd_op:
423 run->mmio.len = 8;
424 break;
425 default:
426 ret = EMULATE_FAIL;
427 break;
428 }
429 break;
430 case 0x38: /* ldx.b/h/w/d, ldx.bu/hu/wu process */
431 rd = inst.reg3_format.rd;
432 opcode = inst.reg3_format.opcode;
433
434 switch (opcode) {
435 case ldxb_op:
436 run->mmio.len = 1;
437 break;
438 case ldxbu_op:
439 run->mmio.len = 1;
440 vcpu->mmio_needed = 1; /* unsigned */
441 break;
442 case ldxh_op:
443 run->mmio.len = 2;
444 break;
445 case ldxhu_op:
446 run->mmio.len = 2;
447 vcpu->mmio_needed = 1; /* unsigned */
448 break;
449 case ldxw_op:
450 run->mmio.len = 4;
451 break;
452 case ldxwu_op:
453 run->mmio.len = 4;
454 vcpu->mmio_needed = 1; /* unsigned */
455 break;
456 case ldxd_op:
457 run->mmio.len = 8;
458 break;
459 default:
460 ret = EMULATE_FAIL;
461 break;
462 }
463 break;
464 default:
465 ret = EMULATE_FAIL;
466 }
467
468 if (ret == EMULATE_DO_MMIO) {
469 trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, run->mmio.phys_addr, NULL);
470
471 vcpu->arch.io_gpr = rd; /* Set for kvm_complete_mmio_read() use */
472
473 /*
474 * If mmio device such as PCH-PIC is emulated in KVM,
475 * it need not return to user space to handle the mmio
476 * exception.
477 */
478 idx = srcu_read_lock(&vcpu->kvm->srcu);
479 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, vcpu->arch.badv,
480 run->mmio.len, run->mmio.data);
481 srcu_read_unlock(&vcpu->kvm->srcu, idx);
482 if (!ret) {
483 kvm_complete_mmio_read(vcpu, run);
484 update_pc(&vcpu->arch);
485 vcpu->mmio_needed = 0;
486 return EMULATE_DONE;
487 }
488
489 run->mmio.is_write = 0;
490 vcpu->mmio_is_write = 0;
491 return EMULATE_DO_MMIO;
492 }
493
494 kvm_err("Read not supported Inst=0x%08x @%lx BadVaddr:%#lx\n",
495 inst.word, vcpu->arch.pc, vcpu->arch.badv);
496 kvm_arch_vcpu_dump_regs(vcpu);
497 vcpu->mmio_needed = 0;
498
499 return ret;
500 }
501
kvm_complete_mmio_read(struct kvm_vcpu * vcpu,struct kvm_run * run)502 int kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
503 {
504 enum emulation_result er = EMULATE_DONE;
505 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
506
507 /* Update with new PC */
508 update_pc(&vcpu->arch);
509 switch (run->mmio.len) {
510 case 1:
511 if (vcpu->mmio_needed == 2)
512 *gpr = *(s8 *)run->mmio.data;
513 else
514 *gpr = *(u8 *)run->mmio.data;
515 break;
516 case 2:
517 if (vcpu->mmio_needed == 2)
518 *gpr = *(s16 *)run->mmio.data;
519 else
520 *gpr = *(u16 *)run->mmio.data;
521 break;
522 case 4:
523 if (vcpu->mmio_needed == 2)
524 *gpr = *(s32 *)run->mmio.data;
525 else
526 *gpr = *(u32 *)run->mmio.data;
527 break;
528 case 8:
529 *gpr = *(s64 *)run->mmio.data;
530 break;
531 default:
532 kvm_err("Bad MMIO length: %d, addr is 0x%lx\n",
533 run->mmio.len, vcpu->arch.badv);
534 er = EMULATE_FAIL;
535 break;
536 }
537
538 trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len,
539 run->mmio.phys_addr, run->mmio.data);
540
541 return er;
542 }
543
kvm_emu_mmio_write(struct kvm_vcpu * vcpu,larch_inst inst)544 int kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst)
545 {
546 int idx, ret;
547 unsigned int rd, op8, opcode;
548 unsigned long curr_pc, rd_val = 0;
549 struct kvm_run *run = vcpu->run;
550 void *data = run->mmio.data;
551
552 /*
553 * Update PC and hold onto current PC in case there is
554 * an error and we want to rollback the PC
555 */
556 curr_pc = vcpu->arch.pc;
557 update_pc(&vcpu->arch);
558
559 op8 = (inst.word >> 24) & 0xff;
560 run->mmio.phys_addr = vcpu->arch.badv;
561 ret = EMULATE_DO_MMIO;
562 switch (op8) {
563 case 0x24 ... 0x27: /* stptr.w/d process */
564 rd = inst.reg2i14_format.rd;
565 opcode = inst.reg2i14_format.opcode;
566
567 switch (opcode) {
568 case stptrw_op:
569 run->mmio.len = 4;
570 *(unsigned int *)data = vcpu->arch.gprs[rd];
571 break;
572 case stptrd_op:
573 run->mmio.len = 8;
574 *(unsigned long *)data = vcpu->arch.gprs[rd];
575 break;
576 default:
577 ret = EMULATE_FAIL;
578 break;
579 }
580 break;
581 case 0x28 ... 0x2e: /* st.b/h/w/d process */
582 rd = inst.reg2i12_format.rd;
583 opcode = inst.reg2i12_format.opcode;
584 rd_val = vcpu->arch.gprs[rd];
585
586 switch (opcode) {
587 case stb_op:
588 run->mmio.len = 1;
589 *(unsigned char *)data = rd_val;
590 break;
591 case sth_op:
592 run->mmio.len = 2;
593 *(unsigned short *)data = rd_val;
594 break;
595 case stw_op:
596 run->mmio.len = 4;
597 *(unsigned int *)data = rd_val;
598 break;
599 case std_op:
600 run->mmio.len = 8;
601 *(unsigned long *)data = rd_val;
602 break;
603 default:
604 ret = EMULATE_FAIL;
605 break;
606 }
607 break;
608 case 0x38: /* stx.b/h/w/d process */
609 rd = inst.reg3_format.rd;
610 opcode = inst.reg3_format.opcode;
611
612 switch (opcode) {
613 case stxb_op:
614 run->mmio.len = 1;
615 *(unsigned char *)data = vcpu->arch.gprs[rd];
616 break;
617 case stxh_op:
618 run->mmio.len = 2;
619 *(unsigned short *)data = vcpu->arch.gprs[rd];
620 break;
621 case stxw_op:
622 run->mmio.len = 4;
623 *(unsigned int *)data = vcpu->arch.gprs[rd];
624 break;
625 case stxd_op:
626 run->mmio.len = 8;
627 *(unsigned long *)data = vcpu->arch.gprs[rd];
628 break;
629 default:
630 ret = EMULATE_FAIL;
631 break;
632 }
633 break;
634 default:
635 ret = EMULATE_FAIL;
636 }
637
638 if (ret == EMULATE_DO_MMIO) {
639 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, run->mmio.phys_addr, data);
640
641 /*
642 * If mmio device such as PCH-PIC is emulated in KVM,
643 * it need not return to user space to handle the mmio
644 * exception.
645 */
646 idx = srcu_read_lock(&vcpu->kvm->srcu);
647 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, run->mmio.len, data);
648 srcu_read_unlock(&vcpu->kvm->srcu, idx);
649 if (!ret)
650 return EMULATE_DONE;
651
652 run->mmio.is_write = 1;
653 vcpu->mmio_needed = 1;
654 vcpu->mmio_is_write = 1;
655 return EMULATE_DO_MMIO;
656 }
657
658 vcpu->arch.pc = curr_pc;
659 kvm_err("Write not supported Inst=0x%08x @%lx BadVaddr:%#lx\n",
660 inst.word, vcpu->arch.pc, vcpu->arch.badv);
661 kvm_arch_vcpu_dump_regs(vcpu);
662 /* Rollback PC if emulation was unsuccessful */
663
664 return ret;
665 }
666
kvm_handle_rdwr_fault(struct kvm_vcpu * vcpu,bool write,int ecode)667 static int kvm_handle_rdwr_fault(struct kvm_vcpu *vcpu, bool write, int ecode)
668 {
669 int ret;
670 larch_inst inst;
671 enum emulation_result er = EMULATE_DONE;
672 struct kvm_run *run = vcpu->run;
673 unsigned long badv = vcpu->arch.badv;
674
675 /* Inject ADE exception if exceed max GPA size */
676 if (unlikely(badv >= vcpu->kvm->arch.gpa_size)) {
677 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM);
678 return RESUME_GUEST;
679 }
680
681 ret = kvm_handle_mm_fault(vcpu, badv, write, ecode);
682 if (ret) {
683 /* Treat as MMIO */
684 inst.word = vcpu->arch.badi;
685 if (write) {
686 er = kvm_emu_mmio_write(vcpu, inst);
687 } else {
688 /* A code fetch fault doesn't count as an MMIO */
689 if (kvm_is_ifetch_fault(&vcpu->arch)) {
690 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEF);
691 return RESUME_GUEST;
692 }
693
694 er = kvm_emu_mmio_read(vcpu, inst);
695 }
696 }
697
698 if (er == EMULATE_DONE) {
699 ret = RESUME_GUEST;
700 } else if (er == EMULATE_DO_MMIO) {
701 run->exit_reason = KVM_EXIT_MMIO;
702 ret = RESUME_HOST;
703 } else {
704 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM);
705 ret = RESUME_GUEST;
706 }
707
708 return ret;
709 }
710
kvm_handle_read_fault(struct kvm_vcpu * vcpu,int ecode)711 static int kvm_handle_read_fault(struct kvm_vcpu *vcpu, int ecode)
712 {
713 return kvm_handle_rdwr_fault(vcpu, false, ecode);
714 }
715
kvm_handle_write_fault(struct kvm_vcpu * vcpu,int ecode)716 static int kvm_handle_write_fault(struct kvm_vcpu *vcpu, int ecode)
717 {
718 return kvm_handle_rdwr_fault(vcpu, true, ecode);
719 }
720
kvm_complete_user_service(struct kvm_vcpu * vcpu,struct kvm_run * run)721 int kvm_complete_user_service(struct kvm_vcpu *vcpu, struct kvm_run *run)
722 {
723 update_pc(&vcpu->arch);
724 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, run->hypercall.ret);
725
726 return 0;
727 }
728
729 /**
730 * kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host
731 * @vcpu: Virtual CPU context.
732 * @ecode: Exception code.
733 *
734 * Handle when the guest attempts to use fpu which hasn't been allowed
735 * by the root context.
736 */
kvm_handle_fpu_disabled(struct kvm_vcpu * vcpu,int ecode)737 static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu, int ecode)
738 {
739 struct kvm_run *run = vcpu->run;
740
741 if (!kvm_guest_has_fpu(&vcpu->arch)) {
742 kvm_queue_exception(vcpu, EXCCODE_INE, 0);
743 return RESUME_GUEST;
744 }
745
746 /*
747 * If guest FPU not present, the FPU operation should have been
748 * treated as a reserved instruction!
749 * If FPU already in use, we shouldn't get this at all.
750 */
751 if (WARN_ON(vcpu->arch.aux_inuse & KVM_LARCH_FPU)) {
752 kvm_err("%s internal error\n", __func__);
753 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
754 return RESUME_HOST;
755 }
756
757 vcpu->arch.aux_ldtype = KVM_LARCH_FPU;
758 kvm_make_request(KVM_REQ_AUX_LOAD, vcpu);
759
760 return RESUME_GUEST;
761 }
762
kvm_save_notify(struct kvm_vcpu * vcpu)763 static long kvm_save_notify(struct kvm_vcpu *vcpu)
764 {
765 unsigned long id, data;
766
767 id = kvm_read_reg(vcpu, LOONGARCH_GPR_A1);
768 data = kvm_read_reg(vcpu, LOONGARCH_GPR_A2);
769 switch (id) {
770 case BIT(KVM_FEATURE_STEAL_TIME):
771 if (data & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID))
772 return KVM_HCALL_INVALID_PARAMETER;
773
774 vcpu->arch.st.guest_addr = data;
775 if (!(data & KVM_STEAL_PHYS_VALID))
776 return 0;
777
778 vcpu->arch.st.last_steal = current->sched_info.run_delay;
779 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
780 return 0;
781 default:
782 return KVM_HCALL_INVALID_CODE;
783 }
784 }
785
786 /*
787 * kvm_handle_lsx_disabled() - Guest used LSX while disabled in root.
788 * @vcpu: Virtual CPU context.
789 * @ecode: Exception code.
790 *
791 * Handle when the guest attempts to use LSX when it is disabled in the root
792 * context.
793 */
kvm_handle_lsx_disabled(struct kvm_vcpu * vcpu,int ecode)794 static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu, int ecode)
795 {
796 if (!kvm_guest_has_lsx(&vcpu->arch))
797 kvm_queue_exception(vcpu, EXCCODE_INE, 0);
798 else {
799 vcpu->arch.aux_ldtype = KVM_LARCH_LSX;
800 kvm_make_request(KVM_REQ_AUX_LOAD, vcpu);
801 }
802
803 return RESUME_GUEST;
804 }
805
806 /*
807 * kvm_handle_lasx_disabled() - Guest used LASX while disabled in root.
808 * @vcpu: Virtual CPU context.
809 * @ecode: Exception code.
810 *
811 * Handle when the guest attempts to use LASX when it is disabled in the root
812 * context.
813 */
kvm_handle_lasx_disabled(struct kvm_vcpu * vcpu,int ecode)814 static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu, int ecode)
815 {
816 if (!kvm_guest_has_lasx(&vcpu->arch))
817 kvm_queue_exception(vcpu, EXCCODE_INE, 0);
818 else {
819 vcpu->arch.aux_ldtype = KVM_LARCH_LASX;
820 kvm_make_request(KVM_REQ_AUX_LOAD, vcpu);
821 }
822
823 return RESUME_GUEST;
824 }
825
kvm_handle_lbt_disabled(struct kvm_vcpu * vcpu,int ecode)826 static int kvm_handle_lbt_disabled(struct kvm_vcpu *vcpu, int ecode)
827 {
828 if (!kvm_guest_has_lbt(&vcpu->arch))
829 kvm_queue_exception(vcpu, EXCCODE_INE, 0);
830 else {
831 vcpu->arch.aux_ldtype = KVM_LARCH_LBT;
832 kvm_make_request(KVM_REQ_AUX_LOAD, vcpu);
833 }
834
835 return RESUME_GUEST;
836 }
837
kvm_send_pv_ipi(struct kvm_vcpu * vcpu)838 static void kvm_send_pv_ipi(struct kvm_vcpu *vcpu)
839 {
840 unsigned int min, cpu;
841 struct kvm_vcpu *dest;
842 DECLARE_BITMAP(ipi_bitmap, BITS_PER_LONG * 2) = {
843 kvm_read_reg(vcpu, LOONGARCH_GPR_A1),
844 kvm_read_reg(vcpu, LOONGARCH_GPR_A2)
845 };
846
847 min = kvm_read_reg(vcpu, LOONGARCH_GPR_A3);
848 for_each_set_bit(cpu, ipi_bitmap, BITS_PER_LONG * 2) {
849 dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min);
850 if (!dest)
851 continue;
852
853 /* Send SWI0 to dest vcpu to emulate IPI interrupt */
854 kvm_queue_irq(dest, INT_SWI0);
855 kvm_vcpu_kick(dest);
856 }
857 }
858
859 /*
860 * Hypercall emulation always return to guest, Caller should check retval.
861 */
kvm_handle_service(struct kvm_vcpu * vcpu)862 static void kvm_handle_service(struct kvm_vcpu *vcpu)
863 {
864 long ret = KVM_HCALL_INVALID_CODE;
865 unsigned long func = kvm_read_reg(vcpu, LOONGARCH_GPR_A0);
866
867 switch (func) {
868 case KVM_HCALL_FUNC_IPI:
869 if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_IPI)) {
870 kvm_send_pv_ipi(vcpu);
871 ret = KVM_HCALL_SUCCESS;
872 }
873 break;
874 case KVM_HCALL_FUNC_NOTIFY:
875 if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME))
876 ret = kvm_save_notify(vcpu);
877 break;
878 default:
879 break;
880 }
881
882 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, ret);
883 }
884
kvm_handle_hypercall(struct kvm_vcpu * vcpu,int ecode)885 static int kvm_handle_hypercall(struct kvm_vcpu *vcpu, int ecode)
886 {
887 int ret;
888 larch_inst inst;
889 unsigned int code;
890
891 inst.word = vcpu->arch.badi;
892 code = inst.reg0i15_format.immediate;
893 ret = RESUME_GUEST;
894
895 switch (code) {
896 case KVM_HCALL_SERVICE:
897 vcpu->stat.hypercall_exits++;
898 kvm_handle_service(vcpu);
899 break;
900 case KVM_HCALL_USER_SERVICE:
901 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_USER_HCALL)) {
902 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE);
903 break;
904 }
905
906 vcpu->stat.hypercall_exits++;
907 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;
908 vcpu->run->hypercall.nr = KVM_HCALL_USER_SERVICE;
909 vcpu->run->hypercall.args[0] = kvm_read_reg(vcpu, LOONGARCH_GPR_A0);
910 vcpu->run->hypercall.args[1] = kvm_read_reg(vcpu, LOONGARCH_GPR_A1);
911 vcpu->run->hypercall.args[2] = kvm_read_reg(vcpu, LOONGARCH_GPR_A2);
912 vcpu->run->hypercall.args[3] = kvm_read_reg(vcpu, LOONGARCH_GPR_A3);
913 vcpu->run->hypercall.args[4] = kvm_read_reg(vcpu, LOONGARCH_GPR_A4);
914 vcpu->run->hypercall.args[5] = kvm_read_reg(vcpu, LOONGARCH_GPR_A5);
915 vcpu->run->hypercall.flags = 0;
916 /*
917 * Set invalid return value by default, let user-mode VMM modify it.
918 */
919 vcpu->run->hypercall.ret = KVM_HCALL_INVALID_CODE;
920 ret = RESUME_HOST;
921 break;
922 case KVM_HCALL_SWDBG:
923 /* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */
924 if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) {
925 vcpu->run->exit_reason = KVM_EXIT_DEBUG;
926 ret = RESUME_HOST;
927 break;
928 }
929 fallthrough;
930 default:
931 /* Treat it as noop intruction, only set return value */
932 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE);
933 break;
934 }
935
936 if (ret == RESUME_GUEST)
937 update_pc(&vcpu->arch);
938
939 return ret;
940 }
941
942 /*
943 * LoongArch KVM callback handling for unimplemented guest exiting
944 */
kvm_fault_ni(struct kvm_vcpu * vcpu,int ecode)945 static int kvm_fault_ni(struct kvm_vcpu *vcpu, int ecode)
946 {
947 unsigned int inst;
948 unsigned long badv;
949
950 /* Fetch the instruction */
951 inst = vcpu->arch.badi;
952 badv = vcpu->arch.badv;
953 kvm_err("ECode: %d PC=%#lx Inst=0x%08x BadVaddr=%#lx ESTAT=%#lx\n",
954 ecode, vcpu->arch.pc, inst, badv, read_gcsr_estat());
955 kvm_arch_vcpu_dump_regs(vcpu);
956 kvm_queue_exception(vcpu, EXCCODE_INE, 0);
957
958 return RESUME_GUEST;
959 }
960
961 static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = {
962 [0 ... EXCCODE_INT_START - 1] = kvm_fault_ni,
963 [EXCCODE_TLBI] = kvm_handle_read_fault,
964 [EXCCODE_TLBL] = kvm_handle_read_fault,
965 [EXCCODE_TLBS] = kvm_handle_write_fault,
966 [EXCCODE_TLBM] = kvm_handle_write_fault,
967 [EXCCODE_FPDIS] = kvm_handle_fpu_disabled,
968 [EXCCODE_LSXDIS] = kvm_handle_lsx_disabled,
969 [EXCCODE_LASXDIS] = kvm_handle_lasx_disabled,
970 [EXCCODE_BTDIS] = kvm_handle_lbt_disabled,
971 [EXCCODE_GSPR] = kvm_handle_gspr,
972 [EXCCODE_HVC] = kvm_handle_hypercall,
973 };
974
kvm_handle_fault(struct kvm_vcpu * vcpu,int fault)975 int kvm_handle_fault(struct kvm_vcpu *vcpu, int fault)
976 {
977 return kvm_fault_tables[fault](vcpu, fault);
978 }
979